Open Stack

Hi,

It happens from time to time, not consistently,
with the latest kolla/centos-binary-neutron-server:ussuri container image.

>From the bottome of traceback, it seems self._ovn is None. Any clues how this can happen?
I don't see other suspicious logging in Neutron and OVN.
=====================
File "/usr/lib/python3.6/site-packages/neutron/services/ovn_l3/plugin.py", line 477, in get_router_availability_zones
ERROR neutron.api.v2.resource     lr = self._ovn.get_lrouter(router['id'])
ERROR neutron.api.v2.resource AttributeError: 'NoneType' object has no attribute 'get_lrouter'
=====================

Here is the full traceback.
=====================
neutron.api.v2.resource [req-bf2230aa-0491-4caa-b5f7-169d37009672 b8de8d37a30147fd98681c22fc515874 b5938e7ebb0645e396b19e99bb7f9868 - default default] index failed: No details.: AttributeError: 'NoneType' object has no attribute 'get_lrouter'
neutron.api.v2.resource Traceback (most recent call last):
neutron.api.v2.resource   File "/usr/lib/python3.6/site-packages/neutron/api/v2/resource.py", line 98, in resource
neutron.api.v2.resource     result = method(request=request, **args)
neutron.api.v2.resource   File "/usr/lib/python3.6/site-packages/neutron_lib/db/api.py", line 139, in wrapped
neutron.api.v2.resource     setattr(e, '_RETRY_EXCEEDED', True)
neutron.api.v2.resource   File "/usr/lib/python3.6/site-packages/oslo_utils/excutils.py", line 220, in __exit__
neutron.api.v2.resource     self.force_reraise()
neutron.api.v2.resource   File "/usr/lib/python3.6/site-packages/oslo_utils/excutils.py", line 196, in force_reraise
neutron.api.v2.resource     six.reraise(self.type_, self.value, self.tb)
neutron.api.v2.resource   File "/usr/lib/python3.6/site-packages/six.py", line 703, in reraise
neutron.api.v2.resource     raise value
neutron.api.v2.resource   File "/usr/lib/python3.6/site-packages/neutron_lib/db/api.py", line 135, in wrapped
neutron.api.v2.resource     return f(*args, **kwargs)
neutron.api.v2.resource   File "/usr/lib/python3.6/site-packages/oslo_db/api.py", line 154, in wrapper
neutron.api.v2.resource     ectxt.value = e.inner_exc
neutron.api.v2.resource   File "/usr/lib/python3.6/site-packages/oslo_utils/excutils.py", line 220, in __exit__
neutron.api.v2.resource     self.force_reraise()
neutron.api.v2.resource   File "/usr/lib/python3.6/site-packages/oslo_utils/excutils.py", line 196, in force_reraise
neutron.api.v2.resource     six.reraise(self.type_, self.value, self.tb)
neutron.api.v2.resource   File "/usr/lib/python3.6/site-packages/six.py", line 703, in reraise
neutron.api.v2.resource     raise value
neutron.api.v2.resource   File "/usr/lib/python3.6/site-packages/oslo_db/api.py", line 142, in wrapper
neutron.api.v2.resource     return f(*args, **kwargs)
neutron.api.v2.resource   File "/usr/lib/python3.6/site-packages/neutron_lib/db/api.py", line 183, in wrapped
neutron.api.v2.resource     LOG.debug("Retry wrapper got retriable exception: %s", e)
neutron.api.v2.resource   File "/usr/lib/python3.6/site-packages/oslo_utils/excutils.py", line 220, in __exit__
neutron.api.v2.resource     self.force_reraise()
neutron.api.v2.resource   File "/usr/lib/python3.6/site-packages/oslo_utils/excutils.py", line 196, in force_reraise
neutron.api.v2.resource     six.reraise(self.type_, self.value, self.tb)
neutron.api.v2.resource   File "/usr/lib/python3.6/site-packages/six.py", line 703, in reraise
neutron.api.v2.resource     raise value
neutron.api.v2.resource   File "/usr/lib/python3.6/site-packages/neutron_lib/db/api.py", line 179, in wrapped
neutron.api.v2.resource     return f(*dup_args, **dup_kwargs)
neutron.api.v2.resource   File "/usr/lib/python3.6/site-packages/neutron/api/v2/base.py", line 369, in index
neutron.api.v2.resource     return self._items(request, True, parent_id)
neutron.api.v2.resource   File "/usr/lib/python3.6/site-packages/neutron/api/v2/base.py", line 304, in _items
neutron.api.v2.resource     obj_list = obj_getter(request.context, **kwargs)
neutron.api.v2.resource   File "/usr/lib/python3.6/site-packages/neutron_lib/db/api.py", line 233, in wrapped
neutron.api.v2.resource     return method(*args, **kwargs)
neutron.api.v2.resource   File "/usr/lib/python3.6/site-packages/neutron_lib/db/api.py", line 139, in wrapped
neutron.api.v2.resource     setattr(e, '_RETRY_EXCEEDED', True)
neutron.api.v2.resource   File "/usr/lib/python3.6/site-packages/oslo_utils/excutils.py", line 220, in __exit__
neutron.api.v2.resource     self.force_reraise()
neutron.api.v2.resource   File "/usr/lib/python3.6/site-packages/oslo_utils/excutils.py", line 196, in force_reraise
neutron.api.v2.resource     six.reraise(self.type_, self.value, self.tb)
neutron.api.v2.resource   File "/usr/lib/python3.6/site-packages/six.py", line 703, in reraise
neutron.api.v2.resource     raise value
neutron.api.v2.resource   File "/usr/lib/python3.6/site-packages/neutron_lib/db/api.py", line 135, in wrapped
neutron.api.v2.resource     return f(*args, **kwargs)
neutron.api.v2.resource   File "/usr/lib/python3.6/site-packages/oslo_db/api.py", line 154, in wrapper
neutron.api.v2.resource     ectxt.value = e.inner_exc
neutron.api.v2.resource   File "/usr/lib/python3.6/site-packages/oslo_utils/excutils.py", line 220, in __exit__
neutron.api.v2.resource     self.force_reraise()
neutron.api.v2.resource   File "/usr/lib/python3.6/site-packages/oslo_utils/excutils.py", line 196, in force_reraise
neutron.api.v2.resource     six.reraise(self.type_, self.value, self.tb)
neutron.api.v2.resource   File "/usr/lib/python3.6/site-packages/six.py", line 703, in reraise
neutron.api.v2.resource     raise value
neutron.api.v2.resource   File "/usr/lib/python3.6/site-packages/oslo_db/api.py", line 142, in wrapper
neutron.api.v2.resource     return f(*args, **kwargs)
neutron.api.v2.resource   File "/usr/lib/python3.6/site-packages/neutron_lib/db/api.py", line 183, in wrapped
neutron.api.v2.resource     LOG.debug("Retry wrapper got retriable exception: %s", e)
neutron.api.v2.resource   File "/usr/lib/python3.6/site-packages/oslo_utils/excutils.py", line 220, in __exit__
neutron.api.v2.resource     self.force_reraise()
neutron.api.v2.resource   File "/usr/lib/python3.6/site-packages/oslo_utils/excutils.py", line 196, in force_reraise
neutron.api.v2.resource     six.reraise(self.type_, self.value, self.tb)
neutron.api.v2.resource   File "/usr/lib/python3.6/site-packages/six.py", line 703, in reraise
neutron.api.v2.resource     raise value
neutron.api.v2.resource   File "/usr/lib/python3.6/site-packages/neutron_lib/db/api.py", line 179, in wrapped
neutron.api.v2.resource     return f(*dup_args, **dup_kwargs)
neutron.api.v2.resource   File "/usr/lib/python3.6/site-packages/neutron/db/l3_db.py", line 547, in get_routers
neutron.api.v2.resource     page_reverse=page_reverse)
neutron.api.v2.resource   File "/usr/lib/python3.6/site-packages/neutron_lib/db/model_query.py", line 317, in get_collection
neutron.api.v2.resource     for c in query
neutron.api.v2.resource   File "/usr/lib/python3.6/site-packages/neutron_lib/db/model_query.py", line 317, in <listcomp>
neutron.api.v2.resource     for c in query
neutron.api.v2.resource   File "/usr/lib/python3.6/site-packages/neutron/db/l3_db.py", line 221, in _make_router_dict
neutron.api.v2.resource     resource_extend.apply_funcs(l3_apidef.ROUTERS, res, router)
neutron.api.v2.resource   File "/usr/lib/python3.6/site-packages/neutron_lib/db/resource_extend.py", line 84, in apply_funcs
neutron.api.v2.resource     resolved_func(response, db_object)
neutron.api.v2.resource   File "/usr/lib/python3.6/site-packages/neutron/db/availability_zone/router.py", line 40, in _add_az_to_response
neutron.api.v2.resource     l3_plugin.get_router_availability_zones(router_db))
neutron.api.v2.resource   File "/usr/lib/python3.6/site-packages/neutron/services/ovn_l3/plugin.py", line 477, in get_router_availability_zones
neutron.api.v2.resource     lr = self._ovn.get_lrouter(router['id'])
neutron.api.v2.resource AttributeError: 'NoneType' object has no attribute 'get_lrouter'
neutron.api.v2.resource 
neutron.wsgi [req-bf2230aa-0491-4caa-b5f7-169d37009672 b8de8d37a30147fd98681c22fc515874 b5938e7ebb0645e396b19e99bb7f9868 - default default] 10.91.1.7,10.250.20.26 "GET /v2.0/routers HTTP/1.1" status: 500  len: 344 time: 0.2660162
=====================

Thanks!
Tony