Open Stack

Folks,

I am running wallaby and Today i had rabbitMQ issue and i rebuild
RabbitMQ. but now my all compute node throwing following error. not
sure what is going on up they are not trying to come up and keep
crashing with following error


2022-02-10 00:52:27.843 3660531 ERROR oslo_service.service
[req-6c3cdaa3-ecf9-4bef-9def-6b6a69dfd30b - - - - -] Error starting
thread.: oslo_messaging.exceptions.MessagingTimeout: Timed out waiting
for a reply to message ID 9510bd6c0d6a48e782ed7bc2887f32c8

   2022-02-10 00:52:27.843 3660531 ERROR oslo_service.service
Traceback (most recent call last):

   2022-02-10 00:52:27.843 3660531 ERROR oslo_service.service   File
"/openstack/venvs/nova-23.1.1/lib/python3.8/site-packages/oslo_messaging/_drivers/amqpdriver.py",
line 433, in get

   2022-02-10 00:52:27.843 3660531 ERROR oslo_service.service
return self._queues[msg_id].get(block=True, timeout=timeout)

   2022-02-10 00:52:27.843 3660531 ERROR oslo_service.service   File
"/openstack/venvs/nova-23.1.1/lib/python3.8/site-packages/eventlet/queue.py",
line 322, in get

   2022-02-10 00:52:27.843 3660531 ERROR oslo_service.service
return waiter.wait()

   2022-02-10 00:52:27.843 3660531 ERROR oslo_service.service   File
"/openstack/venvs/nova-23.1.1/lib/python3.8/site-packages/eventlet/queue.py",
line 141, in wait

   2022-02-10 00:52:27.843 3660531 ERROR oslo_service.service
return get_hub().switch()

   2022-02-10 00:52:27.843 3660531 ERROR oslo_service.service   File
"/openstack/venvs/nova-23.1.1/lib/python3.8/site-packages/eventlet/hubs/hub.py",
line 313, in switch

   2022-02-10 00:52:27.843 3660531 ERROR oslo_service.service
return self.greenlet.switch()

   2022-02-10 00:52:27.843 3660531 ERROR oslo_service.service
_queue.Empty

   2022-02-10 00:52:27.843 3660531 ERROR oslo_service.service

   2022-02-10 00:52:27.843 3660531 ERROR oslo_service.service During
handling of the above exception, another exception occurred:

   2022-02-10 00:52:27.843 3660531 ERROR oslo_service.service

   2022-02-10 00:52:27.843 3660531 ERROR oslo_service.service
Traceback (most recent call last):

   2022-02-10 00:52:27.843 3660531 ERROR oslo_service.service   File
"/openstack/venvs/nova-23.1.1/lib/python3.8/site-packages/oslo_service/service.py",
line 807, in run_service

   2022-02-10 00:52:27.843 3660531 ERROR oslo_service.service
service.start()

   2022-02-10 00:52:27.843 3660531 ERROR oslo_service.service   File
"/openstack/venvs/nova-23.1.1/lib/python3.8/site-packages/nova/service.py",
line 159, in start

   2022-02-10 00:52:27.843 3660531 ERROR oslo_service.service
self.manager.init_host()

   2022-02-10 00:52:27.843 3660531 ERROR oslo_service.service   File
"/openstack/venvs/nova-23.1.1/lib/python3.8/site-packages/nova/compute/manager.py",
line 1413, in init_host

   2022-02-10 00:52:27.843 3660531 ERROR oslo_service.service
self.driver.init_host(host=self.host)

   2022-02-10 00:52:27.843 3660531 ERROR oslo_service.service   File
"/openstack/venvs/nova-23.1.1/lib/python3.8/site-packages/nova/virt/libvirt/driver.py",
line 792, in init_host

   2022-02-10 00:52:27.843 3660531 ERROR oslo_service.service
self._register_instance_machine_type()

   2022-02-10 00:52:27.843 3660531 ERROR oslo_service.service   File
"/openstack/venvs/nova-23.1.1/lib/python3.8/site-packages/nova/virt/libvirt/driver.py",
line 811, in _register_instance_machine_type

   2022-02-10 00:52:27.843 3660531 ERROR oslo_service.service     for
instance in objects.InstanceList.get_by_host(context, hostname):

   2022-02-10 00:52:27.843 3660531 ERROR oslo_service.service   File
"/openstack/venvs/nova-23.1.1/lib/python3.8/site-packages/oslo_versionedobjects/base.py",
line 175, in wrapper

   2022-02-10 00:52:27.843 3660531 ERROR oslo_service.service
result = cls.indirection_api.object_class_action_versions(

   2022-02-10 00:52:27.843 3660531 ERROR oslo_service.service   File
"/openstack/venvs/nova-23.1.1/lib/python3.8/site-packages/nova/conductor/rpcapi.py",
line 240, in object_class_action_versions

   2022-02-10 00:52:27.843 3660531 ERROR oslo_service.service
return cctxt.call(context, 'object_class_action_versions',

   2022-02-10 00:52:27.843 3660531 ERROR oslo_service.service   File
"/openstack/venvs/nova-23.1.1/lib/python3.8/site-packages/oslo_messaging/rpc/client.py",
line 175, in call

   2022-02-10 00:52:27.843 3660531 ERROR oslo_service.service
self.transport._send(self.target, msg_ctxt, msg,

   2022-02-10 00:52:27.843 3660531 ERROR oslo_service.service   File
"/openstack/venvs/nova-23.1.1/lib/python3.8/site-packages/oslo_messaging/transport.py",
line 123, in _send

   2022-02-10 00:52:27.843 3660531 ERROR oslo_service.service
return self._driver.send(target, ctxt, message,

   2022-02-10 00:52:27.843 3660531 ERROR oslo_service.service   File
"/openstack/venvs/nova-23.1.1/lib/python3.8/site-packages/oslo_messaging/_drivers/amqpdriver.py",
line 680, in send

   2022-02-10 00:52:27.843 3660531 ERROR oslo_service.service
return self._send(target, ctxt, message, wait_for_reply, timeout,

   2022-02-10 00:52:27.843 3660531 ERROR oslo_service.service   File
"/openstack/venvs/nova-23.1.1/lib/python3.8/site-packages/oslo_messaging/_drivers/amqpdriver.py",
line 669, in _send

   2022-02-10 00:52:27.843 3660531 ERROR oslo_service.service
result = self._waiter.wait(msg_id, timeout,

   2022-02-10 00:52:27.843 3660531 ERROR oslo_service.service   File
"/openstack/venvs/nova-23.1.1/lib/python3.8/site-packages/oslo_messaging/_drivers/amqpdriver.py",
line 559, in wait

   2022-02-10 00:52:27.843 3660531 ERROR oslo_service.service
message = self.waiters.get(msg_id, timeout=timeout)

   2022-02-10 00:52:27.843 3660531 ERROR oslo_service.service   File
"/openstack/venvs/nova-23.1.1/lib/python3.8/site-packages/oslo_messaging/_drivers/amqpdriver.py",
line 435, in get

   2022-02-10 00:52:27.843 3660531 ERROR oslo_service.service
raise oslo_messaging.MessagingTimeout(

   2022-02-10 00:52:27.843 3660531 ERROR oslo_service.service
oslo_messaging.exceptions.MessagingTimeout: Timed out waiting for a
reply to message ID 9510bd6c0d6a48e782ed7bc2887f32c8

   2022-02-10 00:52:27.843 3660531 ERROR oslo_service.service