Open Stack

Last week (Friday) I was having issue with networking, after some back 
and forth with the list and a massive headache I decided to 'halt' the 
system and pick it back up this week.

Today I went ahead and dropped and re-created all OS DB's (nova, glance, 
keystore) and followed 
http://fedoraproject.org/wiki/Getting_started_with_OpenStack_EPEL to 
re-configure OS.


After a reboot I am now faced with the following stack trace in 
compute.log when booting an instance

2012-06-05 11:02:28 INFO nova.compute.manager 
[req-bac71a83-89aa-4e00-a14b-5dd904025529 None None] Updating bandwidth 
usage cache
2012-06-05 11:02:29 INFO nova.virt.libvirt.connection 
[req-bac71a83-89aa-4e00-a14b-5dd904025529 None None] Compute_service 
record updated for nuvola.potsdam.edu
2012-06-05 11:02:29 INFO nova.rpc.impl_qpid 
[req-bac71a83-89aa-4e00-a14b-5dd904025529 None None] *Connected to AMQP 
server on nuvola.potsdam.edu:5672
2012-06-05 11:02:54 ERROR nova.rpc.impl_qpid 
[req-7478886c-6379-44a0-a06f-bf55963c2a03 
6704a5a7bf6e41c8b607f786de7ff8e3 906d4ac19907429cb66f76ad94196d5c] Timed 
out waiting for RPC response: None*
2012-06-05 11:02:54 TRACE nova.rpc.impl_qpid Traceback (most recent call 
last):
2012-06-05 11:02:54 TRACE nova.rpc.impl_qpid   File 
"/usr/lib/python2.6/site-packages/nova/rpc/impl_qpid.py", line 359, in 
ensure
2012-06-05 11:02:54 TRACE nova.rpc.impl_qpid     return method(*args, 
**kwargs)
2012-06-05 11:02:54 TRACE nova.rpc.impl_qpid   File 
"/usr/lib/python2.6/site-packages/nova/rpc/impl_qpid.py", line 408, in 
_consume
2012-06-05 11:02:54 TRACE nova.rpc.impl_qpid     nxt_receiver = 
self.session.next_receiver(timeout=timeout)
2012-06-05 11:02:54 TRACE nova.rpc.impl_qpid   File "<string>", line 6, 
in next_receiver
2012-06-05 11:02:54 TRACE nova.rpc.impl_qpid   File 
"/usr/lib/python2.6/site-packages/qpid/messaging/endpoints.py", line 
651, in next_receiver
2012-06-05 11:02:54 TRACE nova.rpc.impl_qpid     raise Empty
2012-06-05 11:02:54 TRACE nova.rpc.impl_qpid Empty: None
2012-06-05 11:02:54 TRACE nova.rpc.impl_qpid
*2012-06-05 11:02:54 ERROR nova.rpc.amqp 
[req-7478886c-6379-44a0-a06f-bf55963c2a03 
6704a5a7bf6e41c8b607f786de7ff8e3 906d4ac19907429cb66f76ad94196d5c] 
Exception during message handling*
2012-06-05 11:02:54 TRACE nova.rpc.amqp Traceback (most recent call last):
2012-06-05 11:02:54 TRACE nova.rpc.amqp   File 
"/usr/lib/python2.6/site-packages/nova/rpc/amqp.py", line 253, in 
_process_data
2012-06-05 11:02:54 TRACE nova.rpc.amqp     rval = 
node_func(context=ctxt, **node_args)
2012-06-05 11:02:54 TRACE nova.rpc.amqp   File 
"/usr/lib/python2.6/site-packages/nova/exception.py", line 114, in wrapped
2012-06-05 11:02:54 TRACE nova.rpc.amqp     return f(*args, **kw)
2012-06-05 11:02:54 TRACE nova.rpc.amqp   File 
"/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 159, in 
decorated_function
2012-06-05 11:02:54 TRACE nova.rpc.amqp     function(self, context, 
instance_uuid, *args, **kwargs)
2012-06-05 11:02:54 TRACE nova.rpc.amqp   File 
"/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 183, in 
decorated_function
2012-06-05 11:02:54 TRACE nova.rpc.amqp     sys.exc_info())
2012-06-05 11:02:54 TRACE nova.rpc.amqp   File 
"/usr/lib64/python2.6/contextlib.py", line 23, in __exit__
2012-06-05 11:02:54 TRACE nova.rpc.amqp     self.gen.next()
2012-06-05 11:02:54 TRACE nova.rpc.amqp   File 
"/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 177, in 
decorated_function
2012-06-05 11:02:54 TRACE nova.rpc.amqp     return function(self, 
context, instance_uuid, *args, **kwargs)
2012-06-05 11:02:54 TRACE nova.rpc.amqp   File 
"/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 753, in 
terminate_instance
2012-06-05 11:02:54 TRACE nova.rpc.amqp     do_terminate_instance()
2012-06-05 11:02:54 TRACE nova.rpc.amqp   File 
"/usr/lib/python2.6/site-packages/nova/utils.py", line 946, in inner
2012-06-05 11:02:54 TRACE nova.rpc.amqp     retval = f(*args, **kwargs)
2012-06-05 11:02:54 TRACE nova.rpc.amqp   File 
"/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 744, in 
do_terminate_instance
2012-06-05 11:02:54 TRACE nova.rpc.amqp     
compute_utils.notify_usage_exists(instance, current_period=True)
2012-06-05 11:02:54 TRACE nova.rpc.amqp   File 
"/usr/lib/python2.6/site-packages/nova/compute/utils.py", line 57, in 
notify_usage_exists
2012-06-05 11:02:54 TRACE nova.rpc.amqp     instance_ref)
2012-06-05 11:02:54 TRACE nova.rpc.amqp   File 
"/usr/lib/python2.6/site-packages/nova/network/api.py", line 219, in 
get_instance_nw_info
2012-06-05 11:02:54 TRACE nova.rpc.amqp     'args': args})
2012-06-05 11:02:54 TRACE nova.rpc.amqp   File 
"/usr/lib/python2.6/site-packages/nova/rpc/__init__.py", line 68, in call
2012-06-05 11:02:54 TRACE nova.rpc.amqp     return 
_get_impl().call(context, topic, msg, timeout)
2012-06-05 11:02:54 TRACE nova.rpc.amqp   File 
"/usr/lib/python2.6/site-packages/nova/rpc/impl_qpid.py", line 521, in call
2012-06-05 11:02:54 TRACE nova.rpc.amqp     return 
rpc_amqp.call(context, topic, msg, timeout, Connection.pool)
2012-06-05 11:02:54 TRACE nova.rpc.amqp   File 
"/usr/lib/python2.6/site-packages/nova/rpc/amqp.py", line 343, in call
2012-06-05 11:02:54 TRACE nova.rpc.amqp     rv = list(rv)
2012-06-05 11:02:54 TRACE nova.rpc.amqp   File 
"/usr/lib/python2.6/site-packages/nova/rpc/amqp.py", line 304, in __iter__
2012-06-05 11:02:54 TRACE nova.rpc.amqp     self.done()
2012-06-05 11:02:54 TRACE nova.rpc.amqp   File 
"/usr/lib64/python2.6/contextlib.py", line 23, in __exit__
2012-06-05 11:02:54 TRACE nova.rpc.amqp     self.gen.next()
2012-06-05 11:02:54 TRACE nova.rpc.amqp   File 
"/usr/lib/python2.6/site-packages/nova/rpc/amqp.py", line 301, in __iter__
2012-06-05 11:02:54 TRACE nova.rpc.amqp     self._iterator.next()
2012-06-05 11:02:54 TRACE nova.rpc.amqp   File 
"/usr/lib/python2.6/site-packages/nova/rpc/impl_qpid.py", line 417, in 
iterconsume
2012-06-05 11:02:54 TRACE nova.rpc.amqp     yield 
self.ensure(_error_callback, _consume)
2012-06-05 11:02:54 TRACE nova.rpc.amqp   File 
"/usr/lib/python2.6/site-packages/nova/rpc/impl_qpid.py", line 363, in 
ensure
2012-06-05 11:02:54 TRACE nova.rpc.amqp     error_callback(e)
2012-06-05 11:02:54 TRACE nova.rpc.amqp   File 
"/usr/lib/python2.6/site-packages/nova/rpc/impl_qpid.py", line 402, in 
_error_callback
2012-06-05 11:02:54 TRACE nova.rpc.amqp     raise rpc_common.Timeout()
*2012-06-05 11:02:54 TRACE nova.rpc.amqp Timeout: Timeout while waiting 
on RPC response.*
2012-06-05 11:02:54 TRACE nova.rpc.amqp


Any ideas?  qpidd.conf has auth=no and is running on default port 5672.

grep qpid /etc/nova/nova.conf
rpc_backend = nova.rpc.impl_qpid
qpid_hostname=nuvola.potsdam.edu

rpm -qa | grep openstack
openstack-swift-object-1.4.8-2.el6.noarch
openstack-swift-doc-1.4.8-2.el6.noarch
openstack-nova-2012.1-7.el6.noarch
openstack-glance-2012.1-5.el6.noarch
openstack-dashboard-2012.1-4.el6.noarch
openstack-swift-container-1.4.8-2.el6.noarch
openstack-swift-proxy-1.4.8-2.el6.noarch
openstack-keystone-2012.1-3.el6.noarch
openstack-swift-1.4.8-2.el6.noarch
openstack-swift-account-1.4.8-2.el6.noarch
openstack-quantum-2012.1-4.el6.noarch
openstack-utils-2012.1-1.el6.noarch

rpm -qa | grep qpid
qpid-cpp-server-0.12-6.el6.x86_64
qpid-cpp-client-0.12-6.el6.x86_64
python-qpid-0.14-7.el6_2.noarch


-Ron
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.openstack.org/pipermail/openstack-operators/attachments/20120605/ca704f55/attachment-0002.html>