Open Stack

I just got my cluster up.  I have multiple compute nodes in HA with
VLAN networking.  I have a "controller" backend machine.  All servers
are connected to the same switch.  I had a similar setup using crowbar
so I know VLAN'ing is working on the switch.

When I try to run "nova floating-ip-pool-list" from any node (the
controller node without nova-network or a compute node with
nova-network) I get: ERROR: The server has either erred or is
incapable of performing the requested operation. (HTTP 500)

However, when I run "nova-manage floating list" I see all the IP's in
the floating pool.  Here is the stacktrace I have seen in the
nova-api.log (I also see the dashboard error out as expected)

2012-07-24 11:05:37 ERROR nova.rpc.common
[req-cbc137cd-99e5-46b9-968d-4d8ab7b6bf59
67e970e4b28b4346a077b25d4ab281f6 c61a1d9367254fcea905df473cc5af18]
Timed out waiting for RPC response: timed out
2012-07-24 11:05:37 TRACE nova.rpc.common Traceback (most recent call last):
2012-07-24 11:05:37 TRACE nova.rpc.common   File
"/usr/lib/python2.7/dist-packages/nova/rpc/impl_kombu.py", line 490,
in ensure
2012-07-24 11:05:37 TRACE nova.rpc.common     return method(*args, **kwargs)
2012-07-24 11:05:37 TRACE nova.rpc.common   File
"/usr/lib/python2.7/dist-packages/nova/rpc/impl_kombu.py", line 567,
in _consume
2012-07-24 11:05:37 TRACE nova.rpc.common     return
self.connection.drain_events(timeout=timeout)
2012-07-24 11:05:37 TRACE nova.rpc.common   File
"/usr/lib/python2.7/dist-packages/kombu/connection.py", line 175, in
drain_events
2012-07-24 11:05:37 TRACE nova.rpc.common     return
self.transport.drain_events(self.connection, **kwargs)
2012-07-24 11:05:37 TRACE nova.rpc.common   File
"/usr/lib/python2.7/dist-packages/kombu/transport/pyamqplib.py", line
238, in drain_events
2012-07-24 11:05:37 TRACE nova.rpc.common     return
connection.drain_events(**kwargs)
2012-07-24 11:05:37 TRACE nova.rpc.common   File
"/usr/lib/python2.7/dist-packages/kombu/transport/pyamqplib.py", line
57, in drain_events
2012-07-24 11:05:37 TRACE nova.rpc.common     return
self.wait_multi(self.channels.values(), timeout=timeout)
2012-07-24 11:05:37 TRACE nova.rpc.common   File
"/usr/lib/python2.7/dist-packages/kombu/transport/pyamqplib.py", line
63, in wait_multi
2012-07-24 11:05:37 TRACE nova.rpc.common     chanmap.keys(),
allowed_methods, timeout=timeout)
2012-07-24 11:05:37 TRACE nova.rpc.common   File
"/usr/lib/python2.7/dist-packages/kombu/transport/pyamqplib.py", line
120, in _wait_multiple
2012-07-24 11:05:37 TRACE nova.rpc.common     channel, method_sig,
args, content = read_timeout(timeout)
2012-07-24 11:05:37 TRACE nova.rpc.common   File
"/usr/lib/python2.7/dist-packages/kombu/transport/pyamqplib.py", line
94, in read_timeout
2012-07-24 11:05:37 TRACE nova.rpc.common     return
self.method_reader.read_method()
2012-07-24 11:05:37 TRACE nova.rpc.common   File
"/usr/lib/python2.7/dist-packages/amqplib/client_0_8/method_framing.py",
line 221, in read_method
2012-07-24 11:05:37 TRACE nova.rpc.common     raise m
2012-07-24 11:05:37 TRACE nova.rpc.common timeout: timed out
2012-07-24 11:05:37 TRACE nova.rpc.common
2012-07-24 11:05:37 ERROR nova.api.openstack
[req-cbc137cd-99e5-46b9-968d-4d8ab7b6bf59
67e970e4b28b4346a077b25d4ab281f6 c61a1d9367254fcea905df473cc5af18]
Caught error: Timeout while waiting on RPC response.
2012-07-24 11:05:37 TRACE nova.api.openstack Traceback (most recent call last):
2012-07-24 11:05:37 TRACE nova.api.openstack   File
"/usr/lib/python2.7/dist-packages/nova/api/openstack/__init__.py",
line 41, in __call__
2012-07-24 11:05:37 TRACE nova.api.openstack     return
req.get_response(self.application)
2012-07-24 11:05:37 TRACE nova.api.openstack   File
"/usr/lib/python2.7/dist-packages/webob/request.py", line 1086, in
get_response
2012-07-24 11:05:37 TRACE nova.api.openstack     application,
catch_exc_info=False)
2012-07-24 11:05:37 TRACE nova.api.openstack   File
"/usr/lib/python2.7/dist-packages/webob/request.py", line 1055, in
call_application
2012-07-24 11:05:37 TRACE nova.api.openstack     app_iter =
application(self.environ, start_response)
2012-07-24 11:05:37 TRACE nova.api.openstack   File
"/usr/lib/python2.7/dist-packages/keystone/middleware/auth_token.py",
line 176, in __call__
2012-07-24 11:05:37 TRACE nova.api.openstack     return self.app(env,
start_response)
2012-07-24 11:05:37 TRACE nova.api.openstack   File
"/usr/lib/python2.7/dist-packages/webob/dec.py", line 159, in __call__
2012-07-24 11:05:37 TRACE nova.api.openstack     return resp(environ,
start_response)
2012-07-24 11:05:37 TRACE nova.api.openstack   File
"/usr/lib/python2.7/dist-packages/webob/dec.py", line 159, in __call__
2012-07-24 11:05:37 TRACE nova.api.openstack     return resp(environ,
start_response)
2012-07-24 11:05:37 TRACE nova.api.openstack   File
"/usr/lib/python2.7/dist-packages/webob/dec.py", line 159, in __call__
2012-07-24 11:05:37 TRACE nova.api.openstack     return resp(environ,
start_response)
2012-07-24 11:05:37 TRACE nova.api.openstack   File
"/usr/lib/python2.7/dist-packages/routes/middleware.py", line 131, in
__call__
2012-07-24 11:05:37 TRACE nova.api.openstack     response =
self.app(environ, start_response)
2012-07-24 11:05:37 TRACE nova.api.openstack   File
"/usr/lib/python2.7/dist-packages/webob/dec.py", line 159, in __call__
2012-07-24 11:05:37 TRACE nova.api.openstack     return resp(environ,
start_response)
2012-07-24 11:05:37 TRACE nova.api.openstack   File
"/usr/lib/python2.7/dist-packages/webob/dec.py", line 147, in __call__
2012-07-24 11:05:37 TRACE nova.api.openstack     resp =
self.call_func(req, *args, **self.kwargs)
2012-07-24 11:05:37 TRACE nova.api.openstack   File
"/usr/lib/python2.7/dist-packages/webob/dec.py", line 210, in
call_func
2012-07-24 11:05:37 TRACE nova.api.openstack     return self.func(req,
*args, **kwargs)
2012-07-24 11:05:37 TRACE nova.api.openstack   File
"/usr/lib/python2.7/dist-packages/nova/api/openstack/wsgi.py", line
803, in __call__
2012-07-24 11:05:37 TRACE nova.api.openstack     content_type, body, accept)
2012-07-24 11:05:37 TRACE nova.api.openstack   File
"/usr/lib/python2.7/dist-packages/nova/api/openstack/wsgi.py", line
851, in _process_stack
2012-07-24 11:05:37 TRACE nova.api.openstack     action_result =
self.dispatch(meth, request, action_args)
2012-07-24 11:05:37 TRACE nova.api.openstack   File
"/usr/lib/python2.7/dist-packages/nova/api/openstack/wsgi.py", line
926, in dispatch
2012-07-24 11:05:37 TRACE nova.api.openstack     return
method(req=request, **action_args)
2012-07-24 11:05:37 TRACE nova.api.openstack   File
"/usr/lib/python2.7/dist-packages/nova/api/openstack/compute/contrib/floating_ip_pools.py",
line 74, in index
2012-07-24 11:05:37 TRACE nova.api.openstack     pools =
self.network_api.get_floating_ip_pools(context)
2012-07-24 11:05:37 TRACE nova.api.openstack   File
"/usr/lib/python2.7/dist-packages/nova/network/api.py", line 81, in
get_floating_ip_pools
2012-07-24 11:05:37 TRACE nova.api.openstack     {'method':
'get_floating_pools'})
2012-07-24 11:05:37 TRACE nova.api.openstack   File
"/usr/lib/python2.7/dist-packages/nova/rpc/__init__.py", line 68, in
call
2012-07-24 11:05:37 TRACE nova.api.openstack     return
_get_impl().call(context, topic, msg, timeout)
2012-07-24 11:05:37 TRACE nova.api.openstack   File
"/usr/lib/python2.7/dist-packages/nova/rpc/impl_kombu.py", line 674,
in call
2012-07-24 11:05:37 TRACE nova.api.openstack     return
rpc_amqp.call(context, topic, msg, timeout, Connection.pool)
2012-07-24 11:05:37 TRACE nova.api.openstack   File
"/usr/lib/python2.7/dist-packages/nova/rpc/amqp.py", line 343, i
n call
2012-07-24 11:05:37 TRACE nova.api.openstack     rv = list(rv)
2012-07-24 11:05:37 TRACE nova.api.openstack   File
"/usr/lib/python2.7/dist-packages/nova/rpc/amqp.py", line 304, in
__iter__
2012-07-24 11:05:37 TRACE nova.api.openstack     self.done()
2012-07-24 11:05:37 TRACE nova.api.openstack   File
"/usr/lib/python2.7/contextlib.py", line 24, in __exit__
2012-07-24 11:05:37 TRACE nova.api.openstack     self.gen.next()
2012-07-24 11:05:37 TRACE nova.api.openstack   File
"/usr/lib/python2.7/dist-packages/nova/rpc/amqp.py", line 301, in
__iter__
2012-07-24 11:05:37 TRACE nova.api.openstack     self._iterator.next()
2012-07-24 11:05:37 TRACE nova.api.openstack   File
"/usr/lib/python2.7/dist-packages/nova/rpc/impl_kombu.py", line 572,
in iterconsume
2012-07-24 11:05:37 TRACE nova.api.openstack     yield
self.ensure(_error_callback, _consume)
2012-07-24 11:05:37 TRACE nova.api.openstack   File
"/usr/lib/python2.7/dist-packages/nova/rpc/impl_kombu.py", line 503,
in ensure
2012-07-24 11:05:37 TRACE nova.api.openstack     error_callback(e)
2012-07-24 11:05:37 TRACE nova.api.openstack   File
"/usr/lib/python2.7/dist-packages/nova/rpc/impl_kombu.py", line 553,
in _error_callback
2012-07-24 11:05:37 TRACE nova.api.openstack     raise rpc_common.Timeout()
2012-07-24 11:05:37 TRACE nova.api.openstack Timeout: Timeout while
waiting on RPC response.
2012-07-24 11:05:37 TRACE nova.api.openstack
2012-07-24 11:05:37 INFO nova.api.openstack
[req-cbc137cd-99e5-46b9-968d-4d8ab7b6bf59
67e970e4b28b4346a077b25d4ab281f6 c61a1d9367254fcea905df473cc5af18]
http://192.168.124.15:8774/v2/c61a1d9367254fcea905df473cc5af18/os-floating-ip-pools
returned with HTTP 500


--
Thomas Vachon
Principal Operations Architect
session M