Open Stack

So I emailed yesterday about a strange problem where I was seeing that I 
could create cinder volumes just fine, but when I try to attach them to 
an instance, they get stuck in the "attaching" state and kind of hang 
there forever, it never really attaches.

Again, Icehouse RC1 on RedHat 6.5.

The cinder logs (all types) show nothing.  The only evidence of 
strangeness I see in any logs are in the nova compute log on the server 
that the instance I'm trying to attach to is running on.  I see this:

2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher   File 
"/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 266, in 
decorated_function
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher 
return function(self, context, *args, **kwargs)
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher   File 
"/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 309, in 
decorated_function
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher     e, 
sys.exc_info())
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher   File 
"/usr/lib/python2.6/site-packages/nova/openstack/common/excutils.py", 
line 68, in __exit__
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher 
six.reraise(self.type_, self.value, self.tb)
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher   File 
"/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 296, in 
decorated_function
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher 
return function(self, context, *args, **kwargs)
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher   File 
"/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 4130, 
in attach_volume
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher 
bdm.destroy(context)
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher   File 
"/usr/lib/python2.6/site-packages/nova/openstack/common/excutils.py", 
line 68, in __exit__
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher 
six.reraise(self.type_, self.value, self.tb)
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher   File 
"/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 4127, 
in attach_volume
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher 
return self._attach_volume(context, instance, driver_bdm)
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher   File 
"/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 4148, 
in _attach_volume
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher 
self.volume_api.unreserve_volume(context, bdm.volume_id)
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher   File 
"/usr/lib/python2.6/site-packages/nova/volume/cinder.py", line 173, in 
wrapper
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher     res 
= method(self, ctx, volume_id, *args, **kwargs)
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher   File 
"/usr/lib/python2.6/site-packages/nova/volume/cinder.py", line 249, in 
unreserve_volume
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher 
cinderclient(context).volumes.unreserve(volume_id)
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher   File 
"/usr/lib/python2.6/site-packages/cinderclient/v1/volumes.py", line 293, 
in unreserve
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher 
return self._action('os-unreserve', volume)
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher   File 
"/usr/lib/python2.6/site-packages/cinderclient/v1/volumes.py", line 250, 
in _action
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher 
return self.api.client.post(url, body=body)
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher   File 
"/usr/lib/python2.6/site-packages/cinderclient/client.py", line 210, in post
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher 
return self._cs_request(url, 'POST', **kwargs)
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher   File 
"/usr/lib/python2.6/site-packages/cinderclient/client.py", line 199, in 
_cs_request
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher 
raise exceptions.ConnectionError(msg)
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher 
ConnectionError: Unable to establish connection: [Errno 101] ENETUNREACH
2014-04-09 22:08:29.995 6547 TRACE oslo.messaging.rpc.dispatcher
2014-04-09 22:08:29.998 6547 ERROR oslo.messaging._drivers.common [-] 
Returning exception Unable to establish connection: [Errno 101] 
ENETUNREACH to caller
2014-04-09 22:08:29.998 6547 ERROR oslo.messaging._drivers.common [-] 
['Traceback (most recent call last):\n', '  File 
"/usr/lib/python2.6/site-packages/oslo/messaging/rpc/dispatcher.py", 
line 133, in _dispatch_and_reply\n    incoming.message))\n', '  File 
"/usr/lib/python2.6/site-packages/oslo/messaging/rpc/dispatcher.py", 
line 176, in _dispatch\n    return self._do_dispatch(endpoint, method, 
ctxt, args)\n', '  File 
"/usr/lib/python2.6/site-packages/oslo/messaging/rpc/dispatcher.py", 
line 122, in _do_dispatch\n    result = getattr(endpoint, method)(ctxt, 
**new_args)\n', '  File 
"/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 399, in 
decorated_function\n    return function(self, context, *args, 
**kwargs)\n', '  File 
"/usr/lib/python2.6/site-packages/nova/exception.py", line 88, in 
wrapped\n    payload)\n', '  File 
"/usr/lib/python2.6/site-packages/nova/openstack/common/excutils.py", 
line 68, in __exit__\n    six.reraise(self.type_, self.value, 
self.tb)\n', '  File 
"/usr/lib/python2.6/site-packages/nova/exception.py", line 71, in 
wrapped\n    return f(self, context, *args, **kw)\n', '  File 
"/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 280, in 
decorated_function\n    pass\n', '  File 
"/usr/lib/python2.6/site-packages/nova/openstack/common/excutils.py", 
line 68, in __exit__\n    six.reraise(self.type_, self.value, 
self.tb)\n', '  File 
"/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 266, in 
decorated_function\n    return function(self, context, *args, 
**kwargs)\n', '  File 
"/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 309, in 
decorated_function\n    e, sys.exc_info())\n', '  File 
"/usr/lib/python2.6/site-packages/nova/openstack/common/excutils.py", 
line 68, in __exit__\n    six.reraise(self.type_, self.value, 
self.tb)\n', '  File 
"/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 296, in 
decorated_function\n    return function(self, context, *args, 
**kwargs)\n', '  File 
"/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 4130, 
in attach_volume\n    bdm.destroy(context)\n', '  File 
"/usr/lib/python2.6/site-packages/nova/openstack/common/excutils.py", 
line 68, in __exit__\n    six.reraise(self.type_, self.value, 
self.tb)\n', '  File 
"/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 4127, 
in attach_volume\n    return self._attach_volume(context, instance, 
driver_bdm)\n', '  File 
"/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 4148, 
in _attach_volume\n    self.volume_api.unreserve_volume(context, 
bdm.volume_id)\n', '  File 
"/usr/lib/python2.6/site-packages/nova/volume/cinder.py", line 173, in 
wrapper\n    res = method(self, ctx, volume_id, *args, **kwargs)\n', ' 
File "/usr/lib/python2.6/site-packages/nova/volume/cinder.py", line 249, 
in unreserve_volume\n 
cinderclient(context).volumes.unreserve(volume_id)\n', '  File 
"/usr/lib/python2.6/site-packages/cinderclient/v1/volumes.py", line 293, 
in unreserve\n    return self._action(\'os-unreserve\', volume)\n', ' 
File "/usr/lib/python2.6/site-packages/cinderclient/v1/volumes.py", line 
250, in _action\n    return self.api.client.post(url, body=body)\n', ' 
File "/usr/lib/python2.6/site-packages/cinderclient/client.py", line 
210, in post\n    return self._cs_request(url, \'POST\', **kwargs)\n', ' 
  File "/usr/lib/python2.6/site-packages/cinderclient/client.py", line 
199, in _cs_request\n    raise exceptions.ConnectionError(msg)\n', 
'ConnectionError: Unable to establish connection: [Errno 101] 
ENETUNREACH\n']

So clearly nova is having trouble attaching the volume, and I see 
several "Unable to establish connection" messages.  Does that mean it 
can't connect to the iSCSI port on the cinder server?  No firewall 
there.  Or something else it can't connect to?

Thanks again,
erich