Open Stack

Hi, team,

After restarting my controller and compute1, I couldn't create block
storages any more, always returning an "error" Status immediately when
"cinder list". 

In **cinder-api log**: the requests have been logged, also there is a
**'status': u'error'**;

when I'm checking the **cinder-volume** log, no errors can be found, just
output this:

2014-07-09 01:58:47.356 2740 INFO cinder.volume.manager [-] Updating volume
status

Any suggestions? 
Thanks a lot!


FYI: what I've done so far:

###################################################

root at controller:~# nova-manage shell python

Python 2.7.6 (default, Mar 22 2014, 22:59:56) [GCC 4.8.2] on linux2 Type
"help", "copyright", "credits" or "license" for more information.
(InteractiveConsole)

>>> from nova import version

>>> version.version_string()

'2014.1'

###################################################

root at controller:~# cinder create --display-name 2 2 

+---------------------+--------------------------------------+
|       Property      |                Value                 |
+---------------------+--------------------------------------+
|     attachments     |                  []                  |
|  availability_zone  |                 nova                 |
|       bootable      |                false                 |
|      created_at     |      2014-07-09T08:58:15.871117      |
| display_description |                 None                 |
|     display_name    |                  2                   |
|      encrypted      |                False                 |
|          id         | d89633aa-3444-4e7e-9169-0ac65f4c22a9 |
|       metadata      |                  {}                  |
|         size        |                  2                   |
|     snapshot_id     |                 None                 |
|     source_volid    |                 None                 |
|        status       |               creating               |
|     volume_type     |                 None                 |
+---------------------+--------------------------------------+

###################################################

root at controller:~# cinder list

+--------------------------------------+-----------+--------------+------+--
-----------+----------+--------------------------------------+
|                  ID                  |   Status  | Display Name | Size |
Volume Type | Bootable |             Attached to              |
+--------------------------------------+-----------+--------------+------+--
-----------+----------+--------------------------------------+
| 28f30786-2e12-4eef-958f-bdb7b2135b11 | available |      40      |  40  |
None    |  false   |                                      |
| 2e975633-9259-4dd9-8997-ec1fed12b1b5 |   in-use  |   myVolume   |  1   |
None    |  false   | db8fa420-3a7f-4e3b-b129-3636ef13f87b |
| 35df2c9c-cfc5-49c4-bee8-feb3fccd51ff |   error   |      1       |  1   |
None    |  false   |                                      |
| b2d523b8-e373-4016-9cb0-3105f09c48ea |   in-use  |      50      |  60  |
None    |  false   | 5df1f11c-b7e3-4e20-9048-a9f61ec2671c |
| d89633aa-3444-4e7e-9169-0ac65f4c22a9 |   error   |      2       |  2   |
None    |  false   |                                      |
| e1e12a41-477a-48be-a594-757cc0470c97 | available |    150GB     | 150  |
None    |  false   |                                      |
+--------------------------------------+-----------+--------------+------+--
-----------+----------+--------------------------------------+

###################################################

root at controller:~# grep d89633aa-3444-4e7e-9169-0ac65f4c22a9
/var/log/cinder/*.log

*/var/log/cinder/cinder-api.log*:2014-07-09 01:58:15.995 1361 AUDIT
cinder.api.v1.volumes [req-9eab9bd1-102e-4796-ba7d-1268bd8ff30d
6dec579fdaa246d99a412ccf8285560e 2b3724a8b471467ab077726709cbe6ad - - -]
vol={'migration_status': None, 'availability_zone': 'nova', 'terminated_at':
None, 'reservations': ['9aae859c-318a-4413-8fec-9ffd6cb4dff5',
'93bb3e76-9318-403b-870e-ad97150d176b'], 'updated_at': None,
'provider_geometry': None, 'snapshot_id': None, 'ec2_id': None,
'mountpoint': None, 'deleted_at': None, 'id':
'd89633aa-3444-4e7e-9169-0ac65f4c22a9', 'size': 2, 'user_id':
u'6dec579fdaa246d99a412ccf8285560e', 'attach_time': None, 'attached_host':
None, 'display_description': None, 'encryption_key_id': None, 'project_id':
u'2b3724a8b471467ab077726709cbe6ad', 'launched_at': None, 'scheduled_at':
None, 'status': 'creating', 'volume_type_id': None, 'deleted': False,
'provider_location': None, 'host': None, 'source_volid': None,
'provider_auth': None, 'display_name': u'2', 'instance_uuid': None,
'bootable': False, 'created_at': datetime.datetime(2014, 7, 9, 8, 58, 15,
871117), 'attach_status': 'detached', 'volume_type': None, '_name_id': None,
'volume_metadata': [], 'metadata': {}}

*/var/log/cinder/cinder-api.log:*2014-07-09 01:58:22.532 1361 AUDIT
cinder.api.v1.volumes [req-d1299ff2-6894-4f10-a5c0-5f4e4df9915a
6dec579fdaa246d99a412ccf8285560e 2b3724a8b471467ab077726709cbe6ad - - -]
vol={'migration_status': None, 'availability_zone': u'nova',
'terminated_at': None, 'updated_at': datetime.datetime(2014, 7, 9, 8, 58,
15), 'provider_geometry': None, 'snapshot_id': None, 'ec2_id': None,
'mountpoint': None, 'deleted_at': None, 'id':
u'd89633aa-3444-4e7e-9169-0ac65f4c22a9', 'size': 2L, 'user_id':
u'6dec579fdaa246d99a412ccf8285560e', 'attach_time': None, 'attached_host':
None, 'display_description': None, 'encryption_key_id': None, 'project_id':
u'2b3724a8b471467ab077726709cbe6ad', 'launched_at': None, 'scheduled_at':
None, 'status': u'error', 'volume_type_id': None, 'deleted': False,
'provider_location': None, 'host': None, 'source_volid': None,
'provider_auth': None, 'display_name': u'2', 'instance_uuid': None,
'bootable': False, 'created_at': datetime.datetime(2014, 7, 9, 8, 58, 15),
'attach_status': u'detached', 'volume_type': None, '_name_id': None,
'volume_metadata': []}

###################################################

root at compute1:~# grep d89633aa-3444-4e7e-9169-0ac65f4c22a9
/var/log/cinder/*.log

root at compute1:~# grep d89633aa-3444-4e7e-9169-0ac65f4c22a9
/var/log/cinder/*.log

root at compute1:~# tail /var/log/cinder/cinder-volume.log

2014-07-09 01:51:47.304 2740 INFO cinder.volume.manager [-] Updating volume
status
2014-07-09 01:52:47.364 2740 INFO cinder.volume.manager [-] Updating volume
status
2014-07-09 01:53:47.317 2740 INFO cinder.volume.manager [-] Updating volume
status
2014-07-09 01:54:47.327 2740 INFO cinder.volume.manager [-] Updating volume
status
2014-07-09 01:55:47.337 2740 INFO cinder.volume.manager [-] Updating volume
status
2014-07-09 01:56:47.345 2740 INFO cinder.volume.manager [-] Updating volume
status
2014-07-09 01:57:47.355 2740 INFO cinder.volume.manager [-] Updating volume
status
2014-07-09 01:58:47.356 2740 INFO cinder.volume.manager [-] Updating volume
status
2014-07-09 01:59:47.364 2740 INFO cinder.volume.manager [-] Updating volume
status
2014-07-09 02:00:47.374 2740 INFO cinder.volume.manager [-] Updating volume
status
2014-07-09 02:01:47.381 2740 INFO cinder.volume.manager [-] Updating volume
status
2014-07-09 02:02:47.392 2740 INFO cinder.volume.manager [-] Updating volume
status
2014-07-09 02:03:47.399 2740 INFO cinder.volume.manager [-] Updating volume
status
2014-07-09 02:04:47.409 2740 INFO cinder.volume.manager [-] Updating volume
status
2014-07-09 02:05:47.418 2740 INFO cinder.volume.manager [-] Updating volume
status
2014-07-09 02:06:47.426 2740 INFO cinder.volume.manager [-] Updating volume
status
2014-07-09 02:07:47.437 2740 INFO cinder.volume.manager [-] Updating volume
status

###################################################

root at controller:~# service cinder-scheduler status 

cinder-scheduler start/running, process 1111

root at controller:~# service cinder-api status 

cinder-api start/running, process 1093

root at compute1:~# service cinder-volume status 

cinder-volume start/running, process 1734

root at compute1:~# service tgt status

tgt start/running, process 1745