[cinder] Failed to delete volume

7 Jun 2022

      Hi,

I would like to ask if someone saw issue when volume failed to delete, log
below :
This is happening from time to time.

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server
[req-80b56352-81b5-47b6-841e-94fface5974e
fce19e20993647db8ae39d67094a4294 c8d8a385912c4731832bf1f2cf4727fb - -
-] Exception during message handling: EOFError

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server Traceback
(most recent call last):

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File
"/usr/lib/python3/dist-packages/eventlet/greenio/base.py", line 359,
in _recv_loop

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server
self._read_trampoline()

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File
"/usr/lib/python3/dist-packages/eventlet/greenio/base.py", line 331,
in _read_trampoline

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server
timeout_exc=socket_timeout('timed out'))

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File
"/usr/lib/python3/dist-packages/eventlet/greenio/base.py", line 210,
in _trampoline

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server
mark_as_closed=self._mark_as_closed)

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File
"/usr/lib/python3/dist-packages/eventlet/hubs/__init__.py", line 159,
in trampoline

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server     return
hub.switch()

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File
"/usr/lib/python3/dist-packages/eventlet/hubs/hub.py", line 298, in
switch

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server     return
self.greenlet.switch()

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server
eventlet.hubs.IOClosed: [Errno 107] Operation on closed file

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server During
handling of the above exception, another exception occurred:

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server Traceback
(most recent call last):

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File
"/usr/lib/python3/dist-packages/oslo_messaging/rpc/server.py", line
165, in _process_incoming

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server     res =
self.dispatcher.dispatch(message)

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File
"/usr/lib/python3/dist-packages/oslo_messaging/rpc/dispatcher.py",
line 309, in dispatch

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server     return
self._do_dispatch(endpoint, method, ctxt, args)

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File
"/usr/lib/python3/dist-packages/oslo_messaging/rpc/dispatcher.py",
line 229, in _do_dispatch

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server     result
= func(ctxt, **new_args)

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File
"/usr/lib/python3/dist-packages/osprofiler/profiler.py", line 160, in
wrapper

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server     result
= f(*args, **kwargs)

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File
"/usr/lib/python3/dist-packages/osprofiler/profiler.py", line 160, in
wrapper

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server     result
= f(*args, **kwargs)

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File
"<decorator-gen-754>", line 2, in delete_volume

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File
"/usr/lib/python3/dist-packages/cinder/coordination.py", line 144, in
_synchronized

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server     with
lock(blocking):

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File
"/usr/lib/python3/dist-packages/tooz/locking.py", line 29, in
__enter__

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server     return
self.lock.__enter__(*self.args, **self.kwargs)

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File
"/usr/lib/python3/dist-packages/tooz/locking.py", line 49, in
__enter__

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server
acquired = self.acquire(*args, **kwargs)

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File
"/usr/lib/python3/dist-packages/tooz/drivers/redis.py", line 58, in
wrapper

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server     return
func(*args, **kwargs)

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File
"/usr/lib/python3/dist-packages/tooz/drivers/redis.py", line 110, in
acquire

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server
blocking=blocking, blocking_timeout=timeout)

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File
"/usr/lib/python3/dist-packages/redis/lock.py", line 182, in acquire

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server     if
self.do_acquire(token):

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File
"/usr/lib/python3/dist-packages/redis/lock.py", line 197, in
do_acquire

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server     if
self.redis.set(self.name, token, nx=True, px=timeout):

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File
"/usr/lib/python3/dist-packages/redis/client.py", line 1519, in set

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server     return
self.execute_command('SET', *pieces)

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File
"/usr/lib/python3/dist-packages/redis/client.py", line 839, in
execute_command

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server     return
self.parse_response(conn, command_name, **options)

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File
"/usr/lib/python3/dist-packages/redis/client.py", line 853, in
parse_response

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server
response = connection.read_response()

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File
"/usr/lib/python3/dist-packages/redis/sentinel.py", line 55, in
read_response

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server     return
super(SentinelManagedConnection, self).read_response()

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File
"/usr/lib/python3/dist-packages/redis/connection.py", line 700, in
read_response

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server
response = self._parser.read_response()

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File
"/usr/lib/python3/dist-packages/redis/connection.py", line 310, in
read_response

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server
response = self._buffer.readline()

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File
"/usr/lib/python3/dist-packages/redis/connection.py", line 242, in
readline

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server
self._read_from_socket()

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File
"/usr/lib/python3/dist-packages/redis/connection.py", line 184, in
_read_from_socket

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server     data =
recv(self._sock, socket_read_size)

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File
"/usr/lib/python3/dist-packages/redis/_compat.py", line 71, in recv

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server     return
sock.recv(*args, **kwargs)

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File
"/usr/lib/python3/dist-packages/eventlet/greenio/base.py", line 365,
in recv

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server     return
self._recv_loop(self.fd.recv, b'', bufsize, flags)

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File
"/usr/lib/python3/dist-packages/eventlet/greenio/base.py", line 362,
in _recv_loop

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server     raise EOFError()

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server EOFError

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server

Thanks,

Michal Arbet
Openstack Engineer

Ultimum Technologies a.s.
Na Poříčí 1047/26, 11000 Praha 1
Czech Republic

+420 604 228 897
michal.arbet@ultimum.io
*https://ultimum.io <https://ultimum.io/>*

LinkedIn <https://www.linkedin.com/company/ultimum-technologies> | Twitter
<https://twitter.com/ultimumtech> | Facebook
<https://www.facebook.com/ultimumtechnologies/timeline>

Michal Arbet

tags

participants (1)