Hi,

I would like to ask if someone saw issue when volume failed to delete, log below : 
This is happening from time to time.

2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server [req-80b56352-81b5-47b6-841e-94fface5974e fce19e20993647db8ae39d67094a4294 c8d8a385912c4731832bf1f2cf4727fb - - -] Exception during message handling: EOFError
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server Traceback (most recent call last):
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File "/usr/lib/python3/dist-packages/eventlet/greenio/base.py", line 359, in _recv_loop
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server     self._read_trampoline()
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File "/usr/lib/python3/dist-packages/eventlet/greenio/base.py", line 331, in _read_trampoline
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server     timeout_exc=socket_timeout('timed out'))
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File "/usr/lib/python3/dist-packages/eventlet/greenio/base.py", line 210, in _trampoline
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server     mark_as_closed=self._mark_as_closed)
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File "/usr/lib/python3/dist-packages/eventlet/hubs/__init__.py", line 159, in trampoline
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server     return hub.switch()
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File "/usr/lib/python3/dist-packages/eventlet/hubs/hub.py", line 298, in switch
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server     return self.greenlet.switch()
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server eventlet.hubs.IOClosed: [Errno 107] Operation on closed file
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred:
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server Traceback (most recent call last):
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File "/usr/lib/python3/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server     res = self.dispatcher.dispatch(message)
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File "/usr/lib/python3/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server     return self._do_dispatch(endpoint, method, ctxt, args)
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File "/usr/lib/python3/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server     result = func(ctxt, **new_args)
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File "/usr/lib/python3/dist-packages/osprofiler/profiler.py", line 160, in wrapper
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server     result = f(*args, **kwargs)
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File "/usr/lib/python3/dist-packages/osprofiler/profiler.py", line 160, in wrapper
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server     result = f(*args, **kwargs)
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File "<decorator-gen-754>", line 2, in delete_volume
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File "/usr/lib/python3/dist-packages/cinder/coordination.py", line 144, in _synchronized
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server     with lock(blocking):
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File "/usr/lib/python3/dist-packages/tooz/locking.py", line 29, in __enter__
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server     return self.lock.__enter__(*self.args, **self.kwargs)
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File "/usr/lib/python3/dist-packages/tooz/locking.py", line 49, in __enter__
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server     acquired = self.acquire(*args, **kwargs)
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File "/usr/lib/python3/dist-packages/tooz/drivers/redis.py", line 58, in wrapper
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server     return func(*args, **kwargs)
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File "/usr/lib/python3/dist-packages/tooz/drivers/redis.py", line 110, in acquire
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server     blocking=blocking, blocking_timeout=timeout)
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File "/usr/lib/python3/dist-packages/redis/lock.py", line 182, in acquire
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server     if self.do_acquire(token):
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File "/usr/lib/python3/dist-packages/redis/lock.py", line 197, in do_acquire
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server     if self.redis.set(self.name, token, nx=True, px=timeout):
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File "/usr/lib/python3/dist-packages/redis/client.py", line 1519, in set
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server     return self.execute_command('SET', *pieces)
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File "/usr/lib/python3/dist-packages/redis/client.py", line 839, in execute_command
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server     return self.parse_response(conn, command_name, **options)
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File "/usr/lib/python3/dist-packages/redis/client.py", line 853, in parse_response
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server     response = connection.read_response()
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File "/usr/lib/python3/dist-packages/redis/sentinel.py", line 55, in read_response
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server     return super(SentinelManagedConnection, self).read_response()
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File "/usr/lib/python3/dist-packages/redis/connection.py", line 700, in read_response
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server     response = self._parser.read_response()
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File "/usr/lib/python3/dist-packages/redis/connection.py", line 310, in read_response
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server     response = self._buffer.readline()
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File "/usr/lib/python3/dist-packages/redis/connection.py", line 242, in readline
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server     self._read_from_socket()
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File "/usr/lib/python3/dist-packages/redis/connection.py", line 184, in _read_from_socket
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server     data = recv(self._sock, socket_read_size)
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File "/usr/lib/python3/dist-packages/redis/_compat.py", line 71, in recv
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server     return sock.recv(*args, **kwargs)
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File "/usr/lib/python3/dist-packages/eventlet/greenio/base.py", line 365, in recv
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server     return self._recv_loop(self.fd.recv, b'', bufsize, flags)
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server   File "/usr/lib/python3/dist-packages/eventlet/greenio/base.py", line 362, in _recv_loop
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server     raise EOFError()
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server EOFError
2022-05-23 10:15:24.414 833 ERROR oslo_messaging.rpc.server

Thanks,

Michal Arbet
Openstack Engineer

Ultimum Technologies a.s.
Na Poříčí 1047/26, 11000 Praha 1
Czech Republic

+420 604 228 897 
michal.arbet@ultimum.io
https://ultimum.io