Open Stack

Hi,



The multi_delete in swift3, perform sequential DELETE.  In my 3
storage-node configuration to delete a 1000 objects, it took 30 second.



Following code change to create 100 thread pool to delete 1000 object took
only 12 second. (This may even reduce if more storage nodes in picture).



If the following code change look fine, How can we formally
(propose/review/commit) take this to the swift3 github code base ?





*Code diff:*



diff --git a/swift3/swift-plugin-s3/swift3/controllers/multi_delete.py
b/swift3/swift-plugin-s3/swift3/controllers/multi_delete.py

index 1bfde1d..5140529 100644

--- a/swift3/swift-plugin-s3/swift3/controllers/multi_delete.py

+++ b/swift3/swift-plugin-s3/swift3/controllers/multi_delete.py

@@ -21,9 +21,9 @@ from swift3.response import HTTPOk, S3NotImplemented,
NoSuchKey, \

from swift3.cfg import CONF

from swift3.utils import LOGGER

-# Zadara-Begin

+from eventlet import GreenPool

+import copy

MAX_MULTI_DELETE_BODY_SIZE = 262144

-# Zadara-End



 class MultiObjectDeleteController(Controller):

@@ -44,6 +44,24 @@ class MultiObjectDeleteController(Controller):

         return tostring(elem)

+    def async_delete(self, reqs, key, elem):

+        req = copy.copy(reqs)

+        req.object_name = key

+        try:

+            req.get_response(self.app, method='DELETE')

+        except NoSuchKey:

+            pass

+        except ErrorResponse as e:

+            error = SubElement(elem, 'Error')

+            SubElement(error, 'Key').text = key

+            SubElement(error, 'Code').text = e.__class__.__name__

+            SubElement(error, 'Message').text = e._msg

+            return

+

+        if not self.quiet:

+            deleted = SubElement(elem, 'Deleted')

+            SubElement(deleted, 'Key').text = key

+

     @bucket_operation

     def POST(self, req):

         """

@@ -90,27 +108,17 @@ class MultiObjectDeleteController(Controller):

             body = self._gen_error_body(error, elem, delete_list)

             return HTTPOk(body=body)

+        parallel_delete = 100

+        run_pool = GreenPool(size=parallel_delete)

         for key, version in delete_list:

             if version is not None:

                 # TODO: delete the specific version of the object

                 raise S3NotImplemented()

-            req.object_name = key

-

-            try:

-                req.get_response(self.app, method='DELETE')

-            except NoSuchKey:

-                pass

-            except ErrorResponse as e:

-                error = SubElement(elem, 'Error')

-                SubElement(error, 'Key').text = key

-                SubElement(error, 'Code').text = e.__class__.__name__

-                SubElement(error, 'Message').text = e._msg

-                continue

-

-            if not self.quiet:

-                deleted = SubElement(elem, 'Deleted')

-                SubElement(deleted, 'Key').text = key

+            run_pool.spawn(self.async_delete, req, key, elem)

+

+        # Wait for all the process to complete

+        run_pool.waitall()

         body = tostring(elem)
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.openstack.org/pipermail/openstack-dev/attachments/20160524/702a81c9/attachment.html>