Refactor useful functionality out of OpenStackContainer and into utility class.

diff --git a/src/allmydata/storage/backends/cloud/cloud_common.py b/src/allmydata/storage/backends/cloud/cloud_common.py
index 0a8b12be161aa3931e75baee069451ca6b75a234..98c552bab9cc37e2a6bff5739fe85bfe62f40182 100644 (file)
--- a/src/allmydata/storage/backends/cloud/cloud_common.py
+++ b/src/allmydata/storage/backends/cloud/cloud_common.py
@@ -1,11 +1,17 @@
 
 from collections import deque
 from cStringIO import StringIO
+import urllib
 
 from twisted.internet import defer, reactor, task
 from twisted.python.failure import Failure
 from twisted.web.error import Error
-from twisted.web.client import FileBodyProducer, ResponseDone
+from twisted.web.client import FileBodyProducer, ResponseDone, Agent
+try:
+    from twisted.web.client import HTTPConnectionPool
+except ImportError:
+    # Old version of Twisted
+    HTTPConnectionPool = None
 from twisted.web.http_headers import Headers
 from twisted.internet.protocol import Protocol
 
@@ -688,3 +694,53 @@ class HTTPClientMixin:
             raise self.ServiceError(None, response.code,
                                     message="missing response header %r" % (name,))
         return hs[0]
+
+
+
+class CommonContainerMixin(HTTPClientMixin, ContainerRetryMixin):
+    """
+    Base class for cloud storage providers with similar APIs.
+
+    In particular, OpenStack and Google Storage are very similar (presumably
+    since they both copy S3).
+    """
+
+    def __init__(self, container_name, override_reactor=None):
+        self._container_name = container_name
+        self._reactor = override_reactor or reactor
+        if HTTPConnectionPool:
+            self._agent = Agent(self._reactor, pool=HTTPConnectionPool(self._reactor))
+        else:
+            self._agent = Agent(self._reactor)
+        self.ServiceError = CloudServiceError
+
+    def __repr__(self):
+        return ("<%s %r>" % (self.__class__.__name__, self._container_name,))
+
+    def _make_container_url(self, public_storage_url):
+        return "%s/%s" % (public_storage_url, urllib.quote(self._container_name, safe=''))
+
+    def _make_object_url(self, public_storage_url, object_name):
+        return "%s/%s/%s" % (public_storage_url, urllib.quote(self._container_name, safe=''),
+                             urllib.quote(object_name))
+
+    def create(self):
+        return self._do_request('create container', self._create)
+
+    def delete(self):
+        return self._do_request('delete container', self._delete)
+
+    def list_objects(self, prefix=''):
+        return self._do_request('list objects', self._list_objects, prefix)
+
+    def put_object(self, object_name, data, content_type='application/octet-stream', metadata={}):
+        return self._do_request('PUT object', self._put_object, object_name, data, content_type, metadata)
+
+    def get_object(self, object_name):
+        return self._do_request('GET object', self._get_object, object_name)
+
+    def head_object(self, object_name):
+        return self._do_request('HEAD object', self._head_object, object_name)
+
+    def delete_object(self, object_name):
+        return self._do_request('DELETE object', self._delete_object, object_name)

diff --git a/src/allmydata/storage/backends/cloud/openstack/openstack_container.py b/src/allmydata/storage/backends/cloud/openstack/openstack_container.py
index 26dc006120c3c0b1013ba396e9b66bf8ab7a103e..e022995835f8889722d2f61e13b26625e77a65ff 100644 (file)
--- a/src/allmydata/storage/backends/cloud/openstack/openstack_container.py
+++ b/src/allmydata/storage/backends/cloud/openstack/openstack_container.py
@@ -11,7 +11,7 @@ from zope.interface import implements, Interface
 from allmydata.util import log
 from allmydata.node import InvalidValueError
 from allmydata.storage.backends.cloud.cloud_common import IContainer, \
-     CloudServiceError, ContainerItem, ContainerListing, ContainerRetryMixin, \
+     CloudServiceError, ContainerItem, ContainerListing, CommonContainerMixin, \
      HTTPClientMixin
 
 
@@ -235,35 +235,23 @@ class AuthenticationClient(HTTPClientMixin):
             self._delayed.cancel()
 
 
-class OpenStackContainer(HTTPClientMixin, ContainerRetryMixin):
+class OpenStackContainer(CommonContainerMixin):
     implements(IContainer)
 
     USER_AGENT = "Tahoe-LAFS OpenStack storage client"
 
     def __init__(self, auth_client, container_name, override_reactor=None):
+        CommonContainerMixin.__init__(self, container_name, override_reactor)
         self._auth_client = auth_client
-        self._container_name = container_name
-        self._reactor = override_reactor or reactor
-        self._agent = Agent(self._reactor)
         self.ServiceError = CloudServiceError
 
-    def __repr__(self):
-        return ("<%s %r>" % (self.__class__.__name__, self._container_name,))
-
-    def _make_container_url(self, auth_info):
-        return "%s/%s" % (auth_info.public_storage_url, urllib.quote(self._container_name, safe=''))
-
-    def _make_object_url(self, auth_info, object_name):
-        return "%s/%s/%s" % (auth_info.public_storage_url, urllib.quote(self._container_name, safe=''),
-                             urllib.quote(object_name))
-
     def _react_to_error(self, response_code):
         if response_code == UNAUTHORIZED:
             # Invalidate auth_info and retry.
             self._auth_client.invalidate()
             return True
         else:
-            return ContainerRetryMixin._react_to_error(self, response_code)
+            return CommonContainerMixin._react_to_error(self, response_code)
 
     def _create(self):
         """
@@ -289,7 +277,7 @@ class OpenStackContainer(HTTPClientMixin, ContainerRetryMixin):
             request_headers = {
                 'X-Auth-Token': [auth_info.auth_token],
             }
-            url = self._make_container_url(auth_info)
+            url = self._make_container_url(auth_info.public_storage_url)
             if prefix:
                 url += "?format=json&prefix=%s" % (urllib.quote(prefix, safe=''),)
             return self._http_request("OpenStack list objects", 'GET', url, request_headers,
@@ -335,7 +323,7 @@ class OpenStackContainer(HTTPClientMixin, ContainerRetryMixin):
                 'X-Auth-Token': [auth_info.auth_token],
                 'Content-Type': [content_type],
             }
-            url = self._make_object_url(auth_info, object_name)
+            url = self._make_object_url(auth_info.public_storage_url, object_name)
             return self._http_request("OpenStack put object", 'PUT', url, request_headers, data)
         d.addCallback(_do_put)
         d.addCallback(lambda ign: None)
@@ -350,7 +338,7 @@ class OpenStackContainer(HTTPClientMixin, ContainerRetryMixin):
             request_headers = {
                 'X-Auth-Token': [auth_info.auth_token],
             }
-            url = self._make_object_url(auth_info, object_name)
+            url = self._make_object_url(auth_info.public_storage_url, object_name)
             return self._http_request("OpenStack get object", 'GET', url, request_headers,
                                       need_response_body=True)
         d.addCallback(_do_get)
@@ -366,7 +354,7 @@ class OpenStackContainer(HTTPClientMixin, ContainerRetryMixin):
             request_headers = {
                 'X-Auth-Token': [auth_info.auth_token],
             }
-            url = self._make_object_url(auth_info, object_name)
+            url = self._make_object_url(auth_info.public_storage_url, object_name)
             return self._http_request("OpenStack head object", 'HEAD', url, request_headers)
         d.addCallback(_do_head)
         def _got_head_response( (response, body) ):
@@ -385,29 +373,8 @@ class OpenStackContainer(HTTPClientMixin, ContainerRetryMixin):
             request_headers = {
                 'X-Auth-Token': [auth_info.auth_token],
             }
-            url = self._make_object_url(auth_info, object_name)
+            url = self._make_object_url(auth_info.public_storage_url, object_name)
             return self._http_request("OpenStack delete object", 'DELETE', url, request_headers)
         d.addCallback(_do_delete)
         d.addCallback(lambda ign: None)
         return d
-
-    def create(self):
-        return self._do_request('create container', self._create)
-
-    def delete(self):
-        return self._do_request('delete container', self._delete)
-
-    def list_objects(self, prefix=''):
-        return self._do_request('list objects', self._list_objects, prefix)
-
-    def put_object(self, object_name, data, content_type='application/octet-stream', metadata={}):
-        return self._do_request('PUT object', self._put_object, object_name, data, content_type, metadata)
-
-    def get_object(self, object_name):
-        return self._do_request('GET object', self._get_object, object_name)
-
-    def head_object(self, object_name):
-        return self._do_request('HEAD object', self._head_object, object_name)
-
-    def delete_object(self, object_name):
-        return self._do_request('DELETE object', self._delete_object, object_name)

diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py
index c64d9f32ecd341d15662cc87794702ba946c7f9c..bee5cfeb5f76df23ea1022d1ec8dc3dc5a9cffdc 100644 (file)
--- a/src/allmydata/test/test_storage.py
+++ b/src/allmydata/test/test_storage.py
@@ -449,7 +449,7 @@ class OpenStackCloudBackend(ServiceParentMixin, WorkdirMixin, ShouldFailMixin, u
                 protocol.connectionLost(Failure(ResponseDone()))
 
         class MockAgent(object):
-            def __init__(mock_self, reactor):
+            def __init__(mock_self, reactor, pool=None):
                 pass
 
             def request(mock_self, method, url, headers, bodyProducer=None):
@@ -475,6 +475,7 @@ class OpenStackCloudBackend(ServiceParentMixin, WorkdirMixin, ShouldFailMixin, u
                 return d
 
         self.patch(openstack_container, 'Agent', MockAgent)
+        self.patch(cloud_common, 'Agent', MockAgent)
 
     def _set_request(self, method, url, expected_headers, expected_body,
                            response_code, response_phrase, response_headers, response_body):