From b82146a0cbd58eead9e235aff3fef9fc0772e09f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 4 Mar 2013 14:41:15 -0500 Subject: [PATCH] Refactor useful functionality out of OpenStackContainer and into utility class. --- .../storage/backends/cloud/cloud_common.py | 58 ++++++++++++++++++- .../cloud/openstack/openstack_container.py | 51 +++------------- src/allmydata/test/test_storage.py | 3 +- 3 files changed, 68 insertions(+), 44 deletions(-) diff --git a/src/allmydata/storage/backends/cloud/cloud_common.py b/src/allmydata/storage/backends/cloud/cloud_common.py index 0a8b12be..98c552ba 100644 --- a/src/allmydata/storage/backends/cloud/cloud_common.py +++ b/src/allmydata/storage/backends/cloud/cloud_common.py @@ -1,11 +1,17 @@ from collections import deque from cStringIO import StringIO +import urllib from twisted.internet import defer, reactor, task from twisted.python.failure import Failure from twisted.web.error import Error -from twisted.web.client import FileBodyProducer, ResponseDone +from twisted.web.client import FileBodyProducer, ResponseDone, Agent +try: + from twisted.web.client import HTTPConnectionPool +except ImportError: + # Old version of Twisted + HTTPConnectionPool = None from twisted.web.http_headers import Headers from twisted.internet.protocol import Protocol @@ -688,3 +694,53 @@ class HTTPClientMixin: raise self.ServiceError(None, response.code, message="missing response header %r" % (name,)) return hs[0] + + + +class CommonContainerMixin(HTTPClientMixin, ContainerRetryMixin): + """ + Base class for cloud storage providers with similar APIs. + + In particular, OpenStack and Google Storage are very similar (presumably + since they both copy S3). + """ + + def __init__(self, container_name, override_reactor=None): + self._container_name = container_name + self._reactor = override_reactor or reactor + if HTTPConnectionPool: + self._agent = Agent(self._reactor, pool=HTTPConnectionPool(self._reactor)) + else: + self._agent = Agent(self._reactor) + self.ServiceError = CloudServiceError + + def __repr__(self): + return ("<%s %r>" % (self.__class__.__name__, self._container_name,)) + + def _make_container_url(self, public_storage_url): + return "%s/%s" % (public_storage_url, urllib.quote(self._container_name, safe='')) + + def _make_object_url(self, public_storage_url, object_name): + return "%s/%s/%s" % (public_storage_url, urllib.quote(self._container_name, safe=''), + urllib.quote(object_name)) + + def create(self): + return self._do_request('create container', self._create) + + def delete(self): + return self._do_request('delete container', self._delete) + + def list_objects(self, prefix=''): + return self._do_request('list objects', self._list_objects, prefix) + + def put_object(self, object_name, data, content_type='application/octet-stream', metadata={}): + return self._do_request('PUT object', self._put_object, object_name, data, content_type, metadata) + + def get_object(self, object_name): + return self._do_request('GET object', self._get_object, object_name) + + def head_object(self, object_name): + return self._do_request('HEAD object', self._head_object, object_name) + + def delete_object(self, object_name): + return self._do_request('DELETE object', self._delete_object, object_name) diff --git a/src/allmydata/storage/backends/cloud/openstack/openstack_container.py b/src/allmydata/storage/backends/cloud/openstack/openstack_container.py index 26dc0061..e0229958 100644 --- a/src/allmydata/storage/backends/cloud/openstack/openstack_container.py +++ b/src/allmydata/storage/backends/cloud/openstack/openstack_container.py @@ -11,7 +11,7 @@ from zope.interface import implements, Interface from allmydata.util import log from allmydata.node import InvalidValueError from allmydata.storage.backends.cloud.cloud_common import IContainer, \ - CloudServiceError, ContainerItem, ContainerListing, ContainerRetryMixin, \ + CloudServiceError, ContainerItem, ContainerListing, CommonContainerMixin, \ HTTPClientMixin @@ -235,35 +235,23 @@ class AuthenticationClient(HTTPClientMixin): self._delayed.cancel() -class OpenStackContainer(HTTPClientMixin, ContainerRetryMixin): +class OpenStackContainer(CommonContainerMixin): implements(IContainer) USER_AGENT = "Tahoe-LAFS OpenStack storage client" def __init__(self, auth_client, container_name, override_reactor=None): + CommonContainerMixin.__init__(self, container_name, override_reactor) self._auth_client = auth_client - self._container_name = container_name - self._reactor = override_reactor or reactor - self._agent = Agent(self._reactor) self.ServiceError = CloudServiceError - def __repr__(self): - return ("<%s %r>" % (self.__class__.__name__, self._container_name,)) - - def _make_container_url(self, auth_info): - return "%s/%s" % (auth_info.public_storage_url, urllib.quote(self._container_name, safe='')) - - def _make_object_url(self, auth_info, object_name): - return "%s/%s/%s" % (auth_info.public_storage_url, urllib.quote(self._container_name, safe=''), - urllib.quote(object_name)) - def _react_to_error(self, response_code): if response_code == UNAUTHORIZED: # Invalidate auth_info and retry. self._auth_client.invalidate() return True else: - return ContainerRetryMixin._react_to_error(self, response_code) + return CommonContainerMixin._react_to_error(self, response_code) def _create(self): """ @@ -289,7 +277,7 @@ class OpenStackContainer(HTTPClientMixin, ContainerRetryMixin): request_headers = { 'X-Auth-Token': [auth_info.auth_token], } - url = self._make_container_url(auth_info) + url = self._make_container_url(auth_info.public_storage_url) if prefix: url += "?format=json&prefix=%s" % (urllib.quote(prefix, safe=''),) return self._http_request("OpenStack list objects", 'GET', url, request_headers, @@ -335,7 +323,7 @@ class OpenStackContainer(HTTPClientMixin, ContainerRetryMixin): 'X-Auth-Token': [auth_info.auth_token], 'Content-Type': [content_type], } - url = self._make_object_url(auth_info, object_name) + url = self._make_object_url(auth_info.public_storage_url, object_name) return self._http_request("OpenStack put object", 'PUT', url, request_headers, data) d.addCallback(_do_put) d.addCallback(lambda ign: None) @@ -350,7 +338,7 @@ class OpenStackContainer(HTTPClientMixin, ContainerRetryMixin): request_headers = { 'X-Auth-Token': [auth_info.auth_token], } - url = self._make_object_url(auth_info, object_name) + url = self._make_object_url(auth_info.public_storage_url, object_name) return self._http_request("OpenStack get object", 'GET', url, request_headers, need_response_body=True) d.addCallback(_do_get) @@ -366,7 +354,7 @@ class OpenStackContainer(HTTPClientMixin, ContainerRetryMixin): request_headers = { 'X-Auth-Token': [auth_info.auth_token], } - url = self._make_object_url(auth_info, object_name) + url = self._make_object_url(auth_info.public_storage_url, object_name) return self._http_request("OpenStack head object", 'HEAD', url, request_headers) d.addCallback(_do_head) def _got_head_response( (response, body) ): @@ -385,29 +373,8 @@ class OpenStackContainer(HTTPClientMixin, ContainerRetryMixin): request_headers = { 'X-Auth-Token': [auth_info.auth_token], } - url = self._make_object_url(auth_info, object_name) + url = self._make_object_url(auth_info.public_storage_url, object_name) return self._http_request("OpenStack delete object", 'DELETE', url, request_headers) d.addCallback(_do_delete) d.addCallback(lambda ign: None) return d - - def create(self): - return self._do_request('create container', self._create) - - def delete(self): - return self._do_request('delete container', self._delete) - - def list_objects(self, prefix=''): - return self._do_request('list objects', self._list_objects, prefix) - - def put_object(self, object_name, data, content_type='application/octet-stream', metadata={}): - return self._do_request('PUT object', self._put_object, object_name, data, content_type, metadata) - - def get_object(self, object_name): - return self._do_request('GET object', self._get_object, object_name) - - def head_object(self, object_name): - return self._do_request('HEAD object', self._head_object, object_name) - - def delete_object(self, object_name): - return self._do_request('DELETE object', self._delete_object, object_name) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 1b212f34..654bd706 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -442,7 +442,7 @@ class OpenStackCloudBackend(ServiceParentMixin, WorkdirMixin, ShouldFailMixin, u protocol.connectionLost(Failure(ResponseDone())) class MockAgent(object): - def __init__(mock_self, reactor): + def __init__(mock_self, reactor, pool=None): pass def request(mock_self, method, url, headers, bodyProducer=None): @@ -468,6 +468,7 @@ class OpenStackCloudBackend(ServiceParentMixin, WorkdirMixin, ShouldFailMixin, u return d self.patch(openstack_container, 'Agent', MockAgent) + self.patch(cloud_common, 'Agent', MockAgent) def _set_request(self, method, url, expected_headers, expected_body, response_code, response_phrase, response_headers, response_body): -- 2.45.2