From: David-Sarah Hopwood Date: Sun, 24 Feb 2013 04:08:23 +0000 (+0000) Subject: OpenStack: support HP Cloud Object Storage. X-Git-Url: https://git.rkrishnan.org/components/com_hotproperty//%22%3C?a=commitdiff_plain;h=aa5fdebe117236e1c2cc7de5d8811dafff2d00fd;p=tahoe-lafs%2Ftahoe-lafs.git OpenStack: support HP Cloud Object Storage. Also make the choice of auth protocol for Rackspace configurable via openstack.provider, and change the reauth period to 11 hours. Signed-off-by: David-Sarah Hopwood --- diff --git a/docs/backends/cloud.rst b/docs/backends/cloud.rst index 7c5f0e6d..c74370f1 100644 --- a/docs/backends/cloud.rst +++ b/docs/backends/cloud.rst @@ -60,22 +60,25 @@ user and product token pair is needed by a given storage server. .. _DevPay: http://docs.amazonwebservices.com/AmazonDevPay/latest/DevPayGettingStartedGuide/ -OpenStack and Rackspace Cloud Files -=================================== +OpenStack +========= `OpenStack`_ is an open standard for cloud services, including cloud storage. -Rackspace ( ``__ and ``__ ) -provides a storage service called `Cloud Files`_ based on OpenStack. +The cloud backend currently supports two OpenStack storage providers: + +* Rackspace ( ``__ and ``__ ) + provides a service called `Cloud Files`_. +* HP ( ``__ ) provides a service called + `HP Cloud Object Storage`_. + +Other OpenStack storage providers may be supported in future. .. _OpenStack: https://www.openstack.org/ .. _Cloud Files: http://www.rackspace.com/cloud/files/ +.. _HP Cloud Object Storage: https://www.hpcloud.com/products/object-storage -The authentication service is less precisely specified than other parts of -the OpenStack standards, and so our implementation of it is currently specific -to Rackspace. Other OpenStack storage providers may be supported in future. - -To enable storing shares on Rackspace Cloud Files, add the following keys -to the server's ``tahoe.cfg`` file: +To enable storing shares on one of these services, add the following keys to +the server's ``tahoe.cfg`` file: ``[storage]`` @@ -84,11 +87,32 @@ to the server's ``tahoe.cfg`` file: This turns off the local filesystem backend and enables use of the cloud backend with OpenStack. -``openstack.provider = (string, optional)`` +``openstack.provider = (string, optional, case-insensitive)`` + + The supported providers are ``rackspace.com``, ``rackspace.co.uk``, + ``hpcloud.com west``, and ``hpcloud.com east``. For Rackspace, use the + site on which the Rackspace user account was created. For HP, "west" + and "east" refer to the two storage regions in the United States. - The supported providers are ``rackspace.com`` and ``rackspace.co.uk``. - Use the one corresponding to the site on which the Rackspace user account - was created. The default is ``rackspace.com``. + The default is ``rackspace.com``. + +``openstack.container = (string, required)`` + + This controls which container will be used to hold shares. The Tahoe-LAFS + storage server will only modify and access objects in the configured + container. Multiple storage servers cannot share the same container. + +``openstack.url = (URL string, optional)`` + + This overrides the URL used to access the authentication service. It + does not need to be set when using Rackspace or HP accounts, because the + correct service is chosen based on ``openstack.provider`` by default. + +Authentication is less precisely specified than other parts of the OpenStack +standards, and so the two supported providers require slightly different user +credentials, described below. + +*If using Rackspace:* ``openstack.username = (string, required)`` @@ -99,14 +123,14 @@ to the server's ``tahoe.cfg`` file: The API key should be stored in a separate file named ``private/openstack_api_key``. -``openstack.container = (string, required)`` +*If using HP:* - This controls which container will be used to hold shares. The Tahoe-LAFS - storage server will only modify and access objects in the configured - container. Multiple storage servers cannot share the same container. +``openstack.access_key_id = (string, required)`` -``openstack.url = (URL string, optional)`` +``openstack.tenant_id = (string, required)`` - This overrides the URL used to access the authentication service. It - does not need to be set when using a Rackspace account, because the - correct service is chosen based on ``openstack.provider`` by default. + These are the Access Key ID and Tenant ID (not the tenant name) obtained + by logging in at ``__. + + The secret key, obtained from the same page by clicking SHOW, should + be stored in a separate file named ``private/openstack_secret_key``. diff --git a/src/allmydata/storage/backends/cloud/openstack/openstack_container.py b/src/allmydata/storage/backends/cloud/openstack/openstack_container.py index 617ed163..26dc0061 100644 --- a/src/allmydata/storage/backends/cloud/openstack/openstack_container.py +++ b/src/allmydata/storage/backends/cloud/openstack/openstack_container.py @@ -18,30 +18,51 @@ from allmydata.storage.backends.cloud.cloud_common import IContainer, \ # Enabling this will cause secrets to be logged. UNSAFE_DEBUG = False -#AUTH_PATH = "v1.0" -AUTH_PATH = "v2.0/tokens" DEFAULT_AUTH_URLS = { - "rackspace.com": "https://identity.api.rackspacecloud.com/" + AUTH_PATH, - "rackspace.co.uk": "https://lon.identity.api.rackspacecloud.com/" + AUTH_PATH, + "rackspace.com v1": "https://identity.api.rackspacecloud.com/v1.0", + "rackspace.co.uk v1": "https://lon.identity.api.rackspacecloud.com/v1.0", + "rackspace.com": "https://identity.api.rackspacecloud.com/v2.0/tokens", + "rackspace.co.uk": "https://lon.identity.api.rackspacecloud.com/v2.0/tokens", + "hpcloud.com west": "https://region-a.geo-1.identity.hpcloudsvc.com:35357/v2.0/tokens", + "hpcloud.com east": "https://region-b.geo-1.identity.hpcloudsvc.com:35357/v2.0/tokens", } -USER_AGENT = "Tahoe-LAFS OpenStack client" def configure_openstack_container(storedir, config): - api_key = config.get_or_create_private_config("openstack_api_key") provider = config.get_config("storage", "openstack.provider", "rackspace.com").lower() if provider not in DEFAULT_AUTH_URLS: raise InvalidValueError("[storage]openstack.provider %r is not recognized\n" - "Valid providers are: %s" % (provider, ", ".join(DEFAULT_AUTH_URLS.keys()))) + "Valid providers are: %s" % (provider, ", ".join(sorted(DEFAULT_AUTH_URLS.keys())))) auth_service_url = config.get_config("storage", "openstack.url", DEFAULT_AUTH_URLS[provider]) - username = config.get_config("storage", "openstack.username") container_name = config.get_config("storage", "openstack.container") - reauth_period = 23*60*60 #seconds + reauth_period = 11*60*60 #seconds + + access_key_id = config.get_config("storage", "openstack.access_key_id", None) + if access_key_id is None: + username = config.get_config("storage", "openstack.username") + api_key = config.get_private_config("openstack_api_key") + if auth_service_url.endswith("/v1.0"): + authenticator = AuthenticatorV1(auth_service_url, username, api_key) + else: + authenticator = AuthenticatorV2(auth_service_url, { + 'RAX-KSKEY:apiKeyCredentials': { + 'username': username, + 'apiKey': api_key, + } + }) + else: + tenant_id = config.get_config("storage", "openstack.tenant_id") + secret_key = config.get_private_config("openstack_secret_key") + authenticator = AuthenticatorV2(auth_service_url, { + 'apiAccessKeyCredentials': { + 'accessKey': access_key_id, + 'secretKey': secret_key, + }, + 'tenantId': tenant_id, + }) - AuthenticatorClass = {"v1.0": AuthenticatorV1, "v2.0/tokens": AuthenticatorV2}[AUTH_PATH] - authenticator = AuthenticatorClass(auth_service_url, username, api_key) auth_client = AuthenticationClient(authenticator, reauth_period) return OpenStackContainer(auth_client, container_name) @@ -92,28 +113,17 @@ class AuthenticatorV2(object): """ Authenticates according to V2 protocol as documented by Rackspace: . + + This is also compatible with HP's protocol (using different credentials): + . """ - def __init__(self, auth_service_url, username, api_key): + def __init__(self, auth_service_url, credentials): self._auth_service_url = auth_service_url - self._username = username - self._api_key = api_key - #self._password = password + self._credentials = credentials def make_auth_request(self): - # I suspect that 'RAX-KSKEY:apiKeyCredentials' is Rackspace-specific. - request = { - 'auth': { - # 'passwordCredentials': { - # 'username': self._username, - # 'password': self._password, - # } - 'RAX-KSKEY:apiKeyCredentials': { - 'username': self._username, - 'apiKey': self._api_key, - } - } - } + request = {'auth': self._credentials} json = simplejson.dumps(request) request_headers = { 'Content-Type': ['application/json'], @@ -143,7 +153,7 @@ class AuthenticatorV2(object): for endpoint in endpoints: if not default_region or endpoint['region'] == default_region: public_storage_url = endpoint['publicURL'] - internal_storage_url = endpoint['internalURL'] + internal_storage_url = endpoint.get('internalURL', None) return AuthenticationInfo(auth_token, public_storage_url, internal_storage_url) except KeyError, e: raise CloudServiceError(None, response.code, diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py index e221147d..0888a0b6 100644 --- a/src/allmydata/test/test_client.py +++ b/src/allmydata/test/test_client.py @@ -366,10 +366,10 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase): c = client.Client(basedir) mock_Authenticator.assert_called_with("https://identity.api.rackspacecloud.com/v2.0/tokens", - "alex", "dummy") + {'RAX-KSKEY:apiKeyCredentials': {'username': 'alex', 'apiKey': 'dummy'}}) authclient_call_args = mock_AuthenticationClient.call_args_list self.failUnlessEqual(len(authclient_call_args), 1) - self.failUnlessEqual(authclient_call_args[0][0][1:], (23*60*60,)) + self.failUnlessEqual(authclient_call_args[0][0][1:], (11*60*60,)) container_call_args = mock_OpenStackContainer.call_args_list self.failUnlessEqual(len(container_call_args), 1) self.failUnlessEqual(container_call_args[0][0][1:], ("test",)) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 35363717..2cb7acb9 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -510,7 +510,7 @@ class OpenStackCloudBackend(ServiceParentMixin, WorkdirMixin, ShouldFailMixin, u if default is _None: self.failUnlessIn(option, storage_config) return storage_config.get(option, default) - def get_or_create_private_config(mock_self, filename): + def get_private_config(mock_self, filename): return fileutil.read(os.path.join(privatedir, filename)) self.workdir = self.workdir(name)