From 6ac01fde4cb92c253f7309bb2aab2c99ea9632a4 Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Fri, 11 Jan 2008 04:53:37 -0700 Subject: [PATCH] offloaded: more test coverage on client side, change interfaces a bit --- src/allmydata/interfaces.py | 16 +++++------ src/allmydata/offloaded.py | 14 ++++++---- src/allmydata/test/test_helper.py | 45 ++++++++++++++++++++++++++----- src/allmydata/upload.py | 5 +++- 4 files changed, 59 insertions(+), 21 deletions(-) diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index ec29fdd9..b37e4240 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -1278,7 +1278,7 @@ class RIEncryptedUploadable(RemoteInterface): return Hash -class RIUploadHelper(RemoteInterface): +class RICHKUploadHelper(RemoteInterface): __remote_name__ = "RIUploadHelper.tahoe.allmydata.com" def upload(reader=RIEncryptedUploadable): @@ -1288,7 +1288,7 @@ class RIUploadHelper(RemoteInterface): class RIHelper(RemoteInterface): __remote_name__ = "RIHelper.tahoe.allmydata.com" - def upload(si=StorageIndex): + def upload_chk(si=StorageIndex): """See if a file with a given storage index needs uploading. The helper will ask the appropriate storage servers to see if the file has already been uploaded. If so, the helper will return a set of @@ -1297,10 +1297,10 @@ class RIHelper(RemoteInterface): If the file has not yet been uploaded (or if it was only partially uploaded), the helper will return an empty upload-results dictionary - and also an RIUploadHelper object that will take care of the upload - process. The client should call upload() on this object and pass it a - reference to an RIEncryptedUploadable object that will provide - ciphertext. When the upload is finished, the upload() method will - finish and return the upload results. + and also an RICHKUploadHelper object that will take care of the + upload process. The client should call upload() on this object and + pass it a reference to an RIEncryptedUploadable object that will + provide ciphertext. When the upload is finished, the upload() method + will finish and return the upload results. """ - return (UploadResults, ChoiceOf(RIUploadHelper, None)) + return (UploadResults, ChoiceOf(RICHKUploadHelper, None)) diff --git a/src/allmydata/offloaded.py b/src/allmydata/offloaded.py index 604f306f..be60ad81 100644 --- a/src/allmydata/offloaded.py +++ b/src/allmydata/offloaded.py @@ -13,7 +13,7 @@ class CHKUploadHelper(Referenceable, upload.CHKUploader): peer selection, encoding, and share pushing. I read ciphertext from the remote AssistedUploader. """ - implements(interfaces.RIUploadHelper) + implements(interfaces.RICHKUploadHelper) def __init__(self, storage_index, helper): self._finished = False @@ -30,7 +30,8 @@ class CHKUploadHelper(Referenceable, upload.CHKUploader): def start(self): # determine if we need to upload the file. If so, return ({},self) . # If not, return (UploadResults,None) . - return ({'uri_extension_hash': hashutil.uri_extension_hash("")},self) + #return ({'uri_extension_hash': hashutil.uri_extension_hash("")},self) + return ({}, self) def remote_upload(self, reader): # reader is an RIEncryptedUploadable. I am specified to return an @@ -39,6 +40,7 @@ class CHKUploadHelper(Referenceable, upload.CHKUploader): eu = CiphertextReader(reader, self._storage_index) d = self.start_encrypted(eu) def _done(res): + self.finished(self._storage_index) (uri_extension_hash, needed_shares, total_shares, size) = res return {'uri_extension_hash': uri_extension_hash} d.addCallback(_done) @@ -78,8 +80,10 @@ class CiphertextReader: class Helper(Referenceable, service.MultiService): implements(interfaces.RIHelper) # this is the non-distributed version. When we need to have multiple - # helpers, this object will query the farm to see if anyone has the - # storage_index of interest, and send the request off to them. + # helpers, this object will become the HelperCoordinator, and will query + # the farm of Helpers to see if anyone has the storage_index of interest, + # and send the request off to them. If nobody has it, we'll choose a + # helper at random. chk_upload_helper_class = CHKUploadHelper @@ -93,7 +97,7 @@ class Helper(Referenceable, service.MultiService): kwargs['facility'] = "helper" return self.parent.log(msg, **kwargs) - def remote_upload(self, storage_index): + def remote_upload_chk(self, storage_index): # TODO: look on disk if storage_index in self._active_uploads: uh = self._active_uploads[storage_index] diff --git a/src/allmydata/test/test_helper.py b/src/allmydata/test/test_helper.py index a5d50771..18dfd99d 100644 --- a/src/allmydata/test/test_helper.py +++ b/src/allmydata/test/test_helper.py @@ -8,12 +8,20 @@ from foolscap.logging import log from allmydata import upload, offloaded from allmydata.util import hashutil -class FakeCHKUploadHelper(offloaded.CHKUploadHelper): - def remote_upload(self, reader): - return {'uri_extension_hash': hashutil.uri_extension_hash("")} +class CHKUploadHelper_fake(offloaded.CHKUploadHelper): + def start_encrypted(self, eu): + needed_shares, happy, total_shares = self._encoding_parameters + d = eu.get_size() + def _got_size(size): + return (hashutil.uri_extension_hash(""), + needed_shares, total_shares, size) + d.addCallback(_got_size) + return d -class FakeHelper(offloaded.Helper): - chk_upload_helper_class = FakeCHKUploadHelper +class CHKUploadHelper_already_uploaded(offloaded.CHKUploadHelper): + def start(self): + res = {'uri_extension_hash': hashutil.uri_extension_hash("")} + return (res, None) class FakeClient(service.MultiService): def log(self, msg, **kwargs): @@ -42,8 +50,8 @@ class AssistedUpload(unittest.TestCase): # bogus host/port t.setLocation("bogus:1234") - - h = FakeHelper(".") + self.helper = h = offloaded.Helper(".") + h.chk_upload_helper_class = CHKUploadHelper_fake h.setServiceParent(self.s) self.helper_furl = t.registerReference(h) @@ -75,3 +83,26 @@ class AssistedUpload(unittest.TestCase): return d + + def test_already_uploaded(self): + self.helper.chk_upload_helper_class = CHKUploadHelper_already_uploaded + u = upload.Uploader(self.helper_furl) + u.setServiceParent(self.s) + + # wait a few turns + d = eventual.fireEventually() + d.addCallback(eventual.fireEventually) + d.addCallback(eventual.fireEventually) + + def _ready(res): + assert u._helper + + DATA = "I need help\n" * 1000 + return u.upload_data(DATA) + d.addCallback(_ready) + def _uploaded(uri): + assert "CHK" in uri + d.addCallback(_uploaded) + + return d + diff --git a/src/allmydata/upload.py b/src/allmydata/upload.py index 5c3a491d..30789478 100644 --- a/src/allmydata/upload.py +++ b/src/allmydata/upload.py @@ -611,16 +611,19 @@ class AssistedUploader: self._storage_index = storage_index def _contact_helper(self, res): - d = self._helper.callRemote("upload", self._storage_index) + self.log("contacting helper..") + d = self._helper.callRemote("upload_chk", self._storage_index) d.addCallback(self._contacted_helper) return d def _contacted_helper(self, (upload_results, upload_helper)): if upload_helper: + self.log("helper says we need to upload") # we need to upload the file reu = RemoteEncryptedUploabable(self._encuploadable) d = upload_helper.callRemote("upload", reu) # this Deferred will fire with the upload results return d + self.log("helper says file is already uploaded") return upload_results def _build_readcap(self, upload_results): -- 2.45.2