From: Brian Warner Date: Fri, 11 Jan 2008 12:42:55 +0000 (-0700) Subject: offloaded: add a system test, make it pass. files are now being uploaded through... X-Git-Url: https://git.rkrishnan.org/%5B/%5D%20/uri/flags/reliability?a=commitdiff_plain;h=964edadf44eb878b45ad2f6c9e6cec0a4c3ccb8a;p=tahoe-lafs%2Ftahoe-lafs.git offloaded: add a system test, make it pass. files are now being uploaded through the helper. --- diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 7ef87008..525c719d 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -151,13 +151,13 @@ class Client(node.Node, Referenceable, testutil.PollMixin): return h = Helper(os.path.join(self.basedir, "helper")) h.setServiceParent(self) - helper_furl = self.tub.registerReference(h) # TODO: this is confusing. BASEDIR/private/helper.furl is created by # the helper. BASEDIR/helper.furl is consumed by the client who wants # to use the helper. I like having the filename be the same, since # that makes 'cp' work smoothly, but the difference between config # inputs and generated outputs is hard to see. - self.write_private_config("helper.furl", helper_furl + "\n") + helper_furlfile = os.path.join(self.basedir, "private", "helper.furl") + self.tub.registerReference(h, furlFile=helper_furlfile) def remote_get_versions(self): return str(allmydata.__version__), str(self.OLDEST_SUPPORTED_VERSION) diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index b37e4240..403adf21 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -1269,9 +1269,9 @@ class RIEncryptedUploadable(RemoteInterface): return None def read_encrypted(offset=long, length=long): - return str + return ListOf(str) - def get_plaintext_hashtree_leaves(first=int, last=int): + def get_plaintext_hashtree_leaves(first=int, last=int, num_segments=int): return ListOf(Hash) def get_plaintext_hash(): diff --git a/src/allmydata/offloaded.py b/src/allmydata/offloaded.py index 4f04b8d4..3028c82e 100644 --- a/src/allmydata/offloaded.py +++ b/src/allmydata/offloaded.py @@ -55,6 +55,7 @@ class CiphertextReader: def __init__(self, remote_reader, storage_index): self.rr = remote_reader self.storage_index = storage_index + self._offset = 0 def get_size(self): return self.rr.callRemote("get_size") @@ -65,7 +66,12 @@ class CiphertextReader: def set_serialized_encoding_parameters(self, params): pass # ?? def read_encrypted(self, length): - return self.rr.callRemote("read_encrypted", length) + d = self.rr.callRemote("read_encrypted", self._offset, length) + def _done(strings): + self._offset += sum([len(data) for data in strings]) + return strings + d.addCallback(_done) + return d def get_plaintext_hashtree_leaves(self, first, last, num_segments): return self.rr.callRemote("get_plaintext_hashtree_leaves", first, last, num_segments) diff --git a/src/allmydata/test/test_helper.py b/src/allmydata/test/test_helper.py index 18dfd99d..6f7217a6 100644 --- a/src/allmydata/test/test_helper.py +++ b/src/allmydata/test/test_helper.py @@ -105,4 +105,3 @@ class AssistedUpload(unittest.TestCase): d.addCallback(_uploaded) return d - diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index 5389906b..49fa33b5 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -64,23 +64,40 @@ class SystemTest(testutil.SignalMixin, unittest.TestCase): q = self.introducer self.introducer_furl = q.introducer_url self.clients = [] + basedirs = [] for i in range(self.numclients): basedir = self.getdir("client%d" % i) + basedirs.append(basedir) fileutil.make_dirs(basedir) if i == 0: + # client[0] runs a webserver and a helper open(os.path.join(basedir, "webport"), "w").write("tcp:0:interface=127.0.0.1") + open(os.path.join(basedir, "run_helper"), "w").write("yes\n") if self.createprivdir: fileutil.make_dirs(os.path.join(basedir, "private")) open(os.path.join(basedir, "private", "root_dir.cap"), "w") open(os.path.join(basedir, "introducer.furl"), "w").write(self.introducer_furl) - # this starts all the clients - for i in range(self.numclients): - basedir = self.getdir("client%d" % i) - c = self.add_service(client.Client(basedir=basedir)) - self.clients.append(c) - log.msg("STARTING") - d = self.wait_for_connections() + # start client[0], wait for it's tub to be ready (at which point it + # will have registered the helper furl). + c = self.add_service(client.Client(basedir=basedirs[0])) + self.clients.append(c) + d = c.when_tub_ready() + def _ready(res): + f = open(os.path.join(basedirs[0],"private","helper.furl"), "r") + helper_furl = f.read() + f.close() + f = open(os.path.join(basedirs[3],"helper.furl"), "w") + f.write(helper_furl) + f.close() + + # this starts the rest of the clients + for i in range(1, self.numclients): + c = self.add_service(client.Client(basedir=basedirs[i])) + self.clients.append(c) + log.msg("STARTING") + return self.wait_for_connections() + d.addCallback(_ready) def _connected(res): log.msg("CONNECTED") # now find out where the web port was @@ -239,6 +256,20 @@ class SystemTest(testutil.SignalMixin, unittest.TestCase): d1.addBoth(_baduri_should_fail) return d1 d.addCallback(_download_nonexistent_uri) + + def _upload_with_helper(res): + DATA = "Data that needs help to upload" * 1000 + u = upload.Data(DATA) + d = self.clients[3].upload(u) + def _uploaded(uri): + return self.downloader.download_to_data(uri) + d.addCallback(_uploaded) + def _check(newdata): + self.failUnlessEqual(newdata, DATA) + d.addCallback(_check) + return d + d.addCallback(_upload_with_helper) + return d test_upload_and_download.timeout = 4800 diff --git a/src/allmydata/upload.py b/src/allmydata/upload.py index 30789478..9d565c36 100644 --- a/src/allmydata/upload.py +++ b/src/allmydata/upload.py @@ -555,15 +555,18 @@ class RemoteEncryptedUploabable(Referenceable): def remote_set_segment_size(self, segment_size): self._eu.set_segment_size(segment_size) def remote_read_encrypted(self, offset, length): - assert offset == self._offset # we don't yet implement seek + # we don't yet implement seek + assert offset == self._offset, "%d != %d" % (offset, self._offset) d = self._eu.read_encrypted(length) - def _read(data): - self._offset += len(data) - return data + def _read(strings): + self._offset += sum([len(data) for data in strings]) + return strings d.addCallback(_read) return d - def remote_get_plaintext_hashtree_leaves(self, first, last): - return self._eu.get_plaintext_hashtree_leaves(first, last) + def remote_get_plaintext_hashtree_leaves(self, first, last, num_segments): + d = self._eu.get_plaintext_hashtree_leaves(first, last, num_segments) + d.addCallback(list) + return d def remote_get_plaintext_hash(self): return self._eu.get_plaintext_hash()