From ea24864544499a685c00d1362e9eedc26f27df1b Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Wed, 9 Jan 2008 17:58:47 -0700 Subject: [PATCH] offloaded: more code, fix pyflakes problems, change IEncryptedUploader a bit --- src/allmydata/encode.py | 12 +++++++----- src/allmydata/interfaces.py | 14 +++++++++----- src/allmydata/offloaded.py | 2 +- src/allmydata/upload.py | 38 +++++++++++++++++++++++++++++++------ 4 files changed, 49 insertions(+), 17 deletions(-) diff --git a/src/allmydata/encode.py b/src/allmydata/encode.py index 1aeb93c4..3f51fae1 100644 --- a/src/allmydata/encode.py +++ b/src/allmydata/encode.py @@ -461,12 +461,14 @@ class Encoder(object): d = u.get_plaintext_hash() def _got(plaintext_hash): self.uri_extension_data["plaintext_hash"] = plaintext_hash - return u.get_plaintext_segment_hashtree_nodes(self.num_segments) + return u.get_plaintext_hashtree_leaves(0, self.num_segments, + self.num_segments) d.addCallback(_got) - def _got_hashtree_nodes(t): - self.uri_extension_data["plaintext_root_hash"] = t[0] - self._plaintext_hashtree_nodes = t - d.addCallback(_got_hashtree_nodes) + def _got_hashtree_leaves(leaves): + ht = list(HashTree(list(leaves))) + self.uri_extension_data["plaintext_root_hash"] = ht[0] + self._plaintext_hashtree_nodes = ht + d.addCallback(_got_hashtree_leaves) return d def send_plaintext_hash_tree_to_all_shareholders(self): diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 48de0abe..61b2f07a 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -995,17 +995,21 @@ class IEncryptedUploadable(Interface): instead of plaintext. set_segment_size() must be called before the first call to read_encrypted().""" - def get_plaintext_segment_hashtree_nodes(num_segments): - """Get the nodes of a merkle hash tree over the plaintext segments. + def get_plaintext_hashtree_leaves(first, last, num_segments): + """Get the leaf nodes of a merkle hash tree over the plaintext + segments, i.e. get the tagged hashes of the given segments. - This returns a Deferred which fires with a sequence of hashes. Each - hash is a node of a merkle hash tree, generally obtained from:: + This returns a Deferred which fires with a sequence of hashes, using: - tuple(HashTree(segment_hashes)) + tuple(segment_hashes[first:last]) 'num_segments' is used to assert that the number of segments that the IEncryptedUploadable handled matches the number of segments that the encoder was expecting. + + This method must not be called until the final byte has been read + from read_encrypted(). Once this method is called, read_encrypted() + can never be called again. """ def get_plaintext_hash(): diff --git a/src/allmydata/offloaded.py b/src/allmydata/offloaded.py index 4bb3758b..d23cb492 100644 --- a/src/allmydata/offloaded.py +++ b/src/allmydata/offloaded.py @@ -1,6 +1,6 @@ from foolscap import RemoteInterface -from foolscap.schema import DictOf, ChoiceOf +from foolscap.schema import DictOf, ChoiceOf, ListOf from allmydata.interfaces import StorageIndex, Hash UploadResults = DictOf(str, str) diff --git a/src/allmydata/upload.py b/src/allmydata/upload.py index a997a68f..63e98a11 100644 --- a/src/allmydata/upload.py +++ b/src/allmydata/upload.py @@ -14,6 +14,7 @@ from allmydata import encode, storage, hashtree, uri from allmydata.util import idlib, mathutil from allmydata.util.assertutil import precondition from allmydata.interfaces import IUploadable, IUploader, IEncryptedUploadable +from allmydata.offloaded import RIEncryptedUploadable from pycryptopp.cipher.aes import AES from cStringIO import StringIO @@ -399,7 +400,7 @@ class EncryptAnUploadable: d.addCallback(_got) return d - def get_plaintext_segment_hashtree_nodes(self, num_segments): + def get_plaintext_hashtree_leaves(self, first, last, num_segments): if len(self._plaintext_segment_hashes) < num_segments: # close out the last one assert len(self._plaintext_segment_hashes) == num_segments-1 @@ -407,8 +408,7 @@ class EncryptAnUploadable: self._plaintext_segment_hashes.append(p.digest()) del self._plaintext_segment_hasher assert len(self._plaintext_segment_hashes) == num_segments - ht = hashtree.HashTree(self._plaintext_segment_hashes) - return defer.succeed(list(ht)) + return defer.succeed(tuple(self._plaintext_segment_hashes[first:last])) def get_plaintext_hash(self): h = self._plaintext_hasher.digest() @@ -551,7 +551,32 @@ class LiteralUploader: def close(self): pass -class AssistedUploader(FileUploader): +class RemoteEncryptedUploabable(Referenceable): + implements(RIEncryptedUploadable) + + def __init__(self, encrypted_uploadable): + self._eu = IEncryptedUploadable(encrypted_uploadable) + self._offset = 0 + + def remote_get_size(self): + return self._eu.get_size() + def remote_set_segment_size(self, segment_size): + self._eu.set_segment_size(segment_size) + def remote_read_encrypted(self, offset, length): + assert offset == self._offset # we don't yet implement seek + d = self._eu.read_encrypted(length) + def _read(data): + self._offset += len(data) + return data + d.addCallback(_read) + return d + def remote_get_plaintext_hashtree_leaves(self, first, last): + return self._eu.get_plaintext_hashtree_leaves(first, last) + def remote_get_plaintext_hash(self): + return self._eu.get_plaintext_hash() + + +class AssistedUploader: def __init__(self, helper, options={}): self._helper = helper @@ -562,8 +587,8 @@ class AssistedUploader(FileUploader): pass def start(self, uploadable): - uploadable = IUploadable(uploadable) - eu = IEncryptedUploadable(EncryptAnUploadable(uploadable)) + u = IUploadable(uploadable) + eu = IEncryptedUploadable(EncryptAnUploadable(u)) self._encuploadable = eu d = eu.get_size() d.addCallback(self._got_size) @@ -699,6 +724,7 @@ class Uploader(service.MultiService): def __init__(self, helper_furl=None): self._helper_furl = helper_furl self._helper = None + service.MultiService.__init__(self) def startService(self): service.MultiService.startService(self) -- 2.45.2