From e3a57fca98a5d650ebff4a8fd41b264e31ce65f1 Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Thu, 19 Jul 2007 22:53:29 -0700 Subject: [PATCH] upload: finish refactoring, all unit tests pass now --- src/allmydata/test/test_encode.py | 26 ++++++---------- src/allmydata/test/test_upload.py | 51 +++++++++++++++++++++++++++++++ src/allmydata/upload.py | 6 ++-- 3 files changed, 65 insertions(+), 18 deletions(-) diff --git a/src/allmydata/test/test_encode.py b/src/allmydata/test/test_encode.py index a70bfd6e..ca91614e 100644 --- a/src/allmydata/test/test_encode.py +++ b/src/allmydata/test/test_encode.py @@ -3,7 +3,7 @@ from zope.interface import implements from twisted.trial import unittest from twisted.internet import defer from twisted.python.failure import Failure -from allmydata import encode, download, hashtree +from allmydata import encode, upload, download, hashtree from allmydata.util import hashutil from allmydata.uri import pack_uri from allmydata.Crypto.Cipher import AES @@ -154,10 +154,12 @@ class Encode(unittest.TestCase): # force use of multiple segments options = {"max_segment_size": max_segment_size} e = encode.Encoder(options) + e.set_size(datalen) + e.set_uploadable(upload.Data(data)) nonkey = "\x00" * 16 - e.setup(StringIO(data), nonkey) + e.set_encryption_key(nonkey) + e.setup() assert e.num_shares == NUM_SHARES # else we'll be completely confused - e.setup_codec() # need to rebuild the codec for that change assert (NUM_SEGMENTS-1)*e.segment_size < len(data) <= NUM_SEGMENTS*e.segment_size shareholders = {} all_shareholders = [] @@ -279,11 +281,12 @@ class Roundtrip(unittest.TestCase): options = {"max_segment_size": max_segment_size, "needed_and_happy_and_total_shares": k_and_happy_and_n} e = encode.Encoder(options) + e.set_size(len(data)) + e.set_uploadable(upload.Data(data)) nonkey = "\x00" * 16 - e.setup(StringIO(data), nonkey) - + e.set_encryption_key(nonkey) + e.setup() assert e.num_shares == NUM_SHARES # else we'll be completely confused - e.setup_codec() # need to rebuild the codec for that change shareholders = {} all_peers = [] @@ -292,16 +295,7 @@ class Roundtrip(unittest.TestCase): peer = FakeBucketWriterProxy(mode) shareholders[shnum] = peer e.set_shareholders(shareholders) - plaintext_hasher = hashutil.plaintext_hasher() - plaintext_hasher.update(data) - cryptor = AES.new(key=nonkey, mode=AES.MODE_CTR, - counterstart="\x00"*16) - crypttext_hasher = hashutil.crypttext_hasher() - crypttext_hasher.update(cryptor.encrypt(data)) - - e.set_uri_extension_data({'crypttext_hash': crypttext_hasher.digest(), - 'plaintext_hash': plaintext_hasher.digest(), - }) + d = e.start() def _sent(uri_extension_hash): return (uri_extension_hash, e, shareholders) diff --git a/src/allmydata/test/test_upload.py b/src/allmydata/test/test_upload.py index 050faa67..7b8b4bec 100644 --- a/src/allmydata/test/test_upload.py +++ b/src/allmydata/test/test_upload.py @@ -1,4 +1,5 @@ +import os from twisted.trial import unittest from twisted.python.failure import Failure from twisted.internet import defer @@ -9,6 +10,56 @@ from allmydata.uri import unpack_uri, unpack_lit from allmydata.util.assertutil import precondition from foolscap import eventual +class Uploadable(unittest.TestCase): + def shouldEqual(self, data, expected): + self.failUnless(isinstance(data, list)) + for e in data: + self.failUnless(isinstance(e, str)) + s = "".join(data) + self.failUnlessEqual(s, expected) + + def test_filehandle(self): + s = StringIO("a"*41) + u = upload.FileHandle(s) + d = u.get_size() + d.addCallback(self.failUnlessEqual, 41) + d.addCallback(lambda res: u.read(1)) + d.addCallback(self.shouldEqual, "a") + d.addCallback(lambda res: u.read(80)) + d.addCallback(self.shouldEqual, "a"*40) + d.addCallback(lambda res: u.close()) # this doesn't close the filehandle + d.addCallback(lambda res: s.close()) # that privilege is reserved for us + return d + + def test_filename(self): + basedir = "upload/Uploadable/test_filename" + os.makedirs(basedir) + fn = os.path.join(basedir, "file") + f = open(fn, "w") + f.write("a"*41) + f.close() + u = upload.FileName(fn) + d = u.get_size() + d.addCallback(self.failUnlessEqual, 41) + d.addCallback(lambda res: u.read(1)) + d.addCallback(self.shouldEqual, "a") + d.addCallback(lambda res: u.read(80)) + d.addCallback(self.shouldEqual, "a"*40) + d.addCallback(lambda res: u.close()) + return d + + def test_data(self): + s = "a"*41 + u = upload.Data(s) + d = u.get_size() + d.addCallback(self.failUnlessEqual, 41) + d.addCallback(lambda res: u.read(1)) + d.addCallback(self.shouldEqual, "a") + d.addCallback(lambda res: u.read(80)) + d.addCallback(self.shouldEqual, "a"*40) + d.addCallback(lambda res: u.close()) + return d + class FakePeer: def __init__(self, mode="good"): self.ss = FakeStorageServer(mode) diff --git a/src/allmydata/upload.py b/src/allmydata/upload.py index af37309d..8905da12 100644 --- a/src/allmydata/upload.py +++ b/src/allmydata/upload.py @@ -330,9 +330,11 @@ class CHKUploader: ) def read_this_many_bytes(uploadable, size, prepend_data=[]): + if size == 0: + return defer.succeed([]) d = uploadable.read(size) def _got(data): - assert isinstance(list) + assert isinstance(data, list) bytes = sum([len(piece) for piece in data]) assert bytes > 0 assert bytes <= size @@ -443,7 +445,7 @@ class Uploader(service.MultiService): uploader_class = self.uploader_class if size <= self.URI_LIT_SIZE_THRESHOLD: uploader_class = LiteralUploader - uploader = self.uploader_class(self.parent, uploadable, options) + uploader = uploader_class(self.parent, uploadable, options) uploader.set_params(self.parent.get_encoding_parameters() or self.DEFAULT_ENCODING_PARAMETERS) return uploader.start() -- 2.45.2