From e4463056f333243de107ce3aa2e0eef404ec445f Mon Sep 17 00:00:00 2001 From: Zooko O'Whielacronx Date: Tue, 27 Mar 2007 22:57:15 -0700 Subject: [PATCH] fix test_codec and test_upload to handle current API --- src/allmydata/codec.py | 12 +++++++++--- src/allmydata/test/test_codec.py | 16 ++++++++-------- src/allmydata/test/test_upload.py | 2 +- src/allmydata/upload.py | 16 +++++++++++++--- 4 files changed, 31 insertions(+), 15 deletions(-) diff --git a/src/allmydata/codec.py b/src/allmydata/codec.py index 0d875ebd..910014f9 100644 --- a/src/allmydata/codec.py +++ b/src/allmydata/codec.py @@ -28,7 +28,7 @@ class ReplicatingEncoder(object): ENCODER_TYPE = "rep" def set_params(self, data_size, required_shares, max_shares): - assert required_shares % data_size == 0 + assert data_size % required_shares == 0 assert required_shares <= max_shares self.data_size = data_size self.required_shares = required_shares @@ -44,7 +44,7 @@ class ReplicatingEncoder(object): return self.data_size def encode(self, inshares, desired_shareids=None): - assert isinstance(data, list) + assert isinstance(inshares, list) for inshare in inshares: assert isinstance(inshare, str) assert self.required_shares * len(inshare) == self.data_size @@ -66,7 +66,13 @@ class ReplicatingDecoder(object): def decode(self, some_shares, their_shareids): assert len(some_shares) == self.required_shares assert len(some_shares) == len(their_shareids) - return defer.succeed([some_shares[0]]) + data = some_shares[0] + + chunksize = mathutil.div_ceil(len(data), self.required_shares) + numchunks = mathutil.div_ceil(len(data), chunksize) + l = [ data[i:i+chunksize] for i in range(0, len(data), chunksize) ] + + return defer.succeed(l) class Encoder(object): diff --git a/src/allmydata/test/test_codec.py b/src/allmydata/test/test_codec.py index 59014b06..c9fe13ef 100644 --- a/src/allmydata/test/test_codec.py +++ b/src/allmydata/test/test_codec.py @@ -35,8 +35,14 @@ class Tester: return d1 def _check_data(decoded_shares): + self.failUnlessEqual(len(''.join(decoded_shares)), len(''.join(data0s))) self.failUnlessEqual(len(decoded_shares), len(data0s)) - self.failUnless(tuple(decoded_shares) == tuple(data0s)) + for (i, (x, y)) in enumerate(zip(data0s, decoded_shares)): + self.failUnlessEqual(x, y, "%s: %r != %r.... first share was %r" % (str(i), x, y, data0s[0],)) + self.failUnless(''.join(decoded_shares) == ''.join(data0s), "%s" % ("???",)) + # 0data0sclipped = tuple(data0s) + # data0sclipped[-1] = + # self.failUnless(tuple(decoded_shares) == tuple(data0s)) def _decode_some(res): log.msg("_decode_some") @@ -85,13 +91,7 @@ class Tester: return self.do_test(8, 8, 16) def test_encode2(self): - return self.do_test(123, 25, 100, 90) - - def test_sizes(self): - d = defer.succeed(None) - for i in range(1, 100): - d.addCallback(lambda res,size: self.do_test(size, 4, 10), i) - return d + return self.do_test(125, 25, 100, 90) class Replicating(unittest.TestCase, Tester): enc_class = ReplicatingEncoder diff --git a/src/allmydata/test/test_upload.py b/src/allmydata/test/test_upload.py index c2e92f63..a17c3712 100644 --- a/src/allmydata/test/test_upload.py +++ b/src/allmydata/test/test_upload.py @@ -39,7 +39,7 @@ class FakePeer: def _callRemote(self, methname, **kwargs): assert methname == "allocate_bucket" - assert kwargs["size"] == 100 + #assert kwargs["size"] == 100 assert kwargs["leaser"] == "fakeclient" if self.response == "good": return self diff --git a/src/allmydata/upload.py b/src/allmydata/upload.py index 23da1bc6..234d6fee 100644 --- a/src/allmydata/upload.py +++ b/src/allmydata/upload.py @@ -5,7 +5,7 @@ from twisted.internet import defer from twisted.application import service from foolscap import Referenceable -from allmydata.util import idlib, bencode +from allmydata.util import idlib, bencode, mathutil from allmydata.util.idlib import peerid_to_short_string as shortid from allmydata.util.deferredutil import DeferredListShouldSucceed from allmydata import codec @@ -70,7 +70,9 @@ class FileUploader: needed_shares = self.min_shares self._encoder = codec.ReplicatingEncoder() self._codec_name = self._encoder.get_encoder_type() - self._encoder.set_params(self._size, needed_shares, total_shares) + self._needed_shares = needed_shares + paddedsize = self._size + mathutil.pad_size(self._size, needed_shares) + self._encoder.set_params(paddedsize, needed_shares, total_shares) self._share_size = self._encoder.get_share_size() # first step: who should we upload to? @@ -216,7 +218,15 @@ class FileUploader: assert sorted(self.sharemap.keys()) == range(len(landlords)) # encode all the data at once: this class does not use segmentation data = self._filehandle.read() - d = self._encoder.encode(data, self.sharemap.keys()) + + # xyz i am about to go away anyway. + chunksize = mathutil.div_ceil(len(data), self._needed_shares) + numchunks = mathutil.div_ceil(len(data), chunksize) + l = [ data[i:i+chunksize] for i in range(0, len(data), chunksize) ] + # padding + if len(l[-1]) != len(l[0]): + l[-1] = l[-1] + ('\x00'*(len(l[0])-len(l[-1]))) + d = self._encoder.encode(l, self.sharemap.keys()) d.addCallback(self._send_all_shares) d.addCallback(lambda res: self._encoder.get_serialized_params()) return d -- 2.45.2