From 7d7ce7a05927e690a1a5c06445b4ba0e340bdb62 Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Fri, 30 Mar 2007 11:32:57 -0700 Subject: [PATCH] test_encode: make sure encode_new can produce the data it is supposed to --- src/allmydata/encode_new.py | 20 ++++----- src/allmydata/test/test_encode.py | 70 +++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 11 deletions(-) diff --git a/src/allmydata/encode_new.py b/src/allmydata/encode_new.py index 6500a3bf..2105d260 100644 --- a/src/allmydata/encode_new.py +++ b/src/allmydata/encode_new.py @@ -86,18 +86,22 @@ class Encoder(object): self.required_shares = 25 self.segment_size = min(2*MiB, self.file_size) - self.num_segments = mathutil.div_ceil(self.file_size, self.segment_size) def get_reservation_size(self): - self.num_shares = 100 - self.share_size = mathutil.div_ceil(self.file_size, self.required_shares) + share_size = mathutil.div_ceil(self.file_size, self.required_shares) overhead = self.compute_overhead() - return self.share_size + overhead + return share_size + overhead + def compute_overhead(self): + return 0 def set_shareholders(self, landlords): self.landlords = landlords.copy() def start(self): + self.num_segments = mathutil.div_ceil(self.file_size, + self.segment_size) + self.share_size = mathutil.div_ceil(self.file_size, + self.required_shares) self.setup_encryption() self.setup_encoder() d = defer.succeed(None) @@ -167,10 +171,7 @@ class Encoder(object): return defer.DeferredList(dl) def send_subshare(self, shareid, segment_num, subshare): - #if False: - # offset = hash_size + segment_num * segment_size - # return self.send(shareid, "write", subshare, offset) - return self.send(shareid, "put_subshare", segment_num, subshare) + return self.send(shareid, "put_block", segment_num, subshare) def send(self, shareid, methname, *args, **kwargs): ll = self.landlords[shareid] @@ -191,9 +192,6 @@ class Encoder(object): # all_hashes[1] is the left child, == hash(ah[3]+ah[4]) # all_hashes[n] == hash(all_hashes[2*n+1] + all_hashes[2*n+2]) self.share_root_hashes[shareid] = t[0] - if False: - block = "".join(all_hashes) - return self.send(shareid, "write", block, offset=0) return self.send(shareid, "put_block_hashes", all_hashes) def send_all_share_hash_trees(self): diff --git a/src/allmydata/test/test_encode.py b/src/allmydata/test/test_encode.py index c220ba9a..aa176434 100644 --- a/src/allmydata/test/test_encode.py +++ b/src/allmydata/test/test_encode.py @@ -21,3 +21,73 @@ class Encode(unittest.TestCase): d = e.start() return d +class FakePeer: + def __init__(self): + self.blocks = {} + self.blockhashes = None + self.sharehashes = None + self.closed = False + + def callRemote(self, methname, *args, **kwargs): + def _call(): + meth = getattr(self, methname) + return meth(*args, **kwargs) + return defer.maybeDeferred(_call) + + def put_block(self, segmentnum, data): + assert not self.closed + assert segmentnum not in self.blocks + self.blocks[segmentnum] = data + + def put_block_hashes(self, blockhashes): + assert not self.closed + assert self.blockhashes is None + self.blockhashes = blockhashes + + def put_share_hashes(self, sharehashes): + assert not self.closed + assert self.sharehashes is None + self.sharehashes = sharehashes + + def close(self): + assert not self.closed + self.closed = True + + +class UpDown(unittest.TestCase): + def test_send(self): + e = encode_new.Encoder() + data = "happy happy joy joy" * 4 + e.setup(StringIO(data)) + NUM_SHARES = 100 + assert e.num_shares == NUM_SHARES # else we'll be completely confused + e.segment_size = 25 # force use of multiple segments + NUM_SEGMENTS = 4 + assert (NUM_SEGMENTS-1)*e.segment_size < len(data) <= NUM_SEGMENTS*e.segment_size + shareholders = {} + all_shareholders = [] + for shnum in range(NUM_SHARES): + peer = FakePeer() + shareholders[shnum] = peer + all_shareholders.append(peer) + e.set_shareholders(shareholders) + d = e.start() + def _check(roothash): + self.failUnless(isinstance(roothash, str)) + self.failUnlessEqual(len(roothash), 32) + for i,peer in enumerate(all_shareholders): + self.failUnless(peer.closed) + self.failUnlessEqual(len(peer.blocks), NUM_SEGMENTS) + #self.failUnlessEqual(len(peer.blockhashes), NUM_SEGMENTS) + # that isn't true: each peer gets a full tree, so it's more + # like 2n-1 but with rounding to a power of two + for h in peer.blockhashes: + self.failUnlessEqual(len(h), 32) + #self.failUnlessEqual(len(peer.sharehashes), NUM_SHARES) + # that isn't true: each peer only gets the chain they need + for (hashnum, h) in peer.sharehashes: + self.failUnless(isinstance(hashnum, int)) + self.failUnlessEqual(len(h), 32) + d.addCallback(_check) + + return d -- 2.45.2