From b0315fc549c2a91f7abd7df93e937801c0585ab0 Mon Sep 17 00:00:00 2001 From: Zooko O'Whielacronx Date: Fri, 29 Dec 2006 13:50:53 -0700 Subject: [PATCH] make encode_new use py_ecc for real live erasure coding (This patch is not tested -- I'm working on a Mac which doesn't have gcc installed... (gcc is necessary for the crypto module.) I will now attempt to connect to a better development computer over my mom's staticy, failure-prone, 14.4 K modem...) --- src/allmydata/encode_new.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/src/allmydata/encode_new.py b/src/allmydata/encode_new.py index 50a4f69b..eb6172d2 100644 --- a/src/allmydata/encode_new.py +++ b/src/allmydata/encode_new.py @@ -5,6 +5,8 @@ from twisted.internet import defer from allmydata.chunk import HashTree, roundup_pow2 from Crypto.Cipher import AES import sha +from allmydata.util import mathutil +from allmydata.util.assertutil import _assert, precondition def hash(data): return sha.new(data).digest() @@ -67,14 +69,19 @@ class Encoder(object): infile.seek(0, 2) self.file_size = infile.tell() infile.seek(0, 0) - fsize = 1.0 * self.file_size - self.segment_size = 1024 - self.num_segments = int(math.ceil(fsize / self.segment_size)) self.num_shares = 100 self.required_shares = 25 + + # The segment size needs to be an even multiple of required_shares. + # (See encode_segment().) + self.segment_size = mathutil.next_multiple(1024, self.required_shares) + self.num_segments = mathutil.div_ceil(self.file_size, self.segment_size) + self.share_size = self.file_size / self.required_shares + self.fecer = rs_code.RSCode(self.num_shares, self.required_shares) + def get_reservation_size(self): self.num_shares = 100 self.share_size = self.file_size / self.required_shares @@ -104,8 +111,16 @@ class Encoder(object): return d def encode_segment(self, crypttext): - shares = [crypttext] * self.num_shares - return shares + precondition((len(crypttext) % self.required_shares) == 0, len(crypttext), self.required_shares, len(crypttext) % self.required_shares) + subshares = [[] for x in range(self.num_shares)] + # Note string slices aren't an efficient way to use memory, so when we + # upgrade from the unusably slow py_ecc prototype to a fast ECC we + # should also fix up this memory usage (by using the array module). + for i in range(0, len(crypttext), self.required_shares): + words = self.fecer.Encode(crypttext[i:i+self.required_shares]) + for (subshare, word,) in zip(subshares, words): + subshare.append(word) + return [ ''.join(subshare) for subshare in subshares ] def do_segment(self, segnum): segment_plaintext = self.infile.read(self.segment_size) -- 2.45.2