filefec: Bug fix for the files with certain lengths (see the longer explanation)

Bodecs Bela (bodecsb@vivanet.hu) reported cases where files with
certain sizes, when encoded and then decoded produces slightly longer
files with some extra padded bytes. To reproduce the bug, the input file
sizes needs to be any length in the range:

 (n * 4096 - (k - 1)) < length <= (n * 4096)

Thanks to Bodecs and jg71 for bringing it to my attention.
(https://tahoe-lafs.org/pipermail/tahoe-dev/2013-September/008699.html)

diff --git a/zfec/filefec.py b/zfec/filefec.py
index 48f3d9d790edac6b15b9cf433f99d2af9b1d274f..c79141c71da10ebd2e1af10f81d37a88f8164dc6 100644 (file)
--- a/zfec/filefec.py
+++ b/zfec/filefec.py
@@ -271,8 +271,7 @@ def decode_from_files(outf, infiles, verbose=False):
         if [ch for ch in chunks if len(ch) != len(chunks[-1])]:
             raise CorruptedShareFilesError("Share files were corrupted -- all share files are required to be the same length, but they weren't.")
 
-        if len(chunks[-1]) == CHUNKSIZE:
-            # Then this was a full read, so we're still in the sharefiles.
+        if len(chunks[-1]) > 0:
             resultdata = dec.decode(chunks, shnums, padlen=0)
             outf.write(resultdata)
             byteswritten += len(resultdata)
@@ -280,9 +279,8 @@ def decode_from_files(outf, infiles, verbose=False):
                 if ((byteswritten - len(resultdata)) / (10*MILLION_BYTES)) != (byteswritten / (10*MILLION_BYTES)):
                     print str(byteswritten / MILLION_BYTES) + " MB ...",
         else:
-            # Then this was a short read, so we've reached the end of the sharefiles.
-            resultdata = dec.decode(chunks, shnums, padlen)
-            outf.write(resultdata)
+            if padlen > 0:
+                outf.truncate(byteswritten - padlen)
             return # Done.
     if verbose:
         print

diff --git a/zfec/test/test_zfec.py b/zfec/test/test_zfec.py
index 46bbb1cb6bbdeaf8c12a2982a5c326ceafcf02f4..f2b29d4368990b1aaf8ea90e1c7f25c455fb7209 100755 (executable)
--- a/zfec/test/test_zfec.py
+++ b/zfec/test/test_zfec.py
@@ -321,6 +321,8 @@ class FileFec(unittest.TestCase):
     def test_filefec_min_shares_with_crcrlflf(self, noisy=VERBOSE):
         return self._help_test_filefec("Yellow Whirled!A\r\r\n\n", 3, 8, numshs=3)
 
+    def test_filefec_mul_chunk_size(self):
+        return self._help_test_filefec(randstr(6176761), 13, 16)
 
 class Cmdline(unittest.TestCase):
     def test_basic(self, noisy=VERBOSE):