From: zooko Date: Mon, 1 Oct 2007 03:09:45 +0000 (+0530) Subject: zfec: loosen the interface of the cmdline tool to allow m < 3, and to allow k ==... X-Git-Url: https://git.rkrishnan.org/pf/content/en/seg/module-simplejson.decoder.html?a=commitdiff_plain;h=e0624ee3efff27240a8bc34804cbc6e64ba00d26;p=tahoe-lafs%2Fzfec.git zfec: loosen the interface of the cmdline tool to allow m < 3, and to allow k == 1 and to allow k == m ; WARNING: this breaks data-compatibility with shares produced by earlier versions of zfec! darcs-hash:27c79434d568611d832c31a345377bf97d143105 --- diff --git a/zfec/zfec/cmdline_zfec.py b/zfec/zfec/cmdline_zfec.py index b62d826..accc6b4 100755 --- a/zfec/zfec/cmdline_zfec.py +++ b/zfec/zfec/cmdline_zfec.py @@ -28,6 +28,7 @@ def main(): parser.add_argument('-k', '--requiredshares', help='the number of share files required to reconstruct (default 4)', default=4, type=int, metavar='K') parser.add_argument('-f', '--force', help='overwrite any file which already in place an output file (share file)', action='store_true') parser.add_argument('-v', '--verbose', help='print out messages about progress', action='store_true') + parser.add_argument('-q', '--quiet', help='quiet progress indications and warnings about silly choices of K and M', action='store_true') parser.add_argument('-V', '--version', help='print out version number and exit', action='store_true') args = parser.parse_args() @@ -36,19 +37,23 @@ def main(): if args.prefix == "": args.prefix = "" - if args.totalshares < 3: - print "Invalid parameters, totalshares is required to be >= 3\nPlease see the accompanying documentation." + if args.verbose and args.quiet: + print "Please choose only one of --verbose and --quiet." sys.exit(1) - if args.totalshares > 256: - print "Invalid parameters, totalshares is required to be <= 256\nPlease see the accompanying documentation." + + if args.totalshares > 256 or args.totalshares < 1: + print "Invalid parameters, totalshares is required to be <= 256 and >= 1\nPlease see the accompanying documentation." sys.exit(1) - if args.requiredshares < 2: - print "Invalid parameters, requiredshares is required to be >= 2\nPlease see the accompanying documentation." - sys.exit(1) - if args.requiredshares >= args.totalshares: - print "Invalid parameters, requiredshares is required to be < totalshares\nPlease see the accompanying documentation." + if args.requiredshares > args.totalshares or args.requiredshares < 1: + print "Invalid parameters, requiredshares is required to be <= totalshares and >= 1\nPlease see the accompanying documentation." sys.exit(1) + if not args.quiet: + if args.requiredshares == 1: + print "warning: silly parameters: requiredshares == 1, which means that every share will be a complete copy of the file. You could use \"cp\" for the same effect. But proceeding to do it anyway..." + if args.requiredshares == args.totalshares: + print "warning: silly parameters: requiredshares == totalshares, which means that all shares will be required in order to reconstruct the file. You could use \"split\" for the same effect. But proceeding to do it anyway..." + args.inputfile.seek(0, 2) fsize = args.inputfile.tell() args.inputfile.seek(0, 0) diff --git a/zfec/zfec/filefec.py b/zfec/zfec/filefec.py index 30985a9..35d64c5 100644 --- a/zfec/zfec/filefec.py +++ b/zfec/zfec/filefec.py @@ -23,17 +23,17 @@ class CorruptedShareFilesError(zfec.Error): def _build_header(m, k, pad, sh): """ - @param m: the total number of shares; 3 <= m <= 256 - @param k: the number of shares required to reconstruct; 2 <= k < m + @param m: the total number of shares; 1 <= m <= 256 + @param k: the number of shares required to reconstruct; 1 <= k <= m @param pad: the number of bytes of padding added to the file before encoding; 0 <= pad < k @param sh: the shnum of this share; 0 <= k < m @return: a string (which is hopefully short) encoding m, k, sh, and pad """ - assert m >= 3 + assert m >= 1 assert m <= 2**8 - assert k >= 2 - assert k < m + assert k >= 1 + assert k <= m assert pad >= 0 assert pad < k @@ -43,14 +43,14 @@ def _build_header(m, k, pad, sh): bitsused = 0 val = 0 - val |= (m - 3) + val |= (m - 1) bitsused += 8 # the first 8 bits always encode m - kbits = log_ceil(m-2, 2) # num bits needed to store all possible values of k + kbits = log_ceil(m, 2) # num bits needed to store all possible values of k val <<= kbits bitsused += kbits - val |= (k - 2) + val |= (k - 1) padbits = log_ceil(k, 2) # num bits needed to store all possible values of pad val <<= padbits @@ -64,8 +64,8 @@ def _build_header(m, k, pad, sh): val |= sh - assert bitsused >= 11 - assert bitsused <= 32 + assert bitsused >= 8, bitsused + assert bitsused <= 32, bitsused if bitsused <= 16: val <<= (16-bitsused) @@ -98,17 +98,17 @@ def _parse_header(inf): if not ch: raise CorruptedShareFilesError("Share files were corrupted -- share file %r didn't have a complete metadata header at the front. Perhaps the file was truncated." % (inf.name,)) byte = ord(ch) - m = byte + 3 + m = byte + 1 # The next few bits encode k. - kbits = log_ceil(m-2, 2) # num bits needed to store all possible values of k + kbits = log_ceil(m, 2) # num bits needed to store all possible values of k b2_bits_left = 8-kbits kbitmask = MASK(kbits) << b2_bits_left ch = inf.read(1) if not ch: raise CorruptedShareFilesError("Share files were corrupted -- share file %r didn't have a complete metadata header at the front. Perhaps the file was truncated." % (inf.name,)) byte = ord(ch) - k = ((byte & kbitmask) >> b2_bits_left) + 2 + k = ((byte & kbitmask) >> b2_bits_left) + 1 shbits = log_ceil(m, 2) # num bits needed to store all possible values of shnum padbits = log_ceil(k, 2) # num bits needed to store all possible values of pad diff --git a/zfec/zfec/test/test_zfec.py b/zfec/zfec/test/test_zfec.py index d87d91d..150b7d9 100755 --- a/zfec/zfec/test/test_zfec.py +++ b/zfec/zfec/test/test_zfec.py @@ -98,8 +98,8 @@ class ZFec(unittest.TestCase): class FileFec(unittest.TestCase): def test_filefec_header(self): - for m in [3, 5, 7, 9, 11, 17, 19, 33, 35, 65, 66, 67, 129, 130, 131, 254, 255, 256,]: - for k in [2, 3, 5, 9, 17, 33, 65, 129, 255,]: + for m in [1, 2, 3, 5, 7, 9, 11, 17, 19, 33, 35, 65, 66, 67, 129, 130, 131, 254, 255, 256,]: + for k in [1, 2, 3, 5, 9, 17, 33, 65, 129, 255, 256,]: if k >= m: continue for pad in [0, 1, k-1,]: