From: Zooko O'Whielacronx zooko@zooko.com Date: Sat, 14 Apr 2007 23:00:59 +0000 (+0530) Subject: pyfec: more progress indicators, handling of already-existent outfile X-Git-Url: https://git.rkrishnan.org/vdrive/%22news.html/reliability?a=commitdiff_plain;h=927925230f696f1028d32725922eb6c7ead9b6b6;p=tahoe-lafs%2Fzfec.git pyfec: more progress indicators, handling of already-existent outfile darcs-hash:40abdd9f4629d7ffe462a000e7aebd990e8b04a2 --- diff --git a/pyfec/bin/fec b/pyfec/bin/fec index 4b29862..a70cff4 100755 --- a/pyfec/bin/fec +++ b/pyfec/bin/fec @@ -19,11 +19,11 @@ if '-V' in sys.argv or '--version' in sys.argv: parser = argparse.ArgumentParser(description="Encode a file into a set of share files, a subset of which can later be used to recover the original file.") parser.add_argument('inputfile', help='file to encode or "-" for stdin', type=argparse.FileType('rb'), metavar='INF') -parser.add_argument('-d', '--output-dir', help='directory in which share file names will be created', default='.', metavar='D') +parser.add_argument('-d', '--output-dir', help='directory in which share file names will be created (default ".")', default='.', metavar='D') parser.add_argument('-p', '--prefix', help='prefix for share file names; If omitted, the name of the input file will be used.', metavar='P') -parser.add_argument('-s', '--suffix', help='suffix for share file names', default='.fec', metavar='S') -parser.add_argument('-m', '--totalshares', help='the total number of share files created', default=16, metavar='M') -parser.add_argument('-k', '--requiredshares', help='the number of share files required to reconstruct', default=4, metavar='K') +parser.add_argument('-s', '--suffix', help='suffix for share file names (default ".fec")', default='.fec', metavar='S') +parser.add_argument('-m', '--totalshares', help='the total number of share files created (default 16)', default=16, metavar='M') +parser.add_argument('-k', '--requiredshares', help='the number of share files required to reconstruct (default 4)', default=4, metavar='K') parser.add_argument('-v', '--verbose', help='print out messages about progress', action='store_true') parser.add_argument('-V', '--version', help='print out version number and exit', action='store_true') args = parser.parse_args() diff --git a/pyfec/bin/unfec b/pyfec/bin/unfec index 8ef5c1f..9f21237 100755 --- a/pyfec/bin/unfec +++ b/pyfec/bin/unfec @@ -3,7 +3,7 @@ # import bindann # import bindann.monkeypatch.all -import sys +import os, sys from fec.util import argparse @@ -19,9 +19,10 @@ if '-V' in sys.argv or '--version' in sys.argv: parser = argparse.ArgumentParser(description="Decode data from share files.") -parser.add_argument('outputfile', help='file to write the resulting data to, or "-" for stdout', type=argparse.FileType('wb'), metavar='OUTF') -parser.add_argument('sharefiles', nargs='+', help='shares file to read the encoded data from', type=argparse.FileType('rb'), metavar='SHAREFILE') +parser.add_argument('-o', '--outputfile', required=True, help='file to write the resulting data to, or "-" for stdout', type=str, metavar='OUTF') +parser.add_argument('sharefiles', nargs='*', help='shares file to read the encoded data from', type=argparse.FileType('rb'), metavar='SHAREFILE') parser.add_argument('-v', '--verbose', help='print out messages about progress', action='store_true') +parser.add_argument('-f', '--force', help='overwrite any file which already in place of the output file', action='store_true') parser.add_argument('-V', '--version', help='print out version number and exit', action='store_true') args = parser.parse_args() @@ -29,6 +30,16 @@ if len(args.sharefiles) < 2: print "At least two sharefiles are required." sys.exit(1) -ret = filefec.decode_from_files(args.outputfile, args.sharefiles, args.verbose) +if args.force: + outf = open(args.outputfile, 'wb') +else: + try: + outfd = os.open(args.outputfile, os.O_WRONLY|os.O_CREAT|os.O_EXCL) + except OSError: + print "There is already a file named %r -- aborting. Use --force to overwrite." % (args.outputfile,) + sys.exit(2) + outf = os.fdopen(outfd, "wb") + +ret = filefec.decode_from_files(outf, args.sharefiles, args.verbose) sys.exit(ret) diff --git a/pyfec/fec/filefec.py b/pyfec/fec/filefec.py index f86abce..e83ab2d 100644 --- a/pyfec/fec/filefec.py +++ b/pyfec/fec/filefec.py @@ -188,10 +188,13 @@ def encode_to_files(inf, fsize, dirname, prefix, k, m, suffix=".fec", verbose=Fa fns.append(fn) sumlen = [0] def cb(blocks, length): - if verbose: - print "Writing %d bytes into share files..." % (length,) assert len(blocks) == len(fs) + oldsumlen = sumlen[0] sumlen[0] += length + if verbose: + if int((float(oldsumlen) / fsize) * 10) != int((float(sumlen[0]) / fsize) * 10): + print str(int((float(sumlen[0]) / fsize) * 10) * 10) + "% ...", + if sumlen[0] > fsize: raise IOError("Wrong file size -- possibly the size of the file changed during encoding. Original size: %d, observed size at least: %s" % (fsize, sumlen[0],)) for i in range(len(blocks)): @@ -214,9 +217,16 @@ def encode_to_files(inf, fsize, dirname, prefix, k, m, suffix=".fec", verbose=Fa fileutil.remove_if_possible(fn) return 1 if verbose: + print print "Done!" return 0 +# Note: if you really prefer base-2 and you change this code, then please +# denote 2^20 as "MiB" instead of "MB" in order to avoid ambiguity. +# Thanks. +# http://en.wikipedia.org/wiki/Megabyte +MILLION_BYTES=10**6 + def decode_from_files(outf, infiles, verbose=False): """ Decode from the first k files in infiles, writing the results to outf. @@ -228,6 +238,7 @@ def decode_from_files(outf, infiles, verbose=False): k = None padlen = None + byteswritten = 0 for f in infiles: (nm, nk, npadlen, shnum,) = _parse_header(f) if not (m is None or m == nm): @@ -257,11 +268,18 @@ def decode_from_files(outf, infiles, verbose=False): # Then this was a full read, so we're still in the sharefiles. resultdata = dec.decode(chunks, shnums, padlen=0) outf.write(resultdata) + byteswritten += len(resultdata) + if verbose: + if ((byteswritten - len(resultdata)) / (10*MILLION_BYTES)) != (byteswritten / (10*MILLION_BYTES)): + print str(byteswritten / MILLION_BYTES) + " MB ...", else: # Then this was a short read, so we've reached the end of the sharefiles. resultdata = dec.decode(chunks, shnums, padlen) outf.write(resultdata) return # Done. + if verbose: + print + print "Done!" def encode_file(inf, cb, k, m, chunksize=4096): """