From: Zooko O'Whielacronx zooko@zooko.com <zooko@zooko.com>
Date: Sat, 14 Apr 2007 23:00:59 +0000 (+0530)
Subject: pyfec: more progress indicators, handling of already-existent outfile
X-Git-Url: https://git.rkrishnan.org/components/com_hotproperty/%22doc.html/something?a=commitdiff_plain;h=927925230f696f1028d32725922eb6c7ead9b6b6;p=tahoe-lafs%2Fzfec.git

pyfec: more progress indicators, handling of already-existent outfile

darcs-hash:40abdd9f4629d7ffe462a000e7aebd990e8b04a2
---

diff --git a/pyfec/bin/fec b/pyfec/bin/fec
index 4b29862..a70cff4 100755
--- a/pyfec/bin/fec
+++ b/pyfec/bin/fec
@@ -19,11 +19,11 @@ if '-V' in sys.argv or '--version' in sys.argv:
 parser = argparse.ArgumentParser(description="Encode a file into a set of share files, a subset of which can later be used to recover the original file.")
 
 parser.add_argument('inputfile', help='file to encode or "-" for stdin', type=argparse.FileType('rb'), metavar='INF')
-parser.add_argument('-d', '--output-dir', help='directory in which share file names will be created', default='.', metavar='D')
+parser.add_argument('-d', '--output-dir', help='directory in which share file names will be created (default ".")', default='.', metavar='D')
 parser.add_argument('-p', '--prefix', help='prefix for share file names; If omitted, the name of the input file will be used.', metavar='P')
-parser.add_argument('-s', '--suffix', help='suffix for share file names', default='.fec', metavar='S')
-parser.add_argument('-m', '--totalshares', help='the total number of share files created', default=16, metavar='M')
-parser.add_argument('-k', '--requiredshares', help='the number of share files required to reconstruct', default=4, metavar='K')
+parser.add_argument('-s', '--suffix', help='suffix for share file names (default ".fec")', default='.fec', metavar='S')
+parser.add_argument('-m', '--totalshares', help='the total number of share files created (default 16)', default=16, metavar='M')
+parser.add_argument('-k', '--requiredshares', help='the number of share files required to reconstruct (default 4)', default=4, metavar='K')
 parser.add_argument('-v', '--verbose', help='print out messages about progress', action='store_true')
 parser.add_argument('-V', '--version', help='print out version number and exit', action='store_true')
 args = parser.parse_args()
diff --git a/pyfec/bin/unfec b/pyfec/bin/unfec
index 8ef5c1f..9f21237 100755
--- a/pyfec/bin/unfec
+++ b/pyfec/bin/unfec
@@ -3,7 +3,7 @@
 # import bindann
 # import bindann.monkeypatch.all
 
-import sys
+import os, sys
 
 from fec.util import argparse
 
@@ -19,9 +19,10 @@ if '-V' in sys.argv or '--version' in sys.argv:
   
 parser = argparse.ArgumentParser(description="Decode data from share files.")
 
-parser.add_argument('outputfile', help='file to write the resulting data to, or "-" for stdout', type=argparse.FileType('wb'), metavar='OUTF')
-parser.add_argument('sharefiles', nargs='+', help='shares file to read the encoded data from', type=argparse.FileType('rb'), metavar='SHAREFILE')
+parser.add_argument('-o', '--outputfile', required=True, help='file to write the resulting data to, or "-" for stdout', type=str, metavar='OUTF')
+parser.add_argument('sharefiles', nargs='*', help='shares file to read the encoded data from', type=argparse.FileType('rb'), metavar='SHAREFILE')
 parser.add_argument('-v', '--verbose', help='print out messages about progress', action='store_true')
+parser.add_argument('-f', '--force', help='overwrite any file which already in place of the output file', action='store_true')
 parser.add_argument('-V', '--version', help='print out version number and exit', action='store_true')
 args = parser.parse_args()
 
@@ -29,6 +30,16 @@ if len(args.sharefiles) < 2:
     print "At least two sharefiles are required."
     sys.exit(1)
 
-ret = filefec.decode_from_files(args.outputfile, args.sharefiles, args.verbose)
+if args.force:
+    outf = open(args.outputfile, 'wb')
+else:
+    try:
+        outfd = os.open(args.outputfile, os.O_WRONLY|os.O_CREAT|os.O_EXCL)
+    except OSError:
+        print "There is already a file named %r -- aborting.  Use --force to overwrite." % (args.outputfile,)
+        sys.exit(2)
+    outf = os.fdopen(outfd, "wb")
+
+ret = filefec.decode_from_files(outf, args.sharefiles, args.verbose)
 
 sys.exit(ret)
diff --git a/pyfec/fec/filefec.py b/pyfec/fec/filefec.py
index f86abce..e83ab2d 100644
--- a/pyfec/fec/filefec.py
+++ b/pyfec/fec/filefec.py
@@ -188,10 +188,13 @@ def encode_to_files(inf, fsize, dirname, prefix, k, m, suffix=".fec", verbose=Fa
             fns.append(fn)
         sumlen = [0]
         def cb(blocks, length):
-            if verbose:
-                print "Writing %d bytes into share files..." % (length,)
             assert len(blocks) == len(fs)
+            oldsumlen = sumlen[0]
             sumlen[0] += length
+            if verbose:
+                if int((float(oldsumlen) / fsize) * 10) != int((float(sumlen[0]) / fsize) * 10):
+                    print str(int((float(sumlen[0]) / fsize) * 10) * 10) + "% ...",
+            
             if sumlen[0] > fsize:
                 raise IOError("Wrong file size -- possibly the size of the file changed during encoding.  Original size: %d, observed size at least: %s" % (fsize, sumlen[0],))
             for i in range(len(blocks)):
@@ -214,9 +217,16 @@ def encode_to_files(inf, fsize, dirname, prefix, k, m, suffix=".fec", verbose=Fa
             fileutil.remove_if_possible(fn)
         return 1
     if verbose:
+        print 
         print "Done!"
     return 0
 
+# Note: if you really prefer base-2 and you change this code, then please
+# denote 2^20 as "MiB" instead of "MB" in order to avoid ambiguity.
+# Thanks.
+# http://en.wikipedia.org/wiki/Megabyte
+MILLION_BYTES=10**6
+
 def decode_from_files(outf, infiles, verbose=False):
     """
     Decode from the first k files in infiles, writing the results to outf.
@@ -228,6 +238,7 @@ def decode_from_files(outf, infiles, verbose=False):
     k = None
     padlen = None
 
+    byteswritten = 0
     for f in infiles:
         (nm, nk, npadlen, shnum,) = _parse_header(f)
         if not (m is None or m == nm):
@@ -257,11 +268,18 @@ def decode_from_files(outf, infiles, verbose=False):
             # Then this was a full read, so we're still in the sharefiles.
             resultdata = dec.decode(chunks, shnums, padlen=0)
             outf.write(resultdata)
+            byteswritten += len(resultdata)
+            if verbose:
+                if ((byteswritten - len(resultdata)) / (10*MILLION_BYTES)) != (byteswritten / (10*MILLION_BYTES)):
+                    print str(byteswritten / MILLION_BYTES) + " MB ...",
         else:
             # Then this was a short read, so we've reached the end of the sharefiles.
             resultdata = dec.decode(chunks, shnums, padlen)
             outf.write(resultdata)
             return # Done.
+    if verbose:
+        print
+        print "Done!"
 
 def encode_file(inf, cb, k, m, chunksize=4096):
     """