From: Zooko O'Whielacronx Date: Sat, 21 Apr 2007 02:06:12 +0000 (-0700) Subject: zfec: use setuptools to construct executables, fix bug on Windows -- forgot "BINARY... X-Git-Tag: tahoe_v0.1.0-0-UNSTABLE~8 X-Git-Url: https://git.rkrishnan.org/Site/Content/Exhibitors/class-simplejson.JSONEncoder.html?a=commitdiff_plain;h=4d0a984dd8ddc37a06f4f241313d66cb2decd8aa;p=tahoe-lafs%2Ftahoe-lafs.git zfec: use setuptools to construct executables, fix bug on Windows -- forgot "BINARY" flag to output files --- diff --git a/zfec/cmdline/zfec b/zfec/cmdline/zfec deleted file mode 100644 index 97a516de..00000000 --- a/zfec/cmdline/zfec +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env python - -# zfec -- a fast C implementation of Reed-Solomon erasure coding with -# command-line, C, and Python interfaces - -# import bindann -# import bindann.monkeypatch.all - -import sys - -import zfec -from zfec.util import argparse -from zfec import filefec -from zfec.util.version import Version -__version__ = Version("1.0.0a1-0-STABLE") - -if '-V' in sys.argv or '--version' in sys.argv: - print "zfec library version: ", zfec.__version__ - print "zfec command-line tool version: ", __version__ - sys.exit(0) - -parser = argparse.ArgumentParser(description="Encode a file into a set of share files, a subset of which can later be used to recover the original file.") - -parser.add_argument('inputfile', help='file to encode or "-" for stdin', type=argparse.FileType('rb'), metavar='INF') -parser.add_argument('-d', '--output-dir', help='directory in which share file names will be created (default ".")', default='.', metavar='D') -parser.add_argument('-p', '--prefix', help='prefix for share file names; If omitted, the name of the input file will be used.', metavar='P') -parser.add_argument('-s', '--suffix', help='suffix for share file names (default ".fec")', default='.fec', metavar='S') -parser.add_argument('-m', '--totalshares', help='the total number of share files created (default 16)', default=16, type=int, metavar='M') -parser.add_argument('-k', '--requiredshares', help='the number of share files required to reconstruct (default 4)', default=4, type=int, metavar='K') -parser.add_argument('-f', '--force', help='overwrite any file which already in place an output file (share file)', action='store_true') -parser.add_argument('-v', '--verbose', help='print out messages about progress', action='store_true') -parser.add_argument('-V', '--version', help='print out version number and exit', action='store_true') -args = parser.parse_args() - -if args.prefix is None: - args.prefix = args.inputfile.name - if args.prefix == "": - args.prefix = "" - -if args.totalshares < 3: - print "Invalid parameters, totalshares is required to be >= 3\nPlease see the accompanying documentation." - sys.exit(1) -if args.totalshares > 256: - print "Invalid parameters, totalshares is required to be <= 256\nPlease see the accompanying documentation." - sys.exit(1) -if args.requiredshares < 2: - print "Invalid parameters, requiredshares is required to be >= 2\nPlease see the accompanying documentation." - sys.exit(1) -if args.requiredshares >= args.totalshares: - print "Invalid parameters, requiredshares is required to be < totalshares\nPlease see the accompanying documentation." - sys.exit(1) - -args.inputfile.seek(0, 2) -fsize = args.inputfile.tell() -args.inputfile.seek(0, 0) -ret = filefec.encode_to_files(args.inputfile, fsize, args.output_dir, args.prefix, args.requiredshares, args.totalshares, args.suffix, args.force, args.verbose) - -sys.exit(ret) - -# zfec -- a fast C implementation of Reed-Solomon erasure coding with -# command-line, C, and Python interfaces -# -# Copyright (C) 2007 Allmydata, Inc. -# Author: Zooko Wilcox-O'Hearn -# mailto:zooko@zooko.com -# -# This file is part of zfec. -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation; either version 2 of the License, or (at your option) -# any later version. This package also comes with the added permission that, -# in the case that you are obligated to release a derived work under this -# licence (as per section 2.b of the GPL), you may delay the fulfillment of -# this obligation for up to 12 months. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - diff --git a/zfec/cmdline/zunfec b/zfec/cmdline/zunfec deleted file mode 100644 index 26556b02..00000000 --- a/zfec/cmdline/zunfec +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env python - -# zfec -- a fast C implementation of Reed-Solomon erasure coding with -# command-line, C, and Python interfaces - -# import bindann -# import bindann.monkeypatch.all - -import os, sys - -from zfec.util import argparse - -import zfec -from zfec import filefec -from zfec.util.version import Version -__version__ = Version("1.0.0a1-0-STABLE") - -if '-V' in sys.argv or '--version' in sys.argv: - print "zfec library version: ", zfec.__version__ - print "zunfec command-line tool version: ", __version__ - sys.exit(0) - -parser = argparse.ArgumentParser(description="Decode data from share files.") - -parser.add_argument('-o', '--outputfile', required=True, help='file to write the resulting data to, or "-" for stdout', type=str, metavar='OUTF') -parser.add_argument('sharefiles', nargs='*', help='shares file to read the encoded data from', type=argparse.FileType('rb'), metavar='SHAREFILE') -parser.add_argument('-v', '--verbose', help='print out messages about progress', action='store_true') -parser.add_argument('-f', '--force', help='overwrite any file which already in place of the output file', action='store_true') -parser.add_argument('-V', '--version', help='print out version number and exit', action='store_true') -args = parser.parse_args() - -if len(args.sharefiles) < 2: - print "At least two sharefiles are required." - sys.exit(1) - -if args.force: - outf = open(args.outputfile, 'wb') -else: - try: - outfd = os.open(args.outputfile, os.O_WRONLY|os.O_CREAT|os.O_EXCL) - except OSError: - print "There is already a file named %r -- aborting. Use --force to overwrite." % (args.outputfile,) - sys.exit(2) - outf = os.fdopen(outfd, "wb") - -try: - ret = filefec.decode_from_files(outf, args.sharefiles, args.verbose) -except filefec.InsufficientShareFilesError, e: - print str(e) - sys.exit(3) - -sys.exit(ret) - -# zfec -- a fast C implementation of Reed-Solomon erasure coding with -# command-line, C, and Python interfaces -# -# Copyright (C) 2007 Allmydata, Inc. -# Author: Zooko Wilcox-O'Hearn -# mailto:zooko@zooko.com -# -# This file is part of zfec. -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation; either version 2 of the License, or (at your option) -# any later version. This package also comes with the added permission that, -# in the case that you are obligated to release a derived work under this -# licence (as per section 2.b of the GPL), you may delay the fulfillment of -# this obligation for up to 12 months. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - diff --git a/zfec/setup.py b/zfec/setup.py index 5eafd5ab..b53bb80b 100644 --- a/zfec/setup.py +++ b/zfec/setup.py @@ -66,8 +66,8 @@ setup(name='zfec', url='http://allmydata.com/source/zfec', license='GNU GPL', platform='Any', - packages=['zfec', 'zfec.util', 'zfec.test'], + packages=['zfec', 'zfec.cmdline', 'zfec.util', 'zfec.test'], classifiers=trove_classifiers, - scripts=['cmdline/zfec', 'cmdline/zunfec',], + entry_points = { 'console_scripts': [ 'zfec = zfec.cmdline.zfec:main', 'zunfec = zfec.cmdline.zunfec:main' ] }, ext_modules=[Extension('_fec', ['zfec/fec.c', 'zfec/_fecmodule.c',], extra_link_args=extra_link_args, extra_compile_args=extra_compile_args, undef_macros=undef_macros),], ) diff --git a/zfec/zfec/cmdline/__init__.py b/zfec/zfec/cmdline/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/zfec/zfec/cmdline/zfec.py b/zfec/zfec/cmdline/zfec.py new file mode 100644 index 00000000..5853cb7c --- /dev/null +++ b/zfec/zfec/cmdline/zfec.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python + +# zfec -- a fast C implementation of Reed-Solomon erasure coding with +# command-line, C, and Python interfaces + +# The zfec and zunfec command-line tools require Python 2.5 for relative imports. + +import sys + +from ..util import argparse +from .. import filefec + +from ..zfec import __version__ as libversion +from ..util.version import Version +__version__ = Version("1.0.0a1-0-STABLE") + +def main(): + + if '-V' in sys.argv or '--version' in sys.argv: + print "zfec library version: ", libversion + print "zfec command-line tool version: ", __version__ + sys.exit(0) + + parser = argparse.ArgumentParser(description="Encode a file into a set of share files, a subset of which can later be used to recover the original file.") + + parser.add_argument('inputfile', help='file to encode or "-" for stdin', type=argparse.FileType('rb'), metavar='INF') + parser.add_argument('-d', '--output-dir', help='directory in which share file names will be created (default ".")', default='.', metavar='D') + parser.add_argument('-p', '--prefix', help='prefix for share file names; If omitted, the name of the input file will be used.', metavar='P') + parser.add_argument('-s', '--suffix', help='suffix for share file names (default ".fec")', default='.fec', metavar='S') + parser.add_argument('-m', '--totalshares', help='the total number of share files created (default 16)', default=16, type=int, metavar='M') + parser.add_argument('-k', '--requiredshares', help='the number of share files required to reconstruct (default 4)', default=4, type=int, metavar='K') + parser.add_argument('-f', '--force', help='overwrite any file which already in place an output file (share file)', action='store_true') + parser.add_argument('-v', '--verbose', help='print out messages about progress', action='store_true') + parser.add_argument('-V', '--version', help='print out version number and exit', action='store_true') + args = parser.parse_args() + + if args.prefix is None: + args.prefix = args.inputfile.name + if args.prefix == "": + args.prefix = "" + + if args.totalshares < 3: + print "Invalid parameters, totalshares is required to be >= 3\nPlease see the accompanying documentation." + sys.exit(1) + if args.totalshares > 256: + print "Invalid parameters, totalshares is required to be <= 256\nPlease see the accompanying documentation." + sys.exit(1) + if args.requiredshares < 2: + print "Invalid parameters, requiredshares is required to be >= 2\nPlease see the accompanying documentation." + sys.exit(1) + if args.requiredshares >= args.totalshares: + print "Invalid parameters, requiredshares is required to be < totalshares\nPlease see the accompanying documentation." + sys.exit(1) + + args.inputfile.seek(0, 2) + fsize = args.inputfile.tell() + args.inputfile.seek(0, 0) + return filefec.encode_to_files(args.inputfile, fsize, args.output_dir, args.prefix, args.requiredshares, args.totalshares, args.suffix, args.force, args.verbose) + + +# zfec -- a fast C implementation of Reed-Solomon erasure coding with +# command-line, C, and Python interfaces +# +# Copyright (C) 2007 Allmydata, Inc. +# Author: Zooko Wilcox-O'Hearn +# mailto:zooko@zooko.com +# +# This file is part of zfec. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 2 of the License, or (at your option) +# any later version. This package also comes with the added permission that, +# in the case that you are obligated to release a derived work under this +# licence (as per section 2.b of the GPL), you may delay the fulfillment of +# this obligation for up to 12 months. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + diff --git a/zfec/zfec/cmdline/zunfec.py b/zfec/zfec/cmdline/zunfec.py new file mode 100644 index 00000000..0865adbb --- /dev/null +++ b/zfec/zfec/cmdline/zunfec.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python + +# zfec -- a fast C implementation of Reed-Solomon erasure coding with +# command-line, C, and Python interfaces + +# The zfec and zunfec command-line tools require Python 2.5 for relative imports. + +import os, sys + +from ..util import argparse +from .. import filefec + +from ..zfec import __version__ as libversion +from ..util.version import Version +__version__ = Version("1.0.0a1-0-STABLE") + +def main(): + if '-V' in sys.argv or '--version' in sys.argv: + print "zfec library version: ", libversion + print "zunfec command-line tool version: ", __version__ + return 0 + + parser = argparse.ArgumentParser(description="Decode data from share files.") + + parser.add_argument('-o', '--outputfile', required=True, help='file to write the resulting data to, or "-" for stdout', type=str, metavar='OUTF') + parser.add_argument('sharefiles', nargs='*', help='shares file to read the encoded data from', type=argparse.FileType('rb'), metavar='SHAREFILE') + parser.add_argument('-v', '--verbose', help='print out messages about progress', action='store_true') + parser.add_argument('-f', '--force', help='overwrite any file which already in place of the output file', action='store_true') + parser.add_argument('-V', '--version', help='print out version number and exit', action='store_true') + args = parser.parse_args() + + if len(args.sharefiles) < 2: + print "At least two sharefiles are required." + return 1 + + if args.force: + outf = open(args.outputfile, 'wb') + else: + try: + flags = os.O_WRONLY|os.O_CREAT|os.O_EXCL | (hasattr(os, 'O_BINARY') and os.O_BINARY) + outfd = os.open(args.outputfile, flags) + except OSError: + print "There is already a file named %r -- aborting. Use --force to overwrite." % (args.outputfile,) + return 2 + outf = os.fdopen(outfd, "wb") + + try: + ret = filefec.decode_from_files(outf, args.sharefiles, args.verbose) + except filefec.InsufficientShareFilesError, e: + print str(e) + return 3 + + return 0 + +# zfec -- a fast C implementation of Reed-Solomon erasure coding with +# command-line, C, and Python interfaces +# +# Copyright (C) 2007 Allmydata, Inc. +# Author: Zooko Wilcox-O'Hearn +# mailto:zooko@zooko.com +# +# This file is part of zfec. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 2 of the License, or (at your option) +# any later version. This package also comes with the added permission that, +# in the case that you are obligated to release a derived work under this +# licence (as per section 2.b of the GPL), you may delay the fulfillment of +# this obligation for up to 12 months. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + diff --git a/zfec/zfec/filefec.py b/zfec/zfec/filefec.py index 386f3dcf..efacd12e 100644 --- a/zfec/zfec/filefec.py +++ b/zfec/zfec/filefec.py @@ -174,7 +174,7 @@ def encode_to_files(inf, fsize, dirname, prefix, k, m, suffix=".fec", overwrite= if overwrite: f = open(fn, "wb") else: - fd = os.open(fn, os.O_WRONLY|os.O_CREAT|os.O_EXCL) + fd = os.open(fn, os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_BINARY) f = os.fdopen(fd, "wb") f.write(hdr) fs.append(f) diff --git a/zfec/zfec/test/test_zfec.py b/zfec/zfec/test/test_zfec.py index f4870eb8..8403ab7c 100644 --- a/zfec/zfec/test/test_zfec.py +++ b/zfec/zfec/test/test_zfec.py @@ -164,6 +164,21 @@ class FileFec(unittest.TestCase): def test_filefec_all_shares(self): return self._help_test_filefec("Yellow Whirled!", 3, 8) + def test_filefec_all_shares_2(self): + return self._help_test_filefec("Yellow Whirled", 3, 8) + + def test_filefec_all_shares_3(self): + return self._help_test_filefec("Yellow Whirle", 3, 8) + + def test_filefec_all_shares_3_b(self): + return self._help_test_filefec("Yellow Whirle", 4, 16) + + def test_filefec_all_shares_2_b(self): + return self._help_test_filefec("Yellow Whirled", 4, 16) + + def test_filefec_all_shares_1_b(self): + return self._help_test_filefec("Yellow Whirled!", 4, 16) + def test_filefec_all_shares_with_padding(self, noisy=VERBOSE): return self._help_test_filefec("Yellow Whirled!A", 3, 8)