From: zooko Date: Fri, 9 Nov 2007 04:51:35 +0000 (+0530) Subject: zfec: setup: use argparse and pyutil as separate packages, not by copying their sourc... X-Git-Url: https://git.rkrishnan.org/?a=commitdiff_plain;h=b57aaf145d35738c2728695cebfa5677c5c725dd;p=tahoe-lafs%2Fzfec.git zfec: setup: use argparse and pyutil as separate packages, not by copying their source code into the zfec package darcs-hash:8b55031053efa39c1379a0fbcd036636772cd7f1 --- diff --git a/zfec/setup.py b/zfec/setup.py index 115b4af..6f849f9 100755 --- a/zfec/setup.py +++ b/zfec/setup.py @@ -100,7 +100,6 @@ else: raise RuntimeError("if %s.py exists, it is required to be well-formed" % (VERSIONFILE,)) setup(name='zfec', - # install_requires=['pyutil>=1.0.0',], # It doesn't require pyutil yet. version=verstr, description='a fast erasure code with command-line, C, and Python interfaces', long_description='Fast, portable, programmable erasure coding a.k.a. "forward error correction": the generation of redundant blocks of information such that if some blocks are lost then the original data can be recovered from the remaining blocks.', @@ -108,7 +107,10 @@ setup(name='zfec', author_email='zooko@zooko.com', url='http://allmydata.org/source/zfec', license='GNU GPL', + install_requires=["argparse >= 0.8", "pyutil >= 1.3.5",], packages=find_packages(), + include_package_data=True, + setup_requires=['setuptools_darcs >= 1.0.3',], classifiers=trove_classifiers, entry_points = { 'console_scripts': [ 'zfec = zfec.cmdline_zfec:main', 'zunfec = zfec.cmdline_zunfec:main' ] }, ext_modules=[Extension('zfec/_fec', ['zfec/fec.c', 'zfec/_fecmodule.c',], extra_link_args=extra_link_args, extra_compile_args=extra_compile_args, undef_macros=undef_macros),], diff --git a/zfec/zfec/__init__.py b/zfec/zfec/__init__.py index a445dcc..92c18fa 100644 --- a/zfec/zfec/__init__.py +++ b/zfec/zfec/__init__.py @@ -10,12 +10,12 @@ __version__ = "unknown" try: from _version import __version__ except ImportError: - # we're running in a tree that hasn't run make-version.py, so we don't + # we're running in a tree that hasn't run darcsver.py, so we don't # know what our version is. This should not happen very often. pass from _fec import Encoder, Decoder, Error -import filefec, cmdline_zfec, cmdline_zunfec +import easyfec, filefec, cmdline_zfec, cmdline_zunfec # zfec -- fast forward error correction library with Python interface # diff --git a/zfec/zfec/cmdline_zfec.py b/zfec/zfec/cmdline_zfec.py index accc6b4..2f3d94f 100755 --- a/zfec/zfec/cmdline_zfec.py +++ b/zfec/zfec/cmdline_zfec.py @@ -5,7 +5,7 @@ import sys -from util import argparse +import argparse import filefec from zfec import __version__ as libversion @@ -24,8 +24,8 @@ def main(): parser.add_argument('-d', '--output-dir', help='directory in which share file names will be created (default ".")', default='.', metavar='D') parser.add_argument('-p', '--prefix', help='prefix for share file names; If omitted, the name of the input file will be used.', metavar='P') parser.add_argument('-s', '--suffix', help='suffix for share file names (default ".fec")', default='.fec', metavar='S') - parser.add_argument('-m', '--totalshares', help='the total number of share files created (default 16)', default=16, type=int, metavar='M') - parser.add_argument('-k', '--requiredshares', help='the number of share files required to reconstruct (default 4)', default=4, type=int, metavar='K') + parser.add_argument('-m', '--totalshares', help='the total number of share files created (default 8)', default=8, type=int, metavar='M') + parser.add_argument('-k', '--requiredshares', help='the number of share files required to reconstruct (default 3)', default=3, type=int, metavar='K') parser.add_argument('-f', '--force', help='overwrite any file which already in place an output file (share file)', action='store_true') parser.add_argument('-v', '--verbose', help='print out messages about progress', action='store_true') parser.add_argument('-q', '--quiet', help='quiet progress indications and warnings about silly choices of K and M', action='store_true') diff --git a/zfec/zfec/cmdline_zunfec.py b/zfec/zfec/cmdline_zunfec.py index 06ecd47..e7bb418 100755 --- a/zfec/zfec/cmdline_zunfec.py +++ b/zfec/zfec/cmdline_zunfec.py @@ -5,7 +5,7 @@ import os, sys -from util import argparse +import argparse import filefec from zfec import __version__ as libversion diff --git a/zfec/zfec/filefec.py b/zfec/zfec/filefec.py index 7e502a7..135f4a2 100644 --- a/zfec/zfec/filefec.py +++ b/zfec/zfec/filefec.py @@ -1,6 +1,6 @@ import easyfec, zfec -from util import fileutil -from util.mathutil import log_ceil +from pyutil import fileutil +from pyutil.mathutil import pad_size, log_ceil import array, os, re, struct, traceback @@ -160,7 +160,7 @@ def encode_to_files(inf, fsize, dirname, prefix, k, m, suffix=".fec", overwrite= mlen = len(str(m)) format = FORMAT_FORMAT % (mlen, mlen,) - padbytes = zfec.util.mathutil.pad_size(fsize, k) + padbytes = pad_size(fsize, k) fns = [] fs = [] diff --git a/zfec/zfec/test/test_util.py b/zfec/zfec/test/test_util.py deleted file mode 100644 index a8b183c..0000000 --- a/zfec/zfec/test/test_util.py +++ /dev/null @@ -1,128 +0,0 @@ - -import unittest - -from zfec.util import mathutil - -class Math(unittest.TestCase): - def test_div_ceil(self): - f = mathutil.div_ceil - self.failUnlessEqual(f(0, 1), 0) - self.failUnlessEqual(f(0, 2), 0) - self.failUnlessEqual(f(0, 3), 0) - self.failUnlessEqual(f(1, 3), 1) - self.failUnlessEqual(f(2, 3), 1) - self.failUnlessEqual(f(3, 3), 1) - self.failUnlessEqual(f(4, 3), 2) - self.failUnlessEqual(f(5, 3), 2) - self.failUnlessEqual(f(6, 3), 2) - self.failUnlessEqual(f(7, 3), 3) - - def test_next_multiple(self): - f = mathutil.next_multiple - self.failUnlessEqual(f(5, 1), 5) - self.failUnlessEqual(f(5, 2), 6) - self.failUnlessEqual(f(5, 3), 6) - self.failUnlessEqual(f(5, 4), 8) - self.failUnlessEqual(f(5, 5), 5) - self.failUnlessEqual(f(5, 6), 6) - self.failUnlessEqual(f(32, 1), 32) - self.failUnlessEqual(f(32, 2), 32) - self.failUnlessEqual(f(32, 3), 33) - self.failUnlessEqual(f(32, 4), 32) - self.failUnlessEqual(f(32, 5), 35) - self.failUnlessEqual(f(32, 6), 36) - self.failUnlessEqual(f(32, 7), 35) - self.failUnlessEqual(f(32, 8), 32) - self.failUnlessEqual(f(32, 9), 36) - self.failUnlessEqual(f(32, 10), 40) - self.failUnlessEqual(f(32, 11), 33) - self.failUnlessEqual(f(32, 12), 36) - self.failUnlessEqual(f(32, 13), 39) - self.failUnlessEqual(f(32, 14), 42) - self.failUnlessEqual(f(32, 15), 45) - self.failUnlessEqual(f(32, 16), 32) - self.failUnlessEqual(f(32, 17), 34) - self.failUnlessEqual(f(32, 18), 36) - self.failUnlessEqual(f(32, 589), 589) - - def test_pad_size(self): - f = mathutil.pad_size - self.failUnlessEqual(f(0, 4), 0) - self.failUnlessEqual(f(1, 4), 3) - self.failUnlessEqual(f(2, 4), 2) - self.failUnlessEqual(f(3, 4), 1) - self.failUnlessEqual(f(4, 4), 0) - self.failUnlessEqual(f(5, 4), 3) - - def test_is_power_of_k(self): - f = mathutil.is_power_of_k - for i in range(1, 100): - if i in (1, 2, 4, 8, 16, 32, 64): - self.failUnless(f(i, 2), "but %d *is* a power of 2" % i) - else: - self.failIf(f(i, 2), "but %d is *not* a power of 2" % i) - for i in range(1, 100): - if i in (1, 3, 9, 27, 81): - self.failUnless(f(i, 3), "but %d *is* a power of 3" % i) - else: - self.failIf(f(i, 3), "but %d is *not* a power of 3" % i) - - def test_next_power_of_k(self): - f = mathutil.next_power_of_k - self.failUnlessEqual(f(0,2), 1) - self.failUnlessEqual(f(1,2), 1) - self.failUnlessEqual(f(2,2), 2) - self.failUnlessEqual(f(3,2), 4) - self.failUnlessEqual(f(4,2), 4) - for i in range(5, 8): self.failUnlessEqual(f(i,2), 8, "%d" % i) - for i in range(9, 16): self.failUnlessEqual(f(i,2), 16, "%d" % i) - for i in range(17, 32): self.failUnlessEqual(f(i,2), 32, "%d" % i) - for i in range(33, 64): self.failUnlessEqual(f(i,2), 64, "%d" % i) - for i in range(65, 100): self.failUnlessEqual(f(i,2), 128, "%d" % i) - - self.failUnlessEqual(f(0,3), 1) - self.failUnlessEqual(f(1,3), 1) - self.failUnlessEqual(f(2,3), 3) - self.failUnlessEqual(f(3,3), 3) - for i in range(4, 9): self.failUnlessEqual(f(i,3), 9, "%d" % i) - for i in range(10, 27): self.failUnlessEqual(f(i,3), 27, "%d" % i) - for i in range(28, 81): self.failUnlessEqual(f(i,3), 81, "%d" % i) - for i in range(82, 200): self.failUnlessEqual(f(i,3), 243, "%d" % i) - - def test_ave(self): - f = mathutil.ave - self.failUnlessEqual(f([1,2,3]), 2) - self.failUnlessEqual(f([0,0,0,4]), 1) - self.failUnlessAlmostEqual(f([0.0, 1.0, 1.0]), .666666666666) - - def failUnlessEqualContents(self, a, b): - self.failUnlessEqual(sorted(a), sorted(b)) - - def test_permute(self): - f = mathutil.permute - self.failUnlessEqualContents(f([]), []) - self.failUnlessEqualContents(f([1]), [[1]]) - self.failUnlessEqualContents(f([1,2]), [[1,2], [2,1]]) - self.failUnlessEqualContents(f([1,2,3]), - [[1,2,3], [1,3,2], - [2,1,3], [2,3,1], - [3,1,2], [3,2,1]]) - -# zfec -- fast forward error correction library with Python interface -# -# Copyright (C) 2007 Allmydata, Inc. -# Author: Zooko Wilcox-O'Hearn -# -# This file is part of zfec. -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation; either version 2 of the License, or (at your option) -# any later version, with the added permission that, if you become obligated -# to release a derived work under this licence (as per section 2.b), you may -# delay the fulfillment of this obligation for up to 12 months. See the file -# COPYING for details. -# -# If you would like to inquire about a commercial relationship with Allmydata, -# Inc., please contact partnerships@allmydata.com and visit -# http://allmydata.com/. diff --git a/zfec/zfec/util/__init__.py b/zfec/zfec/util/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/zfec/zfec/util/argparse.py b/zfec/zfec/util/argparse.py deleted file mode 100644 index 33e4427..0000000 --- a/zfec/zfec/util/argparse.py +++ /dev/null @@ -1,1799 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright © 2006 Steven J. Bethard . -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted under the terms of the 3-clause BSD -# license. No warranty expressed or implied. - -"""Command-line parsing library - -This module is an optparse-inspired command-line parsing library that: - -* handles both optional and positional arguments -* produces highly informative usage messages -* supports parsers that dispatch to sub-parsers - -The following is a simple usage example that sums integers from the -command-line and writes the result to a file: - - parser = argparse.ArgumentParser( - description='sum the integers at the command line') - parser.add_argument( - 'integers', metavar='int', nargs='+', type=int, - help='an integer to be summed') - parser.add_argument( - '--log', default=sys.stdout, type=argparse.FileType('w'), - help='the file where the sum should be written') - args = parser.parse_args() - args.log.write('%s' % sum(args.integers)) - args.log.close() - -The module contains the following public classes: - - ArgumentParser -- The main entry point for command-line parsing. As the - example above shows, the add_argument() method is used to populate - the parser with actions for optional and positional arguments. Then - the parse_args() method is invoked to convert the args at the - command-line into an object with attributes. - - ArgumentError -- The exception raised by ArgumentParser objects when - there are errors with the parser's actions. Errors raised while - parsing the command-line are caught by ArgumentParser and emitted - as command-line messages. - - FileType -- A factory for defining types of files to be created. As the - example above shows, instances of FileType are typically passed as - the type= argument of add_argument() calls. - - Action -- The base class for parser actions. Typically actions are - selected by passing strings like 'store_true' or 'append_const' to - the action= argument of add_argument(). However, for greater - customization of ArgumentParser actions, subclasses of Action may - be defined and passed as the action= argument. - - HelpFormatter, RawDescriptionHelpFormatter -- Formatter classes which - may be passed as the formatter_class= argument to the - ArgumentParser constructor. HelpFormatter is the default, while - RawDescriptionHelpFormatter tells the parser not to perform any - line-wrapping on description text. - -All other classes in this module are considered implementation details. -(Also note that HelpFormatter and RawDescriptionHelpFormatter are only -considered public as object names -- the API of the formatter objects is -still considered an implementation detail.) -""" - -import os as _os -import re as _re -import sys as _sys -import textwrap as _textwrap - -from gettext import gettext as _ - -SUPPRESS = '==SUPPRESS==' - -OPTIONAL = '?' -ZERO_OR_MORE = '*' -ONE_OR_MORE = '+' -PARSER = '==PARSER==' - -# ============================= -# Utility functions and classes -# ============================= - -class _AttributeHolder(object): - """Abstract base class that provides __repr__. - - The __repr__ method returns a string in the format: - ClassName(attr=name, attr=name, ...) - The attributes are determined either by a class-level attribute, - '_kwarg_names', or by inspecting the instance __dict__. - """ - - def __repr__(self): - type_name = type(self).__name__ - arg_strings = [] - for arg in self._get_args(): - arg_strings.append(repr(arg)) - for name, value in self._get_kwargs(): - arg_strings.append('%s=%r' % (name, value)) - return '%s(%s)' % (type_name, ', '.join(arg_strings)) - - def _get_kwargs(self): - return sorted(self.__dict__.items()) - - def _get_args(self): - return [] - -def _ensure_value(namespace, name, value): - if getattr(namespace, name, None) is None: - setattr(namespace, name, value) - return getattr(namespace, name) - - - -# =============== -# Formatting Help -# =============== - -class HelpFormatter(object): - - def __init__(self, - prog, - indent_increment=2, - max_help_position=24, - width=None): - - # default setting for width - if width is None: - try: - width = int(_os.environ['COLUMNS']) - except (KeyError, ValueError): - width = 80 - width -= 2 - - self._prog = prog - self._indent_increment = indent_increment - self._max_help_position = max_help_position - self._width = width - - self._current_indent = 0 - self._level = 0 - self._action_max_length = 0 - - self._root_section = self._Section(self, None) - self._current_section = self._root_section - - self._whitespace_matcher = _re.compile(r'\s+') - self._long_break_matcher = _re.compile(r'\n\n\n+') - - # =============================== - # Section and indentation methods - # =============================== - - def _indent(self): - self._current_indent += self._indent_increment - self._level += 1 - - def _dedent(self): - self._current_indent -= self._indent_increment - assert self._current_indent >= 0, 'Indent decreased below 0.' - self._level -= 1 - - class _Section(object): - def __init__(self, formatter, parent, heading=None): - self.formatter = formatter - self.parent = parent - self.heading = heading - self.items = [] - - def format_help(self): - # format the indented section - if self.parent is not None: - self.formatter._indent() - join = self.formatter._join_parts - for func, args in self.items: - func(*args) - item_help = join(func(*args) for func, args in self.items) - if self.parent is not None: - self.formatter._dedent() - - # return nothing if the section was empty - if not item_help: - return '' - - # add the heading if the section was non-empty - if self.heading is not SUPPRESS and self.heading is not None: - current_indent = self.formatter._current_indent - heading = '%*s%s:\n' % (current_indent, '', self.heading) - else: - heading = '' - - # join the section-initial newline, the heading and the help - return join(['\n', heading, item_help, '\n']) - - def _add_item(self, func, args): - self._current_section.items.append((func, args)) - - # ======================== - # Message building methods - # ======================== - - def start_section(self, heading): - self._indent() - section = self._Section(self, self._current_section, heading) - self._add_item(section.format_help, []) - self._current_section = section - - def end_section(self): - self._current_section = self._current_section.parent - self._dedent() - - def add_text(self, text): - if text is not SUPPRESS and text is not None: - self._add_item(self._format_text, [text]) - - def add_usage(self, usage, optionals, positionals, prefix=None): - if usage is not SUPPRESS: - args = usage, optionals, positionals, prefix - self._add_item(self._format_usage, args) - - def add_argument(self, action): - if action.help is not SUPPRESS: - - # update the maximum item length - invocation = self._format_action_invocation(action) - action_length = len(invocation) + self._current_indent - self._action_max_length = max(self._action_max_length, - action_length) - - # add the item to the list - self._add_item(self._format_action, [action]) - - def add_arguments(self, actions): - for action in actions: - self.add_argument(action) - - # ======================= - # Help-formatting methods - # ======================= - - def format_help(self): - help = self._root_section.format_help() % dict(prog=self._prog) - if help: - help = self._long_break_matcher.sub('\n\n', help) - help = help.strip('\n') + '\n' - return help - - def _join_parts(self, part_strings): - return ''.join(part - for part in part_strings - if part and part is not SUPPRESS) - - def _format_usage(self, usage, optionals, positionals, prefix): - if prefix is None: - prefix = _('usage: ') - - # if no optionals or positionals are available, usage is just prog - if usage is None and not optionals and not positionals: - usage = '%(prog)s' - - # if optionals and positionals are available, calculate usage - elif usage is None: - usage = '%(prog)s' % dict(prog=self._prog) - - # determine width of "usage: PROG" and width of text - prefix_width = len(prefix) + len(usage) + 1 - prefix_indent = self._current_indent + prefix_width - text_width = self._width - self._current_indent - - # put them on one line if they're short enough - format = self._format_actions_usage - action_usage = format(optionals + positionals) - if prefix_width + len(action_usage) + 1 < text_width: - usage = '%s %s' % (usage, action_usage) - - # if they're long, wrap optionals and positionals individually - else: - optional_usage = format(optionals) - positional_usage = format(positionals) - indent = ' ' * prefix_indent - - # usage is made of PROG, optionals and positionals - parts = [usage, ' '] - - # options always get added right after PROG - if optional_usage: - parts.append(_textwrap.fill( - optional_usage, text_width, - initial_indent=indent, - subsequent_indent=indent).lstrip()) - - # if there were options, put arguments on the next line - # otherwise, start them right after PROG - if positional_usage: - part = _textwrap.fill( - positional_usage, text_width, - initial_indent=indent, - subsequent_indent=indent).lstrip() - if optional_usage: - part = '\n' + indent + part - parts.append(part) - usage = ''.join(parts) - - # prefix with 'usage:' - return '%s%s\n\n' % (prefix, usage) - - def _format_actions_usage(self, actions): - parts = [] - for action in actions: - if action.help is SUPPRESS: - continue - - # produce all arg strings - if not action.option_strings: - parts.append(self._format_args(action, action.dest)) - - # produce the first way to invoke the option in brackets - else: - option_string = action.option_strings[0] - - # if the Optional doesn't take a value, format is: - # -s or --long - if action.nargs == 0: - part = '%s' % option_string - - # if the Optional takes a value, format is: - # -s ARGS or --long ARGS - else: - default = action.dest.upper() - args_string = self._format_args(action, default) - part = '%s %s' % (option_string, args_string) - - # make it look optional if it's not required - if not action.required: - part = '[%s]' % part - parts.append(part) - - return ' '.join(parts) - - def _format_text(self, text): - text_width = self._width - self._current_indent - indent = ' ' * self._current_indent - return self._fill_text(text, text_width, indent) + '\n\n' - - def _format_action(self, action): - # determine the required width and the entry label - help_position = min(self._action_max_length + 2, - self._max_help_position) - help_width = self._width - help_position - action_width = help_position - self._current_indent - 2 - action_header = self._format_action_invocation(action) - - # ho nelp; start on same line and add a final newline - if not action.help: - tup = self._current_indent, '', action_header - action_header = '%*s%s\n' % tup - - # short action name; start on the same line and pad two spaces - elif len(action_header) <= action_width: - tup = self._current_indent, '', action_width, action_header - action_header = '%*s%-*s ' % tup - indent_first = 0 - - # long action name; start on the next line - else: - tup = self._current_indent, '', action_header - action_header = '%*s%s\n' % tup - indent_first = help_position - - # collect the pieces of the action help - parts = [action_header] - - # if there was help for the action, add lines of help text - if action.help: - help_text = self._expand_help(action) - help_lines = self._split_lines(help_text, help_width) - parts.append('%*s%s\n' % (indent_first, '', help_lines[0])) - for line in help_lines[1:]: - parts.append('%*s%s\n' % (help_position, '', line)) - - # or add a newline if the description doesn't end with one - elif not action_header.endswith('\n'): - parts.append('\n') - - # return a single string - return self._join_parts(parts) - - def _format_action_invocation(self, action): - if not action.option_strings: - return self._format_metavar(action, action.dest) - - else: - parts = [] - - # if the Optional doesn't take a value, format is: - # -s, --long - if action.nargs == 0: - parts.extend(action.option_strings) - - # if the Optional takes a value, format is: - # -s ARGS, --long ARGS - else: - default = action.dest.upper() - args_string = self._format_args(action, default) - for option_string in action.option_strings: - parts.append('%s %s' % (option_string, args_string)) - - return ', '.join(parts) - - def _format_metavar(self, action, default_metavar): - if action.metavar is not None: - name = action.metavar - elif action.choices is not None: - choice_strs = (str(choice) for choice in action.choices) - name = '{%s}' % ','.join(choice_strs) - else: - name = default_metavar - return name - - def _format_args(self, action, default_metavar): - name = self._format_metavar(action, default_metavar) - if action.nargs is None: - result = name - elif action.nargs == OPTIONAL: - result = '[%s]' % name - elif action.nargs == ZERO_OR_MORE: - result = '[%s [%s ...]]' % (name, name) - elif action.nargs == ONE_OR_MORE: - result = '%s [%s ...]' % (name, name) - elif action.nargs is PARSER: - result = '%s ...' % name - else: - result = ' '.join([name] * action.nargs) - return result - - def _expand_help(self, action): - params = dict(vars(action), prog=self._prog) - for name, value in params.items(): - if value is SUPPRESS: - del params[name] - if params.get('choices') is not None: - choices_str = ', '.join(str(c) for c in params['choices']) - params['choices'] = choices_str - return action.help % params - - def _split_lines(self, text, width): - text = self._whitespace_matcher.sub(' ', text).strip() - return _textwrap.wrap(text, width) - - def _fill_text(self, text, width, indent): - text = self._whitespace_matcher.sub(' ', text).strip() - return _textwrap.fill(text, width, initial_indent=indent, - subsequent_indent=indent) - -class RawDescriptionHelpFormatter(HelpFormatter): - - def _fill_text(self, text, width, indent): - return ''.join(indent + line for line in text.splitlines(True)) - -class RawTextHelpFormatter(RawDescriptionHelpFormatter): - - def _split_lines(self, text, width): - return text.splitlines() - -# ===================== -# Options and Arguments -# ===================== - -class ArgumentError(Exception): - """ArgumentError(message, argument) - - Raised whenever there was an error creating or using an argument - (optional or positional). - - The string value of this exception is the message, augmented with - information about the argument that caused it. - """ - - def __init__(self, argument, message): - if argument.option_strings: - self.argument_name = '/'.join(argument.option_strings) - elif argument.metavar not in (None, SUPPRESS): - self.argument_name = argument.metavar - elif argument.dest not in (None, SUPPRESS): - self.argument_name = argument.dest - else: - self.argument_name = None - self.message = message - - def __str__(self): - if self.argument_name is None: - format = '%(message)s' - else: - format = 'argument %(argument_name)s: %(message)s' - return format % dict(message=self.message, - argument_name=self.argument_name) - -# ============== -# Action classes -# ============== - -class Action(_AttributeHolder): - """Action(*strings, **options) - - Action objects hold the information necessary to convert a - set of command-line arguments (possibly including an initial option - string) into the desired Python object(s). - - Keyword Arguments: - - option_strings -- A list of command-line option strings which - should be associated with this action. - - dest -- The name of the attribute to hold the created object(s) - - nargs -- The number of command-line arguments that should be consumed. - By default, one argument will be consumed and a single value will - be produced. Other values include: - * N (an integer) consumes N arguments (and produces a list) - * '?' consumes zero or one arguments - * '*' consumes zero or more arguments (and produces a list) - * '+' consumes one or more arguments (and produces a list) - Note that the difference between the default and nargs=1 is that - with the default, a single value will be produced, while with - nargs=1, a list containing a single value will be produced. - - const -- The value to be produced if the option is specified and the - option uses an action that takes no values. - - default -- The value to be produced if the option is not specified. - - type -- The type which the command-line arguments should be converted - to, should be one of 'string', 'int', 'float', 'complex' or a - callable object that accepts a single string argument. If None, - 'string' is assumed. - - choices -- A container of values that should be allowed. If not None, - after a command-line argument has been converted to the appropriate - type, an exception will be raised if it is not a member of this - collection. - - required -- True if the action must always be specified at the command - line. This is only meaningful for optional command-line arguments. - - help -- The help string describing the argument. - - metavar -- The name to be used for the option's argument with the help - string. If None, the 'dest' value will be used as the name. - """ - - - def __init__(self, - option_strings, - dest, - nargs=None, - const=None, - default=None, - type=None, - choices=None, - required=False, - help=None, - metavar=None): - self.option_strings = option_strings - self.dest = dest - self.nargs = nargs - self.const = const - self.default = default - self.type = type - self.choices = choices - self.required = required - self.help = help - self.metavar = metavar - - def _get_kwargs(self): - names = [ - 'option_strings', - 'dest', - 'nargs', - 'const', - 'default', - 'type', - 'choices', - 'help', - 'metavar' - ] - return [(name, getattr(self, name)) for name in names] - - def __call__(self, parser, namespace, values, option_string=None): - raise NotImplementedError(_('.__call__() not defined')) - -class _StoreAction(Action): - def __init__(self, - option_strings, - dest, - nargs=None, - const=None, - default=None, - type=None, - choices=None, - required=False, - help=None, - metavar=None): - if nargs == 0: - raise ValueError('nargs must be > 0') - if const is not None and nargs != OPTIONAL: - raise ValueError('nargs must be %r to supply const' % OPTIONAL) - super(_StoreAction, self).__init__( - option_strings=option_strings, - dest=dest, - nargs=nargs, - const=const, - default=default, - type=type, - choices=choices, - required=required, - help=help, - metavar=metavar) - - def __call__(self, parser, namespace, values, option_string=None): - setattr(namespace, self.dest, values) - -class _StoreConstAction(Action): - def __init__(self, - option_strings, - dest, - const, - default=None, - required=False, - help=None, - metavar=None): - super(_StoreConstAction, self).__init__( - option_strings=option_strings, - dest=dest, - nargs=0, - const=const, - default=default, - required=required, - help=help) - - def __call__(self, parser, namespace, values, option_string=None): - setattr(namespace, self.dest, self.const) - -class _StoreTrueAction(_StoreConstAction): - def __init__(self, - option_strings, - dest, - default=False, - required=False, - help=None): - super(_StoreTrueAction, self).__init__( - option_strings=option_strings, - dest=dest, - const=True, - default=default, - required=required, - help=help) - -class _StoreFalseAction(_StoreConstAction): - def __init__(self, - option_strings, - dest, - default=True, - required=False, - help=None): - super(_StoreFalseAction, self).__init__( - option_strings=option_strings, - dest=dest, - const=False, - default=default, - required=required, - help=help) - -class _AppendAction(Action): - def __init__(self, - option_strings, - dest, - nargs=None, - const=None, - default=None, - type=None, - choices=None, - required=False, - help=None, - metavar=None): - if nargs == 0: - raise ValueError('nargs must be > 0') - if const is not None and nargs != OPTIONAL: - raise ValueError('nargs must be %r to supply const' % OPTIONAL) - super(_AppendAction, self).__init__( - option_strings=option_strings, - dest=dest, - nargs=nargs, - const=const, - default=default, - type=type, - choices=choices, - required=required, - help=help, - metavar=metavar) - - def __call__(self, parser, namespace, values, option_string=None): - _ensure_value(namespace, self.dest, []).append(values) - -class _AppendConstAction(Action): - def __init__(self, - option_strings, - dest, - const, - default=None, - required=False, - help=None, - metavar=None): - super(_AppendConstAction, self).__init__( - option_strings=option_strings, - dest=dest, - nargs=0, - const=const, - default=default, - required=required, - help=help, - metavar=metavar) - - def __call__(self, parser, namespace, values, option_string=None): - _ensure_value(namespace, self.dest, []).append(self.const) - -class _CountAction(Action): - def __init__(self, - option_strings, - dest, - default=None, - required=False, - help=None): - super(_CountAction, self).__init__( - option_strings=option_strings, - dest=dest, - nargs=0, - default=default, - required=required, - help=help) - - def __call__(self, parser, namespace, values, option_string=None): - new_count = _ensure_value(namespace, self.dest, 0) + 1 - setattr(namespace, self.dest, new_count) - -class _HelpAction(Action): - def __init__(self, - option_strings, - dest, - help=None): - super(_HelpAction, self).__init__( - option_strings=option_strings, - dest=SUPPRESS, - nargs=0, - help=help) - - def __call__(self, parser, namespace, values, option_string=None): - parser.print_help() - parser.exit() - -class _VersionAction(Action): - def __init__(self, - option_strings, - dest, - help=None): - super(_VersionAction, self).__init__( - option_strings=option_strings, - dest=SUPPRESS, - nargs=0, - help=help) - - def __call__(self, parser, namespace, values, option_string=None): - parser.print_version() - parser.exit() - -class _SubParsersAction(Action): - - def __init__(self, - option_strings, - prog, - parser_class, - dest=SUPPRESS, - help=None, - metavar=None): - - self._prog_prefix = prog - self._parser_class = parser_class - self._name_parser_map = {} - - super(_SubParsersAction, self).__init__( - option_strings=option_strings, - dest=dest, - nargs=PARSER, - choices=self._name_parser_map, - help=help, - metavar=metavar) - - def add_parser(self, name, **kwargs): - if kwargs.get('prog') is None: - kwargs['prog'] = '%s %s' % (self._prog_prefix, name) - - parser = self._parser_class(**kwargs) - self._name_parser_map[name] = parser - return parser - - def __call__(self, parser, namespace, values, option_string=None): - parser_name = values[0] - arg_strings = values[1:] - - # set the parser name if requested - if self.dest is not SUPPRESS: - setattr(namespace, self.dest, parser_name) - - # select the parser - try: - parser = self._name_parser_map[parser_name] - except KeyError: - tup = parser_name, ', '.join(self._name_parser_map) - msg = _('unknown parser %r (choices: %s)' % tup) - raise ArgumentError(self, msg) - - # parse all the remaining options into the namespace - parser.parse_args(arg_strings, namespace) - - -# ============== -# Type classes -# ============== - -class FileType(object): - """Factory for creating file object types - - Instances of FileType are typically passed as type= arguments to the - ArgumentParser add_argument() method. - - Keyword Arguments: - mode -- A string indicating how the file is to be opened. Accepts the - same values as the builtin open() function. - bufsize -- The file's desired buffer size. Accepts the same values as - the builtin open() function. - exclusiveopen -- A bool indicating whether the attempt to create the file - should fail if there is already a file present by that name. This is - ignored if 'w' is not in mode. - """ - def __init__(self, mode='r', bufsize=None, exclusivecreate=False): - self._mode = mode - self._bufsize = bufsize - if self._bufsize is None: - self._bufsize = -1 - self._exclusivecreate = exclusivecreate - - def __call__(self, string): - # the special argument "-" means sys.std{in,out} - if string == '-': - if self._mode == 'r': - return _sys.stdin - elif self._mode == 'w': - return _sys.stdout - else: - msg = _('argument "-" with mode %r' % self._mode) - raise ValueError(msg) - - # all other arguments are used as file names - if self._exclusivecreate and ('w' in self._mode): - fd = _os.open(string, _os.O_CREAT|_os.O_EXCL) - return _os.fdopen(fd, self._mode, self._bufsize) - else: - return open(string, self._mode, self._bufsize) - - -# =========================== -# Optional and Positional Parsing -# =========================== - -class Namespace(_AttributeHolder): - - def __init__(self, **kwargs): - for name, value in kwargs.iteritems(): - setattr(self, name, value) - - def __eq__(self, other): - return vars(self) == vars(other) - - def __ne__(self, other): - return not (self == other) - - -class _ActionsContainer(object): - def __init__(self, - description, - conflict_handler): - superinit = super(_ActionsContainer, self).__init__ - superinit() - - self.description = description - self.conflict_handler = conflict_handler - - # set up registries - self._registries = {} - - # register actions - self.register('action', None, _StoreAction) - self.register('action', 'store', _StoreAction) - self.register('action', 'store_const', _StoreConstAction) - self.register('action', 'store_true', _StoreTrueAction) - self.register('action', 'store_false', _StoreFalseAction) - self.register('action', 'append', _AppendAction) - self.register('action', 'append_const', _AppendConstAction) - self.register('action', 'count', _CountAction) - self.register('action', 'help', _HelpAction) - self.register('action', 'version', _VersionAction) - self.register('action', 'parsers', _SubParsersAction) - - # raise an exception if the conflict handler is invalid - self._get_handler() - - # action storage - self._optional_actions_list = [] - self._positional_actions_list = [] - self._positional_actions_full_list = [] - self._option_strings = {} - - # ==================== - # Registration methods - # ==================== - - def register(self, registry_name, value, object): - registry = self._registries.setdefault(registry_name, {}) - registry[value] = object - - def _registry_get(self, registry_name, value, default=None): - return self._registries[registry_name].get(value, default) - - # ======================= - # Adding argument actions - # ======================= - - def add_argument(self, *args, **kwargs): - """ - add_argument(dest, ..., name=value, ...) - add_argument(option_string, option_string, ..., name=value, ...) - """ - - # type='outfile' is deprecated - if kwargs.get('type') == 'outfile': - import warnings - msg = _("use type=FileType('w') instead of type='outfile'") - warnings.warn(msg, DeprecationWarning) - - # if no positional args are supplied or only one is supplied and - # it doesn't look like an option string, parse a positional - # argument - if not args or len(args) == 1 and args[0][0] != '-': - kwargs = self._get_positional_kwargs(*args, **kwargs) - - # otherwise, we're adding an optional argument - else: - kwargs = self._get_optional_kwargs(*args, **kwargs) - - # create the action object, and add it to the parser - action_class = self._pop_action_class(kwargs) - action = action_class(**kwargs) - return self._add_action(action) - - def _add_action(self, action): - # resolve any conflicts - self._check_conflict(action) - - # add to optional or positional list - if action.option_strings: - self._optional_actions_list.append(action) - else: - self._positional_actions_list.append(action) - self._positional_actions_full_list.append(action) - action.container = self - - # index the action by any option strings it has - for option_string in action.option_strings: - self._option_strings[option_string] = action - - # return the created action - return action - - def _add_container_actions(self, container): - for action in container._optional_actions_list: - self._add_action(action) - for action in container._positional_actions_list: - self._add_action(action) - - def _get_positional_kwargs(self, dest, **kwargs): - # make sure required is not specified - if 'required' in kwargs: - msg = _("'required' is an invalid argument for positionals") - raise TypeError(msg) - - # return the keyword arguments with no option strings - return dict(kwargs, dest=dest, option_strings=[]) - - def _get_optional_kwargs(self, *args, **kwargs): - # determine short and long option strings - option_strings = [] - long_option_strings = [] - for option_string in args: - # error on one-or-fewer-character option strings - if len(option_string) < 2: - msg = _('invalid option string %r: ' - 'must be at least two characters long') - raise ValueError(msg % option_string) - - # error on strings that don't start with '-' - if not option_string.startswith('-'): - msg = _('invalid option string %r: ' - 'does not start with "-"') - raise ValueError(msg % option_string) - - # error on strings that are all '-'s - if not option_string.replace('-', ''): - msg = _('invalid option string %r: ' - 'must contain characters other than "-"') - raise ValueError(msg % option_string) - - # strings starting with '--' are long options - option_strings.append(option_string) - if option_string.startswith('--'): - long_option_strings.append(option_string) - - # infer destination, '--foo-bar' -> 'foo_bar' and '-x' -> 'x' - dest = kwargs.pop('dest', None) - if dest is None: - if long_option_strings: - dest_option_string = long_option_strings[0] - else: - dest_option_string = option_strings[0] - dest = dest_option_string.lstrip('-').replace('-', '_') - - # return the updated keyword arguments - return dict(kwargs, dest=dest, option_strings=option_strings) - - def _pop_action_class(self, kwargs, default=None): - action = kwargs.pop('action', default) - return self._registry_get('action', action, action) - - def _get_handler(self): - # determine function from conflict handler string - handler_func_name = '_handle_conflict_%s' % self.conflict_handler - try: - return getattr(self, handler_func_name) - except AttributeError: - msg = _('invalid conflict_resolution value: %r') - raise ValueError(msg % self.conflict_handler) - - def _check_conflict(self, action): - - # find all options that conflict with this option - confl_optionals = [] - for option_string in action.option_strings: - if option_string in self._option_strings: - confl_optional = self._option_strings[option_string] - confl_optionals.append((option_string, confl_optional)) - - # resolve any conflicts - if confl_optionals: - conflict_handler = self._get_handler() - conflict_handler(action, confl_optionals) - - def _handle_conflict_error(self, action, conflicting_actions): - message = _('conflicting option string(s): %s') - conflict_string = ', '.join(option_string - for option_string, action - in conflicting_actions) - raise ArgumentError(action, message % conflict_string) - - def _handle_conflict_resolve(self, action, conflicting_actions): - - # remove all conflicting options - for option_string, action in conflicting_actions: - - # remove the conflicting option - action.option_strings.remove(option_string) - self._option_strings.pop(option_string, None) - - # if the option now has no option string, remove it from the - # container holding it - if not action.option_strings: - action.container._optional_actions_list.remove(action) - - -class _ArgumentGroup(_ActionsContainer): - - def __init__(self, container, title=None, description=None, **kwargs): - # add any missing keyword arguments by checking the container - update = kwargs.setdefault - update('conflict_handler', container.conflict_handler) - superinit = super(_ArgumentGroup, self).__init__ - superinit(description=description, **kwargs) - - self.title = title - self._registries = container._registries - self._positional_actions_full_list = container._positional_actions_full_list - self._option_strings = container._option_strings - - -class ArgumentParser(_AttributeHolder, _ActionsContainer): - - def __init__(self, - prog=None, - usage=None, - description=None, - epilog=None, - version=None, - parents=[], - formatter_class=HelpFormatter, - conflict_handler='error', - add_help=True): - - superinit = super(ArgumentParser, self).__init__ - superinit(description=description, - conflict_handler=conflict_handler) - - # default setting for prog - if prog is None: - prog = _os.path.basename(_sys.argv[0]) - - self.prog = prog - self.usage = usage - self.epilog = epilog - self.version = version - self.formatter_class = formatter_class - self.add_help = add_help - - self._argument_group_class = _ArgumentGroup - self._has_subparsers = False - self._argument_groups = [] - self._defaults = {} - - # register types - def identity(string): - return string - def outfile(string): - if string == '-': - return _sys.stdout - else: - return open(string, 'w') - self.register('type', None, identity) - self.register('type', 'outfile', outfile) - - # add help and version arguments if necessary - if self.add_help: - self._add_help_argument() - if self.version: - self._add_version_argument() - - # add parent arguments and defaults - for parent in parents: - self._add_container_actions(parent) - try: - defaults = parent._defaults - except AttributeError: - pass - else: - self._defaults.update(defaults) - - # determines whether an "option" looks like a negative number - self._negative_number_matcher = _re.compile(r'^-\d+|-\d*.\d+$') - - - # ======================= - # Pretty __repr__ methods - # ======================= - - def _get_kwargs(self): - names = [ - 'prog', - 'usage', - 'description', - 'version', - 'formatter_class', - 'conflict_handler', - 'add_help', - ] - return [(name, getattr(self, name)) for name in names] - - # ================================== - # Namespace default settings methods - # ================================== - - def set_defaults(self, **kwargs): - self._defaults.update(kwargs) - - # ================================== - # Optional/Positional adding methods - # ================================== - - def add_argument_group(self, *args, **kwargs): - group = self._argument_group_class(self, *args, **kwargs) - self._argument_groups.append(group) - return group - - def add_subparsers(self, **kwargs): - if self._has_subparsers: - self.error(_('cannot have multiple subparser arguments')) - - # add the parser class to the arguments if it's not present - kwargs.setdefault('parser_class', type(self)) - - # prog defaults to the usage message of this parser, skipping - # optional arguments and with no "usage:" prefix - if kwargs.get('prog') is None: - formatter = self._get_formatter() - formatter.add_usage(self.usage, [], - self._get_positional_actions(), '') - kwargs['prog'] = formatter.format_help().strip() - - # create the parsers action and add it to the positionals list - parsers_class = self._pop_action_class(kwargs, 'parsers') - action = parsers_class(option_strings=[], **kwargs) - self._positional_actions_list.append(action) - self._positional_actions_full_list.append(action) - self._has_subparsers = True - - # return the created parsers action - return action - - def _add_container_actions(self, container): - super(ArgumentParser, self)._add_container_actions(container) - try: - groups = container._argument_groups - except AttributeError: - pass - else: - for group in groups: - new_group = self.add_argument_group( - title=group.title, - description=group.description, - conflict_handler=group.conflict_handler) - new_group._add_container_actions(group) - - def _get_optional_actions(self): - actions = [] - actions.extend(self._optional_actions_list) - for argument_group in self._argument_groups: - actions.extend(argument_group._optional_actions_list) - return actions - - def _get_positional_actions(self): - return list(self._positional_actions_full_list) - - def _add_help_argument(self): - self.add_argument('-h', '--help', action='help', - help=_('show this help message and exit')) - - def _add_version_argument(self): - self.add_argument('-v', '--version', action='version', - help=_("show program's version number and exit")) - - - # ===================================== - # Command line argument parsing methods - # ===================================== - - def parse_args(self, args=None, namespace=None): - # args default to the system args - if args is None: - args = _sys.argv[1:] - - # default Namespace built from parser defaults - if namespace is None: - namespace = Namespace() - - # add any action defaults that aren't present - optional_actions = self._get_optional_actions() - positional_actions = self._get_positional_actions() - for action in optional_actions + positional_actions: - if action.dest is not SUPPRESS: - if not hasattr(namespace, action.dest): - if action.default is not SUPPRESS: - default = action.default - if isinstance(action.default, basestring): - default = self._get_value(action, default) - setattr(namespace, action.dest, default) - - # add any parser defaults that aren't present - for dest, value in self._defaults.iteritems(): - if not hasattr(namespace, dest): - setattr(namespace, dest, value) - - # parse the arguments and exit if there are any errors - try: - result = self._parse_args(args, namespace) - except ArgumentError, err: - self.error(str(err)) - - # make sure all required optionals are present - for action in self._get_optional_actions(): - if action.required: - if getattr(result, action.dest, None) is None: - opt_strs = '/'.join(action.option_strings) - msg = _('option %s is required' % opt_strs) - self.error(msg) - - # return the parsed arguments - return result - - def _parse_args(self, arg_strings, namespace): - - # find all option indices, and determine the arg_string_pattern - # which has an 'O' if there is an option at an index, - # an 'A' if there is an argument, or a '-' if there is a '--' - option_string_indices = {} - arg_string_pattern_parts = [] - arg_strings_iter = iter(arg_strings) - for i, arg_string in enumerate(arg_strings_iter): - - # all args after -- are non-options - if arg_string == '--': - arg_string_pattern_parts.append('-') - for arg_string in arg_strings_iter: - arg_string_pattern_parts.append('A') - - # otherwise, add the arg to the arg strings - # and note the index if it was an option - else: - option_tuple = self._parse_optional(arg_string) - if option_tuple is None: - pattern = 'A' - else: - option_string_indices[i] = option_tuple - pattern = 'O' - arg_string_pattern_parts.append(pattern) - - # join the pieces together to form the pattern - arg_strings_pattern = ''.join(arg_string_pattern_parts) - - # converts arg strings to the appropriate and then takes the action - def take_action(action, argument_strings, option_string=None): - argument_values = self._get_values(action, argument_strings) - action(self, namespace, argument_values, option_string) - - # function to convert arg_strings into an optional action - def consume_optional(start_index): - - # determine the optional action and parse any explicit - # argument out of the option string - option_tuple = option_string_indices[start_index] - action, option_string, explicit_arg = option_tuple - - # loop because single-dash options can be chained - # (e.g. -xyz is the same as -x -y -z if no args are required) - match_argument = self._match_argument - action_tuples = [] - while True: - - # if we found no optional action, raise an error - if action is None: - self.error(_('no such option: %s') % option_string) - - # if there is an explicit argument, try to match the - # optional's string arguments to only this - if explicit_arg is not None: - arg_count = match_argument(action, 'A') - - # if the action is a single-dash option and takes no - # arguments, try to parse more single-dash options out - # of the tail of the option string - if arg_count == 0 and option_string[1] != '-': - action_tuples.append((action, [], option_string)) - option_string = '-' + explicit_arg - option_tuple = self._parse_optional(option_string) - if option_tuple[0] is None: - msg = _('ignored explicit argument %r') - raise ArgumentError(action, msg % explicit_arg) - - # set the action, etc. for the next loop iteration - action, option_string, explicit_arg = option_tuple - - # if the action expect exactly one argument, we've - # successfully matched the option; exit the loop - elif arg_count == 1: - stop = start_index + 1 - args = [explicit_arg] - action_tuples.append((action, args, option_string)) - break - - # error if a double-dash option did not use the - # explicit argument - else: - msg = _('ignored explicit argument %r') - raise ArgumentError(action, msg % explicit_arg) - - # if there is no explicit argument, try to match the - # optional's string arguments with the following strings - # if successful, exit the loop - else: - start = start_index + 1 - selected_patterns = arg_strings_pattern[start:] - arg_count = match_argument(action, selected_patterns) - stop = start + arg_count - args = arg_strings[start:stop] - action_tuples.append((action, args, option_string)) - break - - # add the Optional to the list and return the index at which - # the Optional's string args stopped - assert action_tuples - for action, args, option_string in action_tuples: - take_action(action, args, option_string) - return stop - - # the list of Positionals left to be parsed; this is modified - # by consume_positionals() - positionals = self._get_positional_actions() - - # function to convert arg_strings into positional actions - def consume_positionals(start_index): - # match as many Positionals as possible - match_partial = self._match_arguments_partial - selected_pattern = arg_strings_pattern[start_index:] - arg_counts = match_partial(positionals, selected_pattern) - - # slice off the appropriate arg strings for each Positional - # and add the Positional and its args to the list - for action, arg_count in zip(positionals, arg_counts): - args = arg_strings[start_index: start_index + arg_count] - start_index += arg_count - take_action(action, args) - - # slice off the Positionals that we just parsed and return the - # index at which the Positionals' string args stopped - positionals[:] = positionals[len(arg_counts):] - return start_index - - # consume Positionals and Optionals alternately, until we have - # passed the last option string - start_index = 0 - if option_string_indices: - max_option_string_index = max(option_string_indices) - else: - max_option_string_index = -1 - while start_index <= max_option_string_index: - - # consume any Positionals preceding the next option - next_option_string_index = min( - index - for index in option_string_indices - if index >= start_index) - if start_index != next_option_string_index: - positionals_end_index = consume_positionals(start_index) - - # only try to parse the next optional if we didn't consume - # the option string during the positionals parsing - if positionals_end_index > start_index: - start_index = positionals_end_index - continue - else: - start_index = positionals_end_index - - # if we consumed all the positionals we could and we're not - # at the index of an option string, there were unparseable - # arguments - if start_index not in option_string_indices: - msg = _('extra arguments found: %s') - extras = arg_strings[start_index:next_option_string_index] - self.error(msg % ' '.join(extras)) - - # consume the next optional and any arguments for it - start_index = consume_optional(start_index) - - # consume any positionals following the last Optional - stop_index = consume_positionals(start_index) - - # if we didn't consume all the argument strings, there were too - # many supplied - if stop_index != len(arg_strings): - extras = arg_strings[stop_index:] - self.error(_('extra arguments found: %s') % ' '.join(extras)) - - # if we didn't use all the Positional objects, there were too few - # arg strings supplied. - if positionals: - self.error(_('too few arguments')) - - # return the updated namespace - return namespace - - def _match_argument(self, action, arg_strings_pattern): - # match the pattern for this action to the arg strings - nargs_pattern = self._get_nargs_pattern(action) - match = _re.match(nargs_pattern, arg_strings_pattern) - - # raise an exception if we weren't able to find a match - if match is None: - nargs_errors = { - None:_('expected one argument'), - OPTIONAL:_('expected at most one argument'), - ONE_OR_MORE:_('expected at least one argument') - } - default = _('expected %s argument(s)') % action.nargs - msg = nargs_errors.get(action.nargs, default) - raise ArgumentError(action, msg) - - # return the number of arguments matched - return len(match.group(1)) - - def _match_arguments_partial(self, actions, arg_strings_pattern): - # progressively shorten the actions list by slicing off the - # final actions until we find a match - result = [] - for i in xrange(len(actions), 0, -1): - actions_slice = actions[:i] - pattern = ''.join(self._get_nargs_pattern(action) - for action in actions_slice) - match = _re.match(pattern, arg_strings_pattern) - if match is not None: - result.extend(len(string) for string in match.groups()) - break - - # return the list of arg string counts - return result - - def _parse_optional(self, arg_string): - # if it doesn't start with a '-', it was meant to be positional - if not arg_string.startswith('-'): - return None - - # if it's just dashes, it was meant to be positional - if not arg_string.strip('-'): - return None - - # if the option string is present in the parser, return the action - if arg_string in self._option_strings: - action = self._option_strings[arg_string] - return action, arg_string, None - - # search through all possible prefixes of the option string - # and all actions in the parser for possible interpretations - option_tuples = [] - prefix_tuples = self._get_option_prefix_tuples(arg_string) - for option_string in self._option_strings: - for option_prefix, explicit_arg in prefix_tuples: - if option_string.startswith(option_prefix): - action = self._option_strings[option_string] - tup = action, option_string, explicit_arg - option_tuples.append(tup) - break - - # if multiple actions match, the option string was ambiguous - if len(option_tuples) > 1: - options = ', '.join(opt_str for _, opt_str, _ in option_tuples) - tup = arg_string, options - self.error(_('ambiguous option: %s could match %s') % tup) - - # if exactly one action matched, this segmentation is good, - # so return the parsed action - elif len(option_tuples) == 1: - option_tuple, = option_tuples - return option_tuple - - # if it was not found as an option, but it looks like a negative - # number, it was meant to be positional - if self._negative_number_matcher.match(arg_string): - return None - - # it was meant to be an optional but there is no such option - # in this parser (though it might be a valid option in a subparser) - return None, arg_string, None - - def _get_option_prefix_tuples(self, option_string): - result = [] - - # option strings starting with '--' are only split at the '=' - if option_string.startswith('--'): - if '=' in option_string: - option_prefix, explicit_arg = option_string.split('=', 1) - else: - option_prefix = option_string - explicit_arg = None - tup = option_prefix, explicit_arg - result.append(tup) - - # option strings starting with '-' are split at all indices - else: - for first_index, char in enumerate(option_string): - if char != '-': - break - for i in xrange(len(option_string), first_index, -1): - tup = option_string[:i], option_string[i:] or None - result.append(tup) - - # return the collected prefix tuples - return result - - def _get_nargs_pattern(self, action): - # in all examples below, we have to allow for '--' args - # which are represented as '-' in the pattern - nargs = action.nargs - - # the default (None) is assumed to be a single argument - if nargs is None: - nargs_pattern = '(-*A-*)' - - # allow zero or one arguments - elif nargs == OPTIONAL: - nargs_pattern = '(-*A?-*)' - - # allow zero or more arguments - elif nargs == ZERO_OR_MORE: - nargs_pattern = '(-*[A-]*)' - - # allow one or more arguments - elif nargs == ONE_OR_MORE: - nargs_pattern = '(-*A[A-]*)' - - # allow one argument followed by any number of options or arguments - elif nargs is PARSER: - nargs_pattern = '(-*A[-AO]*)' - - # all others should be integers - else: - nargs_pattern = '(-*%s-*)' % '-*'.join('A' * nargs) - - # if this is an optional action, -- is not allowed - if action.option_strings: - nargs_pattern = nargs_pattern.replace('-*', '') - nargs_pattern = nargs_pattern.replace('-', '') - - # return the pattern - return nargs_pattern - - # ======================== - # Value conversion methods - # ======================== - - def _get_values(self, action, arg_strings): - # for everything but PARSER args, strip out '--' - if action.nargs is not PARSER: - arg_strings = [s for s in arg_strings if s != '--'] - - # optional argument produces a default when not present - if not arg_strings and action.nargs == OPTIONAL: - if action.option_strings: - value = action.const - else: - value = action.default - if isinstance(value, basestring): - value = self._get_value(action, value) - self._check_value(action, value) - - # when nargs='*' on a positional, if there were no command-line - # args, use the default if it is anything other than None - elif (not arg_strings and action.nargs == ZERO_OR_MORE and - not action.option_strings): - if action.default is not None: - value = action.default - else: - value = arg_strings - self._check_value(action, value) - - # single argument or optional argument produces a single value - elif len(arg_strings) == 1 and action.nargs in [None, OPTIONAL]: - arg_string, = arg_strings - value = self._get_value(action, arg_string) - self._check_value(action, value) - - # PARSER arguments convert all values, but check only the first - elif action.nargs is PARSER: - value = list(self._get_value(action, v) for v in arg_strings) - self._check_value(action, value[0]) - - # all other types of nargs produce a list - else: - value = list(self._get_value(action, v) for v in arg_strings) - for v in value: - self._check_value(action, v) - - # return the converted value - return value - - def _get_value(self, action, arg_string): - type_func = self._registry_get('type', action.type, action.type) - if not callable(type_func): - msg = _('%r is not callable') - raise ArgumentError(action, msg % type_func) - - # convert the value to the appropriate type - try: - result = type_func(arg_string) - - # TypeErrors or ValueErrors indicate errors - except (TypeError, ValueError): - name = getattr(action.type, '__name__', repr(action.type)) - msg = _('invalid %s value: %r') - raise ArgumentError(action, msg % (name, arg_string)) - - # return the converted value - return result - - def _check_value(self, action, value): - # converted value must be one of the choices (if specified) - if action.choices is not None and value not in action.choices: - tup = value, ', '.join(map(repr, action.choices)) - msg = _('invalid choice: %r (choose from %s)') % tup - raise ArgumentError(action, msg) - - - - # ======================= - # Help-formatting methods - # ======================= - - def format_usage(self): - formatter = self._get_formatter() - formatter.add_usage(self.usage, - self._get_optional_actions(), - self._get_positional_actions()) - return formatter.format_help() - - def format_help(self): - formatter = self._get_formatter() - - # usage - formatter.add_usage(self.usage, - self._get_optional_actions(), - self._get_positional_actions()) - - # description - formatter.add_text(self.description) - - # positionals - formatter.start_section(_('positional arguments')) - formatter.add_arguments(self._positional_actions_list) - formatter.end_section() - - # optionals - formatter.start_section(_('optional arguments')) - formatter.add_arguments(self._optional_actions_list) - formatter.end_section() - - # user-defined groups - for argument_group in self._argument_groups: - formatter.start_section(argument_group.title) - formatter.add_text(argument_group.description) - formatter.add_arguments(argument_group._positional_actions_list) - formatter.add_arguments(argument_group._optional_actions_list) - formatter.end_section() - - # epilog - formatter.add_text(self.epilog) - - # determine help from format above - return formatter.format_help() - - def format_version(self): - formatter = self._get_formatter() - formatter.add_text(self.version) - return formatter.format_help() - - def _get_formatter(self): - return self.formatter_class(prog=self.prog) - - # ===================== - # Help-printing methods - # ===================== - - def print_usage(self, file=None): - self._print_message(self.format_usage(), file) - - def print_help(self, file=None): - self._print_message(self.format_help(), file) - - def print_version(self, file=None): - self._print_message(self.format_version(), file) - - def _print_message(self, message, file=None): - if message: - if file is None: - file = _sys.stderr - file.write(message) - - - # =============== - # Exiting methods - # =============== - - def exit(self, status=0, message=None): - if message: - _sys.stderr.write(message) - _sys.exit(status) - - def error(self, message): - """error(message: string) - - Prints a usage message incorporating the message to stderr and - exits. - - If you override this in a subclass, it should not return -- it - should either exit or raise an exception. - """ - self.print_usage(_sys.stderr) - self.exit(2, _('%s: error: %s\n') % (self.prog, message)) diff --git a/zfec/zfec/util/fileutil.py b/zfec/zfec/util/fileutil.py deleted file mode 100644 index 147ecbf..0000000 --- a/zfec/zfec/util/fileutil.py +++ /dev/null @@ -1,257 +0,0 @@ -""" -Futz with files like a pro. -""" - -import exceptions, os, stat, tempfile, time - -try: - from twisted.python import log -except ImportError: - class DummyLog: - def msg(self, *args, **kwargs): - pass - log = DummyLog() - -def rename(src, dst, tries=4, basedelay=0.1): - """ Here is a superkludge to workaround the fact that occasionally on - Windows some other process (e.g. an anti-virus scanner, a local search - engine, etc.) is looking at your file when you want to delete or move it, - and hence you can't. The horrible workaround is to sit and spin, trying - to delete it, for a short time and then give up. - - With the default values of tries and basedelay this can block for less - than a second. - - @param tries: number of tries -- each time after the first we wait twice - as long as the previous wait - @param basedelay: how long to wait before the second try - """ - for i in range(tries-1): - try: - return os.rename(src, dst) - except EnvironmentError, le: - # XXX Tighten this to check if this is a permission denied error (possibly due to another Windows process having the file open and execute the superkludge only in this case. - log.msg("XXX KLUDGE Attempting to move file %s => %s; got %s; sleeping %s seconds" % (src, dst, le, basedelay,)) - time.sleep(basedelay) - basedelay *= 2 - return os.rename(src, dst) # The last try. - -def remove(f, tries=4, basedelay=0.1): - """ Here is a superkludge to workaround the fact that occasionally on - Windows some other process (e.g. an anti-virus scanner, a local search - engine, etc.) is looking at your file when you want to delete or move it, - and hence you can't. The horrible workaround is to sit and spin, trying - to delete it, for a short time and then give up. - - With the default values of tries and basedelay this can block for less - than a second. - - @param tries: number of tries -- each time after the first we wait twice - as long as the previous wait - @param basedelay: how long to wait before the second try - """ - try: - os.chmod(f, stat.S_IWRITE | stat.S_IEXEC | stat.S_IREAD) - except: - pass - for i in range(tries-1): - try: - return os.remove(f) - except EnvironmentError, le: - # XXX Tighten this to check if this is a permission denied error (possibly due to another Windows process having the file open and execute the superkludge only in this case. - if not os.path.exists(f): - return - log.msg("XXX KLUDGE Attempting to remove file %s; got %s; sleeping %s seconds" % (f, le, basedelay,)) - time.sleep(basedelay) - basedelay *= 2 - return os.remove(f) # The last try. - -class _Dir(object): - """ - Hold a set of files and subdirs and clean them all up when asked to. - """ - def __init__(self, name, cleanup=True): - self.name = name - self.cleanup = cleanup - self.files = set() - self.subdirs = set() - - def file(self, fname, mode=None): - """ - Create a file in the tempdir and remember it so as to close() it - before attempting to cleanup the temp dir. - - @rtype: file - """ - ffn = os.path.join(self.name, fname) - if mode is not None: - fo = open(ffn, mode) - else: - fo = open(ffn) - self.register_file(fo) - return fo - - def subdir(self, dirname): - """ - Create a subdirectory in the tempdir and remember it so as to call - shutdown() on it before attempting to clean up. - - @rtype: _Dir instance - """ - ffn = os.path.join(self.name, dirname) - sd = _Dir(ffn, self.cleanup) - self.register_subdir(sd) - - def register_file(self, fileobj): - """ - Remember the file object and call close() on it before attempting to - clean up. - """ - self.files.add(fileobj) - - def register_subdir(self, dirobj): - """ - Remember the _Dir object and call shutdown() on it before attempting - to clean up. - """ - self.subdirs.add(dirobj) - - def shutdown(self): - if self.cleanup: - for subdir in hasattr(self, 'subdirs') and self.subdirs or []: - subdir.shutdown() - for fileobj in hasattr(self, 'files') and self.files or []: - fileobj.close() # "close()" is idempotent so we don't need to catch exceptions here - if hasattr(self, 'name'): - rm_dir(self.name) - - def __repr__(self): - return "<%s instance at %x %s>" % (self.__class__.__name__, id(self), self.name) - - def __str__(self): - return self.__repr__() - - def __del__(self): - try: - self.shutdown() - except: - import traceback - traceback.print_exc() - -class NamedTemporaryDirectory(_Dir): - """ - Call tempfile.mkdtemp(), store the name of the dir in self.name, and - rm_dir() when it gets garbage collected or "shutdown()". - - Also keep track of file objects for files within the tempdir and call - close() on them before rm_dir(). This is a convenient way to open temp - files within the directory, and it is very helpful on Windows because you - can't delete a directory which contains a file which is currently open. - """ - - def __init__(self, cleanup=True, *args, **kwargs): - """ If cleanup, then the directory will be rmrf'ed when the object is shutdown. """ - name = tempfile.mkdtemp(*args, **kwargs) - _Dir.__init__(self, name, cleanup) - -def make_dirs(dirname, mode=0777, strictmode=False): - """ - A threadsafe and idempotent version of os.makedirs(). If the dir already - exists, do nothing and return without raising an exception. If this call - creates the dir, return without raising an exception. If there is an - error that prevents creation or if the directory gets deleted after - make_dirs() creates it and before make_dirs() checks that it exists, raise - an exception. - - @param strictmode if true, then make_dirs() will raise an exception if the - directory doesn't have the desired mode. For example, if the - directory already exists, and has a different mode than the one - specified by the mode parameter, then if strictmode is true, - make_dirs() will raise an exception, else it will ignore the - discrepancy. - """ - tx = None - try: - os.makedirs(dirname, mode) - except OSError, x: - tx = x - - if not os.path.isdir(dirname): - if tx: - raise tx - raise exceptions.IOError, "unknown error prevented creation of directory, or deleted the directory immediately after creation: %s" % dirname # careful not to construct an IOError with a 2-tuple, as that has a special meaning... - - tx = None - if hasattr(os, 'chmod'): - try: - os.chmod(dirname, mode) - except OSError, x: - tx = x - - if strictmode and hasattr(os, 'stat'): - s = os.stat(dirname) - resmode = stat.S_IMODE(s.st_mode) - if resmode != mode: - if tx: - raise tx - raise exceptions.IOError, "unknown error prevented setting correct mode of directory, or changed mode of the directory immediately after creation. dirname: %s, mode: %04o, resmode: %04o" % (dirname, mode, resmode,) # careful not to construct an IOError with a 2-tuple, as that has a special meaning... - -def rm_dir(dirname): - """ - A threadsafe and idempotent version of shutil.rmtree(). If the dir is - already gone, do nothing and return without raising an exception. If this - call removes the dir, return without raising an exception. If there is an - error that prevents deletion or if the directory gets created again after - rm_dir() deletes it and before rm_dir() checks that it is gone, raise an - exception. - """ - excs = [] - try: - os.chmod(dirname, stat.S_IWRITE | stat.S_IEXEC | stat.S_IREAD) - for f in os.listdir(dirname): - fullname = os.path.join(dirname, f) - if os.path.isdir(fullname): - rm_dir(fullname) - else: - remove(fullname) - os.rmdir(dirname) - except Exception, le: - # Ignore "No such file or directory" - if (not isinstance(le, OSError)) or le.args[0] != 2: - excs.append(le) - - # Okay, now we've recursively removed everything, ignoring any "No - # such file or directory" errors, and collecting any other errors. - - if os.path.exists(dirname): - if len(excs) == 1: - raise excs[0] - if len(excs) == 0: - raise OSError, "Failed to remove dir for unknown reason." - raise OSError, excs - - -def remove_if_possible(f): - try: - remove(f) - except: - pass - -# zfec -- fast forward error correction library with Python interface -# -# Copyright (C) 2007 Allmydata, Inc. -# Author: Zooko Wilcox-O'Hearn -# -# This file is part of zfec. -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation; either version 2 of the License, or (at your option) -# any later version, with the added permission that, if you become obligated -# to release a derived work under this licence (as per section 2.b), you may -# delay the fulfillment of this obligation for up to 12 months. See the file -# COPYING for details. -# -# If you would like to inquire about a commercial relationship with Allmydata, -# Inc., please contact partnerships@allmydata.com and visit -# http://allmydata.com/. diff --git a/zfec/zfec/util/mathutil.py b/zfec/zfec/util/mathutil.py deleted file mode 100644 index caf7b1e..0000000 --- a/zfec/zfec/util/mathutil.py +++ /dev/null @@ -1,91 +0,0 @@ -""" -A few commonly needed functions. -""" - -import math - -def div_ceil(n, d): - """ - The smallest integer k such that k*d >= n. - """ - return (n/d) + (n%d != 0) - -def next_multiple(n, k): - """ - The smallest multiple of k which is >= n. - """ - return div_ceil(n, k) * k - -def pad_size(n, k): - """ - The smallest number that has to be added to n so that n is a multiple of k. - """ - if n%k: - return k - n%k - else: - return 0 - -def is_power_of_k(n, k): - return k**int(math.log(n, k) + 0.5) == n - -def next_power_of_k(n, k): - p = 1 - while p < n: - p *= k - return p - -def ave(l): - return sum(l) / len(l) - -def log_ceil(n, b): - """ - The smallest integer k such that b^k >= n. - - log_ceil(n, 2) is the number of bits needed to store any of n values, e.g. - the number of bits needed to store any of 128 possible values is 7. - """ - p = 1 - k = 0 - while p < n: - p *= b - k += 1 - return k - -def permute(l): - """ - Return all possible permutations of l. - - @type l: sequence - @rtype: a list of sequences - """ - if len(l) == 1: - return [l,] - - res = [] - for i in range(len(l)): - l2 = list(l[:]) - x = l2.pop(i) - for l3 in permute(l2): - l3.append(x) - res.append(l3) - - return res - -# zfec -- fast forward error correction library with Python interface -# -# Copyright (C) 2007 Allmydata, Inc. -# Author: Zooko Wilcox-O'Hearn -# -# This file is part of zfec. -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation; either version 2 of the License, or (at your option) -# any later version, with the added permission that, if you become obligated -# to release a derived work under this licence (as per section 2.b), you may -# delay the fulfillment of this obligation for up to 12 months. See the file -# COPYING for details. -# -# If you would like to inquire about a commercial relationship with Allmydata, -# Inc., please contact partnerships@allmydata.com and visit -# http://allmydata.com/. diff --git a/zfec/zfec/util/version_class.py b/zfec/zfec/util/version_class.py deleted file mode 100644 index 1e1e2fc..0000000 --- a/zfec/zfec/util/version_class.py +++ /dev/null @@ -1,141 +0,0 @@ -# Copyright (c) 2004-2007 Bryce "Zooko" Wilcox-O'Hearn -# mailto:zooko@zooko.com -# http://zooko.com/repos/pyutil -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this work to deal in this work without restriction (including the rights -# to use, modify, distribute, sublicense, and/or sell copies). - -""" -extended version number class -""" - -# from setuptools, but intended to be included in future version of Python Standard Library (PEP 365) -try: - import pkg_resources -except ImportError: - import distutils.version - def cmp_version(v1, v2): - return cmp(distutils.version.LooseVersion(str(v1)), distutils.version.LooseVersion(str(v2))) -else: - def cmp_version(v1, v2): - return cmp(pkg_resources.parse_version(str(v1)), pkg_resources.parse_version(str(v2))) - -# bindann, by Nathan Wilcox (needed only for debugging) -try: - import bindann - bindann.install_exception_handler() -except ImportError: - pass - -# Python Standard Library -import re - -# End users see version strings like this: - -# "1.0.0" -# ^ ^ ^ -# | | | -# | | '- micro version number -# | '- minor version number -# '- major version number - -# The first number is "major version number". The second number is the "minor -# version number" -- it gets bumped whenever we make a new release that adds or -# changes functionality. The third version is the "micro version number" -- it -# gets bumped whenever we make a new release that doesn't add or change -# functionality, but just fixes bugs (including performance issues). - -# Early-adopter end users see version strings like this: - -# "1.0.0a1" -# ^ ^ ^^^ -# | | ||| -# | | ||'- release number -# | | |'- a=alpha, b=beta, c=release candidate, or none -# | | '- micro version number -# | '- minor version number -# '- major version number - -# The optional "a" or "b" stands for "alpha release" or "beta release" -# respectively. The number after "a" or "b" gets bumped every time we -# make a new alpha or beta release. This has the same form and the same -# meaning as version numbers of releases of Python. - -# Developers see "full version strings", like this: - -# "1.0.0a1-55" -# ^ ^ ^^^ ^ -# | | ||| | -# | | ||| '- nano version number -# | | ||'- release number -# | | |'- a=alpha, b=beta, c=release candidate or none -# | | '- micro version number -# | '- minor version number -# '- major version number - -# The presence of the nano version number means that this is a development -# version. There are no guarantees about compatibility, etc. This version is -# considered to be more recent than the version without this field -# (e.g. "1.0.0a1"). - -# The nano version number is meaningful only to developers. It gets generated -# automatically from darcs revision control history by "make-version.py". It -# is the count of patches that have been applied since the last version number -# tag was applied. - -VERSION_BASE_RE_STR="(\d+)\.(\d+)(\.(\d+))?((a|b|c)(\d+))?" -VERSION_RE_STR=VERSION_BASE_RE_STR + "(-(\d+))?" -VERSION_RE=re.compile("^" + VERSION_RE_STR + "$") - -class Version(object): - def __init__(self, vstring=None): - self.major = None - self.minor = None - self.micro = None - self.prereleasetag = None - self.prerelease = None - self.nano = None - self.leftovers = '' - if vstring: - try: - self.parse(vstring) - except ValueError, le: - le.args = tuple(le.args + ('vstring:', vstring,)) - raise - - def parse(self, vstring): - mo = VERSION_RE.search(vstring) - if not mo: - raise ValueError, "Not a valid version string for pyutil.version_class.Version(): %r" % (vstring,) - - self.major = int(mo.group(1)) - self.minor = int(mo.group(2)) - self.micro = int(mo.group(4)) - reltag = mo.group(5) - if reltag: - reltagnum = int(mo.group(6)) - self.prereleasetag = reltag - self.prerelease = reltagnum - - if mo.group(8): - self.nano = int(mo.group(9)) - - self.fullstr = "%d.%d.%d%s%s" % (self.major, self.minor, self.micro, self.prereleasetag and "%s%d" % (self.prereleasetag, self.prerelease,) or "", self.nano and "-%d" % (self.nano,) or "",) - - def user_str(self): - return self.strictversion.__str__() - - def full_str(self): - if hasattr(self, 'fullstr'): - return self.fullstr - else: - return 'None' - - def __str__(self): - return self.full_str() - - def __repr__(self): - return self.__str__() - - def __cmp__ (self, other): - return cmp_version(self, other)