From: Brian Warner Date: Wed, 29 Jul 2015 00:38:09 +0000 (-0700) Subject: Merge branch '2436.less-scary.3' X-Git-Tag: allmydata-tahoe-1.10.2b1~3 X-Git-Url: https://git.rkrishnan.org/?a=commitdiff_plain;h=28399b689203a0841880cc03b378dcc0a696d8a3;hp=29ab496bd75c8f9c1891f7a97ff9e682c79277e4;p=tahoe-lafs%2Ftahoe-lafs.git Merge branch '2436.less-scary.3' This reverts and improves the earlier fix for ticket:2436 --- diff --git a/src/allmydata/__init__.py b/src/allmydata/__init__.py index 5262773a..0b0c5f81 100644 --- a/src/allmydata/__init__.py +++ b/src/allmydata/__init__.py @@ -178,7 +178,7 @@ def extract_openssl_version(ssl_module): def get_package_versions_and_locations(): import warnings from _auto_deps import package_imports, global_deprecation_messages, deprecation_messages, \ - runtime_warning_messages, warning_imports + runtime_warning_messages, warning_imports, ignorable def package_dir(srcfile): return os.path.dirname(os.path.dirname(os.path.normcase(os.path.realpath(srcfile)))) @@ -246,7 +246,26 @@ def get_package_versions_and_locations(): elif pkgname == 'OpenSSL': packages.append( (pkgname, get_openssl_version()) ) - return packages + cross_check_errors = [] + + if not hasattr(sys, 'frozen'): + import pkg_resources + from _auto_deps import install_requires + + pkg_resources_vers_and_locs = dict([(p.project_name.lower(), (str(p.version), p.location)) + for p in pkg_resources.require(install_requires)]) + + imported_packages = set([p.lower() for (p, _) in packages]) + extra_packages = [] + + for pr_name, (pr_ver, pr_loc) in pkg_resources_vers_and_locs.iteritems(): + if pr_name not in imported_packages and pr_name not in ignorable: + extra_packages.append( (pr_name, (pr_ver, pr_loc, "according to pkg_resources")) ) + + cross_check_errors = cross_check(pkg_resources_vers_and_locs, packages) + packages += extra_packages + + return packages, cross_check_errors def check_requirement(req, vers_and_locs): @@ -300,25 +319,10 @@ def match_requirement(req, reqlist, actualver): return True -_vers_and_locs_list = get_package_versions_and_locations() - - -def cross_check_pkg_resources_versus_import(): - """This function returns a list of errors due to any failed cross-checks.""" - - import pkg_resources - from _auto_deps import install_requires - - pkg_resources_vers_and_locs = dict([(p.project_name.lower(), (str(p.version), p.location)) - for p in pkg_resources.require(install_requires)]) - - return cross_check(pkg_resources_vers_and_locs, _vers_and_locs_list) - - def cross_check(pkg_resources_vers_and_locs, imported_vers_and_locs_list): """This function returns a list of errors due to any failed cross-checks.""" - from _auto_deps import not_import_versionable, ignorable + from _auto_deps import not_import_versionable errors = [] not_pkg_resourceable = ['python', 'platform', __appname__.lower(), 'openssl'] @@ -376,13 +380,10 @@ def cross_check(pkg_resources_vers_and_locs, imported_vers_and_locs_list): "by pkg_resources, but version %r (normalized to %r, from %r) by import." % (name, pr_ver, str(pr_normver), pr_loc, imp_ver, str(imp_normver), imp_loc)) - imported_packages = set([p.lower() for (p, _) in imported_vers_and_locs_list]) - extra_vers_and_locs_list = [] - for pr_name, (pr_ver, pr_loc) in pkg_resources_vers_and_locs.iteritems(): - if pr_name not in imported_packages and pr_name not in ignorable: - extra_vers_and_locs_list.append( (pr_name, (pr_ver, pr_loc, "according to pkg_resources")) ) + return errors - return errors, extra_vers_and_locs_list + +_vers_and_locs_list, _cross_check_errors = get_package_versions_and_locations() def get_error_string(errors, debug=False): @@ -405,7 +406,7 @@ def check_all_requirements(): from allmydata._auto_deps import install_requires - errors = [] + fatal_errors = [] # We require at least 2.6 on all platforms. # (On Python 3, we'll have failed long before this point.) @@ -414,18 +415,18 @@ def check_all_requirements(): version_string = ".".join(map(str, sys.version_info)) except Exception: version_string = repr(sys.version_info) - errors.append("Tahoe-LAFS currently requires Python v2.6 or greater (but less than v3), not %s" - % (version_string,)) + fatal_errors.append("Tahoe-LAFS currently requires Python v2.6 or greater (but less than v3), not %s" + % (version_string,)) vers_and_locs = dict(_vers_and_locs_list) for requirement in install_requires: try: check_requirement(requirement, vers_and_locs) except (ImportError, PackagingError), e: - errors.append("%s: %s" % (e.__class__.__name__, e)) + fatal_errors.append("%s: %s" % (e.__class__.__name__, e)) - if errors: - raise PackagingError(get_error_string(errors, debug=True)) + if fatal_errors: + raise PackagingError(get_error_string(fatal_errors + _cross_check_errors, debug=True)) check_all_requirements() @@ -437,12 +438,6 @@ def get_package_locations(): return dict([(k, l) for k, (v, l, c) in _vers_and_locs_list]) def get_package_versions_string(show_paths=False, debug=False): - errors = [] - if not hasattr(sys, 'frozen'): - global _vers_and_locs_list - errors, extra_vers_and_locs_list = cross_check_pkg_resources_versus_import() - _vers_and_locs_list += extra_vers_and_locs_list - res = [] for p, (v, loc, comment) in _vers_and_locs_list: info = str(p) + ": " + str(v) @@ -454,7 +449,7 @@ def get_package_versions_string(show_paths=False, debug=False): output = "\n".join(res) + "\n" - if errors: - output += get_error_string(errors, debug=debug) + if _cross_check_errors: + output += get_error_string(_cross_check_errors, debug=debug) return output diff --git a/src/allmydata/test/test_import.py b/src/allmydata/test/test_import.py index 287c4f5a..b34012d6 100644 --- a/src/allmydata/test/test_import.py +++ b/src/allmydata/test/test_import.py @@ -8,19 +8,22 @@ import __builtin__ class T(unittest.TestCase): def test_report_import_error(self): + marker = "wheeeyo" real_import_func = __import__ def raiseIE_from_this_particular_func(name, *args): if name == "foolscap": - marker = "wheeeyo" raise ImportError(marker + " foolscap cant be imported") else: return real_import_func(name, *args) # Let's run as little code as possible with __import__ patched. patcher = MonkeyPatcher((__builtin__, '__import__', raiseIE_from_this_particular_func)) - vers_and_locs = patcher.runWithPatches(allmydata.get_package_versions_and_locations) + vers_and_locs, errors = patcher.runWithPatches(allmydata.get_package_versions_and_locations) - for (pkgname, stuff) in vers_and_locs: - if pkgname == 'foolscap': - self.failUnless('wheeeyo' in str(stuff[2]), stuff) - self.failUnless('raiseIE_from_this_particular_func' in str(stuff[2]), stuff) + foolscap_stuffs = [stuff for (pkg, stuff) in vers_and_locs if pkg == 'foolscap'] + self.failUnlessEqual(len(foolscap_stuffs), 1) + comment = str(foolscap_stuffs[0][2]) + self.failUnlessIn(marker, comment) + self.failUnlessIn('raiseIE_from_this_particular_func', comment) + + self.failUnless([e for e in errors if "dependency \'foolscap\' could not be imported" in e]) diff --git a/src/allmydata/test/test_version.py b/src/allmydata/test/test_version.py index 076e6189..7851f91a 100644 --- a/src/allmydata/test/test_version.py +++ b/src/allmydata/test/test_version.py @@ -1,9 +1,12 @@ +import sys +import pkg_resources from pkg_resources import Requirement from twisted.trial import unittest -from allmydata import check_requirement, cross_check, extract_openssl_version, PackagingError +from allmydata import check_requirement, cross_check, get_package_versions_and_locations, \ + extract_openssl_version, PackagingError from allmydata.util.verlib import NormalizedVersion as V, \ IrrationalVersionError, \ suggest_normalized_version as suggest @@ -69,75 +72,87 @@ class CheckRequirement(unittest.TestCase): for pkg, ver in vers_and_locs.items(): self.failIf(ver[0] in Requirement.parse(req), str((ver, req))) + def test_packages_from_pkg_resources(self): + if hasattr(sys, 'frozen'): + raise unittest.SkipTest("This test doesn't apply to frozen builds.") + + class MockPackage(object): + def __init__(self, project_name, version, location): + self.project_name = project_name + self.version = version + self.location = location + + def call_pkg_resources_require(*args): + return [MockPackage("Foo", "1.0", "/path")] + self.patch(pkg_resources, 'require', call_pkg_resources_require) + + (packages, errors) = get_package_versions_and_locations() + self.failUnlessIn(("foo", ("1.0", "/path", "according to pkg_resources")), packages) + self.failIfEqual(errors, []) + self.failUnlessEqual([e for e in errors if "was not found by pkg_resources" not in e], []) + def test_cross_check_ticket_1355(self): # The bug in #1355 is triggered when a version string from either pkg_resources or import # is not parseable at all by normalized_version. - (errors, extras) = cross_check({"foo": ("unparseable", "")}, [("foo", ("1.0", "", None))]) - self.failUnlessEqual(extras, []) - self.failUnlessEqual(len(errors), 1) - self.failUnlessIn("by pkg_resources could not be parsed", errors[0]) + res = cross_check({"foo": ("unparseable", "")}, [("foo", ("1.0", "", None))]) + self.failUnlessEqual(len(res), 1) + self.failUnlessIn("by pkg_resources could not be parsed", res[0]) - (errors, extras) = cross_check({"foo": ("1.0", "")}, [("foo", ("unparseable", "", None))]) - self.failUnlessEqual(extras, []) - self.failUnlessEqual(len(errors), 1) - self.failUnlessIn(") could not be parsed", errors[0]) + res = cross_check({"foo": ("1.0", "")}, [("foo", ("unparseable", "", None))]) + self.failUnlessEqual(len(res), 1) + self.failUnlessIn(") could not be parsed", res[0]) def test_cross_check(self): res = cross_check({}, []) - self.failUnlessEqual(res, ([], [])) + self.failUnlessEqual(res, []) res = cross_check({}, [("allmydata-tahoe", ("1.0", "", "blah"))]) - self.failUnlessEqual(res, ([], [])) + self.failUnlessEqual(res, []) res = cross_check({"foo": ("unparseable", "")}, []) - self.failUnlessEqual(res, ([], [("foo", ("unparseable", "", "according to pkg_resources"))])) + self.failUnlessEqual(res, []) res = cross_check({"argparse": ("unparseable", "")}, []) - self.failUnlessEqual(res, ([], [])) + self.failUnlessEqual(res, []) - (errors, extras) = cross_check({}, [("foo", ("unparseable", "", None))]) - self.failUnlessEqual(extras, []) - self.failUnlessEqual(len(errors), 1) - self.failUnlessIn("was not found by pkg_resources", errors[0]) + res = cross_check({}, [("foo", ("unparseable", "", None))]) + self.failUnlessEqual(len(res), 1) + self.failUnlessIn("not found by pkg_resources", res[0]) res = cross_check({"distribute": ("1.0", "/somewhere")}, [("setuptools", ("2.0", "/somewhere", "distribute"))]) - self.failUnlessEqual(res, ([], [])) + self.failUnlessEqual(res, []) - (errors, extras) = cross_check({"distribute": ("1.0", "/somewhere")}, [("setuptools", ("2.0", "/somewhere", None))]) - self.failUnlessEqual(extras, []) - self.failUnlessEqual(len(errors), 1) - self.failUnlessIn("location mismatch", errors[0]) + res = cross_check({"distribute": ("1.0", "/somewhere")}, [("setuptools", ("2.0", "/somewhere", None))]) + self.failUnlessEqual(len(res), 1) + self.failUnlessIn("location mismatch", res[0]) - (errors, extras) = cross_check({"distribute": ("1.0", "/somewhere")}, [("setuptools", ("2.0", "/somewhere_different", None))]) - self.failUnlessEqual(extras, []) - self.failUnlessEqual(len(errors), 1) - self.failUnlessIn("location mismatch", errors[0]) + res = cross_check({"distribute": ("1.0", "/somewhere")}, [("setuptools", ("2.0", "/somewhere_different", None))]) + self.failUnlessEqual(len(res), 1) + self.failUnlessIn("location mismatch", res[0]) res = cross_check({"zope.interface": ("1.0", "")}, [("zope.interface", ("unknown", "", None))]) - self.failUnlessEqual(res, ([], [])) + self.failUnlessEqual(res, []) - (errors, extras) = cross_check({"foo": ("1.0", "")}, [("foo", ("unknown", "", None))]) - self.failUnlessEqual(extras, []) - self.failUnlessEqual(len(errors), 1) - self.failUnlessIn("could not find a version number", errors[0]) + res = cross_check({"foo": ("1.0", "")}, [("foo", ("unknown", "", None))]) + self.failUnlessEqual(len(res), 1) + self.failUnlessIn("could not find a version number", res[0]) # When pkg_resources and import both find a package, there is only a warning if both # the version and the path fail to match. res = cross_check({"foo": ("1.0", "/somewhere")}, [("foo", ("2.0", "/somewhere", None))]) - self.failUnlessEqual(res, ([], [])) + self.failUnlessEqual(res, []) res = cross_check({"foo": ("1.0", "/somewhere")}, [("foo", ("1.0", "/somewhere_different", None))]) - self.failUnlessEqual(res, ([], [])) + self.failUnlessEqual(res, []) res = cross_check({"foo": ("1.0-r123", "/somewhere")}, [("foo", ("1.0.post123", "/somewhere_different", None))]) - self.failUnlessEqual(res, ([], [])) + self.failUnlessEqual(res, []) - (errors, extras) = cross_check({"foo": ("1.0", "/somewhere")}, [("foo", ("2.0", "/somewhere_different", None))]) - self.failUnlessEqual(extras, []) - self.failUnlessEqual(len(errors), 1) - self.failUnlessIn("but version '2.0'", errors[0]) + res = cross_check({"foo": ("1.0", "/somewhere")}, [("foo", ("2.0", "/somewhere_different", None))]) + self.failUnlessEqual(len(res), 1) + self.failUnlessIn("but version '2.0'", res[0]) def test_extract_openssl_version(self): self.failUnlessEqual(extract_openssl_version(MockSSL("")),