From: Daira Hopwood Date: Tue, 24 Feb 2015 17:38:19 +0000 (+0000) Subject: WIP X-Git-Url: https://git.rkrishnan.org/%3C?a=commitdiff_plain;h=7a5900aaaf77dfe731484030303a6ce2d0761747;p=tahoe-lafs%2Ftahoe-lafs.git WIP --- diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 35ccb355..516b0f22 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -11,7 +11,8 @@ from allmydata.util import log from allmydata.util import fileutil, iputil, observer from allmydata.util.assertutil import precondition, _assert from allmydata.util.fileutil import abspath_expanduser_unicode -from allmydata.util.encodingutil import get_filesystem_encoding, quote_output +from allmydata.util.encodingutil import get_filesystem_encoding, quote_output, \ + quote_local_unicode_path # Add our application versions to the data that Foolscap's LogPublisher # reports. @@ -50,7 +51,7 @@ class OldConfigError(Exception): return ("Found pre-Tahoe-LAFS-v1.3 configuration file(s):\n" "%s\n" "See docs/historical/configuration.rst." - % "\n".join([quote_output(fname) for fname in self.args[0]])) + % "\n".join([quote_local_unicode_path(fname) for fname in self.args[0]])) class OldConfigOptionError(Exception): pass @@ -74,8 +75,8 @@ class Node(service.MultiService): self.basedir = abspath_expanduser_unicode(unicode(basedir)) self._portnumfile = os.path.join(self.basedir, self.PORTNUMFILE) self._tub_ready_observerlist = observer.OneShotObserverList() - fileutil.make_dirs(os.path.join(self.basedir, "private"), 0700) - open(os.path.join(self.basedir, "private", "README"), "w").write(PRIV_README) + fileutil.make_dirs(os.path.join(self.basedir, u"private"), 0700) + open(os.path.join(self.basedir, u"private", u"README"), "w").write(PRIV_README) # creates self.config self.read_config() @@ -143,7 +144,7 @@ class Node(service.MultiService): self.error_about_old_config_files() self.config = ConfigParser.SafeConfigParser() - tahoe_cfg = os.path.join(self.basedir, "tahoe.cfg") + tahoe_cfg = os.path.join(self.basedir, u"tahoe.cfg") try: f = open(tahoe_cfg, "rb") try: @@ -181,7 +182,7 @@ class Node(service.MultiService): 'no_storage', 'readonly_storage', 'sizelimit', 'debug_discard_storage', 'run_helper']: if name not in self.GENERATED_FILES: - fullfname = os.path.join(self.basedir, name) + fullfname = os.path.join(self.basedir, unicode(name)) if os.path.exists(fullfname): oldfnames.add(fullfname) if oldfnames: @@ -190,7 +191,7 @@ class Node(service.MultiService): raise e def create_tub(self): - certfile = os.path.join(self.basedir, "private", self.CERTFILE) + certfile = os.path.join(self.basedir, u"private", self.CERTFILE) self.tub = Tub(certFile=certfile) self.tub.setOption("logLocalFailures", True) self.tub.setOption("logRemoteFailures", True) @@ -248,7 +249,7 @@ class Node(service.MultiService): config file that resides within the subdirectory named 'private'), and return it. """ - privname = os.path.join(self.basedir, "private", name) + privname = os.path.join(self.basedir, u"private", name) open(privname, "w").write(value) def get_private_config(self, name, default=_None): @@ -257,7 +258,7 @@ class Node(service.MultiService): and return it. Return a default, or raise an error if one was not given. """ - privname = os.path.join(self.basedir, "private", name) + privname = os.path.join(self.basedir, u"private", name) try: return fileutil.read(privname) except EnvironmentError: @@ -280,7 +281,7 @@ class Node(service.MultiService): If 'default' is a string, use it as a default value. If not, treat it as a zero-argument callable that is expected to return a string. """ - privname = os.path.join(self.basedir, "private", name) + privname = os.path.join(self.basedir, u"private", name) try: value = fileutil.read(privname) except EnvironmentError: @@ -373,16 +374,16 @@ class Node(service.MultiService): ob.formatTime = newmeth # TODO: twisted >2.5.0 offers maxRotatedFiles=50 - lgfurl_file = os.path.join(self.basedir, "private", "logport.furl").encode(get_filesystem_encoding()) - self.tub.setOption("logport-furlfile", lgfurl_file) + logport_furl_file = os.path.join(self.basedir, u"private", u"logport.furl") + self.tub.setOption("logport-furlfile", logport_furl_file.encode(get_filesystem_encoding())) lgfurl = self.get_config("node", "log_gatherer.furl", "") if lgfurl: # this is in addition to the contents of log-gatherer-furlfile self.tub.setOption("log-gatherer-furl", lgfurl) - self.tub.setOption("log-gatherer-furlfile", - os.path.join(self.basedir, "log_gatherer.furl")) + log_gatherer_furl_file = os.path.join(self.basedir, u"log_gatherer.furl") + self.tub.setOption("log-gatherer-furlfile",log_gatherer_furl_file.encode(get_filesystem_encoding())) self.tub.setOption("bridge-twisted-logs", True) - incident_dir = os.path.join(self.basedir, "logs", "incidents") + incident_dir = os.path.join(self.basedir, u"logs", u"incidents") foolscap.logging.log.setLogDir(incident_dir.encode(get_filesystem_encoding())) def log(self, *args, **kwargs): diff --git a/src/allmydata/scripts/cli.py b/src/allmydata/scripts/cli.py index e240c9ec..1fb194e4 100644 --- a/src/allmydata/scripts/cli.py +++ b/src/allmydata/scripts/cli.py @@ -1,8 +1,9 @@ import os.path, re, fnmatch from twisted.python import usage from allmydata.scripts.common import get_aliases, get_default_nodedir, \ - DEFAULT_ALIAS, BaseOptions -from allmydata.util.encodingutil import argv_to_unicode, argv_to_abspath, quote_local_unicode_path + DEFAULT_ALIAS, BaseOptions +from allmydata.util.encodingutil import argv_to_unicode, argv_to_abspath, quote_output, \ + quote_local_unicode_path NODEURL_RE=re.compile("http(s?)://([^:]*)(:([1-9][0-9]*))?") @@ -528,7 +529,7 @@ def get(options): pass else: print >>options.stderr, "%s retrieved and written to %s" % \ - (options.from_file, options.to_file) + (quote_output(options.from_file), quote_local_unicode_path(options.to_file)) return rc def put(options): diff --git a/src/allmydata/scripts/debug.py b/src/allmydata/scripts/debug.py index fedd6985..1ba8e7ad 100644 --- a/src/allmydata/scripts/debug.py +++ b/src/allmydata/scripts/debug.py @@ -37,12 +37,12 @@ verify-cap for the file that uses the share. def dump_share(options): from allmydata.storage.mutable import MutableShareFile - from allmydata.util.encodingutil import quote_output + from allmydata.util.encodingutil import quote_local_unicode_path out = options.stdout # check the version, to see if we have a mutable or immutable share - print >>out, "share filename: %s" % quote_output(options['filename']) + print >>out, "share filename: %s" % quote_local_unicode_path(options['filename']) f = open(options['filename'], "rb") prefix = f.read(32) @@ -649,11 +649,11 @@ def find_shares(options): out = options.stdout sharedir = storage_index_to_dir(si_a2b(options.si_s)) - for d in options.nodedirs: - d = os.path.join(d, "storage", "shares", sharedir) - if os.path.exists(d): - for shnum in listdir_unicode(d): - print >>out, quote_local_unicode_path(os.path.join(d, shnum), quotemarks=False) + for nodedir in options.nodedirs: + abs_sharedir = os.path.join(nodedir, u"storage", u"shares", sharedir) + if os.path.exists(abs_sharedir): + for shnum in listdir_unicode(abs_sharedir): + print >>out, quote_local_unicode_path(os.path.join(abs_sharedir, shnum), quotemarks=False) return 0 @@ -712,7 +712,7 @@ def describe_share(abs_sharefile, si_s, shnum_s, now, out): from allmydata.mutable.common import NeedMoreDataError from allmydata.immutable.layout import ReadBucketProxy from allmydata.util import base32 - from allmydata.util.encodingutil import quote_output + from allmydata.util.encodingutil import quote_local_unicode_path import struct f = open(abs_sharefile, "rb") @@ -755,7 +755,7 @@ def describe_share(abs_sharefile, si_s, shnum_s, now, out): print >>out, "SDMF %s %d/%d %d #%d:%s %d %s" % \ (si_s, k, N, datalen, seqnum, base32.b2a(root_hash), - expiration, quote_output(abs_sharefile)) + expiration, quote_local_unicode_path(abs_sharefile)) elif share_type == "MDMF": from allmydata.mutable.layout import MDMFSlotReadProxy fake_shnum = 0 @@ -784,9 +784,9 @@ def describe_share(abs_sharefile, si_s, shnum_s, now, out): print >>out, "MDMF %s %d/%d %d #%d:%s %d %s" % \ (si_s, k, N, datalen, seqnum, base32.b2a(root_hash), - expiration, quote_output(abs_sharefile)) + expiration, quote_local_unicode_path(abs_sharefile)) else: - print >>out, "UNKNOWN mutable %s" % quote_output(abs_sharefile) + print >>out, "UNKNOWN mutable %s" % quote_local_unicode_path(abs_sharefile) elif struct.unpack(">L", prefix[:4]) == (1,): # immutable @@ -818,10 +818,10 @@ def describe_share(abs_sharefile, si_s, shnum_s, now, out): print >>out, "CHK %s %d/%d %d %s %d %s" % (si_s, k, N, filesize, ueb_hash, expiration, - quote_output(abs_sharefile)) + quote_local_unicode_path(abs_sharefile)) else: - print >>out, "UNKNOWN really-unknown %s" % quote_output(abs_sharefile) + print >>out, "UNKNOWN really-unknown %s" % quote_local_unicode_path(abs_sharefile) f.close() @@ -831,18 +831,18 @@ def catalog_shares(options): out = options.stdout err = options.stderr now = time.time() - for d in options.nodedirs: - d = os.path.join(d, "storage", "shares") + for nodedir in options.nodedirs: + abs_sharedir = os.path.join(nodedir, u"storage", u"shares") try: - abbrevs = listdir_unicode(d) + abbrevs = listdir_unicode(abs_sharedir) except EnvironmentError: # ignore nodes that have storage turned off altogether pass else: for abbrevdir in sorted(abbrevs): - if abbrevdir == "incoming": + if abbrevdir == u"incoming": continue - abbrevdir = os.path.join(d, abbrevdir) + abbrevdir = os.path.join(nodedir, abbrevdir) # this tool may get run against bad disks, so we can't assume # that listdir_unicode will always succeed. Try to catalog as much # as possible. @@ -864,7 +864,7 @@ def _as_number(s): return "not int" def catalog_shares_one_abbrevdir(si_s, si_dir, now, out, err): - from allmydata.util.encodingutil import listdir_unicode, quote_output + from allmydata.util.encodingutil import listdir_unicode, quote_local_unicode_path try: for shnum_s in sorted(listdir_unicode(si_dir), key=_as_number): @@ -874,10 +874,10 @@ def catalog_shares_one_abbrevdir(si_s, si_dir, now, out, err): describe_share(abs_sharefile, si_s, shnum_s, now, out) except: - print >>err, "Error processing %s" % quote_output(abs_sharefile) + print >>err, "Error processing %s" % quote_local_unicode_path(abs_sharefile) failure.Failure().printTraceback(err) except: - print >>err, "Error processing %s" % quote_output(si_dir) + print >>err, "Error processing %s" % quote_local_unicode_path(si_dir) failure.Failure().printTraceback(err) class CorruptShareOptions(BaseOptions): diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index 1a6de258..a3978dc0 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -4,6 +4,8 @@ from cStringIO import StringIO from twisted.python import usage +from allmydata.util.assertutil import precondition + from allmydata.scripts.common import get_default_nodedir from allmydata.scripts import debug, create_node, startstop_node, cli, keygen, stats_gatherer, admin from allmydata.util.encodingutil import quote_output, quote_local_unicode_path, get_io_encoding @@ -88,6 +90,8 @@ def runner(argv, run_by_human=True, stdin=None, stdout=None, stderr=None, install_node_control=True, additional_commands=None): + for arg in argv: + precondition(isinstance(arg, str), argv) stdin = stdin or sys.stdin stdout = stdout or sys.stdout diff --git a/src/allmydata/storage/common.py b/src/allmydata/storage/common.py index 865275bc..1c4cc9bd 100644 --- a/src/allmydata/storage/common.py +++ b/src/allmydata/storage/common.py @@ -18,4 +18,4 @@ def si_a2b(ascii_storageindex): def storage_index_to_dir(storageindex): sia = si_b2a(storageindex) - return os.path.join(sia[:2], sia) + return unicode(os.path.join(sia[:2], sia)) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 1de4b22f..7439f16f 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -6,6 +6,7 @@ from twisted.application import service from zope.interface import implements from allmydata.interfaces import RIStorageServer, IStatsProducer from allmydata.util import fileutil, idlib, log, time_format +from allmydata.util.assertutil import precondition import allmydata # for __full_version__ from allmydata.storage.common import si_b2a, si_a2b, storage_index_to_dir @@ -47,23 +48,25 @@ class StorageServer(service.MultiService, Referenceable): expiration_cutoff_date=None, expiration_sharetypes=("mutable", "immutable")): service.MultiService.__init__(self) - assert isinstance(nodeid, str) - assert len(nodeid) == 20 + + precondition(isinstance(nodeid, str), nodeid) + precondition(len(nodeid) == 20, nodeid) + self.my_nodeid = nodeid - self.storedir = storedir - sharedir = os.path.join(storedir, "shares") + self.storedir = unicode(storedir) + sharedir = os.path.join(storedir, u"shares") fileutil.make_dirs(sharedir) self.sharedir = sharedir # we don't actually create the corruption-advisory dir until necessary self.corruption_advisory_dir = os.path.join(storedir, - "corruption-advisories") + u"corruption-advisories") self.reserved_space = int(reserved_space) self.no_storage = discard_storage self.readonly_storage = readonly_storage self.stats_provider = stats_provider if self.stats_provider: self.stats_provider.register_producer(self) - self.incomingdir = os.path.join(sharedir, 'incoming') + self.incomingdir = os.path.join(sharedir, u"incoming") self._clean_incomplete() fileutil.make_dirs(self.incomingdir) self._active_writers = weakref.WeakKeyDictionary() @@ -87,8 +90,8 @@ class StorageServer(service.MultiService, Referenceable): } self.add_bucket_counter() - statefile = os.path.join(self.storedir, "lease_checker.state") - historyfile = os.path.join(self.storedir, "lease_checker.history") + statefile = os.path.join(self.storedir, u"lease_checker.state") + historyfile = os.path.join(self.storedir, u"lease_checker.history") klass = self.LeaseCheckerClass self.lease_checker = klass(self, statefile, historyfile, expiration_enabled, expiration_mode, @@ -106,7 +109,7 @@ class StorageServer(service.MultiService, Referenceable): return bool(set(os.listdir(self.sharedir)) - set(["incoming"])) def add_bucket_counter(self): - statefile = os.path.join(self.storedir, "bucket_counter.state") + statefile = os.path.join(self.storedir, u"bucket_counter.state") self.bucket_counter = BucketCountingCrawler(self, statefile) self.bucket_counter.setServiceParent(self) @@ -283,8 +286,8 @@ class StorageServer(service.MultiService, Referenceable): sf.add_or_renew_lease(lease_info) for shnum in sharenums: - incominghome = os.path.join(self.incomingdir, si_dir, "%d" % shnum) - finalhome = os.path.join(self.sharedir, si_dir, "%d" % shnum) + incominghome = os.path.join(self.incomingdir, si_dir, u"%d" % shnum) + finalhome = os.path.join(self.sharedir, si_dir, u"%d" % shnum) if os.path.exists(finalhome): # great! we already have it. easy. pass @@ -491,7 +494,7 @@ class StorageServer(service.MultiService, Referenceable): (write_enabler, renew_secret, cancel_secret) = secrets my_nodeid = self.my_nodeid fileutil.make_dirs(bucketdir) - filename = os.path.join(bucketdir, "%d" % sharenum) + filename = os.path.join(bucketdir, u"%d" % sharenum) share = create_mutable_sharefile(filename, my_nodeid, write_enabler, self) return share @@ -530,7 +533,7 @@ class StorageServer(service.MultiService, Referenceable): si_s = si_b2a(storage_index) # windows can't handle colons in the filename fn = os.path.join(self.corruption_advisory_dir, - "%s--%s-%d" % (now, si_s, shnum)).replace(":","") + (u"%s--%s-%d" % (now, si_s, shnum)).replace(u":", u"")) f = open(fn, "w") f.write("report: Share Corruption\n") f.write("type: %s\n" % share_type) diff --git a/src/allmydata/test/no_network.py b/src/allmydata/test/no_network.py index 8dd9a2f9..ad818262 100644 --- a/src/allmydata/test/no_network.py +++ b/src/allmydata/test/no_network.py @@ -24,6 +24,7 @@ from allmydata import uri as tahoe_uri from allmydata.client import Client from allmydata.storage.server import StorageServer, storage_index_to_dir from allmydata.util import fileutil, idlib, hashutil +from allmydata.util.assertutil import precondition from allmydata.util.hashutil import sha1 from allmydata.test.common_web import HTTPClientGETFactory from allmydata.interfaces import IStorageBroker, IServer @@ -224,6 +225,8 @@ class NoNetworkGrid(service.MultiService): def __init__(self, basedir, num_clients=1, num_servers=10, client_config_hooks={}): service.MultiService.__init__(self) + precondition(isinstance(basedir, unicode), basedir) + self.basedir = basedir fileutil.make_dirs(basedir) @@ -266,8 +269,8 @@ class NoNetworkGrid(service.MultiService): def make_server(self, i, readonly=False): serverid = hashutil.tagged_hash("serverid", str(i))[:20] - serverdir = os.path.join(self.basedir, "servers", - idlib.shortnodeid_b2a(serverid), "storage") + serverdir = os.path.join(self.basedir, u"servers", + unicode(idlib.shortnodeid_b2a(serverid)), u"storage") fileutil.make_dirs(serverdir) ss = StorageServer(serverdir, serverid, stats_provider=SimpleStats(), readonly_storage=readonly) @@ -345,7 +348,7 @@ class GridTestMixin: def set_up_grid(self, num_clients=1, num_servers=10, client_config_hooks={}): # self.basedir must be set - self.g = NoNetworkGrid(self.basedir, + self.g = NoNetworkGrid(unicode(self.basedir), num_clients=num_clients, num_servers=num_servers, client_config_hooks=client_config_hooks) diff --git a/src/allmydata/test/test_cli.py b/src/allmydata/test/test_cli.py index 2eb6dd9e..68c6f12f 100644 --- a/src/allmydata/test/test_cli.py +++ b/src/allmydata/test/test_cli.py @@ -6,6 +6,8 @@ import urllib, sys from mock import Mock, call +from allmydata.util.assertutil import precondition + import allmydata from allmydata.util import fileutil, hashutil, base32, keyutil from allmydata import uri @@ -33,9 +35,8 @@ from allmydata.test.no_network import GridTestMixin from twisted.internet import threads # CLI tests use deferToThread from twisted.python import usage -from allmydata.util.assertutil import precondition from allmydata.util.encodingutil import listdir_unicode, unicode_platform, \ - get_io_encoding, get_filesystem_encoding + get_io_encoding, get_filesystem_encoding, unicode_to_argv timeout = 480 # deep_check takes 360s on Zandr's linksys box, others take > 240s @@ -52,6 +53,11 @@ class CLITestMixin(ReallyEqualMixin): "--node-directory", unicode_to_argv(self.get_clientdir()), ] argv = nodeargs + [verb] + list(args) + + # runner.runner will also check this, but in another thread; this gives a better traceback + for arg in argv: + precondition(isinstance(arg, str), argv) + stdin = kwargs.get("stdin", "") stdout, stderr = StringIO(), StringIO() d = threads.deferToThread(runner.runner, argv, run_by_human=False, diff --git a/src/allmydata/test/test_cli_backup.py b/src/allmydata/test/test_cli_backup.py index 6d1dbe92..41556fd9 100644 --- a/src/allmydata/test/test_cli_backup.py +++ b/src/allmydata/test/test_cli_backup.py @@ -136,7 +136,7 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): # timestamp to force a check on all files def _reset_last_checked(res): dbfile = os.path.join(self.get_clientdir(), - "private", "backupdb.sqlite") + u"private", u"backupdb.sqlite") self.failUnless(os.path.exists(dbfile), dbfile) bdb = backupdb.get_backupdb(dbfile) bdb.cursor.execute("UPDATE last_upload SET last_checked=0") diff --git a/src/allmydata/test/test_cli_create_alias.py b/src/allmydata/test/test_cli_create_alias.py index ae3cd5bd..b3d73565 100644 --- a/src/allmydata/test/test_cli_create_alias.py +++ b/src/allmydata/test/test_cli_create_alias.py @@ -5,7 +5,7 @@ from allmydata.util import fileutil from allmydata.scripts.common import get_aliases from allmydata.scripts import cli, runner from allmydata.test.no_network import GridTestMixin -from allmydata.util.encodingutil import quote_output, get_io_encoding +from allmydata.util.encodingutil import quote_output, get_io_encoding, unicode_to_argv from .test_cli import CLITestMixin timeout = 480 # deep_check takes 360s on Zandr's linksys box, others take > 240s @@ -14,7 +14,7 @@ class CreateAlias(GridTestMixin, CLITestMixin, unittest.TestCase): def _test_webopen(self, args, expected_url): o = runner.Options() - o.parseOptions(["--node-directory", self.get_clientdir(), "webopen"] + o.parseOptions(["--node-directory", unicode_to_argv(self.get_clientdir()), "webopen"] + list(args)) urls = [] rc = cli.webopen(o, urls.append) @@ -25,7 +25,7 @@ class CreateAlias(GridTestMixin, CLITestMixin, unittest.TestCase): def test_create(self): self.basedir = "cli/CreateAlias/create" self.set_up_grid() - aliasfile = os.path.join(self.get_clientdir(), "private", "aliases") + aliasfile = os.path.join(self.get_clientdir(), u"private", u"aliases") d = self.do_cli("create-alias", "tahoe") def _done((rc,stdout,stderr)): @@ -39,7 +39,7 @@ class CreateAlias(GridTestMixin, CLITestMixin, unittest.TestCase): def _stash_urls(res): aliases = get_aliases(self.get_clientdir()) - node_url_file = os.path.join(self.get_clientdir(), "node.url") + node_url_file = os.path.join(self.get_clientdir(), u"node.url") nodeurl = fileutil.read(node_url_file).strip() self.welcome_url = nodeurl uribase = nodeurl + "uri/" diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py index d6cd7d0d..3a3aaa2f 100644 --- a/src/allmydata/test/test_client.py +++ b/src/allmydata/test/test_client.py @@ -1,4 +1,4 @@ -import os, sys +import os from twisted.trial import unittest from twisted.application import service @@ -70,7 +70,7 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase): fileutil.write(os.path.join(basedir, "debug_discard_storage"), "") e = self.failUnlessRaises(OldConfigError, client.Client, basedir) - abs_basedir = fileutil.abspath_expanduser_unicode(unicode(basedir)).encode(sys.getfilesystemencoding()) + abs_basedir = fileutil.abspath_expanduser_unicode(unicode(basedir)) self.failUnlessIn(os.path.join(abs_basedir, "introducer.furl"), e.args[0]) self.failUnlessIn(os.path.join(abs_basedir, "no_storage"), e.args[0]) self.failUnlessIn(os.path.join(abs_basedir, "readonly_storage"), e.args[0]) diff --git a/src/allmydata/test/test_deepcheck.py b/src/allmydata/test/test_deepcheck.py index fd9db7ec..9b6bf718 100644 --- a/src/allmydata/test/test_deepcheck.py +++ b/src/allmydata/test/test_deepcheck.py @@ -4,11 +4,15 @@ from cStringIO import StringIO from twisted.trial import unittest from twisted.internet import defer from twisted.internet import threads # CLI tests use deferToThread + +from allmydata.util.assertutil import precondition + from allmydata.immutable import upload from allmydata.mutable.common import UnrecoverableFileError from allmydata.mutable.publish import MutableData from allmydata.util import idlib from allmydata.util import base32 +from allmydata.util.encodingutil import unicode_to_argv from allmydata.scripts import runner from allmydata.interfaces import ICheckResults, ICheckAndRepairResults, \ IDeepCheckResults, IDeepCheckAndRepairResults @@ -25,9 +29,10 @@ timeout = 2400 # One of these took 1046.091s on Zandr's ARM box. class MutableChecker(GridTestMixin, unittest.TestCase, ErrorMixin): def _run_cli(self, argv): + precondition(argv[0] == "debug", argv) + stdout, stderr = StringIO(), StringIO() # this can only do synchronous operations - assert argv[0] == "debug" runner.runner(argv, run_by_human=False, stdout=stdout, stderr=stderr) return stdout.getvalue() @@ -728,6 +733,10 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): return d def _run_cli(self, argv, stdin=""): + # runner.runner will also check this, but in another thread; this gives a better traceback + for arg in argv: + precondition(isinstance(arg, str), argv) + #print "CLI:", argv stdout, stderr = StringIO(), StringIO() d = threads.deferToThread(runner.runner, argv, run_by_human=False, @@ -758,7 +767,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): def do_cli_manifest_stream1(self): basedir = self.get_clientdir(0) - d = self._run_cli(["--node-directory", basedir, + d = self._run_cli(["--node-directory", unicode_to_argv(basedir), "manifest", self.root_uri]) def _check((out,err)): @@ -786,7 +795,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): def do_cli_manifest_stream2(self): basedir = self.get_clientdir(0) - d = self._run_cli(["--node-directory", basedir, + d = self._run_cli(["--node-directory", unicode_to_argv(basedir), "manifest", "--raw", self.root_uri]) @@ -799,7 +808,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): def do_cli_manifest_stream3(self): basedir = self.get_clientdir(0) - d = self._run_cli(["--node-directory", basedir, + d = self._run_cli(["--node-directory", unicode_to_argv(basedir), "manifest", "--storage-index", self.root_uri]) @@ -811,7 +820,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): def do_cli_manifest_stream4(self): basedir = self.get_clientdir(0) - d = self._run_cli(["--node-directory", basedir, + d = self._run_cli(["--node-directory", unicode_to_argv(basedir), "manifest", "--verify-cap", self.root_uri]) @@ -827,7 +836,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): def do_cli_manifest_stream5(self): basedir = self.get_clientdir(0) - d = self._run_cli(["--node-directory", basedir, + d = self._run_cli(["--node-directory", unicode_to_argv(basedir), "manifest", "--repair-cap", self.root_uri]) @@ -843,7 +852,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): def do_cli_stats1(self): basedir = self.get_clientdir(0) - d = self._run_cli(["--node-directory", basedir, + d = self._run_cli(["--node-directory", unicode_to_argv(basedir), "stats", self.root_uri]) def _check3((out,err)): @@ -863,7 +872,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): def do_cli_stats2(self): basedir = self.get_clientdir(0) - d = self._run_cli(["--node-directory", basedir, + d = self._run_cli(["--node-directory", unicode_to_argv(basedir), "stats", "--raw", self.root_uri]) @@ -984,9 +993,10 @@ class DeepCheckWebBad(DeepCheckBase, unittest.TestCase): return d def _run_cli(self, argv): + precondition(argv[0] == "debug", argv) + stdout, stderr = StringIO(), StringIO() # this can only do synchronous operations - assert argv[0] == "debug" runner.runner(argv, run_by_human=False, stdout=stdout, stderr=stderr) return stdout.getvalue() @@ -996,7 +1006,7 @@ class DeepCheckWebBad(DeepCheckBase, unittest.TestCase): def _corrupt_some_shares(self, node): for (shnum, serverid, sharefile) in self.find_uri_shares(node.get_uri()): if shnum in (0,1): - self._run_cli(["debug", "corrupt-share", sharefile]) + self._run_cli(["debug", "corrupt-share", unicode_to_argv(sharefile)]) def _delete_most_shares(self, node): self.delete_shares_numbered(node.get_uri(), range(1,10)) diff --git a/src/allmydata/test/test_mutable.py b/src/allmydata/test/test_mutable.py index 26c7f244..fc3f6caa 100644 --- a/src/allmydata/test/test_mutable.py +++ b/src/allmydata/test/test_mutable.py @@ -3113,7 +3113,7 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ fso = debug.FindSharesOptions() storage_index = base32.b2a(n.get_storage_index()) fso.si_s = storage_index - fso.nodedirs = [os.path.dirname(abspath_expanduser_unicode(unicode(storedir))) + fso.nodedirs = [os.path.dirname(abspath_expanduser_unicode(storedir)) for (i,ss,storedir) in self.iterate_servers()] fso.stdout = StringIO() diff --git a/src/allmydata/test/test_no_network.py b/src/allmydata/test/test_no_network.py index 345662f7..476e31de 100644 --- a/src/allmydata/test/test_no_network.py +++ b/src/allmydata/test/test_no_network.py @@ -7,6 +7,7 @@ from allmydata.test.no_network import NoNetworkGrid from allmydata.immutable.upload import Data from allmydata.util.consumer import download_to_data + class Harness(unittest.TestCase): def setUp(self): self.s = service.MultiService() @@ -16,13 +17,13 @@ class Harness(unittest.TestCase): return self.s.stopService() def test_create(self): - basedir = "no_network/Harness/create" + basedir = u"no_network/Harness/create" g = NoNetworkGrid(basedir) g.startService() return g.stopService() def test_upload(self): - basedir = "no_network/Harness/upload" + basedir = u"no_network/Harness/upload" g = NoNetworkGrid(basedir) g.setServiceParent(self.s) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 9692466c..2d0cdbd3 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -1414,7 +1414,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): def workdir(self, name): - basedir = os.path.join("storage", "MutableServer", name) + basedir = os.path.join("storage", "MDMFProxies", name) return basedir @@ -2793,7 +2793,7 @@ class Stats(unittest.TestCase): return self.sparent.stopService() def workdir(self, name): - basedir = os.path.join("storage", "Server", name) + basedir = os.path.join("storage", "Stats", name) return basedir def create(self, name): diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index 3b23b2f6..f323db06 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -16,7 +16,8 @@ from allmydata.immutable.filenode import ImmutableFileNode from allmydata.util import idlib, mathutil from allmydata.util import log, base32 from allmydata.util.verlib import NormalizedVersion -from allmydata.util.encodingutil import quote_output, unicode_to_argv, get_filesystem_encoding +from allmydata.util.encodingutil import quote_local_unicode_path, \ + unicode_to_argv, get_filesystem_encoding from allmydata.util.fileutil import abspath_expanduser_unicode from allmydata.util.consumer import MemoryConsumer, download_to_data from allmydata.scripts import runner @@ -730,6 +731,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): d.addCallback(self.log, "starting publish") d.addCallback(self._do_publish1) d.addCallback(self._test_runner) + return d d.addCallback(self._do_publish2) # at this point, we have the following filesystem (where "R" denotes # self._root_directory_uri): @@ -1314,14 +1316,14 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): # find a share for (dirpath, dirnames, filenames) in os.walk(unicode(self.basedir)): - if "storage" not in dirpath: + if u"storage" not in dirpath: continue if not filenames: continue pieces = dirpath.split(os.sep) if (len(pieces) >= 4 - and pieces[-4] == "storage" - and pieces[-3] == "shares"): + and pieces[-4] == u"storage" + and pieces[-3] == u"shares"): # we're sitting in .../storage/shares/$START/$SINDEX , and there # are sharefiles here filename = os.path.join(dirpath, filenames[0]) @@ -1343,7 +1345,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): # we only upload a single file, so we can assert some things about # its size and shares. - self.failUnlessIn("share filename: %s" % quote_output(abspath_expanduser_unicode(filename)), output) + self.failUnlessIn("share filename: %s" % quote_local_unicode_path(abspath_expanduser_unicode(filename)), output) self.failUnlessIn("size: %d\n" % len(self.data), output) self.failUnlessIn("num_segments: 1\n", output) # segment_size is always a multiple of needed_shares @@ -1377,11 +1379,12 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): out,err = StringIO(), StringIO() nodedirs = [self.getdir("client%d" % i) for i in range(self.numclients)] cmd = ["debug", "catalog-shares"] + nodedirs + #import pdb; pdb.set_trace() rc = runner.runner(cmd, stdout=out, stderr=err) self.failUnlessEqual(rc, 0) out.seek(0) descriptions = [sfn.strip() for sfn in out.readlines()] - self.failUnlessEqual(len(descriptions), 30) + self.failUnlessEqual(len(descriptions), 30, descriptions) matching = [line for line in descriptions if line.startswith("CHK %s " % storage_index_s)] diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index a4dcbd8c..8ae959de 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -175,6 +175,7 @@ class Math(unittest.TestCase): f = mathutil.round_sigfigs self.failUnlessEqual(f(22.0/3, 4), 7.3330000000000002) + class Statistics(unittest.TestCase): def should_assert(self, msg, func, *args, **kwargs): try: @@ -372,6 +373,7 @@ class Asserts(unittest.TestCase): m = self.should_assert(f, False, othermsg="message2") self.failUnlessEqual("postcondition: othermsg: 'message2' ", m) + class FileUtil(ReallyEqualMixin, unittest.TestCase): def mkdir(self, basedir, path, mode=0777): fn = os.path.join(basedir, path) @@ -554,6 +556,7 @@ class FileUtil(ReallyEqualMixin, unittest.TestCase): disk = fileutil.get_disk_stats(u".", 2**128) self.failUnlessEqual(disk['avail'], 0) + class PollMixinTests(unittest.TestCase): def setUp(self): self.pm = pollmixin.PollMixin() @@ -581,6 +584,7 @@ class PollMixinTests(unittest.TestCase): d.addCallbacks(_suc, _err) return d + class DeferredUtilTests(unittest.TestCase): def test_gather_results(self): d1 = defer.Deferred() @@ -621,8 +625,8 @@ class DeferredUtilTests(unittest.TestCase): self.failUnless(isinstance(f, Failure)) self.failUnless(f.check(ValueError)) -class HashUtilTests(unittest.TestCase): +class HashUtilTests(unittest.TestCase): def test_random_key(self): k = hashutil.random_key() self.failUnlessEqual(len(k), hashutil.KEYLEN) @@ -815,6 +819,7 @@ class Abbreviate(unittest.TestCase): e = self.failUnlessRaises(ValueError, p, "fhtagn") self.failUnlessIn("fhtagn", str(e)) + class Limiter(unittest.TestCase): timeout = 480 # This takes longer than 240 seconds on Francois's arm box. @@ -890,6 +895,7 @@ class Limiter(unittest.TestCase): d.addCallback(_all_done) return d + class TimeFormat(unittest.TestCase): def test_epoch(self): return self._help_test_epoch() @@ -978,6 +984,7 @@ class TimeFormat(unittest.TestCase): def test_parse_date(self): self.failUnlessEqual(time_format.parse_date("2010-02-21"), 1266710400) + class CacheDir(unittest.TestCase): def test_basic(self): basedir = "test_util/CacheDir/test_basic" @@ -1042,6 +1049,7 @@ class CacheDir(unittest.TestCase): _failUnlessExists("c") del b2 + ctr = [0] class EqButNotIs: def __init__(self, x): @@ -1065,6 +1073,7 @@ class EqButNotIs: def __eq__(self, other): return self.x == other + class DictUtil(unittest.TestCase): def _help_test_empty_dict(self, klass): d1 = klass() @@ -1440,6 +1449,7 @@ class DictUtil(unittest.TestCase): self.failUnlessEqual(d["one"], 1) self.failUnlessEqual(d.get_aux("one"), None) + class Pipeline(unittest.TestCase): def pause(self, *args, **kwargs): d = defer.Deferred() @@ -1614,9 +1624,11 @@ class Pipeline(unittest.TestCase): del d1,d2,d3,d4 + class SampleError(Exception): pass + class Log(unittest.TestCase): def test_err(self): if not hasattr(self, "flushLoggedErrors"): @@ -1716,6 +1728,7 @@ class SimpleSpans: return False return True + class ByteSpans(unittest.TestCase): def test_basic(self): s = Spans() @@ -1950,6 +1963,7 @@ class ByteSpans(unittest.TestCase): out.write(" ") out.write("\n") + def extend(s, start, length, fill): if len(s) >= start+length: return s