From 80252debcd94fc28d877f59952dba0c462b7401a Mon Sep 17 00:00:00 2001 From: david-sarah Date: Sun, 6 Jun 2010 18:02:15 -0700 Subject: [PATCH] Unicode fixes. --- src/allmydata/scripts/cli.py | 1 + src/allmydata/scripts/common.py | 65 +- src/allmydata/scripts/common_http.py | 19 +- src/allmydata/scripts/consolidate.py | 135 ++- src/allmydata/scripts/debug.py | 3 +- src/allmydata/scripts/slow_operation.py | 14 +- src/allmydata/scripts/tahoe_add_alias.py | 26 +- src/allmydata/scripts/tahoe_backup.py | 62 +- src/allmydata/scripts/tahoe_check.py | 35 +- src/allmydata/scripts/tahoe_cp.py | 157 ++-- src/allmydata/scripts/tahoe_get.py | 10 +- src/allmydata/scripts/tahoe_ls.py | 69 +- src/allmydata/scripts/tahoe_manifest.py | 50 +- src/allmydata/scripts/tahoe_mkdir.py | 12 +- src/allmydata/scripts/tahoe_mv.py | 23 +- src/allmydata/scripts/tahoe_put.py | 21 +- src/allmydata/scripts/tahoe_rm.py | 9 +- src/allmydata/scripts/tahoe_webopen.py | 2 +- src/allmydata/test/common_util.py | 9 +- src/allmydata/test/test_cli.py | 1097 ++++++++++++---------- src/allmydata/test/test_consolidate.py | 83 +- src/allmydata/test/test_sftp.py | 3 +- src/allmydata/test/test_stringutils.py | 189 ++-- src/allmydata/test/test_system.py | 2 +- src/allmydata/util/stringutils.py | 198 +++- windows/tahoe.py | 14 +- 26 files changed, 1344 insertions(+), 964 deletions(-) diff --git a/src/allmydata/scripts/cli.py b/src/allmydata/scripts/cli.py index 80e102b9..6419f0a9 100644 --- a/src/allmydata/scripts/cli.py +++ b/src/allmydata/scripts/cli.py @@ -22,6 +22,7 @@ class VDriveOptions(BaseOptions, usage.Options): ] def postOptions(self): + # FIXME: allow Unicode node-dir # compute a node-url from the existing options, put in self['node-url'] if self['node-directory']: if sys.platform == 'win32' and self['node-directory'] == '~/.tahoe': diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py index 38937133..1116e1db 100644 --- a/src/allmydata/scripts/common.py +++ b/src/allmydata/scripts/common.py @@ -2,7 +2,7 @@ import os, sys, urllib import codecs from twisted.python import usage -from allmydata.util.stringutils import unicode_to_url +from allmydata.util.stringutils import unicode_to_url, quote_output from allmydata.util.assertutil import precondition class BaseOptions: @@ -19,12 +19,12 @@ class BaseOptions: def opt_version(self): import allmydata - print allmydata.get_package_versions_string() + print >>self.stdout, allmydata.get_package_versions_string() sys.exit(0) def opt_version_and_path(self): import allmydata - print allmydata.get_package_versions_string(show_paths=True) + print >>self.stdout, allmydata.get_package_versions_string(show_paths=True) sys.exit(0) @@ -84,7 +84,7 @@ class NoDefaultBasedirMixin(BasedirMixin): if not self.basedirs: raise usage.UsageError("--basedir must be provided") -DEFAULT_ALIAS = "tahoe" +DEFAULT_ALIAS = u"tahoe" def get_aliases(nodedir): @@ -96,7 +96,7 @@ def get_aliases(nodedir): f = open(rootfile, "r") rootcap = f.read().strip() if rootcap: - aliases["tahoe"] = uri.from_string_dirnode(rootcap).to_string() + aliases[u"tahoe"] = uri.from_string_dirnode(rootcap).to_string() except EnvironmentError: pass try: @@ -105,7 +105,7 @@ def get_aliases(nodedir): line = line.strip() if line.startswith("#") or not line: continue - name, cap = line.split(":", 1) + name, cap = line.split(u":", 1) # normalize it: remove http: prefix, urldecode cap = cap.strip().encode('utf-8') aliases[name] = uri.from_string_dirnode(cap).to_string() @@ -124,21 +124,37 @@ def platform_uses_lettercolon_drivename(): return True return False -class UnknownAliasError(Exception): - pass -def get_alias(aliases, path, default): +class TahoeError(Exception): + def __init__(self, msg): + Exception.__init__(self, msg) + self.msg = msg + + def display(self, err): + print >>err, self.msg + + +class UnknownAliasError(TahoeError): + def __init__(self, msg): + TahoeError.__init__(self, "error: " + msg) + + +def get_alias(aliases, path_unicode, default): + """ + Transform u"work:path/filename" into (aliases[u"work"], u"path/filename".encode('utf-8')). + If default=None, then an empty alias is indicated by returning + DefaultAliasMarker. We special-case strings with a recognized cap URI + prefix, to make it easy to access specific files/directories by their + caps. + If the transformed alias is either not found in aliases, or is blank + and default is not found in aliases, an UnknownAliasError is + raised. + """ + precondition(isinstance(path_unicode, unicode), path_unicode) + from allmydata import uri - # transform "work:path/filename" into (aliases["work"], "path/filename"). - # If default=None, then an empty alias is indicated by returning - # DefaultAliasMarker. We special-case strings with a recognized cap URI - # prefix, to make it easy to access specific files/directories by their - # caps. - # If the transformed alias is either not found in aliases, or is blank - # and default is not found in aliases, an UnknownAliasError is - # raised. - path = path.strip() - if uri.has_uri_prefix(path.encode('utf-8')): + path = path_unicode.encode('utf-8').strip(" ") + if uri.has_uri_prefix(path): # We used to require "URI:blah:./foo" in order to get a subpath, # stripping out the ":./" sequence. We still allow that for compatibility, # but now also allow just "URI:blah/foo". @@ -159,12 +175,14 @@ def get_alias(aliases, path, default): "'tahoe' alias doesn't exist. To create " "it, use 'tahoe create-alias tahoe'.") return aliases[default], path - if colon == 1 and default == None and platform_uses_lettercolon_drivename(): + if colon == 1 and default is None and platform_uses_lettercolon_drivename(): # treat C:\why\must\windows\be\so\weird as a local path, not a tahoe # file in the "C:" alias return DefaultAliasMarker, path - alias = path[:colon] - if "/" in alias: + + # decoding must succeed because path is valid UTF-8 and colon & space are ASCII + alias = path[:colon].decode('utf-8') + if u"/" in alias: # no alias, but there's a colon in a dirname/filename, like # "foo/bar:7" if default == None: @@ -175,7 +193,8 @@ def get_alias(aliases, path, default): "it, use 'tahoe create-alias tahoe'.") return aliases[default], path if alias not in aliases: - raise UnknownAliasError("Unknown alias '%s', please create it with 'tahoe add-alias' or 'tahoe create-alias'." % alias) + raise UnknownAliasError("Unknown alias %s, please create it with 'tahoe add-alias' or 'tahoe create-alias'." % + quote_output(alias)) return aliases[alias], path[colon+1:] def escape_path(path): diff --git a/src/allmydata/scripts/common_http.py b/src/allmydata/scripts/common_http.py index cc145b02..089b4573 100644 --- a/src/allmydata/scripts/common_http.py +++ b/src/allmydata/scripts/common_http.py @@ -3,6 +3,10 @@ from cStringIO import StringIO import urlparse, httplib import allmydata # for __full_version__ +from allmydata.util.stringutils import quote_output +from allmydata.scripts.common import TahoeError + + # copied from twisted/web/client.py def parse_url(url, defaultPort=None): url = url.strip() @@ -63,7 +67,20 @@ def do_http(method, url, body=""): return c.getresponse() + +def format_http_success(resp): + return "%s %s" % (resp.status, quote_output(resp.reason, quotemarks=False)) + +def format_http_error(msg, resp): + return "%s: %s %s\n%s" % (msg, resp.status, quote_output(resp.reason, quotemarks=False), + quote_output(resp.read(), quotemarks=False)) + def check_http_error(resp, stderr): if resp.status < 200 or resp.status >= 300: - print >>stderr, "error %d during HTTP request" % resp.status + print >>stderr, format_http_error("Error during HTTP request", resp) return 1 + + +class HTTPError(TahoeError): + def __init__(self, msg, resp): + TahoeError.__init__(format_http_error(msg, resp)) diff --git a/src/allmydata/scripts/consolidate.py b/src/allmydata/scripts/consolidate.py index 5088b666..da0252fa 100644 --- a/src/allmydata/scripts/consolidate.py +++ b/src/allmydata/scripts/consolidate.py @@ -4,12 +4,61 @@ import sqlite3 as sqlite import urllib import simplejson -from allmydata.scripts.common_http import do_http -from allmydata.scripts.tahoe_backup import parse_old_timestamp, readonly, \ - raiseHTTPError, HTTPError -from allmydata.util import hashutil, base32 +from allmydata.scripts.common_http import do_http, HTTPError +from allmydata.util import hashutil, base32, time_format +from allmydata.util.stringutils import to_str, quote_output, quote_path from allmydata.util.netstring import netstring from allmydata.scripts.common import get_alias, DEFAULT_ALIAS +from allmydata import uri + + +def readonly(writedircap): + return uri.from_string_dirnode(writedircap).get_readonly().to_string() + +def parse_old_timestamp(s, options): + try: + if not s.endswith("Z"): + raise ValueError + # This returns seconds-since-epoch for an ISO-8601-ish-formatted UTC + # time string. This might raise ValueError if the string is not in the + # right format. + when = time_format.iso_utc_time_to_seconds(s[:-1]) + return when + except ValueError: + pass + + try: + # "2008-11-16 10.34 PM" (localtime) + if s[-3:] in (" AM", " PM"): + # this might raise ValueError + when = time.strptime(s[:-3], "%Y-%m-%d %I.%M") + if s[-3:] == "PM": + when += 12*60*60 + return when + except ValueError: + pass + + try: + # "2008-11-16 10.34.56 PM" (localtime) + if s[-3:] in (" AM", " PM"): + # this might raise ValueError + when = time.strptime(s[:-3], "%Y-%m-%d %I.%M.%S") + if s[-3:] == "PM": + when += 12*60*60 + return when + except ValueError: + pass + + try: + # "2008-12-31 18.21.43" + when = time.strptime(s, "%Y-%m-%d %H.%M.%S") + return when + except ValueError: + pass + + print >>options.stderr, "unable to parse old timestamp '%s', ignoring" % s + return None + TAG = "consolidator_dirhash_v1" @@ -23,6 +72,7 @@ class Consolidator: self.rootcap, path = get_alias(options.aliases, options.where, DEFAULT_ALIAS) assert path == "" + # TODO: allow dbfile and backupfile to be Unicode self.dbfile = options["dbfile"] assert self.dbfile, "--dbfile is required" self.backupfile = options["backupfile"] @@ -47,7 +97,7 @@ class Consolidator: url = self.nodeurl + "uri/%s?t=json" % urllib.quote(dircap) resp = do_http("GET", url) if resp.status != 200: - raiseHTTPError("Error during directory GET", resp) + raise HTTPError("Error during directory GET", resp) jd = simplejson.load(resp) ntype, ndata = jd if ntype != "dirnode": @@ -72,18 +122,25 @@ class Consolidator: for (childname, (childtype, childdata)) in kids.items(): if childtype != "dirnode": continue - potential_systems[childname] = str(childdata["rw_uri"]) + if "rw_uri" not in childdata: + self.msg("%s: not writeable" % quote_output(childname)) + continue + potential_systems[childname] = to_str(childdata["rw_uri"]) backup_data = {"Backups": data, "systems": {}, "archives": {}} systems = {} for name, sdircap in potential_systems.items(): sdata = self.read_directory_json(sdircap) kids = sdata["children"] if not u"Archives" in kids and not u"Latest Backup" in kids: - self.msg("%s: not a backupdir, no 'Archives' and 'Latest'" % name) + self.msg("%s: not a backupdir, no 'Archives' and 'Latest'" % quote_output(name)) + continue + archives_capdata = kids[u"Archives"][1] + if "rw_uri" not in archives_capdata: + self.msg("%s: /Archives is not writeable" % quote_output(name)) continue - self.msg("%s is a system" % name) + self.msg("%s is a system" % quote_output(name)) backup_data["systems"][name] = sdata - archives_dircap = kids[u"Archives"][1]["rw_uri"] + archives_dircap = to_str(archives_capdata["rw_uri"]) archives_data = self.read_directory_json(archives_dircap) backup_data["archives"][name] = archives_data systems[name] = archives_dircap @@ -136,7 +193,7 @@ class Consolidator: # [$NAME, writecap, $NAME-readonly, readcap] : processed, not replaced # [None, None, $NAME, readcap] : processed and replaced - self.msg("consolidating system %s" % system_name) + self.msg("consolidating system %s" % quote_output(system_name)) self.directories_reused = 0 self.directories_used_as_is = 0 self.directories_created = 0 @@ -149,11 +206,11 @@ class Consolidator: children = sorted(data["children"].items()) for i, (childname, (childtype, childdata)) in enumerate(children): if childtype != "dirnode": - self.msg("non-dirnode %s in Archives/" % childname) + self.msg("non-dirnode %s in Archives/" % quote_output(childname)) continue - timename = childname - if childname.endswith("-readonly"): - timename = childname[:-len("-readonly")] + timename = to_str(childname) + if timename.endswith("-readonly"): + timename = timename[:-len("-readonly")] timestamp = parse_old_timestamp(timename, self.options) assert timestamp is not None, timename snapshots.setdefault(timestamp, [None, None, None, None]) @@ -161,15 +218,15 @@ class Consolidator: # need to re-scan it is_readonly = not childdata.has_key("rw_uri") if is_readonly: - readcap = str(childdata["ro_uri"]) + readcap = to_str(childdata["ro_uri"]) if self.must_rescan_readonly_snapshots: self.msg(" scanning old %s (%d/%d)" % - (childname, i+1, len(children))) - self.scan_old_directory(str(childdata["ro_uri"])) + (quote_output(childname), i+1, len(children))) + self.scan_old_directory(to_str(childdata["ro_uri"])) snapshots[timestamp][2] = childname snapshots[timestamp][3] = readcap else: - writecap = str(childdata["rw_uri"]) + writecap = to_str(childdata["rw_uri"]) snapshots[timestamp][0] = childname snapshots[timestamp][1] = writecap snapshots = [ [timestamp] + values @@ -197,7 +254,7 @@ class Consolidator: assert roname assert not rwname first_snapshot = False - self.msg(" %s already readonly" % roname) + self.msg(" %s already readonly" % quote_output(roname)) continue if readcap and writecap: # we've processed it, creating a -readonly version, but we @@ -205,9 +262,9 @@ class Consolidator: assert roname assert rwname first_snapshot = False - self.msg(" %s processed but not yet replaced" % roname) + self.msg(" %s processed but not yet replaced" % quote_output(roname)) if self.options["really"]: - self.msg(" replacing %s with %s" % (rwname, roname)) + self.msg(" replacing %s with %s" % (quote_output(rwname), quote_output(roname))) self.put_child(archives_dircap, rwname, readcap) self.delete_child(archives_dircap, roname) continue @@ -221,29 +278,29 @@ class Consolidator: first_snapshot = False readcap = readonly(writecap) self.directories_used_as_is += 1 - self.msg(" %s: oldest snapshot, using as-is" % rwname) + self.msg(" %s: oldest snapshot, using as-is" % quote_output(rwname)) self.scan_old_directory(readcap) else: # for the others, we must scan their contents and build up a new # readonly directory (which shares common subdirs with previous # backups) - self.msg(" %s: processing (%d/%d)" % (rwname, i+1, len(snapshots))) + self.msg(" %s: processing (%d/%d)" % (quote_output(rwname), i+1, len(snapshots))) started = time.time() readcap = self.process_directory(readonly(writecap), (rwname,)) elapsed = time.time() - started eta = "%ds" % (elapsed * (len(snapshots) - i-1)) if self.options["really"]: - self.msg(" replaced %s" % rwname) + self.msg(" replaced %s" % quote_output(rwname)) self.put_child(archives_dircap, rwname, readcap) else: - self.msg(" created %s" % roname) + self.msg(" created %s" % quote_output(roname)) self.put_child(archives_dircap, roname, readcap) snapshot_created = self.directories_created - start_created snapshot_used_as_is = self.directories_used_as_is - start_used_as_is snapshot_reused = self.directories_reused - start_reused self.msg(" %s: done: %d dirs created, %d used as-is, %d reused, eta %s" - % (rwname, + % (quote_output(rwname), snapshot_created, snapshot_used_as_is, snapshot_reused, eta)) # done! @@ -259,7 +316,7 @@ class Consolidator: # for my contents. In all cases I return a directory readcap that # points to my contents. - assert isinstance(readcap, str) + readcap = to_str(readcap) self.directories_seen.add(readcap) # build up contents to pass to mkdir() (which uses t=set_children) @@ -271,13 +328,13 @@ class Consolidator: for (childname, (childtype, childdata)) in sorted(data["children"].items()): if childtype == "dirnode": childpath = path + (childname,) - old_childcap = str(childdata["ro_uri"]) + old_childcap = to_str(childdata["ro_uri"]) childcap = self.process_directory(old_childcap, childpath) if childcap != old_childcap: children_modified = True contents[childname] = ("dirnode", childcap, None) else: - childcap = str(childdata["ro_uri"]) + childcap = to_str(childdata["ro_uri"]) contents[childname] = (childtype, childcap, None) hashkids.append( (childname, childcap) ) @@ -285,7 +342,7 @@ class Consolidator: old_dircap = self.get_old_dirhash(dirhash) if old_dircap: if self.options["verbose"]: - self.msg(" %r: reused" % "/".join(path)) + self.msg(" %s: reused" % quote_path(path)) assert isinstance(old_dircap, str) self.directories_reused += 1 self.directories_used.add(old_dircap) @@ -293,7 +350,7 @@ class Consolidator: if not children_modified: # we're allowed to use this directory as-is if self.options["verbose"]: - self.msg(" %r: used as-is" % "/".join(path)) + self.msg(" %s: used as-is" % quote_path(path)) new_dircap = readonly(readcap) assert isinstance(new_dircap, str) self.store_dirhash(dirhash, new_dircap) @@ -302,7 +359,7 @@ class Consolidator: return new_dircap # otherwise, we need to create a new directory if self.options["verbose"]: - self.msg(" %r: created" % "/".join(path)) + self.msg(" %s: created" % quote_path(path)) new_dircap = readonly(self.mkdir(contents)) assert isinstance(new_dircap, str) self.store_dirhash(dirhash, new_dircap) @@ -315,21 +372,21 @@ class Consolidator: urllib.quote(childname)) resp = do_http("PUT", url, childcap) if resp.status not in (200, 201): - raiseHTTPError("error during put_child", resp) + raise HTTPError("Error during put_child", resp) def delete_child(self, dircap, childname): url = self.nodeurl + "uri/%s/%s" % (urllib.quote(dircap), urllib.quote(childname)) resp = do_http("DELETE", url) if resp.status not in (200, 201): - raiseHTTPError("error during delete_child", resp) + raise HTTPError("Error during delete_child", resp) def mkdir(self, contents): url = self.nodeurl + "uri?t=mkdir" resp = do_http("POST", url) if resp.status < 200 or resp.status >= 300: - raiseHTTPError("error during mkdir", resp) - dircap = str(resp.read().strip()) + raise HTTPError("Error during mkdir", resp) + dircap = to_str(resp.read().strip()) url = self.nodeurl + "uri/%s?t=set_children" % urllib.quote(dircap) body = dict([ (childname, (contents[childname][0], {"ro_uri": contents[childname][1], @@ -339,7 +396,7 @@ class Consolidator: ]) resp = do_http("POST", url, simplejson.dumps(body)) if resp.status != 200: - raiseHTTPError("error during set_children", resp) + raise HTTPError("Error during set_children", resp) return dircap def scan_old_directory(self, dircap, ancestors=()): @@ -358,7 +415,7 @@ class Consolidator: data = self.read_directory_json(dircap) kids = [] for (childname, (childtype, childdata)) in data["children"].items(): - childcap = str(childdata["ro_uri"]) + childcap = to_str(childdata["ro_uri"]) if childtype == "dirnode": self.scan_old_directory(childcap, ancestors) kids.append( (childname, childcap) ) @@ -368,7 +425,7 @@ class Consolidator: def hash_directory_contents(self, kids): kids.sort() - s = "".join([netstring(childname.encode("utf-8"))+netstring(childcap) + s = "".join([netstring(to_str(childname))+netstring(childcap) for (childname, childcap) in kids]) return hashutil.tagged_hash(TAG, s) diff --git a/src/allmydata/scripts/debug.py b/src/allmydata/scripts/debug.py index d18d3cb1..0af308f6 100644 --- a/src/allmydata/scripts/debug.py +++ b/src/allmydata/scripts/debug.py @@ -5,6 +5,7 @@ import struct, time, os from twisted.python import usage, failure from twisted.internet import defer from allmydata.scripts.cli import VDriveOptions +from allmydata.util.stringutils import argv_to_unicode class DumpOptions(usage.Options): def getSynopsis(self): @@ -768,7 +769,7 @@ class ConsolidateOptions(VDriveOptions): ("verbose", "v", "Emit a line for every directory examined"), ] def parseArgs(self, where): - self.where = where + self.where = argv_to_unicode(where) def consolidate(options): from allmydata.scripts.consolidate import main diff --git a/src/allmydata/scripts/slow_operation.py b/src/allmydata/scripts/slow_operation.py index 3f0a9c59..f47c0412 100644 --- a/src/allmydata/scripts/slow_operation.py +++ b/src/allmydata/scripts/slow_operation.py @@ -2,8 +2,9 @@ import os, time from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \ UnknownAliasError -from allmydata.scripts.common_http import do_http +from allmydata.scripts.common_http import do_http, format_http_error from allmydata.util import base32 +from allmydata.util.stringutils import quote_output, is_printable_ascii import urllib import simplejson @@ -21,7 +22,7 @@ class SlowOperationRunner: try: rootcap, path = get_alias(options.aliases, where, DEFAULT_ALIAS) except UnknownAliasError, e: - print >>stderr, "error: %s" % e.args[0] + e.display(stderr) return 1 if path == '/': path = '' @@ -32,7 +33,7 @@ class SlowOperationRunner: url = self.make_url(url, ophandle) resp = do_http("POST", url) if resp.status not in (200, 302): - print >>stderr, "ERROR", resp.status, resp.reason, resp.read() + print >>stderr, format_http_error("ERROR", resp) return 1 # now we poll for results. We nominally poll at t=1, 5, 10, 30, 60, # 90, k*120 seconds, but if the poll takes non-zero time, that will @@ -65,14 +66,17 @@ class SlowOperationRunner: stderr = self.options.stderr resp = do_http("GET", url) if resp.status != 200: - print >>stderr, "ERROR", resp.status, resp.reason, resp.read() + print >>stderr, format_http_error("ERROR", resp) return True jdata = resp.read() data = simplejson.loads(jdata) if not data["finished"]: return False if self.options.get("raw"): - print >>stdout, jdata + if is_printable_ascii(jdata): + print >>stdout, jdata + else: + print >>stderr, "The JSON response contained unprintable characters:\n%s" % quote_output(jdata) return True self.write_results(data) return True diff --git a/src/allmydata/scripts/tahoe_add_alias.py b/src/allmydata/scripts/tahoe_add_alias.py index 1c03ad5a..984719a4 100644 --- a/src/allmydata/scripts/tahoe_add_alias.py +++ b/src/allmydata/scripts/tahoe_add_alias.py @@ -5,7 +5,7 @@ from allmydata import uri from allmydata.scripts.common_http import do_http, check_http_error from allmydata.scripts.common import get_aliases from allmydata.util.fileutil import move_into_place -from allmydata.util.stringutils import unicode_to_stdout +from allmydata.util.stringutils import unicode_to_output, quote_output def add_line_to_aliasfile(aliasfile, alias, cap): @@ -37,14 +37,14 @@ def add_alias(options): old_aliases = get_aliases(nodedir) if alias in old_aliases: - print >>stderr, "Alias '%s' already exists!" % alias + print >>stderr, "Alias %s already exists!" % quote_output(alias) return 1 aliasfile = os.path.join(nodedir, "private", "aliases") cap = uri.from_string_dirnode(cap).to_string() add_line_to_aliasfile(aliasfile, alias, cap) - print >>stdout, "Alias '%s' added" % (unicode_to_stdout(alias),) + print >>stdout, "Alias %s added" % quote_output(alias) return 0 def create_alias(options): @@ -58,7 +58,7 @@ def create_alias(options): old_aliases = get_aliases(nodedir) if alias in old_aliases: - print >>stderr, "Alias '%s' already exists!" % alias + print >>stderr, "Alias %s already exists!" % quote_output(alias) return 1 aliasfile = os.path.join(nodedir, "private", "aliases") @@ -77,16 +77,26 @@ def create_alias(options): add_line_to_aliasfile(aliasfile, alias, new_uri) - print >>stdout, "Alias '%s' created" % (unicode_to_stdout(alias),) + print >>stdout, "Alias %s created" % (quote_output(alias),) return 0 def list_aliases(options): nodedir = options['node-directory'] stdout = options.stdout + stderr = options.stderr aliases = get_aliases(nodedir) alias_names = sorted(aliases.keys()) - max_width = max([len(name) for name in alias_names] + [0]) + max_width = max([len(quote_output(name)) for name in alias_names] + [0]) fmt = "%" + str(max_width) + "s: %s" + rc = 0 for name in alias_names: - print >>stdout, fmt % (name, aliases[name]) - + try: + print >>stdout, fmt % (unicode_to_output(name), unicode_to_output(aliases[name].decode('utf-8'))) + except (UnicodeEncodeError, UnicodeDecodeError): + print >>stderr, fmt % (quote_output(name), quote_output(aliases[name])) + rc = 1 + + if rc == 1: + print >>stderr, "\nThis listing included aliases or caps that could not be converted to the terminal" \ + "\noutput encoding. These are shown using backslash escapes and in quotes." + return rc diff --git a/src/allmydata/scripts/tahoe_backup.py b/src/allmydata/scripts/tahoe_backup.py index 425e8eba..e332b9f5 100644 --- a/src/allmydata/scripts/tahoe_backup.py +++ b/src/allmydata/scripts/tahoe_backup.py @@ -6,20 +6,13 @@ import simplejson import datetime from allmydata.scripts.common import get_alias, escape_path, DEFAULT_ALIAS, \ UnknownAliasError -from allmydata.scripts.common_http import do_http +from allmydata.scripts.common_http import do_http, HTTPError, format_http_error from allmydata.util import time_format from allmydata.scripts import backupdb -from allmydata.util.stringutils import unicode_to_stdout, listdir_unicode, open_unicode +from allmydata.util.stringutils import listdir_unicode, open_unicode, quote_output, to_str from allmydata.util.assertutil import precondition -class HTTPError(Exception): - pass - -def raiseHTTPError(msg, resp): - msg = msg + ": %s %s %s" % (resp.status, resp.reason, resp.read()) - raise HTTPError(msg) - def get_local_metadata(path): metadata = {} @@ -49,8 +42,9 @@ def mkdir(contents, options): url = options['node-url'] + "uri?t=mkdir-immutable" resp = do_http("POST", url, body) if resp.status < 200 or resp.status >= 300: - raiseHTTPError("error during mkdir", resp) - dircap = str(resp.read().strip()) + raise HTTPError("Error during mkdir", resp) + + dircap = to_str(resp.read().strip()) return dircap def put_child(dirurl, childname, childcap): @@ -58,7 +52,7 @@ def put_child(dirurl, childname, childcap): url = dirurl + urllib.quote(childname) + "?t=uri" resp = do_http("PUT", url, childcap) if resp.status not in (200, 201): - raiseHTTPError("error during put_child", resp) + raise HTTPError("Error during put_child", resp) class BackupProcessingError(Exception): pass @@ -99,7 +93,7 @@ class BackerUpper: try: rootcap, path = get_alias(options.aliases, options.to_dir, DEFAULT_ALIAS) except UnknownAliasError, e: - print >>stderr, "error: %s" % e.args[0] + e.display(stderr) return 1 to_url = nodeurl + "uri/%s/" % urllib.quote(rootcap) if path: @@ -115,8 +109,7 @@ class BackerUpper: if resp.status == 404: resp = do_http("POST", archives_url + "?t=mkdir") if resp.status != 200: - print >>stderr, "Unable to create target directory: %s %s %s" % \ - (resp.status, resp.reason, resp.read()) + print >>stderr, format_http_error("Unable to create target directory", resp) return 1 # second step: process the tree @@ -156,20 +149,19 @@ class BackerUpper: return 0 def verboseprint(self, msg): + precondition(isinstance(msg, str), msg) if self.verbosity >= 2: - if isinstance(msg, unicode): - msg = unicode_to_stdout(msg) - print >>self.options.stdout, msg def warn(self, msg): + precondition(isinstance(msg, str), msg) print >>self.options.stderr, msg def process(self, localpath): precondition(isinstance(localpath, unicode), localpath) # returns newdircap - self.verboseprint("processing %s" % localpath) + self.verboseprint("processing %s" % quote_output(localpath)) create_contents = {} # childname -> (type, rocap, metadata) compare_contents = {} # childname -> rocap @@ -177,7 +169,7 @@ class BackerUpper: children = listdir_unicode(localpath) except EnvironmentError: self.directories_skipped += 1 - self.warn("WARNING: permission denied on directory %s" % localpath) + self.warn("WARNING: permission denied on directory %s" % quote_output(localpath)) children = [] for child in self.options.filter_listdir(children): @@ -199,17 +191,17 @@ class BackerUpper: compare_contents[child] = childcap except EnvironmentError: self.files_skipped += 1 - self.warn("WARNING: permission denied on file %s" % childpath) + self.warn("WARNING: permission denied on file %s" % quote_output(childpath)) else: self.files_skipped += 1 if os.path.islink(childpath): - self.warn("WARNING: cannot backup symlink %s" % childpath) + self.warn("WARNING: cannot backup symlink %s" % quote_output(childpath)) else: - self.warn("WARNING: cannot backup special file %s" % childpath) + self.warn("WARNING: cannot backup special file %s" % quote_output(childpath)) must_create, r = self.check_backupdb_directory(compare_contents) if must_create: - self.verboseprint(" creating directory for %s" % localpath) + self.verboseprint(" creating directory for %s" % quote_output(localpath)) newdircap = mkdir(create_contents, self.options) assert isinstance(newdircap, str) if r: @@ -217,7 +209,7 @@ class BackerUpper: self.directories_created += 1 return newdircap else: - self.verboseprint(" re-using old directory for %s" % localpath) + self.verboseprint(" re-using old directory for %s" % quote_output(localpath)) self.directories_reused += 1 return r.was_created() @@ -237,7 +229,7 @@ class BackerUpper: # we must check the file before using the results filecap = r.was_uploaded() - self.verboseprint("checking %s" % filecap) + self.verboseprint("checking %s" % quote_output(filecap)) nodeurl = self.options['node-url'] checkurl = nodeurl + "uri/%s?t=check&output=JSON" % urllib.quote(filecap) self.files_checked += 1 @@ -270,7 +262,7 @@ class BackerUpper: # we must check the directory before re-using it dircap = r.was_created() - self.verboseprint("checking %s" % dircap) + self.verboseprint("checking %s" % quote_output(dircap)) nodeurl = self.options['node-url'] checkurl = nodeurl + "uri/%s?t=check&output=JSON" % urllib.quote(dircap) self.directories_checked += 1 @@ -292,22 +284,24 @@ class BackerUpper: def upload(self, childpath): precondition(isinstance(childpath, unicode), childpath) - #self.verboseprint("uploading %s.." % childpath) + #self.verboseprint("uploading %s.." % quote_output(childpath)) metadata = get_local_metadata(childpath) # we can use the backupdb here must_upload, bdb_results = self.check_backupdb_file(childpath) if must_upload: - self.verboseprint("uploading %s.." % childpath) - infileobj = open_unicode(os.path.expanduser(childpath), "rb") + self.verboseprint("uploading %s.." % quote_output(childpath)) + infileobj = open_unicode(childpath, "rb") url = self.options['node-url'] + "uri" resp = do_http("PUT", url, infileobj) if resp.status not in (200, 201): - raiseHTTPError("Error during file PUT", resp) + raise HTTPError("Error during file PUT", resp) + filecap = resp.read().strip() - self.verboseprint(" %s -> %s" % (childpath, filecap)) - #self.verboseprint(" metadata: %s" % (metadata,)) + self.verboseprint(" %s -> %s" % (quote_output(childpath, quotemarks=False), + quote_output(filecap, quotemarks=False))) + #self.verboseprint(" metadata: %s" % (quote_output(metadata, quotemarks=False),)) if bdb_results: bdb_results.did_upload(filecap) @@ -316,7 +310,7 @@ class BackerUpper: return filecap, metadata else: - self.verboseprint("skipping %s.." % childpath) + self.verboseprint("skipping %s.." % quote_output(childpath)) self.files_reused += 1 return bdb_results.was_uploaded(), metadata diff --git a/src/allmydata/scripts/tahoe_check.py b/src/allmydata/scripts/tahoe_check.py index 6fdd9013..7191c9da 100644 --- a/src/allmydata/scripts/tahoe_check.py +++ b/src/allmydata/scripts/tahoe_check.py @@ -4,7 +4,8 @@ import simplejson from twisted.protocols.basic import LineOnlyReceiver from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \ UnknownAliasError -from allmydata.scripts.common_http import do_http +from allmydata.scripts.common_http import do_http, format_http_error +from allmydata.util.stringutils import quote_output, quote_path class Checker: pass @@ -19,7 +20,7 @@ def check(options): try: rootcap, path = get_alias(options.aliases, where, DEFAULT_ALIAS) except UnknownAliasError, e: - print >>stderr, "error: %s" % e.args[0] + e.display(stderr) return 1 if path == '/': path = '' @@ -37,7 +38,7 @@ def check(options): resp = do_http("POST", url) if resp.status != 200: - print >>stderr, "ERROR", resp.status, resp.reason, resp.read() + print >>stderr, format_http_error("ERROR", resp) return 1 jdata = resp.read() if options.get("raw"): @@ -108,12 +109,12 @@ class DeepCheckOutput(LineOnlyReceiver): def lineReceived(self, line): if self.in_error: - print >>self.stderr, line + print >>self.stderr, quote_output(line, quotemarks=False) return if line.startswith("ERROR:"): self.in_error = True self.streamer.rc = 1 - print >>self.stderr, line + print >>self.stderr, quote_output(line, quotemarks=False) return d = simplejson.loads(line) @@ -135,12 +136,8 @@ class DeepCheckOutput(LineOnlyReceiver): if not path: path = [""] summary = cr.get("summary", "Healthy (LIT)") - try: - print >>stdout, "%s: %s" % ("/".join(path), summary) - except UnicodeEncodeError: - print >>stdout, "%s: %s" % ("/".join([p.encode("utf-8") - for p in path]), - summary) + print >>stdout, "%s: %s" % (quote_path(path), summary) + # always print out corrupt shares for shareloc in cr["results"].get("list-corrupt-shares", []): (serverid, storage_index, sharenum) = shareloc @@ -174,12 +171,12 @@ class DeepCheckAndRepairOutput(LineOnlyReceiver): def lineReceived(self, line): if self.in_error: - print >>self.stderr, line + print >>self.stderr, quote_output(line, quotemarks=False) return if line.startswith("ERROR:"): self.in_error = True self.streamer.rc = 1 - print >>self.stderr, line + print >>self.stderr, quote_output(line, quotemarks=False) return d = simplejson.loads(line) @@ -221,12 +218,8 @@ class DeepCheckAndRepairOutput(LineOnlyReceiver): summary = "healthy" else: summary = "not healthy" - try: - print >>stdout, "%s: %s" % ("/".join(path), summary) - except UnicodeEncodeError: - print >>stdout, "%s: %s" % ("/".join([p.encode("utf-8") - for p in path]), - summary) + print >>stdout, "%s: %s" % (quote_path(path), summary) + # always print out corrupt shares prr = crr.get("pre-repair-results", {}) for shareloc in prr.get("results", {}).get("list-corrupt-shares", []): @@ -272,7 +265,7 @@ class DeepCheckStreamer(LineOnlyReceiver): try: rootcap, path = get_alias(options.aliases, where, DEFAULT_ALIAS) except UnknownAliasError, e: - print >>stderr, "error: %s" % e.args[0] + e.display(stderr) return 1 if path == '/': path = '' @@ -292,7 +285,7 @@ class DeepCheckStreamer(LineOnlyReceiver): url += "&add-lease=true" resp = do_http("POST", url) if resp.status not in (200, 302): - print >>stderr, "ERROR", resp.status, resp.reason, resp.read() + print >>stderr, format_http_error("ERROR", resp) return 1 # use Twisted to split this into lines diff --git a/src/allmydata/scripts/tahoe_cp.py b/src/allmydata/scripts/tahoe_cp.py index 02790d08..2a264119 100644 --- a/src/allmydata/scripts/tahoe_cp.py +++ b/src/allmydata/scripts/tahoe_cp.py @@ -5,37 +5,38 @@ import simplejson from cStringIO import StringIO from twisted.python.failure import Failure from allmydata.scripts.common import get_alias, escape_path, \ - DefaultAliasMarker, UnknownAliasError -from allmydata.scripts.common_http import do_http + DefaultAliasMarker, TahoeError +from allmydata.scripts.common_http import do_http, HTTPError from allmydata import uri -from allmydata.util.stringutils import unicode_to_url, listdir_unicode, open_unicode +from allmydata.util.stringutils import unicode_to_url, listdir_unicode, open_unicode, \ + abspath_expanduser_unicode, quote_output, to_str from allmydata.util.assertutil import precondition -def ascii_or_none(s): - if s is None: - return s - return str(s) +def _put_local_file(pathname, inf): + # TODO: create temporary file and move into place? + # TODO: move this to fileutil. + outf = open_unicode(pathname, "wb") + try: + while True: + data = inf.read(32768) + if not data: + break + outf.write(data) + finally: + outf.close() -class TahoeError(Exception): - def __init__(self, msg, resp): - self.msg = msg - self.status = resp.status - self.reason = resp.reason - self.body = resp.read() - def display(self, err): - print >>err, "%s: %s %s" % (self.msg, self.status, self.reason) - print >>err, self.body +class MissingSourceError(TahoeError): + def __init__(self, name): + TahoeError.__init__("No such file or directory %s" % quote_output(name)) -class MissingSourceError(Exception): - pass def GET_to_file(url): resp = do_http("GET", url) if resp.status == 200: return resp - raise TahoeError("Error during GET", resp) + raise HTTPError("Error during GET", resp) def GET_to_string(url): f = GET_to_file(url) @@ -45,20 +46,20 @@ def PUT(url, data): resp = do_http("PUT", url, data) if resp.status in (200, 201): return resp.read() - raise TahoeError("Error during PUT", resp) + raise HTTPError("Error during PUT", resp) def POST(url, data): resp = do_http("POST", url, data) if resp.status in (200, 201): return resp.read() - raise TahoeError("Error during POST", resp) + raise HTTPError("Error during POST", resp) def mkdir(targeturl): url = targeturl + "?t=mkdir" resp = do_http("POST", url) if resp.status in (200, 201): return resp.read().strip() - raise TahoeError("Error during mkdir", resp) + raise HTTPError("Error during mkdir", resp) def make_tahoe_subdirectory(nodeurl, parent_writecap, name): url = nodeurl + "/".join(["uri", @@ -68,7 +69,7 @@ def make_tahoe_subdirectory(nodeurl, parent_writecap, name): resp = do_http("POST", url) if resp.status in (200, 201): return resp.read().strip() - raise TahoeError("Error during mkdir", resp) + raise HTTPError("Error during mkdir", resp) class LocalFileSource: @@ -80,20 +81,17 @@ class LocalFileSource: return True def open(self, caps_only): - return open(self.pathname, "rb") + return open_unicode(self.pathname, "rb") + class LocalFileTarget: def __init__(self, pathname): precondition(isinstance(pathname, unicode), pathname) self.pathname = pathname + def put_file(self, inf): - outf = open(self.pathname, "wb") - while True: - data = inf.read(32768) - if not data: - break - outf.write(data) - outf.close() + _put_local_file(self.pathname, inf) + class LocalMissingTarget: def __init__(self, pathname): @@ -101,13 +99,8 @@ class LocalMissingTarget: self.pathname = pathname def put_file(self, inf): - outf = open(self.pathname, "wb") - while True: - data = inf.read(32768) - if not data: - break - outf.write(data) - outf.close() + _put_local_file(self.pathname, inf) + class LocalDirectorySource: def __init__(self, progressfunc, pathname): @@ -134,6 +127,7 @@ class LocalDirectorySource: self.children[n] = LocalFileSource(pn) else: # Could be dangling symlink; probably not copy-able. + # TODO: output a warning pass class LocalDirectoryTarget: @@ -151,6 +145,7 @@ class LocalDirectoryTarget: children = listdir_unicode(self.pathname) for i,n in enumerate(children): self.progressfunc("examining %d of %d" % (i, len(children))) + n = unicode(n) pn = os.path.join(self.pathname, n) if os.path.isdir(pn): child = LocalDirectoryTarget(self.progressfunc, pn) @@ -173,13 +168,7 @@ class LocalDirectoryTarget: def put_file(self, name, inf): precondition(isinstance(name, unicode), name) pathname = os.path.join(self.pathname, name) - outf = open_unicode(pathname, "wb") - while True: - data = inf.read(32768) - if not data: - break - outf.write(data) - outf.close() + _put_local_file(pathname, inf) def set_children(self): pass @@ -238,7 +227,7 @@ class TahoeDirectorySource: url = self.nodeurl + "uri/%s" % urllib.quote(bestcap) resp = do_http("GET", url + "?t=json") if resp.status != 200: - raise TahoeError("Error examining source directory", resp) + raise HTTPError("Error examining source directory", resp) parsed = simplejson.loads(resp.read()) nodetype, d = parsed assert nodetype == "dirnode" @@ -250,8 +239,8 @@ class TahoeDirectorySource: def init_from_parsed(self, parsed): nodetype, d = parsed - self.writecap = ascii_or_none(d.get("rw_uri")) - self.readcap = ascii_or_none(d.get("ro_uri")) + self.writecap = to_str(d.get("rw_uri")) + self.readcap = to_str(d.get("ro_uri")) self.mutable = d.get("mutable", False) # older nodes don't provide it self.children_d = dict( [(unicode(name),value) for (name,value) @@ -266,13 +255,13 @@ class TahoeDirectorySource: self.progressfunc("examining %d of %d" % (i, len(self.children_d))) if data[0] == "filenode": mutable = data[1].get("mutable", False) - writecap = ascii_or_none(data[1].get("rw_uri")) - readcap = ascii_or_none(data[1].get("ro_uri")) + writecap = to_str(data[1].get("rw_uri")) + readcap = to_str(data[1].get("ro_uri")) self.children[name] = TahoeFileSource(self.nodeurl, mutable, writecap, readcap) elif data[0] == "dirnode": - writecap = ascii_or_none(data[1].get("rw_uri")) - readcap = ascii_or_none(data[1].get("ro_uri")) + writecap = to_str(data[1].get("rw_uri")) + readcap = to_str(data[1].get("ro_uri")) if writecap and writecap in self.cache: child = self.cache[writecap] elif readcap and readcap in self.cache: @@ -320,8 +309,8 @@ class TahoeDirectoryTarget: def init_from_parsed(self, parsed): nodetype, d = parsed - self.writecap = ascii_or_none(d.get("rw_uri")) - self.readcap = ascii_or_none(d.get("ro_uri")) + self.writecap = to_str(d.get("rw_uri")) + self.readcap = to_str(d.get("ro_uri")) self.mutable = d.get("mutable", False) # older nodes don't provide it self.children_d = dict( [(unicode(name),value) for (name,value) @@ -335,7 +324,7 @@ class TahoeDirectoryTarget: url = self.nodeurl + "uri/%s" % urllib.quote(bestcap) resp = do_http("GET", url + "?t=json") if resp.status != 200: - raise TahoeError("Error examining target directory", resp) + raise HTTPError("Error examining target directory", resp) parsed = simplejson.loads(resp.read()) nodetype, d = parsed assert nodetype == "dirnode" @@ -360,8 +349,8 @@ class TahoeDirectoryTarget: self.progressfunc("examining %d of %d" % (i, len(self.children_d))) if data[0] == "filenode": mutable = data[1].get("mutable", False) - writecap = ascii_or_none(data[1].get("rw_uri")) - readcap = ascii_or_none(data[1].get("ro_uri")) + writecap = to_str(data[1].get("rw_uri")) + readcap = to_str(data[1].get("ro_uri")) url = None if self.writecap: url = self.nodeurl + "/".join(["uri", @@ -370,8 +359,8 @@ class TahoeDirectoryTarget: self.children[name] = TahoeFileTarget(self.nodeurl, mutable, writecap, readcap, url) elif data[0] == "dirnode": - writecap = ascii_or_none(data[1].get("rw_uri")) - readcap = ascii_or_none(data[1].get("ro_uri")) + writecap = to_str(data[1].get("rw_uri")) + readcap = to_str(data[1].get("ro_uri")) if writecap and writecap in self.cache: child = self.cache[writecap] elif readcap and readcap in self.cache: @@ -466,8 +455,9 @@ class Copier: status = self.try_copy() return status except TahoeError, te: - Failure().printTraceback(self.stderr) - print >>self.stderr + if verbosity >= 2: + Failure().printTraceback(self.stderr) + print >>self.stderr te.display(self.stderr) return 1 @@ -476,23 +466,12 @@ class Copier: destination_spec = self.options.destination recursive = self.options["recursive"] - try: - target = self.get_target_info(destination_spec) - except UnknownAliasError, e: - self.to_stderr("error: %s" % e.args[0]) - return 1 + target = self.get_target_info(destination_spec) - try: - sources = [] # list of (name, source object) - for ss in source_specs: - name, source = self.get_source_info(ss) - sources.append( (name, source) ) - except MissingSourceError, e: - self.to_stderr("No such file or directory %s" % e.args[0]) - return 1 - except UnknownAliasError, e: - self.to_stderr("error: %s" % e.args[0]) - return 1 + sources = [] # list of (name, source object) + for ss in source_specs: + name, source = self.get_source_info(ss) + sources.append( (name, source) ) have_source_dirs = bool([s for (name,s) in sources if isinstance(s, (LocalDirectorySource, @@ -506,7 +485,7 @@ class Copier: # cp STUFF foo.txt, where foo.txt already exists. This limits the # possibilities considerably. if len(sources) > 1: - self.to_stderr("target '%s' is not a directory" % destination_spec) + self.to_stderr("target %s is not a directory" % quote_output(destination_spec)) return 1 if have_source_dirs: self.to_stderr("cannot copy directory into a file") @@ -546,7 +525,7 @@ class Copier: rootcap, path = get_alias(self.aliases, destination_spec, None) if rootcap == DefaultAliasMarker: # no alias, so this is a local file - pathname = os.path.abspath(os.path.expanduser(path)) + pathname = abspath_expanduser_unicode(path.decode('utf-8')) if not os.path.exists(pathname): t = LocalMissingTarget(pathname) elif os.path.isdir(pathname): @@ -572,21 +551,21 @@ class Copier: self.progress) t.init_from_parsed(parsed) else: - writecap = ascii_or_none(d.get("rw_uri")) - readcap = ascii_or_none(d.get("ro_uri")) + writecap = to_str(d.get("rw_uri")) + readcap = to_str(d.get("ro_uri")) mutable = d.get("mutable", False) t = TahoeFileTarget(self.nodeurl, mutable, writecap, readcap, url) else: - raise TahoeError("Error examining target '%s'" - % destination_spec, resp) + raise HTTPError("Error examining target %s" + % quote_output(destination_spec), resp) return t def get_source_info(self, source_spec): rootcap, path = get_alias(self.aliases, source_spec, None) if rootcap == DefaultAliasMarker: # no alias, so this is a local file - pathname = os.path.abspath(os.path.expanduser(path)) + pathname = abspath_expanduser_unicode(path.decode('utf-8')) name = os.path.basename(pathname) if not os.path.exists(pathname): raise MissingSourceError(source_spec) @@ -610,8 +589,8 @@ class Copier: if resp.status == 404: raise MissingSourceError(source_spec) elif resp.status != 200: - raise TahoeError("Error examining source '%s'" % source_spec, - resp) + raise HTTPError("Error examining source %s" % quote_output(source_spec), + resp) parsed = simplejson.loads(resp.read()) nodetype, d = parsed if nodetype == "dirnode": @@ -619,8 +598,8 @@ class Copier: self.progress) t.init_from_parsed(parsed) else: - writecap = ascii_or_none(d.get("rw_uri")) - readcap = ascii_or_none(d.get("ro_uri")) + writecap = to_str(d.get("rw_uri")) + readcap = to_str(d.get("ro_uri")) mutable = d.get("mutable", False) # older nodes don't provide it if source_spec.rfind('/') != -1: name = source_spec[source_spec.rfind('/')+1:] @@ -630,7 +609,7 @@ class Copier: def dump_graph(self, s, indent=" "): for name, child in s.children.items(): - print indent + name + ":" + str(child) + print "%s%s: %r" % (indent, quote_output(name), child) if isinstance(child, (LocalDirectorySource, TahoeDirectorySource)): self.dump_graph(child, indent+" ") diff --git a/src/allmydata/scripts/tahoe_get.py b/src/allmydata/scripts/tahoe_get.py index 5b709569..dfc36644 100644 --- a/src/allmydata/scripts/tahoe_get.py +++ b/src/allmydata/scripts/tahoe_get.py @@ -2,7 +2,8 @@ import urllib from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \ UnknownAliasError -from allmydata.scripts.common_http import do_http +from allmydata.scripts.common_http import do_http, format_http_error +from allmydata.util.stringutils import open_unicode def get(options): nodeurl = options['node-url'] @@ -17,7 +18,7 @@ def get(options): try: rootcap, path = get_alias(aliases, from_file, DEFAULT_ALIAS) except UnknownAliasError, e: - print >>stderr, "error: %s" % e.args[0] + e.display(stderr) return 1 url = nodeurl + "uri/%s" % urllib.quote(rootcap) if path: @@ -26,7 +27,7 @@ def get(options): resp = do_http("GET", url) if resp.status in (200, 201,): if to_file: - outf = open(to_file, "wb") + outf = open_unicode(to_file, "wb") else: outf = stdout while True: @@ -38,8 +39,7 @@ def get(options): outf.close() rc = 0 else: - print >>stderr, "Error, got %s %s" % (resp.status, resp.reason) - print >>stderr, resp.read() + print >>stderr, format_http_error("Error during GET", resp) rc = 1 return rc diff --git a/src/allmydata/scripts/tahoe_ls.py b/src/allmydata/scripts/tahoe_ls.py index 4e42f2fb..d7c7ef83 100644 --- a/src/allmydata/scripts/tahoe_ls.py +++ b/src/allmydata/scripts/tahoe_ls.py @@ -3,8 +3,8 @@ import urllib, time import simplejson from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \ UnknownAliasError -from allmydata.scripts.common_http import do_http -from allmydata.util.stringutils import unicode_to_stdout +from allmydata.scripts.common_http import do_http, format_http_error +from allmydata.util.stringutils import unicode_to_output, quote_output, is_printable_ascii, to_str def list(options): nodeurl = options['node-url'] @@ -20,7 +20,7 @@ def list(options): try: rootcap, path = get_alias(aliases, where, DEFAULT_ALIAS) except UnknownAliasError, e: - print >>stderr, "error: %s" % e.args[0] + e.display(stderr) return 1 url = nodeurl + "uri/%s" % urllib.quote(rootcap) if path: @@ -33,9 +33,7 @@ def list(options): print >>stderr, "No such file or directory" return 2 if resp.status != 200: - print >>stderr, "Error during GET: %s %s %s" % (resp.status, - resp.reason, - resp.read()) + print >>stderr, format_http_error("Error during GET", resp) if resp.status == 0: return 3 else: @@ -44,20 +42,28 @@ def list(options): data = resp.read() if options['json']: - print >>stdout, data - return + # The webapi server should always output printable ASCII. + if is_printable_ascii(data): + print >>stdout, data + return 0 + else: + print >>stderr, "The JSON response contained unprintable characters:\n%s" % quote_output(data) + return 1 try: parsed = simplejson.loads(data) - except Exception, le: - le.args = tuple(le.args + (data,)) - raise + except Exception, e: + print >>stderr, "error: %s" % quote_output(e.args[0], quotemarks=False) + print >>stderr, "Could not parse JSON response:\n%s" % quote_output(data) + return 1 + nodetype, d = parsed children = {} if nodetype == "dirnode": children = d['children'] else: - childname = path.split("/")[-1] + # paths returned from get_alias are always valid UTF-8 + childname = path.split("/")[-1].decode('utf-8') children = {childname: (nodetype, d)} if "metadata" not in d: d["metadata"] = {} @@ -71,8 +77,8 @@ def list(options): has_unknowns = False for name in childnames: - name = unicode(name) child = children[name] + name = unicode(name) childtype = child[0] # See webapi.txt for a discussion of the meanings of unix local @@ -85,8 +91,8 @@ def list(options): mtime = child[1].get("metadata", {}).get('tahoe', {}).get("linkmotime") if not mtime: mtime = child[1]["metadata"].get("mtime") - rw_uri = child[1].get("rw_uri") - ro_uri = child[1].get("ro_uri") + rw_uri = to_str(child[1].get("rw_uri")) + ro_uri = to_str(child[1].get("ro_uri")) if ctime: # match for formatting that GNU 'ls' does if (now - ctime) > 6*30*24*60*60: @@ -131,17 +137,24 @@ def list(options): line.append(ctime_s) if not options["classify"]: classify = "" - line.append(unicode_to_stdout(name) + classify) + + encoding_error = False + try: + line.append(unicode_to_output(name) + classify) + except UnicodeEncodeError: + encoding_error = True + line.append(quote_output(name) + classify) + if options["uri"]: line.append(uri) if options["readonly-uri"]: - line.append(ro_uri or "-") + line.append(quote_output(ro_uri or "-", quotemarks=False)) - rows.append(line) + rows.append((encoding_error, line)) max_widths = [] left_justifys = [] - for row in rows: + for (encoding_error, row) in rows: for i,cell in enumerate(row): while len(max_widths) <= i: max_widths.append(0) @@ -161,14 +174,20 @@ def list(options): piece += "s" fmt_pieces.append(piece) fmt = " ".join(fmt_pieces) - for row in rows: - print >>stdout, (fmt % tuple(row)).rstrip() + + rc = 0 + for (encoding_error, row) in rows: + if encoding_error: + print >>stderr, (fmt % tuple(row)).rstrip() + rc = 1 + else: + print >>stdout, (fmt % tuple(row)).rstrip() + if rc == 1: + print >>stderr, "\nThis listing included files whose names could not be converted to the terminal" \ + "\noutput encoding. Their names are shown using backslash escapes and in quotes." if has_unknowns: print >>stderr, "\nThis listing included unknown objects. Using a webapi server that supports" \ "\na later version of Tahoe may help." - return 0 - -# error cases that need improvement: -# list-one-file: tahoe ls my:docs/Makefile + return rc diff --git a/src/allmydata/scripts/tahoe_manifest.py b/src/allmydata/scripts/tahoe_manifest.py index 88a578a9..db422082 100644 --- a/src/allmydata/scripts/tahoe_manifest.py +++ b/src/allmydata/scripts/tahoe_manifest.py @@ -5,8 +5,8 @@ from allmydata.util.abbreviate import abbreviate_space_both from allmydata.scripts.slow_operation import SlowOperationRunner from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \ UnknownAliasError -from allmydata.scripts.common_http import do_http -from allmydata.util.stringutils import unicode_to_stdout +from allmydata.scripts.common_http import do_http, format_http_error +from allmydata.util.stringutils import quote_output, quote_path class FakeTransport: disconnecting = False @@ -30,7 +30,7 @@ class ManifestStreamer(LineOnlyReceiver): try: rootcap, path = get_alias(options.aliases, where, DEFAULT_ALIAS) except UnknownAliasError, e: - print >>stderr, "error: %s" % e.args[0] + e.display(stderr) return 1 if path == '/': path = '' @@ -41,7 +41,7 @@ class ManifestStreamer(LineOnlyReceiver): url += "?t=stream-manifest" resp = do_http("POST", url) if resp.status not in (200, 302): - print >>stderr, "ERROR", resp.status, resp.reason, resp.read() + print >>stderr, format_http_error("ERROR", resp) return 1 #print "RESP", dir(resp) # use Twisted to split this into lines @@ -60,31 +60,35 @@ class ManifestStreamer(LineOnlyReceiver): stdout = self.options.stdout stderr = self.options.stderr if self.in_error: - print >>stderr, line + print >>stderr, quote_output(line, quotemarks=False) return if line.startswith("ERROR:"): self.in_error = True self.rc = 1 - print >>stderr, line + print >>stderr, quote_output(line, quotemarks=False) return - d = simplejson.loads(line.decode('utf-8')) - if d["type"] in ("file", "directory"): - if self.options["storage-index"]: - si = d["storage-index"] - if si: - print >>stdout, si - elif self.options["verify-cap"]: - vc = d["verifycap"] - if vc: - print >>stdout, vc - elif self.options["repair-cap"]: - vc = d["repaircap"] - if vc: - print >>stdout, vc - else: - print >>stdout, d["cap"], "/".join([unicode_to_stdout(p) - for p in d["path"]]) + try: + d = simplejson.loads(line.decode('utf-8')) + except Exception, e: + print >>stderr, "ERROR could not decode/parse %s\nERROR %r" % (quote_output(line), e) + else: + if d["type"] in ("file", "directory"): + if self.options["storage-index"]: + si = d.get("storage-index", None) + if si: + print >>stdout, quote_output(si, quotemarks=False) + elif self.options["verify-cap"]: + vc = d.get("verifycap", None) + if vc: + print >>stdout, quote_output(vc, quotemarks=False) + elif self.options["repair-cap"]: + vc = d.get("repaircap", None) + if vc: + print >>stdout, quote_output(vc, quotemarks=False) + else: + print >>stdout, "%s %s" % (quote_output(d["cap"], quotemarks=False), + quote_path(d["path"], quotemarks=False)) def manifest(options): return ManifestStreamer().run(options) diff --git a/src/allmydata/scripts/tahoe_mkdir.py b/src/allmydata/scripts/tahoe_mkdir.py index cc4b0688..7ff8a99a 100644 --- a/src/allmydata/scripts/tahoe_mkdir.py +++ b/src/allmydata/scripts/tahoe_mkdir.py @@ -2,7 +2,7 @@ import urllib from allmydata.scripts.common_http import do_http, check_http_error from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, UnknownAliasError -from allmydata.util.stringutils import unicode_to_url +from allmydata.util.stringutils import quote_output def mkdir(options): nodeurl = options['node-url'] @@ -16,7 +16,7 @@ def mkdir(options): try: rootcap, path = get_alias(aliases, where, DEFAULT_ALIAS) except UnknownAliasError, e: - print >>stderr, "error: %s" % e.args[0] + e.display(stderr) return 1 if not where or not path: @@ -28,17 +28,17 @@ def mkdir(options): return rc new_uri = resp.read().strip() # emit its write-cap - print >>stdout, new_uri + print >>stdout, quote_output(new_uri, quotemarks=False) return 0 # create a new directory at the given location if path.endswith("/"): path = path[:-1] - # path (in argv) must be "/".join([s.encode("utf-8") for s in segments]) + # path must be "/".join([s.encode("utf-8") for s in segments]) url = nodeurl + "uri/%s/%s?t=mkdir" % (urllib.quote(rootcap), - urllib.quote(unicode_to_url(path))) + urllib.quote(path)) resp = do_http("POST", url) check_http_error(resp, stderr) new_uri = resp.read().strip() - print >>stdout, new_uri + print >>stdout, quote_output(new_uri, quotemarks=False) return 0 diff --git a/src/allmydata/scripts/tahoe_mv.py b/src/allmydata/scripts/tahoe_mv.py index 208b720e..d336d804 100644 --- a/src/allmydata/scripts/tahoe_mv.py +++ b/src/allmydata/scripts/tahoe_mv.py @@ -4,7 +4,8 @@ import urllib import simplejson from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \ UnknownAliasError -from allmydata.scripts.common_http import do_http +from allmydata.scripts.common_http import do_http, format_http_error +from allmydata.util.stringutils import to_str # this script is used for both 'mv' and 'ln' @@ -21,7 +22,7 @@ def mv(options, mode="move"): try: rootcap, from_path = get_alias(aliases, from_file, DEFAULT_ALIAS) except UnknownAliasError, e: - print >>stderr, "error: %s" % e.args[0] + e.display(stderr) return 1 from_url = nodeurl + "uri/%s" % urllib.quote(rootcap) if from_path: @@ -29,16 +30,13 @@ def mv(options, mode="move"): # figure out the source cap data = urllib.urlopen(from_url + "?t=json").read() nodetype, attrs = simplejson.loads(data) - cap = attrs.get("rw_uri") or attrs["ro_uri"] - # simplejson sometimes returns unicode, but we know that it's really just - # an ASCII file-cap. - cap = str(cap) + cap = to_str(attrs.get("rw_uri") or attrs["ro_uri"]) # now get the target try: rootcap, path = get_alias(aliases, to_file, DEFAULT_ALIAS) except UnknownAliasError, e: - print >>stderr, "error: %s" % e.args[0] + e.display(stderr) return 1 to_url = nodeurl + "uri/%s" % urllib.quote(rootcap) if path: @@ -56,18 +54,17 @@ def mv(options, mode="move"): if status == 409: print >>stderr, "Error: You can't overwrite a directory with a file" else: - print >>stderr, "error, got %s %s" % (resp.status, resp.reason) - print >>stderr, resp.read() + print >>stderr, format_http_error("Error", resp) if mode == "move": print >>stderr, "NOT removing the original" - return + return 1 if mode == "move": # now remove the original resp = do_http("DELETE", from_url) if not re.search(r'^2\d\d$', str(status)): - print >>stderr, "error, got %s %s" % (resp.status, resp.reason) - print >>stderr, resp.read() + print >>stderr, format_http_error("Error deleting original after move", resp) + return 2 print >>stdout, "OK" - return + return 0 diff --git a/src/allmydata/scripts/tahoe_put.py b/src/allmydata/scripts/tahoe_put.py index 24a7e3ef..82a9e4cc 100644 --- a/src/allmydata/scripts/tahoe_put.py +++ b/src/allmydata/scripts/tahoe_put.py @@ -1,10 +1,10 @@ from cStringIO import StringIO -import os.path import urllib -from allmydata.scripts.common_http import do_http +from allmydata.scripts.common_http import do_http, format_http_success, format_http_error from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \ UnknownAliasError +from allmydata.util.stringutils import quote_output, open_unicode def put(options): """ @@ -48,12 +48,12 @@ def put(options): try: rootcap, path = get_alias(aliases, to_file, DEFAULT_ALIAS) except UnknownAliasError, e: - print >>stderr, "error: %s" % e.args[0] + e.display(stderr) return 1 if path.startswith("/"): - suggestion = to_file.replace("/", "", 1) - print >>stderr, "ERROR: The remote filename must not start with a slash" - print >>stderr, "Please try again, perhaps with:", suggestion + suggestion = to_file.replace(u"/", u"", 1) + print >>stderr, "Error: The remote filename must not start with a slash" + print >>stderr, "Please try again, perhaps with %s" % quote_output(suggestion) return 1 url = nodeurl + "uri/%s/" % urllib.quote(rootcap) if path: @@ -64,7 +64,7 @@ def put(options): if mutable: url += "?mutable=true" if from_file: - infileobj = open(os.path.expanduser(from_file), "rb") + infileobj = open_unicode(from_file, "rb") else: # do_http() can't use stdin directly: for one thing, we need a # Content-Length field. So we currently must copy it. @@ -76,10 +76,9 @@ def put(options): resp = do_http("PUT", url, infileobj) if resp.status in (200, 201,): - print >>stderr, "%s %s" % (resp.status, resp.reason) - print >>stdout, resp.read() + print >>stderr, format_http_success(resp) + print >>stdout, quote_output(resp.read(), quotemarks=False) return 0 - print >>stderr, "error, got %s %s" % (resp.status, resp.reason) - print >>stderr, resp.read() + print >>stderr, format_http_error("Error", resp) return 1 diff --git a/src/allmydata/scripts/tahoe_rm.py b/src/allmydata/scripts/tahoe_rm.py index ba557b36..3b133d08 100644 --- a/src/allmydata/scripts/tahoe_rm.py +++ b/src/allmydata/scripts/tahoe_rm.py @@ -1,6 +1,6 @@ import urllib -from allmydata.scripts.common_http import do_http +from allmydata.scripts.common_http import do_http, format_http_success, format_http_error from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \ UnknownAliasError @@ -19,7 +19,7 @@ def rm(options): try: rootcap, path = get_alias(aliases, where, DEFAULT_ALIAS) except UnknownAliasError, e: - print >>stderr, "error: %s" % e.args[0] + e.display(stderr) return 1 assert path url = nodeurl + "uri/%s" % urllib.quote(rootcap) @@ -28,9 +28,8 @@ def rm(options): resp = do_http("DELETE", url) if resp.status in (200,): - print >>stdout, "%s %s" % (resp.status, resp.reason) + print >>stdout, format_http_success(resp) return 0 - print >>stderr, "error, got %s %s" % (resp.status, resp.reason) - print >>stderr, resp.read() + print >>stderr, format_http_error("ERROR", resp) return 1 diff --git a/src/allmydata/scripts/tahoe_webopen.py b/src/allmydata/scripts/tahoe_webopen.py index 891d8db0..03500dc7 100644 --- a/src/allmydata/scripts/tahoe_webopen.py +++ b/src/allmydata/scripts/tahoe_webopen.py @@ -13,7 +13,7 @@ def webopen(options, opener=None): try: rootcap, path = get_alias(options.aliases, where, DEFAULT_ALIAS) except UnknownAliasError, e: - print >>stderr, "error: %s" % e.args[0] + e.display(stderr) return 1 if path == '/': path = '' diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index 0b7b919d..41361671 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -25,6 +25,13 @@ def flip_one_bit(s, offset=0, size=None): assert result != s, "Internal error -- flip_one_bit() produced the same string as its input: %s == %s" % (result, s) return result + +class ReallyEqualMixin: + def failUnlessReallyEqual(self, a, b, msg=None): + self.failUnlessEqual(a, b, msg=msg) + self.failUnlessEqual(type(a), type(b), msg=msg) + + class SignalMixin: # This class is necessary for any code which wants to use Processes # outside the usual reactor.run() environment. It is copied from @@ -131,7 +138,7 @@ class TestMixin(SignalMixin): if p.active(): p.cancel() else: - print "WEIRNESS! pending timed call not active+!" + print "WEIRDNESS! pending timed call not active!" if required_to_quiesce and active: self.fail("Reactor was still active when it was required to be quiescent.") diff --git a/src/allmydata/test/test_cli.py b/src/allmydata/test/test_cli.py index 8d8e954f..76d1f115 100644 --- a/src/allmydata/test/test_cli.py +++ b/src/allmydata/test/test_cli.py @@ -3,41 +3,68 @@ import os.path from twisted.trial import unittest from cStringIO import StringIO -import locale, urllib, re, sys +import urllib, re import simplejson -from allmydata.util import fileutil, hashutil, base32, stringutils +from allmydata.util import fileutil, hashutil, base32 from allmydata import uri from allmydata.immutable import upload # Test that the scripts can be imported -- although the actual tests of their # functionality are done by invoking them in a subprocess. -from allmydata.scripts import tahoe_ls, tahoe_get, tahoe_put, tahoe_rm, tahoe_cp -_hush_pyflakes = [tahoe_ls, tahoe_get, tahoe_put, tahoe_rm, tahoe_cp] +from allmydata.scripts import create_node, debug, keygen, startstop_node, \ + tahoe_add_alias, tahoe_backup, tahoe_check, tahoe_cp, tahoe_get, tahoe_ls, \ + tahoe_manifest, tahoe_mkdir, tahoe_mv, tahoe_put, tahoe_rm, tahoe_webopen +_hush_pyflakes = [create_node, debug, keygen, startstop_node, + tahoe_add_alias, tahoe_backup, tahoe_check, tahoe_cp, tahoe_get, tahoe_ls, + tahoe_manifest, tahoe_mkdir, tahoe_mv, tahoe_put, tahoe_rm, tahoe_webopen] from allmydata.scripts import common from allmydata.scripts.common import DEFAULT_ALIAS, get_aliases, get_alias, \ DefaultAliasMarker from allmydata.scripts import cli, debug, runner, backupdb -from allmydata.test.common_util import StallMixin +from allmydata.test.common_util import StallMixin, ReallyEqualMixin from allmydata.test.no_network import GridTestMixin from twisted.internet import threads # CLI tests use deferToThread from twisted.python import usage -from allmydata.util.stringutils import listdir_unicode, unicode_platform, FilenameEncodingError +from allmydata.util.assertutil import precondition +from allmydata.util.stringutils import listdir_unicode, open_unicode, unicode_platform, \ + quote_output, get_output_encoding, get_argv_encoding, get_filesystem_encoding, \ + unicode_to_output, FilenameEncodingError timeout = 480 # deep_check takes 360s on Zandr's linksys box, others take > 240s -def skip_non_unicode_fs(): - if sys.getfilesystemencoding().lower() not in ('utf-8', 'mbcs', 'utf_16', 'utf_16_be', 'utf_16_le', 'utf_32', 'utf_32_be', 'utf_32_le', 'utf_7', 'utf_8', 'utf_8_sig',): - raise unittest.SkipTest("Arbitrary filenames are not supported by this platform") -def skip_non_unicode_stdout(): - if stringutils.get_term_encoding().lower() not in ('utf-8', 'mbcs', 'utf_16', 'utf_16_be', 'utf_16_le', 'utf_32', 'utf_32_be', 'utf_32_le', 'utf_7', 'utf_8', 'utf_8_sig',): - raise unittest.SkipTest("Arbitrary command-line arguments (argv) are not supported by this platform") +class CLITestMixin(ReallyEqualMixin): + def do_cli(self, verb, *args, **kwargs): + nodeargs = [ + "--node-directory", self.get_clientdir(), + ] + argv = [verb] + nodeargs + list(args) + stdin = kwargs.get("stdin", "") + stdout, stderr = StringIO(), StringIO() + d = threads.deferToThread(runner.runner, argv, run_by_human=False, + stdin=StringIO(stdin), + stdout=stdout, stderr=stderr) + def _done(rc): + return rc, stdout.getvalue(), stderr.getvalue() + d.addCallback(_done) + return d + + def skip_if_cannot_represent_filename(self, u): + precondition(isinstance(u, unicode)) -class CLI(unittest.TestCase): + enc = get_filesystem_encoding() + if not unicode_platform(): + try: + u.encode(enc) + except UnicodeEncodeError: + raise unittest.SkipTest("A non-ASCII filename %r could not be encoded as %s" (u, enc)) + + +class CLI(CLITestMixin, unittest.TestCase): # this test case only looks at argument-processing and simple stuff. def test_options(self): fileutil.rm_dir("cli/test_options") @@ -50,23 +77,23 @@ class CLI(unittest.TestCase): fileutil.write("cli/test_options/private/root_dir.cap", private_uri + "\n") o = cli.ListOptions() o.parseOptions(["--node-directory", "cli/test_options"]) - self.failUnlessEqual(o['node-url'], "http://localhost:8080/") - self.failUnlessEqual(o.aliases[DEFAULT_ALIAS], private_uri) - self.failUnlessEqual(o.where, "") + self.failUnlessReallyEqual(o['node-url'], "http://localhost:8080/") + self.failUnlessReallyEqual(o.aliases[DEFAULT_ALIAS], private_uri) + self.failUnlessReallyEqual(o.where, u"") o = cli.ListOptions() o.parseOptions(["--node-directory", "cli/test_options", "--node-url", "http://example.org:8111/"]) - self.failUnlessEqual(o['node-url'], "http://example.org:8111/") - self.failUnlessEqual(o.aliases[DEFAULT_ALIAS], private_uri) - self.failUnlessEqual(o.where, "") + self.failUnlessReallyEqual(o['node-url'], "http://example.org:8111/") + self.failUnlessReallyEqual(o.aliases[DEFAULT_ALIAS], private_uri) + self.failUnlessReallyEqual(o.where, u"") o = cli.ListOptions() o.parseOptions(["--node-directory", "cli/test_options", "--dir-cap", "root"]) - self.failUnlessEqual(o['node-url'], "http://localhost:8080/") - self.failUnlessEqual(o.aliases[DEFAULT_ALIAS], "root") - self.failUnlessEqual(o.where, "") + self.failUnlessReallyEqual(o['node-url'], "http://localhost:8080/") + self.failUnlessReallyEqual(o.aliases[DEFAULT_ALIAS], "root") + self.failUnlessReallyEqual(o.where, u"") o = cli.ListOptions() other_filenode_uri = uri.WriteableSSKFileURI(writekey="\x11"*16, @@ -74,16 +101,16 @@ class CLI(unittest.TestCase): other_uri = uri.DirectoryURI(other_filenode_uri).to_string() o.parseOptions(["--node-directory", "cli/test_options", "--dir-cap", other_uri]) - self.failUnlessEqual(o['node-url'], "http://localhost:8080/") - self.failUnlessEqual(o.aliases[DEFAULT_ALIAS], other_uri) - self.failUnlessEqual(o.where, "") + self.failUnlessReallyEqual(o['node-url'], "http://localhost:8080/") + self.failUnlessReallyEqual(o.aliases[DEFAULT_ALIAS], other_uri) + self.failUnlessReallyEqual(o.where, u"") o = cli.ListOptions() o.parseOptions(["--node-directory", "cli/test_options", "--dir-cap", other_uri, "subdir"]) - self.failUnlessEqual(o['node-url'], "http://localhost:8080/") - self.failUnlessEqual(o.aliases[DEFAULT_ALIAS], other_uri) - self.failUnlessEqual(o.where, "subdir") + self.failUnlessReallyEqual(o['node-url'], "http://localhost:8080/") + self.failUnlessReallyEqual(o.aliases[DEFAULT_ALIAS], other_uri) + self.failUnlessReallyEqual(o.where, u"subdir") o = cli.ListOptions() self.failUnlessRaises(usage.UsageError, @@ -94,12 +121,12 @@ class CLI(unittest.TestCase): o = cli.ListOptions() o.parseOptions(["--node-directory", "cli/test_options", "--node-url", "http://localhost:8080"]) - self.failUnlessEqual(o["node-url"], "http://localhost:8080/") + self.failUnlessReallyEqual(o["node-url"], "http://localhost:8080/") o = cli.ListOptions() o.parseOptions(["--node-directory", "cli/test_options", "--node-url", "https://localhost/"]) - self.failUnlessEqual(o["node-url"], "https://localhost/") + self.failUnlessReallyEqual(o["node-url"], "https://localhost/") def _dump_cap(self, *args): config = debug.DumpCapOptions() @@ -275,7 +302,7 @@ class CLI(unittest.TestCase): # now make sure that the 'catalog-shares' commands survives the error out, err = self._catalog_shares(nodedir1, nodedir2) - self.failUnlessEqual(out, "", out) + self.failUnlessReallyEqual(out, "", out) self.failUnless("Error processing " in err, "didn't see 'error processing' in '%s'" % err) #self.failUnless(nodedir1 in err, @@ -294,63 +321,63 @@ class CLI(unittest.TestCase): def ga1(path): return get_alias(aliases, path, u"tahoe") uses_lettercolon = common.platform_uses_lettercolon_drivename() - self.failUnlessEqual(ga1("bare"), ("TA", "bare")) - self.failUnlessEqual(ga1("baredir/file"), ("TA", "baredir/file")) - self.failUnlessEqual(ga1("baredir/file:7"), ("TA", "baredir/file:7")) - self.failUnlessEqual(ga1("tahoe:"), ("TA", "")) - self.failUnlessEqual(ga1("tahoe:file"), ("TA", "file")) - self.failUnlessEqual(ga1("tahoe:dir/file"), ("TA", "dir/file")) - self.failUnlessEqual(ga1("work:"), ("WA", "")) - self.failUnlessEqual(ga1("work:file"), ("WA", "file")) - self.failUnlessEqual(ga1("work:dir/file"), ("WA", "dir/file")) + self.failUnlessReallyEqual(ga1(u"bare"), ("TA", "bare")) + self.failUnlessReallyEqual(ga1(u"baredir/file"), ("TA", "baredir/file")) + self.failUnlessReallyEqual(ga1(u"baredir/file:7"), ("TA", "baredir/file:7")) + self.failUnlessReallyEqual(ga1(u"tahoe:"), ("TA", "")) + self.failUnlessReallyEqual(ga1(u"tahoe:file"), ("TA", "file")) + self.failUnlessReallyEqual(ga1(u"tahoe:dir/file"), ("TA", "dir/file")) + self.failUnlessReallyEqual(ga1(u"work:"), ("WA", "")) + self.failUnlessReallyEqual(ga1(u"work:file"), ("WA", "file")) + self.failUnlessReallyEqual(ga1(u"work:dir/file"), ("WA", "dir/file")) # default != None means we really expect a tahoe path, regardless of # whether we're on windows or not. This is what 'tahoe get' uses. - self.failUnlessEqual(ga1("c:"), ("CA", "")) - self.failUnlessEqual(ga1("c:file"), ("CA", "file")) - self.failUnlessEqual(ga1("c:dir/file"), ("CA", "dir/file")) - self.failUnlessEqual(ga1("URI:stuff"), ("URI:stuff", "")) - self.failUnlessEqual(ga1("URI:stuff/file"), ("URI:stuff", "file")) - self.failUnlessEqual(ga1("URI:stuff:./file"), ("URI:stuff", "file")) - self.failUnlessEqual(ga1("URI:stuff/dir/file"), ("URI:stuff", "dir/file")) - self.failUnlessEqual(ga1("URI:stuff:./dir/file"), ("URI:stuff", "dir/file")) - self.failUnlessRaises(common.UnknownAliasError, ga1, "missing:") - self.failUnlessRaises(common.UnknownAliasError, ga1, "missing:dir") - self.failUnlessRaises(common.UnknownAliasError, ga1, "missing:dir/file") + self.failUnlessReallyEqual(ga1(u"c:"), ("CA", "")) + self.failUnlessReallyEqual(ga1(u"c:file"), ("CA", "file")) + self.failUnlessReallyEqual(ga1(u"c:dir/file"), ("CA", "dir/file")) + self.failUnlessReallyEqual(ga1(u"URI:stuff"), ("URI:stuff", "")) + self.failUnlessReallyEqual(ga1(u"URI:stuff/file"), ("URI:stuff", "file")) + self.failUnlessReallyEqual(ga1(u"URI:stuff:./file"), ("URI:stuff", "file")) + self.failUnlessReallyEqual(ga1(u"URI:stuff/dir/file"), ("URI:stuff", "dir/file")) + self.failUnlessReallyEqual(ga1(u"URI:stuff:./dir/file"), ("URI:stuff", "dir/file")) + self.failUnlessRaises(common.UnknownAliasError, ga1, u"missing:") + self.failUnlessRaises(common.UnknownAliasError, ga1, u"missing:dir") + self.failUnlessRaises(common.UnknownAliasError, ga1, u"missing:dir/file") def ga2(path): return get_alias(aliases, path, None) - self.failUnlessEqual(ga2("bare"), (DefaultAliasMarker, "bare")) - self.failUnlessEqual(ga2("baredir/file"), + self.failUnlessReallyEqual(ga2(u"bare"), (DefaultAliasMarker, "bare")) + self.failUnlessReallyEqual(ga2(u"baredir/file"), (DefaultAliasMarker, "baredir/file")) - self.failUnlessEqual(ga2("baredir/file:7"), + self.failUnlessReallyEqual(ga2(u"baredir/file:7"), (DefaultAliasMarker, "baredir/file:7")) - self.failUnlessEqual(ga2("baredir/sub:1/file:7"), + self.failUnlessReallyEqual(ga2(u"baredir/sub:1/file:7"), (DefaultAliasMarker, "baredir/sub:1/file:7")) - self.failUnlessEqual(ga2("tahoe:"), ("TA", "")) - self.failUnlessEqual(ga2("tahoe:file"), ("TA", "file")) - self.failUnlessEqual(ga2("tahoe:dir/file"), ("TA", "dir/file")) + self.failUnlessReallyEqual(ga2(u"tahoe:"), ("TA", "")) + self.failUnlessReallyEqual(ga2(u"tahoe:file"), ("TA", "file")) + self.failUnlessReallyEqual(ga2(u"tahoe:dir/file"), ("TA", "dir/file")) # on windows, we really want c:foo to indicate a local file. # default==None is what 'tahoe cp' uses. if uses_lettercolon: - self.failUnlessEqual(ga2("c:"), (DefaultAliasMarker, "c:")) - self.failUnlessEqual(ga2("c:file"), (DefaultAliasMarker, "c:file")) - self.failUnlessEqual(ga2("c:dir/file"), + self.failUnlessReallyEqual(ga2(u"c:"), (DefaultAliasMarker, "c:")) + self.failUnlessReallyEqual(ga2(u"c:file"), (DefaultAliasMarker, "c:file")) + self.failUnlessReallyEqual(ga2(u"c:dir/file"), (DefaultAliasMarker, "c:dir/file")) else: - self.failUnlessEqual(ga2("c:"), ("CA", "")) - self.failUnlessEqual(ga2("c:file"), ("CA", "file")) - self.failUnlessEqual(ga2("c:dir/file"), ("CA", "dir/file")) - self.failUnlessEqual(ga2("work:"), ("WA", "")) - self.failUnlessEqual(ga2("work:file"), ("WA", "file")) - self.failUnlessEqual(ga2("work:dir/file"), ("WA", "dir/file")) - self.failUnlessEqual(ga2("URI:stuff"), ("URI:stuff", "")) - self.failUnlessEqual(ga2("URI:stuff/file"), ("URI:stuff", "file")) - self.failUnlessEqual(ga2("URI:stuff:./file"), ("URI:stuff", "file")) - self.failUnlessEqual(ga2("URI:stuff/dir/file"), ("URI:stuff", "dir/file")) - self.failUnlessEqual(ga2("URI:stuff:./dir/file"), ("URI:stuff", "dir/file")) - self.failUnlessRaises(common.UnknownAliasError, ga2, "missing:") - self.failUnlessRaises(common.UnknownAliasError, ga2, "missing:dir") - self.failUnlessRaises(common.UnknownAliasError, ga2, "missing:dir/file") + self.failUnlessReallyEqual(ga2(u"c:"), ("CA", "")) + self.failUnlessReallyEqual(ga2(u"c:file"), ("CA", "file")) + self.failUnlessReallyEqual(ga2(u"c:dir/file"), ("CA", "dir/file")) + self.failUnlessReallyEqual(ga2(u"work:"), ("WA", "")) + self.failUnlessReallyEqual(ga2(u"work:file"), ("WA", "file")) + self.failUnlessReallyEqual(ga2(u"work:dir/file"), ("WA", "dir/file")) + self.failUnlessReallyEqual(ga2(u"URI:stuff"), ("URI:stuff", "")) + self.failUnlessReallyEqual(ga2(u"URI:stuff/file"), ("URI:stuff", "file")) + self.failUnlessReallyEqual(ga2(u"URI:stuff:./file"), ("URI:stuff", "file")) + self.failUnlessReallyEqual(ga2(u"URI:stuff/dir/file"), ("URI:stuff", "dir/file")) + self.failUnlessReallyEqual(ga2(u"URI:stuff:./dir/file"), ("URI:stuff", "dir/file")) + self.failUnlessRaises(common.UnknownAliasError, ga2, u"missing:") + self.failUnlessRaises(common.UnknownAliasError, ga2, u"missing:dir") + self.failUnlessRaises(common.UnknownAliasError, ga2, u"missing:dir/file") def ga3(path): old = common.pretend_platform_uses_lettercolon @@ -360,36 +387,36 @@ class CLI(unittest.TestCase): finally: common.pretend_platform_uses_lettercolon = old return retval - self.failUnlessEqual(ga3("bare"), (DefaultAliasMarker, "bare")) - self.failUnlessEqual(ga3("baredir/file"), + self.failUnlessReallyEqual(ga3(u"bare"), (DefaultAliasMarker, "bare")) + self.failUnlessReallyEqual(ga3(u"baredir/file"), (DefaultAliasMarker, "baredir/file")) - self.failUnlessEqual(ga3("baredir/file:7"), + self.failUnlessReallyEqual(ga3(u"baredir/file:7"), (DefaultAliasMarker, "baredir/file:7")) - self.failUnlessEqual(ga3("baredir/sub:1/file:7"), + self.failUnlessReallyEqual(ga3(u"baredir/sub:1/file:7"), (DefaultAliasMarker, "baredir/sub:1/file:7")) - self.failUnlessEqual(ga3("tahoe:"), ("TA", "")) - self.failUnlessEqual(ga3("tahoe:file"), ("TA", "file")) - self.failUnlessEqual(ga3("tahoe:dir/file"), ("TA", "dir/file")) - self.failUnlessEqual(ga3("c:"), (DefaultAliasMarker, "c:")) - self.failUnlessEqual(ga3("c:file"), (DefaultAliasMarker, "c:file")) - self.failUnlessEqual(ga3("c:dir/file"), + self.failUnlessReallyEqual(ga3(u"tahoe:"), ("TA", "")) + self.failUnlessReallyEqual(ga3(u"tahoe:file"), ("TA", "file")) + self.failUnlessReallyEqual(ga3(u"tahoe:dir/file"), ("TA", "dir/file")) + self.failUnlessReallyEqual(ga3(u"c:"), (DefaultAliasMarker, "c:")) + self.failUnlessReallyEqual(ga3(u"c:file"), (DefaultAliasMarker, "c:file")) + self.failUnlessReallyEqual(ga3(u"c:dir/file"), (DefaultAliasMarker, "c:dir/file")) - self.failUnlessEqual(ga3("work:"), ("WA", "")) - self.failUnlessEqual(ga3("work:file"), ("WA", "file")) - self.failUnlessEqual(ga3("work:dir/file"), ("WA", "dir/file")) - self.failUnlessEqual(ga3("URI:stuff"), ("URI:stuff", "")) - self.failUnlessEqual(ga3("URI:stuff:./file"), ("URI:stuff", "file")) - self.failUnlessEqual(ga3("URI:stuff:./dir/file"), ("URI:stuff", "dir/file")) - self.failUnlessRaises(common.UnknownAliasError, ga3, "missing:") - self.failUnlessRaises(common.UnknownAliasError, ga3, "missing:dir") - self.failUnlessRaises(common.UnknownAliasError, ga3, "missing:dir/file") + self.failUnlessReallyEqual(ga3(u"work:"), ("WA", "")) + self.failUnlessReallyEqual(ga3(u"work:file"), ("WA", "file")) + self.failUnlessReallyEqual(ga3(u"work:dir/file"), ("WA", "dir/file")) + self.failUnlessReallyEqual(ga3(u"URI:stuff"), ("URI:stuff", "")) + self.failUnlessReallyEqual(ga3(u"URI:stuff:./file"), ("URI:stuff", "file")) + self.failUnlessReallyEqual(ga3(u"URI:stuff:./dir/file"), ("URI:stuff", "dir/file")) + self.failUnlessRaises(common.UnknownAliasError, ga3, u"missing:") + self.failUnlessRaises(common.UnknownAliasError, ga3, u"missing:dir") + self.failUnlessRaises(common.UnknownAliasError, ga3, u"missing:dir/file") # calling get_alias with a path that doesn't include an alias and # default set to something that isn't in the aliases argument should # raise an UnknownAliasError. def ga4(path): return get_alias(aliases, path, u"badddefault:") - self.failUnlessRaises(common.UnknownAliasError, ga4, "afile") - self.failUnlessRaises(common.UnknownAliasError, ga4, "a/dir/path/") + self.failUnlessRaises(common.UnknownAliasError, ga4, u"afile") + self.failUnlessRaises(common.UnknownAliasError, ga4, u"a/dir/path/") def ga5(path): old = common.pretend_platform_uses_lettercolon @@ -399,48 +426,52 @@ class CLI(unittest.TestCase): finally: common.pretend_platform_uses_lettercolon = old return retval - self.failUnlessRaises(common.UnknownAliasError, ga5, "C:\\Windows") + self.failUnlessRaises(common.UnknownAliasError, ga5, u"C:\\Windows") def test_listdir_unicode_good(self): - files = (u'Lôzane', u'Bern', u'Genève') - enc = sys.getfilesystemencoding() or 'ascii' + filenames = [u'Lôzane', u'Bern', u'Genève'] - # Ensure that our test filenames can actually be represented by the - # current filesystem encoding - try: - [f.encode(enc) for f in files] - except UnicodeEncodeError: - raise unittest.SkipTest("Cannot represent non-ASCII filenames on this filesystem") + for name in filenames: + self.skip_if_cannot_represent_filename(name) - basedir = u"cli/common/listdir_unicode_good" + basedir = "cli/common/listdir_unicode_good" fileutil.make_dirs(basedir) - for file in files: - open(os.path.join(basedir, file), "w").close() + for name in filenames: + open_unicode(os.path.join(unicode(basedir), name), "wb").close() - for file in listdir_unicode(basedir): - self.failUnlessEqual(file in files, True) + for file in listdir_unicode(unicode(basedir)): + self.failUnlessIn(file, filenames) def test_listdir_unicode_bad(self): - if unicode_platform(): - raise unittest.SkipTest("This test doesn't make any sense on architecture which handle filenames natively as Unicode entities.") - - basedir = u"cli/common/listdir_unicode_bad" + basedir = "cli/common/listdir_unicode_bad" fileutil.make_dirs(basedir) - files = (u'Lôzane', u'Bern', u'Genève') + filenames = [name.encode('latin1') for name in [u'Lôzane', u'Bern', u'Genève']] + enc = get_filesystem_encoding() + def is_decodable(u): + try: + u.decode(enc) + return True + except UnicodeDecodeError: + return False - # We use a wrong encoding on purpose - if sys.getfilesystemencoding() == 'UTF-8': - encoding = 'latin1' - else: - encoding = 'UTF-8' + if all(map(is_decodable, filenames)): + raise unittest.SkipTest("To perform this test, we must know a filename that is " + "not decodable in the platform's filesystem encoding.") - for file in files: - path = os.path.join(basedir, file).encode(encoding) - open(path, "w").close() + try: + for name in filenames: + path = os.path.join(basedir, name) + open(path, "wb").close() + except EnvironmentError, e: + # Maybe the OS or Python wouldn't let us create a file at the badly encoded path, + # which is entirely reasonable. + raise unittest.SkipTest("This test is only applicable to platforms that allow " + "creating files at badly encoded paths.\n%r" % (e,)) + + self.failUnlessRaises(FilenameEncodingError, listdir_unicode, unicode(basedir)) - self.failUnlessRaises(FilenameEncodingError, listdir_unicode, basedir) class Help(unittest.TestCase): @@ -499,21 +530,6 @@ class Help(unittest.TestCase): help = str(cli.AddAliasOptions()) self.failUnless("add-alias ALIAS DIRCAP" in help, help) -class CLITestMixin: - def do_cli(self, verb, *args, **kwargs): - nodeargs = [ - "--node-directory", self.get_clientdir(), - ] - argv = [verb] + nodeargs + list(args) - stdin = kwargs.get("stdin", "") - stdout, stderr = StringIO(), StringIO() - d = threads.deferToThread(runner.runner, argv, run_by_human=False, - stdin=StringIO(stdin), - stdout=stdout, stderr=stderr) - def _done(rc): - return rc, stdout.getvalue(), stderr.getvalue() - d.addCallback(_done) - return d class CreateAlias(GridTestMixin, CLITestMixin, unittest.TestCase): @@ -523,9 +539,9 @@ class CreateAlias(GridTestMixin, CLITestMixin, unittest.TestCase): woo.parseOptions(all_args) urls = [] rc = cli.webopen(woo, urls.append) - self.failUnlessEqual(rc, 0) - self.failUnlessEqual(len(urls), 1) - self.failUnlessEqual(urls[0], expected_url) + self.failUnlessReallyEqual(rc, 0) + self.failUnlessReallyEqual(len(urls), 1) + self.failUnlessReallyEqual(urls[0], expected_url) def test_create(self): self.basedir = "cli/CreateAlias/create" @@ -559,12 +575,12 @@ class CreateAlias(GridTestMixin, CLITestMixin, unittest.TestCase): self.failIfEqual(rc, 0) self.failUnless("Alias 'two' already exists!" in stderr) aliases = get_aliases(self.get_clientdir()) - self.failUnlessEqual(aliases["two"], self.two_uri) + self.failUnlessReallyEqual(aliases["two"], self.two_uri) d.addCallback(_check_create_duplicate) d.addCallback(lambda res: self.do_cli("add-alias", "added", self.two_uri)) def _check_add((rc,stdout,stderr)): - self.failUnlessEqual(rc, 0) + self.failUnlessReallyEqual(rc, 0) self.failUnless("Alias 'added' added" in stdout) d.addCallback(_check_add) @@ -574,7 +590,7 @@ class CreateAlias(GridTestMixin, CLITestMixin, unittest.TestCase): self.failIfEqual(rc, 0) self.failUnless("Alias 'two' already exists!" in stderr) aliases = get_aliases(self.get_clientdir()) - self.failUnlessEqual(aliases["two"], self.two_uri) + self.failUnlessReallyEqual(aliases["two"], self.two_uri) d.addCallback(_check_add_duplicate) def _test_urls(junk): @@ -640,77 +656,70 @@ class CreateAlias(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnless(aliases["un-corrupted2"].startswith("URI:DIR2:")) d.addCallback(_check_not_corrupted) - def test_create_unicode(self): - skip_non_unicode_fs() - skip_non_unicode_stdout() - self.basedir = "cli/CreateAlias/create_unicode" self.set_up_grid() - d = self.do_cli("create-alias", "études") - def _check_create_unicode((rc,stdout,stderr)): - self.failUnlessEqual(rc, 0) - self.failIf(stderr) + try: + etudes_arg = u"études".encode(get_argv_encoding()) + lumiere_arg = u"lumière.txt".encode(get_argv_encoding()) + except UnicodeEncodeError, e: + raise unittest.SkipTest("A non-ASCII test argument could not be encoded as %s:\n%r" (get_argv_encoding(), e)) - # If stdout only supports ascii, accentuated characters are - # being replaced by '?' - if sys.stdout.encoding == "ANSI_X3.4-1968": - self.failUnless("Alias '?tudes' created" in stdout) - else: - self.failUnless("Alias 'études' created" in stdout) + d = self.do_cli("create-alias", etudes_arg) + def _check_create_unicode((rc, out, err)): + self.failUnlessReallyEqual(rc, 0) + self.failUnlessReallyEqual(err, "") + self.failUnlessIn("Alias %s created" % quote_output(u"études"), out) aliases = get_aliases(self.get_clientdir()) self.failUnless(aliases[u"études"].startswith("URI:DIR2:")) d.addCallback(_check_create_unicode) - d.addCallback(lambda res: self.do_cli("ls", "études:")) - def _check_ls1((rc, stdout, stderr)): - self.failUnlessEqual(rc, 0) - self.failIf(stderr) - - self.failUnlessEqual(stdout, "") + d.addCallback(lambda res: self.do_cli("ls", etudes_arg + ":")) + def _check_ls1((rc, out, err)): + self.failUnlessReallyEqual(rc, 0) + self.failUnlessReallyEqual(err, "") + self.failUnlessReallyEqual(out, "") d.addCallback(_check_ls1) - d.addCallback(lambda res: self.do_cli("put", "-", "études:uploaded.txt", - stdin="Blah blah blah")) + d.addCallback(lambda res: self.do_cli("put", "-", etudes_arg + ":uploaded.txt", + stdin="Blah blah blah")) - d.addCallback(lambda res: self.do_cli("ls", "études:")) - def _check_ls2((rc, stdout, stderr)): - self.failUnlessEqual(rc, 0) - self.failIf(stderr) - - self.failUnlessEqual(stdout, "uploaded.txt\n") + d.addCallback(lambda res: self.do_cli("ls", etudes_arg + ":")) + def _check_ls2((rc, out, err)): + self.failUnlessReallyEqual(rc, 0) + self.failUnlessReallyEqual(err, "") + self.failUnlessReallyEqual(out, "uploaded.txt\n") d.addCallback(_check_ls2) - d.addCallback(lambda res: self.do_cli("get", "études:uploaded.txt")) - def _check_get((rc, stdout, stderr)): - self.failUnlessEqual(rc, 0) - self.failIf(stderr) - self.failUnlessEqual(stdout, "Blah blah blah") + d.addCallback(lambda res: self.do_cli("get", etudes_arg + ":uploaded.txt")) + def _check_get((rc, out, err)): + self.failUnlessReallyEqual(rc, 0) + self.failUnlessReallyEqual(err, "") + self.failUnlessReallyEqual(out, "Blah blah blah") d.addCallback(_check_get) # Ensure that an Unicode filename in an Unicode alias works as expected - d.addCallback(lambda res: self.do_cli("put", "-", "études:lumière.txt", - stdin="Let the sunshine In!")) + d.addCallback(lambda res: self.do_cli("put", "-", etudes_arg + ":" + lumiere_arg, + stdin="Let the sunshine In!")) d.addCallback(lambda res: self.do_cli("get", - get_aliases(self.get_clientdir())[u"études"] + "/lumière.txt")) - def _check_get2((rc, stdout, stderr)): - self.failUnlessEqual(rc, 0) - self.failIf(stderr) - self.failUnlessEqual(stdout, "Let the sunshine In!") + get_aliases(self.get_clientdir())[u"études"] + "/" + lumiere_arg)) + def _check_get2((rc, out, err)): + self.failUnlessReallyEqual(rc, 0) + self.failUnlessReallyEqual(err, "") + self.failUnlessReallyEqual(out, "Let the sunshine In!") d.addCallback(_check_get2) return d + class Ln(GridTestMixin, CLITestMixin, unittest.TestCase): def _create_test_file(self): data = "puppies" * 1000 path = os.path.join(self.basedir, "datafile") - f = open(path, 'wb') - f.write(data) - f.close() + fileutil.write(path, data) self.datafile = path def test_ln_without_alias(self): @@ -721,8 +730,9 @@ class Ln(GridTestMixin, CLITestMixin, unittest.TestCase): self.set_up_grid() d = self.do_cli("ln", "from", "to") def _check((rc, out, err)): - self.failUnlessEqual(rc, 1) + self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) + self.failUnlessReallyEqual(out, "") d.addCallback(_check) # Make sure that validation extends to the "to" parameter d.addCallback(lambda ign: self.do_cli("create-alias", "havasu")) @@ -740,7 +750,7 @@ class Ln(GridTestMixin, CLITestMixin, unittest.TestCase): self.set_up_grid() d = self.do_cli("ln", "havasu:from", "havasu:to") def _check((rc, out, err)): - self.failUnlessEqual(rc, 1) + self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) d.addCallback(_check) # Make sure that validation occurs on the to parameter if the @@ -764,21 +774,21 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): self.set_up_grid() d = self.do_cli("put", stdin=DATA) def _uploaded(res): - (rc, stdout, stderr) = res - self.failUnless("waiting for file data on stdin.." in stderr) - self.failUnless("200 OK" in stderr, stderr) - self.readcap = stdout + (rc, out, err) = res + self.failUnlessIn("waiting for file data on stdin..", err) + self.failUnlessIn("200 OK", err) + self.readcap = out self.failUnless(self.readcap.startswith("URI:CHK:")) d.addCallback(_uploaded) d.addCallback(lambda res: self.do_cli("get", self.readcap)) def _downloaded(res): - (rc, stdout, stderr) = res - self.failUnlessEqual(stderr, "") - self.failUnlessEqual(stdout, DATA) + (rc, out, err) = res + self.failUnlessReallyEqual(err, "") + self.failUnlessReallyEqual(out, DATA) d.addCallback(_downloaded) d.addCallback(lambda res: self.do_cli("put", "-", stdin=DATA)) - d.addCallback(lambda (rc,stdout,stderr): - self.failUnlessEqual(stdout, self.readcap)) + d.addCallback(lambda (rc, out, err): + self.failUnlessReallyEqual(out, self.readcap)) return d def test_unlinked_immutable_from_file(self): @@ -794,17 +804,17 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): # we make the file small enough to fit in a LIT file, for speed fileutil.write(rel_fn, "short file") d = self.do_cli("put", rel_fn) - def _uploaded((rc,stdout,stderr)): - readcap = stdout - self.failUnless(readcap.startswith("URI:LIT:")) + def _uploaded((rc, out, err)): + readcap = out + self.failUnless(readcap.startswith("URI:LIT:"), readcap) self.readcap = readcap d.addCallback(_uploaded) d.addCallback(lambda res: self.do_cli("put", "./" + rel_fn)) d.addCallback(lambda (rc,stdout,stderr): - self.failUnlessEqual(stdout, self.readcap)) + self.failUnlessReallyEqual(stdout, self.readcap)) d.addCallback(lambda res: self.do_cli("put", abs_fn)) d.addCallback(lambda (rc,stdout,stderr): - self.failUnlessEqual(stdout, self.readcap)) + self.failUnlessReallyEqual(stdout, self.readcap)) # we just have to assume that ~ is handled properly return d @@ -829,43 +839,43 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(lambda res: self.do_cli("put", rel_fn, "uploaded.txt")) - def _uploaded((rc,stdout,stderr)): - readcap = stdout.strip() - self.failUnless(readcap.startswith("URI:LIT:")) - self.failUnless("201 Created" in stderr, stderr) + def _uploaded((rc, out, err)): + readcap = out.strip() + self.failUnless(readcap.startswith("URI:LIT:"), readcap) + self.failUnlessIn("201 Created", err) self.readcap = readcap d.addCallback(_uploaded) d.addCallback(lambda res: self.do_cli("get", "tahoe:uploaded.txt")) d.addCallback(lambda (rc,stdout,stderr): - self.failUnlessEqual(stdout, DATA)) + self.failUnlessReallyEqual(stdout, DATA)) d.addCallback(lambda res: self.do_cli("put", "-", "uploaded.txt", stdin=DATA2)) - def _replaced((rc,stdout,stderr)): - readcap = stdout.strip() - self.failUnless(readcap.startswith("URI:LIT:")) - self.failUnless("200 OK" in stderr, stderr) + def _replaced((rc, out, err)): + readcap = out.strip() + self.failUnless(readcap.startswith("URI:LIT:"), readcap) + self.failUnlessIn("200 OK", err) d.addCallback(_replaced) d.addCallback(lambda res: self.do_cli("put", rel_fn, "subdir/uploaded2.txt")) d.addCallback(lambda res: self.do_cli("get", "subdir/uploaded2.txt")) d.addCallback(lambda (rc,stdout,stderr): - self.failUnlessEqual(stdout, DATA)) + self.failUnlessReallyEqual(stdout, DATA)) d.addCallback(lambda res: self.do_cli("put", rel_fn, "tahoe:uploaded3.txt")) d.addCallback(lambda res: self.do_cli("get", "tahoe:uploaded3.txt")) d.addCallback(lambda (rc,stdout,stderr): - self.failUnlessEqual(stdout, DATA)) + self.failUnlessReallyEqual(stdout, DATA)) d.addCallback(lambda res: self.do_cli("put", rel_fn, "tahoe:subdir/uploaded4.txt")) d.addCallback(lambda res: self.do_cli("get", "tahoe:subdir/uploaded4.txt")) d.addCallback(lambda (rc,stdout,stderr): - self.failUnlessEqual(stdout, DATA)) + self.failUnlessReallyEqual(stdout, DATA)) def _get_dircap(res): self.dircap = get_aliases(self.get_clientdir())["tahoe"] @@ -877,7 +887,7 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(lambda res: self.do_cli("get", "tahoe:uploaded5.txt")) d.addCallback(lambda (rc,stdout,stderr): - self.failUnlessEqual(stdout, DATA)) + self.failUnlessReallyEqual(stdout, DATA)) d.addCallback(lambda res: self.do_cli("put", rel_fn, @@ -885,7 +895,7 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(lambda res: self.do_cli("get", "tahoe:subdir/uploaded6.txt")) d.addCallback(lambda (rc,stdout,stderr): - self.failUnlessEqual(stdout, DATA)) + self.failUnlessReallyEqual(stdout, DATA)) return d @@ -906,33 +916,33 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): d = self.do_cli("put", "--mutable", stdin=DATA) def _created(res): - (rc, stdout, stderr) = res - self.failUnless("waiting for file data on stdin.." in stderr) - self.failUnless("200 OK" in stderr) - self.filecap = stdout - self.failUnless(self.filecap.startswith("URI:SSK:")) + (rc, out, err) = res + self.failUnlessIn("waiting for file data on stdin..", err) + self.failUnlessIn("200 OK", err) + self.filecap = out + self.failUnless(self.filecap.startswith("URI:SSK:"), self.filecap) d.addCallback(_created) d.addCallback(lambda res: self.do_cli("get", self.filecap)) - d.addCallback(lambda (rc,out,err): self.failUnlessEqual(out, DATA)) + d.addCallback(lambda (rc,out,err): self.failUnlessReallyEqual(out, DATA)) d.addCallback(lambda res: self.do_cli("put", "-", self.filecap, stdin=DATA2)) def _replaced(res): - (rc, stdout, stderr) = res - self.failUnless("waiting for file data on stdin.." in stderr) - self.failUnless("200 OK" in stderr) - self.failUnlessEqual(self.filecap, stdout) + (rc, out, err) = res + self.failUnlessIn("waiting for file data on stdin..", err) + self.failUnlessIn("200 OK", err) + self.failUnlessReallyEqual(self.filecap, out) d.addCallback(_replaced) d.addCallback(lambda res: self.do_cli("get", self.filecap)) - d.addCallback(lambda (rc,out,err): self.failUnlessEqual(out, DATA2)) + d.addCallback(lambda (rc,out,err): self.failUnlessReallyEqual(out, DATA2)) d.addCallback(lambda res: self.do_cli("put", rel_fn, self.filecap)) def _replaced2(res): - (rc, stdout, stderr) = res - self.failUnless("200 OK" in stderr) - self.failUnlessEqual(self.filecap, stdout) + (rc, out, err) = res + self.failUnlessIn("200 OK", err) + self.failUnlessReallyEqual(self.filecap, out) d.addCallback(_replaced2) d.addCallback(lambda res: self.do_cli("get", self.filecap)) - d.addCallback(lambda (rc,out,err): self.failUnlessEqual(out, DATA3)) + d.addCallback(lambda (rc,out,err): self.failUnlessReallyEqual(out, DATA3)) return d @@ -958,7 +968,7 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): self.do_cli("put", fn2, "tahoe:uploaded.txt")) d.addCallback(lambda res: self.do_cli("get", "tahoe:uploaded.txt")) - d.addCallback(lambda (rc,out,err): self.failUnlessEqual(out, DATA2)) + d.addCallback(lambda (rc,out,err): self.failUnlessReallyEqual(out, DATA2)) return d def test_put_with_nonexistent_alias(self): @@ -968,41 +978,49 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): self.set_up_grid() d = self.do_cli("put", "somefile", "fake:afile") def _check((rc, out, err)): - self.failUnlessEqual(rc, 1) + self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) + self.failUnlessReallyEqual(out, "") d.addCallback(_check) return d - def test_immutable_from_file_unicode(self): - skip_non_unicode_stdout() - - # tahoe put file.txt "à trier.txt" - self.basedir = os.path.dirname(self.mktemp()) + # tahoe put "à trier.txt" "à trier.txt" + + try: + a_trier_arg = u"à trier.txt".encode(get_argv_encoding()) + except UnicodeEncodeError, e: + raise unittest.SkipTest("A non-ASCII command argument could not be encoded as %s:\n%r" (get_argv_encoding(), e)) + + self.skip_if_cannot_represent_filename(u"à trier.txt") + + self.basedir = "cli/Put/immutable_from_file_unicode" self.set_up_grid() - rel_fn = os.path.join(self.basedir, "DATAFILE") + rel_fn = os.path.join(unicode(self.basedir), u"à trier.txt") # we make the file small enough to fit in a LIT file, for speed DATA = "short file" - f = open(rel_fn, "w") - f.write(DATA) - f.close() + f = open_unicode(rel_fn, "wb") + try: + f.write(DATA) + finally: + f.close() d = self.do_cli("create-alias", "tahoe") d.addCallback(lambda res: - self.do_cli("put", rel_fn, "à trier.txt")) - def _uploaded((rc,stdout,stderr)): - readcap = stdout.strip() - self.failUnless(readcap.startswith("URI:LIT:")) - self.failUnless("201 Created" in stderr, stderr) + self.do_cli("put", rel_fn.encode(get_argv_encoding()), a_trier_arg)) + def _uploaded((rc, out, err)): + readcap = out.strip() + self.failUnless(readcap.startswith("URI:LIT:"), readcap) + self.failUnlessIn("201 Created", err) self.readcap = readcap d.addCallback(_uploaded) d.addCallback(lambda res: - self.do_cli("get", "tahoe:à trier.txt")) - d.addCallback(lambda (rc,stdout,stderr): - self.failUnlessEqual(stdout, DATA)) + self.do_cli("get", "tahoe:" + a_trier_arg)) + d.addCallback(lambda (rc, out, err): + self.failUnlessReallyEqual(out, DATA)) return d @@ -1012,6 +1030,19 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase): self.set_up_grid() c0 = self.g.clients[0] small = "small" + + # u"gööd" might not be representable in the argv and/or output encodings. + # It is initially included in the directory in any case. + try: + good_arg = u"gööd".encode(get_argv_encoding()) + except UnicodeEncodeError: + good_arg = None + + try: + good_out = u"gööd".encode(get_output_encoding()) + except UnicodeEncodeError: + good_out = None + d = c0.create_dirnode() def _stash_root_and_create_file(n): self.rootnode = n @@ -1031,51 +1062,92 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase): self.do_cli("add-alias", "tahoe", self.rooturi)) d.addCallback(lambda ign: self.do_cli("ls")) def _check1((rc,out,err)): - self.failUnlessEqual(err, "") - self.failUnlessEqual(rc, 0) - outstrs = out.splitlines() - outunics = [ outstr.decode(stringutils.get_term_encoding()) for outstr in outstrs ] - self.failUnlessEqual(outunics, [u"0share", u"1share", u"gööd"]) + if good_out is None: + self.failUnlessReallyEqual(rc, 1) + self.failUnlessIn("could not be encoded", err) + self.failUnlessReallyEqual(out, "") + else: + self.failUnlessReallyEqual(rc, 0) + self.failUnlessReallyEqual(err, "") + outstrs = out.splitlines() + self.failUnlessReallyEqual(outstrs, ["0share", "1share", good_out]) d.addCallback(_check1) d.addCallback(lambda ign: self.do_cli("ls", "missing")) def _check2((rc,out,err)): self.failIfEqual(rc, 0) - self.failUnlessEqual(err.strip(), "No such file or directory") - self.failUnlessEqual(out, "") + self.failUnlessReallyEqual(err.strip(), "No such file or directory") + self.failUnlessReallyEqual(out, "") d.addCallback(_check2) d.addCallback(lambda ign: self.do_cli("ls", "1share")) def _check3((rc,out,err)): self.failIfEqual(rc, 0) - self.failUnlessIn("Error during GET: 410 Gone ", err) + self.failUnlessIn("Error during GET: 410 Gone", err) self.failUnlessIn("UnrecoverableFileError:", err) self.failUnlessIn("could not be retrieved, because there were " "insufficient good shares.", err) - self.failUnlessEqual(out, "") + self.failUnlessReallyEqual(out, "") d.addCallback(_check3) d.addCallback(lambda ign: self.do_cli("ls", "0share")) d.addCallback(_check3) def _check4((rc, out, err)): + if good_out is None: + self.failUnlessReallyEqual(rc, 1) + self.failUnlessIn("could not be encoded", err) + self.failUnlessReallyEqual(out, "") + else: + # listing a file (as dir/filename) should have the edge metadata, + # including the filename + self.failUnlessReallyEqual(rc, 0) + self.failUnlessIn(good_out, out) + self.failIfIn("-r-- %d -" % len(small), out, + "trailing hyphen means unknown date") + + if good_arg is not None: + d.addCallback(lambda ign: self.do_cli("ls", "-l", good_arg)) + d.addCallback(_check4) + # listing a file as $DIRCAP/filename should work just like dir/filename + d.addCallback(lambda ign: self.do_cli("ls", "-l", self.rooturi + "/" + good_arg)) + d.addCallback(_check4) + # and similarly for $DIRCAP:./filename + d.addCallback(lambda ign: self.do_cli("ls", "-l", self.rooturi + ":./" + good_arg)) + d.addCallback(_check4) + + def _check5((rc, out, err)): + # listing a raw filecap should not explode, but it will have no + # metadata, just the size + self.failUnlessReallyEqual(rc, 0) + self.failUnlessReallyEqual("-r-- %d -" % len(small), out.strip()) + d.addCallback(lambda ign: self.do_cli("ls", "-l", self.goodcap)) + d.addCallback(_check5) + + # Now rename 'gööd' to 'good' and repeat the tests that might have been skipped due + # to encoding problems. + d.addCallback(lambda ign: self.rootnode.move_child_to(u"gööd", self.rootnode, u"good")) + + d.addCallback(lambda ign: self.do_cli("ls")) + def _check1_ascii((rc,out,err)): + self.failUnlessReallyEqual(rc, 0) + self.failUnlessReallyEqual(err, "") + outstrs = out.splitlines() + self.failUnlessReallyEqual(outstrs, ["0share", "1share", "good"]) + d.addCallback(_check1_ascii) + def _check4_ascii((rc, out, err)): # listing a file (as dir/filename) should have the edge metadata, # including the filename - self.failUnlessEqual(rc, 0) - self.failUnlessIn(u"gööd", out.decode(stringutils.get_term_encoding())) + self.failUnlessReallyEqual(rc, 0) + self.failUnlessIn("good", out) self.failIfIn("-r-- %d -" % len(small), out, "trailing hyphen means unknown date") - d.addCallback(lambda ign: self.do_cli("ls", "-l", u"gööd".encode(locale.getpreferredencoding()))) - d.addCallback(_check4) + + d.addCallback(lambda ign: self.do_cli("ls", "-l", "good")) + d.addCallback(_check4_ascii) # listing a file as $DIRCAP/filename should work just like dir/filename - d.addCallback(lambda ign: self.do_cli("ls", "-l", self.rooturi + u"/gööd".encode(locale.getpreferredencoding()))) - d.addCallback(_check4) + d.addCallback(lambda ign: self.do_cli("ls", "-l", self.rooturi + "/good")) + d.addCallback(_check4_ascii) # and similarly for $DIRCAP:./filename - d.addCallback(lambda ign: self.do_cli("ls", "-l", self.rooturi + u":./gööd".encode(locale.getpreferredencoding()))) - d.addCallback(_check4) - def _check5((rc, out, err)): - # listing a raw filecap should not explode, but it will have no - # metadata, just the size - self.failUnlessEqual(rc, 0) - self.failUnlessEqual("-r-- %d -" % len(small), out.strip()) - d.addCallback(lambda ign: self.do_cli("ls", "-l", self.goodcap)) - d.addCallback(_check5) + d.addCallback(lambda ign: self.do_cli("ls", "-l", self.rooturi + ":./good")) + d.addCallback(_check4_ascii) + unknown_immcap = "imm.URI:unknown" def _create_unknown(ign): nm = c0.nodemaker @@ -1086,7 +1158,7 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase): def _check6((rc, out, err)): # listing a directory referencing an unknown object should print # an extra message to stderr - self.failUnlessEqual(rc, 0) + self.failUnlessReallyEqual(rc, 0) self.failUnlessIn("?r-- ? - unknownchild-imm\n", out) self.failUnlessIn("included unknown objects", err) d.addCallback(lambda ign: self.do_cli("ls", "-l", "unknown")) @@ -1096,7 +1168,7 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase): # to stderr (currently this only works if the URI starts with 'URI:' # after any 'ro.' or 'imm.' prefix, otherwise it will be confused # with an alias). - self.failUnlessEqual(rc, 0) + self.failUnlessReallyEqual(rc, 0) self.failUnlessIn("?r-- ? -\n", out) self.failUnlessIn("included unknown objects", err) d.addCallback(lambda ign: self.do_cli("ls", "-l", unknown_immcap)) @@ -1110,8 +1182,9 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase): self.set_up_grid() d = self.do_cli("ls") def _check((rc, out, err)): - self.failUnlessEqual(rc, 1) + self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) + self.failUnlessReallyEqual(out, "") d.addCallback(_check) return d @@ -1122,9 +1195,10 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase): self.set_up_grid() d = self.do_cli("ls", "nonexistent:") def _check((rc, out, err)): - self.failUnlessEqual(rc, 1) + self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) self.failUnlessIn("nonexistent", err) + self.failUnlessReallyEqual(out, "") d.addCallback(_check) return d @@ -1232,8 +1306,9 @@ class Mv(GridTestMixin, CLITestMixin, unittest.TestCase): self.set_up_grid() d = self.do_cli("mv", "afile", "anotherfile") def _check((rc, out, err)): - self.failUnlessEqual(rc, 1) + self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) + self.failUnlessReallyEqual(out, "") d.addCallback(_check) # check to see that the validation extends to the # target argument by making an alias that will work with the first @@ -1241,9 +1316,7 @@ class Mv(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(lambda ign: self.do_cli("create-alias", "havasu")) def _create_a_test_file(ign): self.test_file_path = os.path.join(self.basedir, "afile") - f = open(self.test_file_path, "wb") - f.write("puppies" * 100) - f.close() + fileutil.write(self.test_file_path, "puppies" * 100) d.addCallback(_create_a_test_file) d.addCallback(lambda ign: self.do_cli("put", self.test_file_path, "havasu:afile")) @@ -1259,9 +1332,10 @@ class Mv(GridTestMixin, CLITestMixin, unittest.TestCase): self.set_up_grid() d = self.do_cli("mv", "fake:afile", "fake:anotherfile") def _check((rc, out, err)): - self.failUnlessEqual(rc, 1) + self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) self.failUnlessIn("fake", err) + self.failUnlessReallyEqual(out, "") d.addCallback(_check) # check to see that the validation extends to the # target argument by making an alias that will work with the first @@ -1269,9 +1343,7 @@ class Mv(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(lambda ign: self.do_cli("create-alias", "havasu")) def _create_a_test_file(ign): self.test_file_path = os.path.join(self.basedir, "afile") - f = open(self.test_file_path, "wb") - f.write("puppies" * 100) - f.close() + fileutil.write(self.test_file_path, "puppies" * 100) d.addCallback(_create_a_test_file) d.addCallback(lambda ign: self.do_cli("put", self.test_file_path, "havasu:afile")) @@ -1289,40 +1361,63 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase): o.parseOptions, ["onearg"]) def test_unicode_filename(self): - skip_non_unicode_fs() - skip_non_unicode_stdout() - self.basedir = "cli/Cp/unicode_filename" - self.set_up_grid() - d = self.do_cli("create-alias", "tahoe") - # Use unicode strings when calling os functions - fn1 = os.path.join(self.basedir, u"Ärtonwall") - DATA1 = "unicode file content" - fileutil.write(fn1, DATA1) + fn1 = os.path.join(unicode(self.basedir), u"Ärtonwall") + try: + fn1_arg = fn1.encode(get_argv_encoding()) + artonwall_arg = u"Ärtonwall".encode(get_argv_encoding()) + except UnicodeEncodeError, e: + raise unittest.SkipTest("A non-ASCII command argument could not be encoded as %s:\n%r" (get_argv_encoding(), e)) - d.addCallback(lambda res: self.do_cli("cp", fn1.encode('utf-8'), "tahoe:")) + self.skip_if_cannot_represent_filename(fn1) - d.addCallback(lambda res: self.do_cli("get", "tahoe:Ärtonwall")) - d.addCallback(lambda (rc,out,err): self.failUnlessEqual(out, DATA1)) + self.set_up_grid() - fn2 = os.path.join(self.basedir, u"Metallica") + DATA1 = "unicode file content" + f = open_unicode(fn1, "wb") + try: + f.write(DATA1) + finally: + f.close() + + fn2 = os.path.join(self.basedir, "Metallica") DATA2 = "non-unicode file content" fileutil.write(fn2, DATA2) - d.addCallback(lambda res: self.do_cli("cp", fn2.encode('utf-8'), "tahoe:")) + d = self.do_cli("create-alias", "tahoe") + + d.addCallback(lambda res: self.do_cli("cp", fn1_arg, "tahoe:")) + + d.addCallback(lambda res: self.do_cli("get", "tahoe:" + artonwall_arg)) + d.addCallback(lambda (rc,out,err): self.failUnlessReallyEqual(out, DATA1)) + + d.addCallback(lambda res: self.do_cli("cp", fn2, "tahoe:")) d.addCallback(lambda res: self.do_cli("get", "tahoe:Metallica")) - d.addCallback(lambda (rc,out,err): self.failUnlessEqual(out, DATA2)) + d.addCallback(lambda (rc,out,err): self.failUnlessReallyEqual(out, DATA2)) d.addCallback(lambda res: self.do_cli("ls", "tahoe:")) - d.addCallback(lambda (rc,out,err): self.failUnlessEqual(out, "Metallica\nÄrtonwall\n")) + def _check((rc, out, err)): + try: + unicode_to_output(u"Ärtonwall") + except UnicodeEncodeError: + self.failUnlessReallyEqual(rc, 1) + self.failUnlessReallyEqual(out, "Metallica\n") + self.failUnlessIn(quote_output(u"Ärtonwall"), err) + self.failUnlessIn("files whose names could not be converted", err) + else: + self.failUnlessReallyEqual(rc, 0) + self.failUnlessReallyEqual(out.decode(get_output_encoding()), u"Metallica\nÄrtonwall\n") + self.failUnlessReallyEqual(err, "") + d.addCallback(_check) return d def test_dangling_symlink_vs_recursion(self): if not hasattr(os, 'symlink'): - raise unittest.SkipTest("There is no symlink on this platform.") + raise unittest.SkipTest("Symlinks are not supported by Python on this platform.") + # cp -r on a directory containing a dangling symlink shouldn't assert self.basedir = "cli/Cp/dangling_symlink_vs_recursion" self.set_up_grid() @@ -1351,7 +1446,8 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase): d = self.do_cli("create-alias", "tahoe") d.addCallback(lambda ign: self.do_cli("put", fn1)) def _put_file((rc, out, err)): - self.failUnlessEqual(rc, 0) + self.failUnlessReallyEqual(rc, 0) + self.failUnlessIn("200 OK", err) # keep track of the filecap self.filecap = out.strip() d.addCallback(_put_file) @@ -1360,48 +1456,49 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase): # cp FILECAP filename d.addCallback(lambda ign: self.do_cli("cp", self.filecap, fn2)) def _copy_file((rc, out, err)): - self.failUnlessEqual(rc, 0) + self.failUnlessReallyEqual(rc, 0) results = fileutil.read(fn2) - self.failUnlessEqual(results, DATA1) + self.failUnlessReallyEqual(results, DATA1) d.addCallback(_copy_file) # Test with ./ (see #761) # cp FILECAP localdir d.addCallback(lambda ign: self.do_cli("cp", self.filecap, outdir)) def _resp((rc, out, err)): - self.failUnlessEqual(rc, 1) + self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error: you must specify a destination filename", err) + self.failUnlessReallyEqual(out, "") d.addCallback(_resp) # Create a directory, linked at tahoe:test d.addCallback(lambda ign: self.do_cli("mkdir", "tahoe:test")) def _get_dir((rc, out, err)): - self.failUnlessEqual(rc, 0) + self.failUnlessReallyEqual(rc, 0) self.dircap = out.strip() d.addCallback(_get_dir) # Upload a file to the directory d.addCallback(lambda ign: self.do_cli("put", fn1, "tahoe:test/test_file")) - d.addCallback(lambda (rc, out, err): self.failUnlessEqual(rc, 0)) + d.addCallback(lambda (rc, out, err): self.failUnlessReallyEqual(rc, 0)) # cp DIRCAP/filename localdir d.addCallback(lambda ign: self.do_cli("cp", self.dircap + "/test_file", outdir)) def _get_resp((rc, out, err)): - self.failUnlessEqual(rc, 0) + self.failUnlessReallyEqual(rc, 0) results = fileutil.read(os.path.join(outdir, "test_file")) - self.failUnlessEqual(results, DATA1) + self.failUnlessReallyEqual(results, DATA1) d.addCallback(_get_resp) # cp -r DIRCAP/filename filename2 d.addCallback(lambda ign: self.do_cli("cp", self.dircap + "/test_file", fn3)) def _get_resp2((rc, out, err)): - self.failUnlessEqual(rc, 0) + self.failUnlessReallyEqual(rc, 0) results = fileutil.read(fn3) - self.failUnlessEqual(results, DATA1) + self.failUnlessReallyEqual(results, DATA1) d.addCallback(_get_resp2) return d @@ -1412,7 +1509,7 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase): self.set_up_grid() d = self.do_cli("cp", "fake:file1", "fake:file2") def _check((rc, out, err)): - self.failUnlessEqual(rc, 1) + self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) d.addCallback(_check) # 'tahoe cp' actually processes the target argument first, so we need @@ -1425,17 +1522,6 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase): return d -class Mkdir(GridTestMixin, CLITestMixin, unittest.TestCase): - def test_unicode_mkdir(self): - self.basedir = os.path.dirname(self.mktemp()) - self.set_up_grid() - - d = self.do_cli("create-alias", "tahoe") - d.addCallback(lambda res: self.do_cli("mkdir", "tahoe:Motörhead")) - - return d - - class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): def writeto(self, path, data): @@ -1490,54 +1576,54 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): d.addCallback(lambda res: do_backup()) def _check0((rc, out, err)): - self.failUnlessEqual(err, "") - self.failUnlessEqual(rc, 0) + self.failUnlessReallyEqual(err, "") + self.failUnlessReallyEqual(rc, 0) fu, fr, fs, dc, dr, ds = self.count_output(out) # foo.txt, bar.txt, blah.txt - self.failUnlessEqual(fu, 3) - self.failUnlessEqual(fr, 0) - self.failUnlessEqual(fs, 0) + self.failUnlessReallyEqual(fu, 3) + self.failUnlessReallyEqual(fr, 0) + self.failUnlessReallyEqual(fs, 0) # empty, home, home/parent, home/parent/subdir - self.failUnlessEqual(dc, 4) - self.failUnlessEqual(dr, 0) - self.failUnlessEqual(ds, 0) + self.failUnlessReallyEqual(dc, 4) + self.failUnlessReallyEqual(dr, 0) + self.failUnlessReallyEqual(ds, 0) d.addCallback(_check0) d.addCallback(lambda res: self.do_cli("ls", "--uri", "tahoe:backups")) def _check1((rc, out, err)): - self.failUnlessEqual(err, "") - self.failUnlessEqual(rc, 0) + self.failUnlessReallyEqual(err, "") + self.failUnlessReallyEqual(rc, 0) lines = out.split("\n") children = dict([line.split() for line in lines if line]) latest_uri = children["Latest"] self.failUnless(latest_uri.startswith("URI:DIR2-CHK:"), latest_uri) childnames = children.keys() - self.failUnlessEqual(sorted(childnames), ["Archives", "Latest"]) + self.failUnlessReallyEqual(sorted(childnames), ["Archives", "Latest"]) d.addCallback(_check1) d.addCallback(lambda res: self.do_cli("ls", "tahoe:backups/Latest")) def _check2((rc, out, err)): - self.failUnlessEqual(err, "") - self.failUnlessEqual(rc, 0) - self.failUnlessEqual(sorted(out.split()), ["empty", "parent"]) + self.failUnlessReallyEqual(err, "") + self.failUnlessReallyEqual(rc, 0) + self.failUnlessReallyEqual(sorted(out.split()), ["empty", "parent"]) d.addCallback(_check2) d.addCallback(lambda res: self.do_cli("ls", "tahoe:backups/Latest/empty")) def _check2a((rc, out, err)): - self.failUnlessEqual(err, "") - self.failUnlessEqual(rc, 0) - self.failUnlessEqual(out.strip(), "") + self.failUnlessReallyEqual(err, "") + self.failUnlessReallyEqual(rc, 0) + self.failUnlessReallyEqual(out.strip(), "") d.addCallback(_check2a) d.addCallback(lambda res: self.do_cli("get", "tahoe:backups/Latest/parent/subdir/foo.txt")) def _check3((rc, out, err)): - self.failUnlessEqual(err, "") - self.failUnlessEqual(rc, 0) - self.failUnlessEqual(out, "foo") + self.failUnlessReallyEqual(err, "") + self.failUnlessReallyEqual(rc, 0) + self.failUnlessReallyEqual(out, "foo") d.addCallback(_check3) d.addCallback(lambda res: self.do_cli("ls", "tahoe:backups/Archives")) def _check4((rc, out, err)): - self.failUnlessEqual(err, "") - self.failUnlessEqual(rc, 0) + self.failUnlessReallyEqual(err, "") + self.failUnlessReallyEqual(rc, 0) self.old_archives = out.split() - self.failUnlessEqual(len(self.old_archives), 1) + self.failUnlessReallyEqual(len(self.old_archives), 1) d.addCallback(_check4) @@ -1546,18 +1632,18 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): def _check4a((rc, out, err)): # second backup should reuse everything, if the backupdb is # available - self.failUnlessEqual(err, "") - self.failUnlessEqual(rc, 0) + self.failUnlessReallyEqual(err, "") + self.failUnlessReallyEqual(rc, 0) if have_bdb: fu, fr, fs, dc, dr, ds = self.count_output(out) # foo.txt, bar.txt, blah.txt - self.failUnlessEqual(fu, 0) - self.failUnlessEqual(fr, 3) - self.failUnlessEqual(fs, 0) + self.failUnlessReallyEqual(fu, 0) + self.failUnlessReallyEqual(fr, 3) + self.failUnlessReallyEqual(fs, 0) # empty, home, home/parent, home/parent/subdir - self.failUnlessEqual(dc, 0) - self.failUnlessEqual(dr, 4) - self.failUnlessEqual(ds, 0) + self.failUnlessReallyEqual(dc, 0) + self.failUnlessReallyEqual(dr, 4) + self.failUnlessReallyEqual(ds, 0) d.addCallback(_check4a) if have_bdb: @@ -1580,32 +1666,32 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): # we should check all files, and re-use all of them. None of # the directories should have been changed, so we should # re-use all of them too. - self.failUnlessEqual(err, "") - self.failUnlessEqual(rc, 0) + self.failUnlessReallyEqual(err, "") + self.failUnlessReallyEqual(rc, 0) fu, fr, fs, dc, dr, ds = self.count_output(out) fchecked, dchecked = self.count_output2(out) - self.failUnlessEqual(fchecked, 3) - self.failUnlessEqual(fu, 0) - self.failUnlessEqual(fr, 3) - self.failUnlessEqual(fs, 0) - self.failUnlessEqual(dchecked, 4) - self.failUnlessEqual(dc, 0) - self.failUnlessEqual(dr, 4) - self.failUnlessEqual(ds, 0) + self.failUnlessReallyEqual(fchecked, 3) + self.failUnlessReallyEqual(fu, 0) + self.failUnlessReallyEqual(fr, 3) + self.failUnlessReallyEqual(fs, 0) + self.failUnlessReallyEqual(dchecked, 4) + self.failUnlessReallyEqual(dc, 0) + self.failUnlessReallyEqual(dr, 4) + self.failUnlessReallyEqual(ds, 0) d.addCallback(_check4b) d.addCallback(lambda res: self.do_cli("ls", "tahoe:backups/Archives")) def _check5((rc, out, err)): - self.failUnlessEqual(err, "") - self.failUnlessEqual(rc, 0) + self.failUnlessReallyEqual(err, "") + self.failUnlessReallyEqual(rc, 0) self.new_archives = out.split() expected_new = 2 if have_bdb: expected_new += 1 - self.failUnlessEqual(len(self.new_archives), expected_new, out) + self.failUnlessReallyEqual(len(self.new_archives), expected_new, out) # the original backup should still be the oldest (i.e. sorts # alphabetically towards the beginning) - self.failUnlessEqual(sorted(self.new_archives)[0], + self.failUnlessReallyEqual(sorted(self.new_archives)[0], self.old_archives[0]) d.addCallback(_check5) @@ -1625,44 +1711,44 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): def _check5a((rc, out, err)): # second backup should reuse bar.txt (if backupdb is available), # and upload the rest. None of the directories can be reused. - self.failUnlessEqual(err, "") - self.failUnlessEqual(rc, 0) + self.failUnlessReallyEqual(err, "") + self.failUnlessReallyEqual(rc, 0) if have_bdb: fu, fr, fs, dc, dr, ds = self.count_output(out) # new foo.txt, surprise file, subfile, empty - self.failUnlessEqual(fu, 4) + self.failUnlessReallyEqual(fu, 4) # old bar.txt - self.failUnlessEqual(fr, 1) - self.failUnlessEqual(fs, 0) + self.failUnlessReallyEqual(fr, 1) + self.failUnlessReallyEqual(fs, 0) # home, parent, subdir, blah.txt, surprisedir - self.failUnlessEqual(dc, 5) - self.failUnlessEqual(dr, 0) - self.failUnlessEqual(ds, 0) + self.failUnlessReallyEqual(dc, 5) + self.failUnlessReallyEqual(dr, 0) + self.failUnlessReallyEqual(ds, 0) d.addCallback(_check5a) d.addCallback(lambda res: self.do_cli("ls", "tahoe:backups/Archives")) def _check6((rc, out, err)): - self.failUnlessEqual(err, "") - self.failUnlessEqual(rc, 0) + self.failUnlessReallyEqual(err, "") + self.failUnlessReallyEqual(rc, 0) self.new_archives = out.split() expected_new = 3 if have_bdb: expected_new += 1 - self.failUnlessEqual(len(self.new_archives), expected_new) - self.failUnlessEqual(sorted(self.new_archives)[0], + self.failUnlessReallyEqual(len(self.new_archives), expected_new) + self.failUnlessReallyEqual(sorted(self.new_archives)[0], self.old_archives[0]) d.addCallback(_check6) d.addCallback(lambda res: self.do_cli("get", "tahoe:backups/Latest/parent/subdir/foo.txt")) def _check7((rc, out, err)): - self.failUnlessEqual(err, "") - self.failUnlessEqual(rc, 0) - self.failUnlessEqual(out, "FOOF!") + self.failUnlessReallyEqual(err, "") + self.failUnlessReallyEqual(rc, 0) + self.failUnlessReallyEqual(out, "FOOF!") # the old snapshot should not be modified return self.do_cli("get", "tahoe:backups/Archives/%s/parent/subdir/foo.txt" % self.old_archives[0]) d.addCallback(_check7) def _check8((rc, out, err)): - self.failUnlessEqual(err, "") - self.failUnlessEqual(rc, 0) - self.failUnlessEqual(out, "foo") + self.failUnlessReallyEqual(err, "") + self.failUnlessReallyEqual(rc, 0) + self.failUnlessReallyEqual(out, "foo") d.addCallback(_check8) return d @@ -1687,8 +1773,8 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): all = set(all) included = set(included) excluded = set(excluded) - self.failUnlessEqual(filtered, included) - self.failUnlessEqual(all.difference(filtered), excluded) + self.failUnlessReallyEqual(filtered, included) + self.failUnlessReallyEqual(all.difference(filtered), excluded) # test simple exclude backup_options = cli.BackupOptions() @@ -1737,7 +1823,7 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): def test_ignore_symlinks(self): if not hasattr(os, 'symlink'): - raise unittest.SkipTest("There is no symlink on this platform.") + raise unittest.SkipTest("Symlinks are not supported by Python on this platform.") self.basedir = os.path.dirname(self.mktemp()) self.set_up_grid() @@ -1750,20 +1836,20 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): d.addCallback(lambda res: self.do_cli("backup", "--verbose", source, "tahoe:test")) def _check((rc, out, err)): - self.failUnlessEqual(rc, 2) + self.failUnlessReallyEqual(rc, 2) foo2 = os.path.join(source, "foo2.txt") - self.failUnlessEqual(err, "WARNING: cannot backup symlink %s\n" % foo2) + self.failUnlessReallyEqual(err, "WARNING: cannot backup symlink %s\n" % foo2) fu, fr, fs, dc, dr, ds = self.count_output(out) # foo.txt - self.failUnlessEqual(fu, 1) - self.failUnlessEqual(fr, 0) + self.failUnlessReallyEqual(fu, 1) + self.failUnlessReallyEqual(fr, 0) # foo2.txt - self.failUnlessEqual(fs, 1) + self.failUnlessReallyEqual(fs, 1) # home - self.failUnlessEqual(dc, 1) - self.failUnlessEqual(dr, 0) - self.failUnlessEqual(ds, 0) + self.failUnlessReallyEqual(dc, 1) + self.failUnlessReallyEqual(dr, 0) + self.failUnlessReallyEqual(ds, 0) d.addCallback(_check) return d @@ -1780,18 +1866,18 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): d.addCallback(lambda res: self.do_cli("backup", source, "tahoe:test")) def _check((rc, out, err)): - self.failUnlessEqual(rc, 2) - self.failUnlessEqual(err, "WARNING: permission denied on file %s\n" % os.path.join(source, "foo.txt")) + self.failUnlessReallyEqual(rc, 2) + self.failUnlessReallyEqual(err, "WARNING: permission denied on file %s\n" % os.path.join(source, "foo.txt")) fu, fr, fs, dc, dr, ds = self.count_output(out) - self.failUnlessEqual(fu, 0) - self.failUnlessEqual(fr, 0) + self.failUnlessReallyEqual(fu, 0) + self.failUnlessReallyEqual(fr, 0) # foo.txt - self.failUnlessEqual(fs, 1) + self.failUnlessReallyEqual(fs, 1) # home - self.failUnlessEqual(dc, 1) - self.failUnlessEqual(dr, 0) - self.failUnlessEqual(ds, 0) + self.failUnlessReallyEqual(dc, 1) + self.failUnlessReallyEqual(dr, 0) + self.failUnlessReallyEqual(ds, 0) d.addCallback(_check) # This is necessary for the temp files to be correctly removed @@ -1815,18 +1901,18 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): d.addCallback(lambda res: self.do_cli("backup", source, "tahoe:test")) def _check((rc, out, err)): - self.failUnlessEqual(rc, 2) - self.failUnlessEqual(err, "WARNING: permission denied on directory %s\n" % os.path.join(source, "test")) + self.failUnlessReallyEqual(rc, 2) + self.failUnlessReallyEqual(err, "WARNING: permission denied on directory %s\n" % os.path.join(source, "test")) fu, fr, fs, dc, dr, ds = self.count_output(out) - self.failUnlessEqual(fu, 0) - self.failUnlessEqual(fr, 0) - self.failUnlessEqual(fs, 0) + self.failUnlessReallyEqual(fu, 0) + self.failUnlessReallyEqual(fr, 0) + self.failUnlessReallyEqual(fs, 0) # home, test - self.failUnlessEqual(dc, 2) - self.failUnlessEqual(dr, 0) + self.failUnlessReallyEqual(dc, 2) + self.failUnlessReallyEqual(dr, 0) # test - self.failUnlessEqual(ds, 1) + self.failUnlessReallyEqual(ds, 1) d.addCallback(_check) # This is necessary for the temp files to be correctly removed @@ -1844,8 +1930,9 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): source = os.path.join(self.basedir, "file1") d = self.do_cli('backup', source, source) def _check((rc, out, err)): - self.failUnlessEqual(rc, 1) + self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) + self.failUnlessReallyEqual(out, "") d.addCallback(_check) return d @@ -1857,9 +1944,10 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): source = os.path.join(self.basedir, "file1") d = self.do_cli("backup", source, "nonexistent:" + source) def _check((rc, out, err)): - self.failUnlessEqual(rc, 1) + self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) self.failUnlessIn("nonexistent", err) + self.failUnlessReallyEqual(out, "") d.addCallback(_check) return d @@ -1878,8 +1966,8 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(lambda ign: self.do_cli("check", self.uri)) def _check1((rc, out, err)): - self.failUnlessEqual(err, "") - self.failUnlessEqual(rc, 0) + self.failUnlessReallyEqual(err, "") + self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() self.failUnless("Summary: Healthy" in lines, out) self.failUnless(" good-shares: 10 (encoding is 3-of-10)" in lines, out) @@ -1887,16 +1975,16 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(lambda ign: self.do_cli("check", "--raw", self.uri)) def _check2((rc, out, err)): - self.failUnlessEqual(err, "") - self.failUnlessEqual(rc, 0) + self.failUnlessReallyEqual(err, "") + self.failUnlessReallyEqual(rc, 0) data = simplejson.loads(out) - self.failUnlessEqual(data["summary"], "Healthy") + self.failUnlessReallyEqual(data["summary"], "Healthy") d.addCallback(_check2) def _clobber_shares(ignored): # delete one, corrupt a second shares = self.find_shares(self.uri) - self.failUnlessEqual(len(shares), 10) + self.failUnlessReallyEqual(len(shares), 10) os.unlink(shares[0][2]) cso = debug.CorruptShareOptions() cso.stdout = StringIO() @@ -1911,8 +1999,8 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(lambda ign: self.do_cli("check", "--verify", self.uri)) def _check3((rc, out, err)): - self.failUnlessEqual(err, "") - self.failUnlessEqual(rc, 0) + self.failUnlessReallyEqual(err, "") + self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() summary = [l for l in lines if l.startswith("Summary")][0] self.failUnless("Summary: Unhealthy: 8 shares (enc 3-of-10)" @@ -1925,8 +2013,8 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(lambda ign: self.do_cli("check", "--verify", "--repair", self.uri)) def _check4((rc, out, err)): - self.failUnlessEqual(err, "") - self.failUnlessEqual(rc, 0) + self.failUnlessReallyEqual(err, "") + self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() self.failUnless("Summary: not healthy" in lines, out) self.failUnless(" good-shares: 8 (encoding is 3-of-10)" in lines, out) @@ -1938,8 +2026,8 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(lambda ign: self.do_cli("check", "--verify", "--repair", self.uri)) def _check5((rc, out, err)): - self.failUnlessEqual(err, "") - self.failUnlessEqual(rc, 0) + self.failUnlessReallyEqual(err, "") + self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() self.failUnless("Summary: healthy" in lines, out) self.failUnless(" good-shares: 10 (encoding is 3-of-10)" in lines, out) @@ -1955,6 +2043,8 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): self.uris = {} self.fileurls = {} DATA = "data" * 100 + quoted_good = quote_output(u"gööd") + d = c0.create_dirnode() def _stash_root_and_create_file(n): self.rootnode = n @@ -1976,8 +2066,8 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(lambda ign: self.do_cli("deep-check", self.rooturi)) def _check1((rc, out, err)): - self.failUnlessEqual(err, "") - self.failUnlessEqual(rc, 0) + self.failUnlessReallyEqual(err, "") + self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() self.failUnless("done: 4 objects checked, 4 healthy, 0 unhealthy" in lines, out) @@ -1991,22 +2081,21 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(lambda ign: self.do_cli("deep-check", "--verbose", self.rooturi)) def _check2((rc, out, err)): - self.failUnlessEqual(err, "") - self.failUnlessEqual(rc, 0) + self.failUnlessReallyEqual(err, "") + self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() - linesu = [ line.decode(stringutils.get_term_encoding()) for line in lines] - self.failUnless(": Healthy" in lines, out) - self.failUnless("small: Healthy (LIT)" in lines, out) - self.failUnless(u"gööd: Healthy" in linesu, out) - self.failUnless("mutable: Healthy" in lines, out) + self.failUnless("'': Healthy" in lines, out) + self.failUnless("'small': Healthy (LIT)" in lines, out) + self.failUnless((quoted_good + ": Healthy") in lines, out) + self.failUnless("'mutable': Healthy" in lines, out) self.failUnless("done: 4 objects checked, 4 healthy, 0 unhealthy" in lines, out) d.addCallback(_check2) d.addCallback(lambda ign: self.do_cli("stats", self.rooturi)) def _check_stats((rc, out, err)): - self.failUnlessEqual(err, "") - self.failUnlessEqual(rc, 0) + self.failUnlessReallyEqual(err, "") + self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() self.failUnlessIn(" count-immutable-files: 1", lines) self.failUnlessIn(" count-mutable-files: 1", lines) @@ -2020,7 +2109,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): def _clobber_shares(ignored): shares = self.find_shares(self.uris[u"gööd"]) - self.failUnlessEqual(len(shares), 10) + self.failUnlessReallyEqual(len(shares), 10) os.unlink(shares[0][2]) shares = self.find_shares(self.uris["mutable"]) @@ -2043,14 +2132,13 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(lambda ign: self.do_cli("deep-check", "--verbose", self.rooturi)) def _check3((rc, out, err)): - self.failUnlessEqual(err, "") - self.failUnlessEqual(rc, 0) + self.failUnlessReallyEqual(err, "") + self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() - linesu = [ line.decode(stringutils.get_term_encoding()) for line in lines] - self.failUnless(": Healthy" in lines, out) - self.failUnless("small: Healthy (LIT)" in lines, out) - self.failUnless("mutable: Healthy" in lines, out) # needs verifier - self.failUnless(u"gööd: Not Healthy: 9 shares (enc 3-of-10)" in linesu, out) + self.failUnless("'': Healthy" in lines, out) + self.failUnless("'small': Healthy (LIT)" in lines, out) + self.failUnless("'mutable': Healthy" in lines, out) # needs verifier + self.failUnless((quoted_good + ": Not Healthy: 9 shares (enc 3-of-10)") in lines, out) self.failIf(self._corrupt_share_line in lines, out) self.failUnless("done: 4 objects checked, 3 healthy, 1 unhealthy" in lines, out) @@ -2060,17 +2148,16 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): self.do_cli("deep-check", "--verbose", "--verify", self.rooturi)) def _check4((rc, out, err)): - self.failUnlessEqual(err, "") - self.failUnlessEqual(rc, 0) + self.failUnlessReallyEqual(err, "") + self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() - linesu = [ line.decode(stringutils.get_term_encoding()) for line in lines] - self.failUnless(": Healthy" in lines, out) - self.failUnless("small: Healthy (LIT)" in lines, out) - mutable = [l for l in lines if l.startswith("mutable")][0] - self.failUnless(mutable.startswith("mutable: Unhealthy: 9 shares (enc 3-of-10)"), + self.failUnless("'': Healthy" in lines, out) + self.failUnless("'small': Healthy (LIT)" in lines, out) + mutable = [l for l in lines if l.startswith("'mutable'")][0] + self.failUnless(mutable.startswith("'mutable': Unhealthy: 9 shares (enc 3-of-10)"), mutable) self.failUnless(self._corrupt_share_line in lines, out) - self.failUnless(u"gööd: Not Healthy: 9 shares (enc 3-of-10)" in linesu, out) + self.failUnless((quoted_good + ": Not Healthy: 9 shares (enc 3-of-10)") in lines, out) self.failUnless("done: 4 objects checked, 2 healthy, 2 unhealthy" in lines, out) d.addCallback(_check4) @@ -2079,12 +2166,12 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): self.do_cli("deep-check", "--raw", self.rooturi)) def _check5((rc, out, err)): - self.failUnlessEqual(err, "") - self.failUnlessEqual(rc, 0) + self.failUnlessReallyEqual(err, "") + self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() units = [simplejson.loads(line) for line in lines] # root, small, gööd, mutable, stats - self.failUnlessEqual(len(units), 4+1) + self.failUnlessReallyEqual(len(units), 4+1) d.addCallback(_check5) d.addCallback(lambda ign: @@ -2092,15 +2179,14 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): "--verbose", "--verify", "--repair", self.rooturi)) def _check6((rc, out, err)): - self.failUnlessEqual(err, "") - self.failUnlessEqual(rc, 0) + self.failUnlessReallyEqual(err, "") + self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() - linesu = [ line.decode(stringutils.get_term_encoding()) for line in lines] - self.failUnless(": healthy" in lines, out) - self.failUnless("small: healthy" in lines, out) - self.failUnless("mutable: not healthy" in lines, out) + self.failUnless("'': healthy" in lines, out) + self.failUnless("'small': healthy" in lines, out) + self.failUnless("'mutable': not healthy" in lines, out) self.failUnless(self._corrupt_share_line in lines, out) - self.failUnless(u"gööd: not healthy" in linesu, out) + self.failUnless((quoted_good + ": not healthy") in lines, out) self.failUnless("done: 4 objects checked" in lines, out) self.failUnless(" pre-repair: 2 healthy, 2 unhealthy" in lines, out) self.failUnless(" 2 repairs attempted, 2 successful, 0 failed" @@ -2166,8 +2252,9 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): self.set_up_grid() d = self.do_cli("check") def _check((rc, out, err)): - self.failUnlessEqual(rc, 1) + self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) + self.failUnlessReallyEqual(out, "") d.addCallback(_check) d.addCallback(lambda ign: self.do_cli("deep-check")) d.addCallback(_check) @@ -2180,9 +2267,10 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): self.set_up_grid() d = self.do_cli("check", "nonexistent:") def _check((rc, out, err)): - self.failUnlessEqual(rc, 1) + self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) self.failUnlessIn("nonexistent", err) + self.failUnlessReallyEqual(out, "") d.addCallback(_check) return d @@ -2230,8 +2318,9 @@ class Get(GridTestMixin, CLITestMixin, unittest.TestCase): self.set_up_grid() d = self.do_cli('get', 'file') def _check((rc, out, err)): - self.failUnlessEqual(rc, 1) + self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) + self.failUnlessReallyEqual(out, "") d.addCallback(_check) return d @@ -2242,9 +2331,10 @@ class Get(GridTestMixin, CLITestMixin, unittest.TestCase): self.set_up_grid() d = self.do_cli("get", "nonexistent:file") def _check((rc, out, err)): - self.failUnlessEqual(rc, 1) + self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) self.failUnlessIn("nonexistent", err) + self.failUnlessReallyEqual(out, "") d.addCallback(_check) return d @@ -2258,8 +2348,9 @@ class Manifest(GridTestMixin, CLITestMixin, unittest.TestCase): self.set_up_grid() d = self.do_cli("manifest") def _check((rc, out, err)): - self.failUnlessEqual(rc, 1) + self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) + self.failUnlessReallyEqual(out, "") d.addCallback(_check) return d @@ -2270,14 +2361,50 @@ class Manifest(GridTestMixin, CLITestMixin, unittest.TestCase): self.set_up_grid() d = self.do_cli("manifest", "nonexistent:") def _check((rc, out, err)): - self.failUnlessEqual(rc, 1) + self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) self.failUnlessIn("nonexistent", err) + self.failUnlessReallyEqual(out, "") d.addCallback(_check) return d class Mkdir(GridTestMixin, CLITestMixin, unittest.TestCase): + def test_mkdir(self): + self.basedir = os.path.dirname(self.mktemp()) + self.set_up_grid() + + d = self.do_cli("create-alias", "tahoe") + d.addCallback(lambda res: self.do_cli("mkdir", "test")) + def _check((rc, out, err)): + self.failUnlessReallyEqual(rc, 0) + self.failUnlessReallyEqual(err, "") + #self.failUnlessIn(..., out) + d.addCallback(_check) + #d.addCallback(lambda ign: ...) + + return d + + def test_mkdir_unicode(self): + self.basedir = os.path.dirname(self.mktemp()) + self.set_up_grid() + + try: + motorhead_arg = u"tahoe:Motörhead".encode(get_argv_encoding()) + except UnicodeEncodeError, e: + raise unittest.SkipTest("A non-ASCII command argument could not be encoded as %s:\n%r" (get_argv_encoding(), e)) + + d = self.do_cli("create-alias", "tahoe") + d.addCallback(lambda res: self.do_cli("mkdir", motorhead_arg)) + def _check((rc, out, err)): + self.failUnlessReallyEqual(rc, 0) + self.failUnlessReallyEqual(err, "") + #self.failUnlessIn(..., out) + d.addCallback(_check) + #d.addCallback(lambda ign: ...) + + return d + def test_mkdir_with_nonexistent_alias(self): # when invoked with an alias that doesn't exist, 'tahoe mkdir' should # output a sensible error message rather than a stack trace. @@ -2285,8 +2412,9 @@ class Mkdir(GridTestMixin, CLITestMixin, unittest.TestCase): self.set_up_grid() d = self.do_cli("mkdir", "havasu:") def _check((rc, out, err)): - self.failUnlessEqual(rc, 1) + self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) + self.failUnlessReallyEqual(out, "") d.addCallback(_check) return d @@ -2299,8 +2427,9 @@ class Rm(GridTestMixin, CLITestMixin, unittest.TestCase): self.set_up_grid() d = self.do_cli("rm", "afile") def _check((rc, out, err)): + self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) - self.failUnlessEqual(rc, 1) + self.failUnlessReallyEqual(out, "") d.addCallback(_check) return d @@ -2311,9 +2440,10 @@ class Rm(GridTestMixin, CLITestMixin, unittest.TestCase): self.set_up_grid() d = self.do_cli("rm", "nonexistent:afile") def _check((rc, out, err)): - self.failUnlessEqual(rc, 1) + self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) self.failUnlessIn("nonexistent", err) + self.failUnlessReallyEqual(out, "") d.addCallback(_check) return d @@ -2333,8 +2463,8 @@ class Stats(GridTestMixin, CLITestMixin, unittest.TestCase): # make sure we can get stats on an empty directory too d.addCallback(lambda ign: self.do_cli("stats", self.rooturi)) def _check_stats((rc, out, err)): - self.failUnlessEqual(err, "") - self.failUnlessEqual(rc, 0) + self.failUnlessReallyEqual(err, "") + self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() self.failUnlessIn(" count-immutable-files: 0", lines) self.failUnlessIn(" count-mutable-files: 0", lines) @@ -2354,8 +2484,9 @@ class Stats(GridTestMixin, CLITestMixin, unittest.TestCase): self.set_up_grid() d = self.do_cli("stats") def _check((rc, out, err)): - self.failUnlessEqual(rc, 1) + self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) + self.failUnlessReallyEqual(out, "") d.addCallback(_check) return d @@ -2366,8 +2497,9 @@ class Stats(GridTestMixin, CLITestMixin, unittest.TestCase): self.set_up_grid() d = self.do_cli("stats", "havasu:") def _check((rc, out, err)): - self.failUnlessEqual(rc, 1) + self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) + self.failUnlessReallyEqual(out, "") d.addCallback(_check) return d @@ -2381,7 +2513,8 @@ class Webopen(GridTestMixin, CLITestMixin, unittest.TestCase): self.set_up_grid() d = self.do_cli("webopen", "fake:") def _check((rc, out, err)): - self.failUnlessEqual(rc, 1) + self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) + self.failUnlessReallyEqual(out, "") d.addCallback(_check) return d diff --git a/src/allmydata/test/test_consolidate.py b/src/allmydata/test/test_consolidate.py index d1562a6f..991e8bdf 100644 --- a/src/allmydata/test/test_consolidate.py +++ b/src/allmydata/test/test_consolidate.py @@ -4,12 +4,23 @@ from cStringIO import StringIO import pickle from twisted.trial import unittest from allmydata.test.no_network import GridTestMixin +from allmydata.test.common_util import ReallyEqualMixin from allmydata.util import fileutil from allmydata.scripts import runner, debug from allmydata.scripts.common import get_aliases from twisted.internet import defer, threads # CLI tests use deferToThread from allmydata.interfaces import IDirectoryNode +have_sqlite3 = False +try: + import sqlite3 + sqlite3 # hush pyflakes + have_sqlite3 = True +except ImportError: + pass +else: + from allmydata.scripts import consolidate + class CLITestMixin: def do_cli(self, verb, *args, **kwargs): @@ -30,7 +41,7 @@ class CLITestMixin: d.addCallback(_done) return d -class Consolidate(GridTestMixin, CLITestMixin, unittest.TestCase): +class Consolidate(GridTestMixin, CLITestMixin, ReallyEqualMixin, unittest.TestCase): def writeto(self, path, data): d = os.path.dirname(os.path.join(self.basedir, "home", path)) @@ -46,8 +57,8 @@ class Consolidate(GridTestMixin, CLITestMixin, unittest.TestCase): def do_cli_good(self, verb, *args, **kwargs): d = self.do_cli(verb, *args, **kwargs) def _check((rc,out,err)): - self.failUnlessEqual(err, "", verb) - self.failUnlessEqual(rc, 0, verb) + self.failUnlessReallyEqual(err, "", verb) + self.failUnlessReallyEqual(rc, 0, verb) return out d.addCallback(_check) return d @@ -59,29 +70,14 @@ class Consolidate(GridTestMixin, CLITestMixin, unittest.TestCase): co.parseOptions(["--node-directory", self.get_clientdir(), "--dbfile", "foo.db", "--backupfile", "backup", "--really", "URI:DIR2:foo"]) - self.failUnlessEqual(co["dbfile"], "foo.db") - self.failUnlessEqual(co["backupfile"], "backup") + self.failUnlessReallyEqual(co["dbfile"], "foo.db") + self.failUnlessReallyEqual(co["backupfile"], "backup") self.failUnless(co["really"]) - self.failUnlessEqual(co.where, "URI:DIR2:foo") + self.failUnlessReallyEqual(co.where, u"URI:DIR2:foo") - def OFF_test_basic(self): - # rename this method to enable the test. I've disabled it because, in - # my opinion: - # - # 1: 'tahoe debug consolidate' is useful enough to include in trunk, - # but not useful enough justify a lot of compatibility effort or - # extra test time - # 2: it requires sqlite3; I did not bother to make it work with - # pysqlite, nor did I bother making it fail gracefully when - # sqlite3 is not available - # 3: this test takes 30 seconds to run on my workstation, and it likely - # to take several minutes on the old slow dapper buildslave - # 4: I don't want other folks to see a SkipTest and wonder "oh no, what - # did I do wrong to not allow this test to run" - # - # These may not be strong arguments: I welcome feedback. In particular, - # this command may be more suitable for a plugin of some sort, if we - # had plugins of some sort. -warner 12-Mar-09 + def test_basic(self): + if not have_sqlite3: + raise unittest.SkipTest("'tahoe debug consolidate' is not supported because sqlite3 is not available.") self.basedir = "consolidate/Consolidate/basic" self.set_up_grid(num_clients=1) @@ -175,7 +171,7 @@ class Consolidate(GridTestMixin, CLITestMixin, unittest.TestCase): def _check_consolidate_output1(out): lines = out.splitlines() last = lines[-1] - self.failUnlessEqual(last.strip(), + self.failUnlessReallyEqual(last.strip(), "system done, dircounts: " "25/12 seen/used, 7 created, 2 as-is, 13 reused") self.failUnless(os.path.exists(dbfile)) @@ -185,7 +181,7 @@ class Consolidate(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnless(u"fluxx" in backup["archives"]) adata = backup["archives"]["fluxx"] kids = adata[u"children"] - self.failUnlessEqual(str(kids[u"2009-03-01 01.01.01"][1][u"rw_uri"]), + self.failUnlessReallyEqual(str(kids[u"2009-03-01 01.01.01"][1][u"rw_uri"]), c("1-b-start")) d.addCallback(_check_consolidate_output1) d.addCallback(lambda ign: @@ -196,11 +192,11 @@ class Consolidate(GridTestMixin, CLITestMixin, unittest.TestCase): def _check_consolidate_output2(out): lines = out.splitlines() last = lines[-1] - self.failUnlessEqual(last.strip(), + self.failUnlessReallyEqual(last.strip(), "system done, dircounts: " "0/0 seen/used, 0 created, 0 as-is, 0 reused") backup = pickle.load(open(backupfile, "rb")) - self.failUnlessEqual(backup, self.first_backup) + self.failUnlessReallyEqual(backup, self.first_backup) self.failUnless(os.path.exists(backupfile + ".0")) d.addCallback(_check_consolidate_output2) @@ -214,14 +210,13 @@ class Consolidate(GridTestMixin, CLITestMixin, unittest.TestCase): # self.manifests[which][path]) # last snapshot should be untouched - self.failUnlessEqual(c("7-b"), c("7-b-start")) + self.failUnlessReallyEqual(c("7-b"), c("7-b-start")) # first snapshot should be a readonly form of the original - from allmydata.scripts.tahoe_backup import readonly - self.failUnlessEqual(c("1-b-finish"), readonly(c("1-b-start"))) - self.failUnlessEqual(c("1-bp-finish"), readonly(c("1-bp-start"))) - self.failUnlessEqual(c("1-bps1-finish"), readonly(c("1-bps1-start"))) - self.failUnlessEqual(c("1-bps2-finish"), readonly(c("1-bps2-start"))) + self.failUnlessReallyEqual(c("1-b-finish"), consolidate.readonly(c("1-b-start"))) + self.failUnlessReallyEqual(c("1-bp-finish"), consolidate.readonly(c("1-bp-start"))) + self.failUnlessReallyEqual(c("1-bps1-finish"), consolidate.readonly(c("1-bps1-start"))) + self.failUnlessReallyEqual(c("1-bps2-finish"), consolidate.readonly(c("1-bps2-start"))) # new directories should be different than the old ones self.failIfEqual(c("1-b"), c("1-b-start")) @@ -246,33 +241,33 @@ class Consolidate(GridTestMixin, CLITestMixin, unittest.TestCase): self.failIfEqual(c("5-bps2"), c("5-bps2-start")) # snapshot 1 and snapshot 2 should be identical - self.failUnlessEqual(c("2-b"), c("1-b")) + self.failUnlessReallyEqual(c("2-b"), c("1-b")) # snapshot 3 modified a file underneath parent/ self.failIfEqual(c("3-b"), c("2-b")) # 3 modified a file self.failIfEqual(c("3-bp"), c("2-bp")) # but the subdirs are the same - self.failUnlessEqual(c("3-bps1"), c("2-bps1")) - self.failUnlessEqual(c("3-bps2"), c("2-bps2")) + self.failUnlessReallyEqual(c("3-bps1"), c("2-bps1")) + self.failUnlessReallyEqual(c("3-bps2"), c("2-bps2")) # snapshot 4 should be the same as 2 - self.failUnlessEqual(c("4-b"), c("2-b")) - self.failUnlessEqual(c("4-bp"), c("2-bp")) - self.failUnlessEqual(c("4-bps1"), c("2-bps1")) - self.failUnlessEqual(c("4-bps2"), c("2-bps2")) + self.failUnlessReallyEqual(c("4-b"), c("2-b")) + self.failUnlessReallyEqual(c("4-bp"), c("2-bp")) + self.failUnlessReallyEqual(c("4-bps1"), c("2-bps1")) + self.failUnlessReallyEqual(c("4-bps2"), c("2-bps2")) # snapshot 5 added a file under subdir1 self.failIfEqual(c("5-b"), c("4-b")) self.failIfEqual(c("5-bp"), c("4-bp")) self.failIfEqual(c("5-bps1"), c("4-bps1")) - self.failUnlessEqual(c("5-bps2"), c("4-bps2")) + self.failUnlessReallyEqual(c("5-bps2"), c("4-bps2")) # snapshot 6 copied a directory-it should be shared self.failIfEqual(c("6-b"), c("5-b")) self.failIfEqual(c("6-bp"), c("5-bp")) - self.failUnlessEqual(c("6-bps1"), c("5-bps1")) + self.failUnlessReallyEqual(c("6-bps1"), c("5-bps1")) self.failIfEqual(c("6-bps2"), c("5-bps2")) - self.failUnlessEqual(c("6-bps2c1"), c("6-bps1")) + self.failUnlessReallyEqual(c("6-bps2c1"), c("6-bps1")) d.addCallback(check_consolidation) diff --git a/src/allmydata/test/test_sftp.py b/src/allmydata/test/test_sftp.py index 87c6c899..efba99e2 100644 --- a/src/allmydata/test/test_sftp.py +++ b/src/allmydata/test/test_sftp.py @@ -30,10 +30,11 @@ from allmydata.util.consumer import download_to_data from allmydata.immutable import upload from allmydata.test.no_network import GridTestMixin from allmydata.test.common import ShouldFailMixin +from allmydata.test.common_util import ReallyEqualMixin timeout = 240 -class Handler(GridTestMixin, ShouldFailMixin, unittest.TestCase): +class Handler(GridTestMixin, ShouldFailMixin, ReallyEqualMixin, unittest.TestCase): """This is a no-network unit test of the SFTPUserHandler and the abstractions it uses.""" if not have_pycrypto: diff --git a/src/allmydata/test/test_stringutils.py b/src/allmydata/test/test_stringutils.py index 08311e7d..f3e57bb1 100644 --- a/src/allmydata/test/test_stringutils.py +++ b/src/allmydata/test/test_stringutils.py @@ -14,7 +14,7 @@ if __name__ == "__main__": import tempfile import shutil import platform - + if len(sys.argv) != 2: print "Usage: %s lumière" % sys.argv[0] sys.exit(1) @@ -22,10 +22,12 @@ if __name__ == "__main__": print print "class MyWeirdOS(StringUtils, unittest.TestCase):" print " uname = '%s'" % ' '.join(platform.uname()) - print " argv = %s" % repr(sys.argv[1]) + if sys.platform != "win32": + print " argv = %s" % repr(sys.argv[1]) print " platform = '%s'" % sys.platform - print " filesystemencoding = '%s'" % sys.getfilesystemencoding() - print " stdoutencoding = '%s'" % sys.stdout.encoding + print " filesystem_encoding = '%s'" % sys.getfilesystemencoding() + print " output_encoding = '%s'" % sys.stdout.encoding + print " argv_encoding = '%s'" % (sys.platform == "win32" and 'utf-8' or sys.stdout.encoding) try: tmpdir = tempfile.mkdtemp() @@ -48,47 +50,65 @@ if __name__ == "__main__": from twisted.trial import unittest from mock import patch -import locale, sys +import sys +from allmydata.test.common_util import ReallyEqualMixin from allmydata.util.stringutils import argv_to_unicode, unicode_to_url, \ - unicode_to_stdout, unicode_platform, listdir_unicode, open_unicode, \ - FilenameEncodingError, get_term_encoding + unicode_to_output, unicode_platform, listdir_unicode, open_unicode, \ + FilenameEncodingError, get_output_encoding, _reload + from twisted.python import usage -class StringUtilsErrors(unittest.TestCase): +class StringUtilsErrors(ReallyEqualMixin, unittest.TestCase): + def tearDown(self): + _reload() + @patch('sys.stdout') - def test_get_term_encoding(self, mock): - mock.encoding = None + def test_get_output_encoding(self, mock_stdout): + mock_stdout.encoding = 'UTF-8' + _reload() + self.failUnlessReallyEqual(get_output_encoding(), 'utf-8') + + mock_stdout.encoding = 'cp65001' + _reload() + self.failUnlessReallyEqual(get_output_encoding(), 'utf-8') + + mock_stdout.encoding = 'koi8-r' + _reload() + self.failUnlessReallyEqual(get_output_encoding(), 'koi8-r') + + mock_stdout.encoding = 'nonexistent_encoding' + self.failUnlessRaises(AssertionError, _reload) - self.failUnlessEqual(get_term_encoding().lower(), locale.getpreferredencoding().lower()) + # TODO: mock_stdout.encoding = None @patch('sys.stdout') def test_argv_to_unicode(self, mock): mock.encoding = 'utf-8' + _reload() self.failUnlessRaises(usage.UsageError, argv_to_unicode, u'lumière'.encode('latin1')) - def test_unicode_to_url(self): - pass - @patch('sys.stdout') - def test_unicode_to_stdout(self, mock): + def test_unicode_to_output(self, mock): # Encoding koi8-r cannot represent 'è' mock.encoding = 'koi8-r' - self.failUnlessEqual(unicode_to_stdout(u'lumière'), 'lumi?re') + _reload() + self.failUnlessRaises(UnicodeEncodeError, unicode_to_output, u'lumière') @patch('os.listdir') def test_unicode_normalization(self, mock): - # Pretend to run on an Unicode platform such as Windows + # Pretend to run on an Unicode platform orig_platform = sys.platform - sys.platform = 'win32' - - mock.return_value = [u'A\u0308rtonwall.mp3'] - self.failUnlessEqual(listdir_unicode(u'/dummy'), [u'\xc4rtonwall.mp3']) - - sys.platform = orig_platform + try: + sys.platform = 'darwin' + mock.return_value = [u'A\u0308rtonwall.mp3'] + _reload() + self.failUnlessReallyEqual(listdir_unicode(u'/dummy'), [u'\xc4rtonwall.mp3']) + finally: + sys.platform = orig_platform # The following tests applies only to platforms which don't store filenames as # Unicode entities on the filesystem. @@ -100,18 +120,19 @@ class StringUtilsNonUnicodePlatform(unittest.TestCase): def tearDown(self): sys.platform = self.original_platform + _reload() @patch('sys.getfilesystemencoding') @patch('os.listdir') def test_listdir_unicode(self, mock_listdir, mock_getfilesystemencoding): - # What happen if a latin1-encoded filenames is encountered on an UTF-8 + # What happens if latin1-encoded filenames are encountered on an UTF-8 # filesystem? mock_listdir.return_value = [ u'lumière'.encode('utf-8'), u'lumière'.encode('latin1')] mock_getfilesystemencoding.return_value = 'utf-8' - + _reload() self.failUnlessRaises(FilenameEncodingError, listdir_unicode, u'/dummy') @@ -119,6 +140,7 @@ class StringUtilsNonUnicodePlatform(unittest.TestCase): # We're trying to list a directory whose name cannot be represented in # the filesystem encoding. This should fail. mock_getfilesystemencoding.return_value = 'ascii' + _reload() self.failUnlessRaises(FilenameEncodingError, listdir_unicode, u'/lumière') @@ -126,12 +148,12 @@ class StringUtilsNonUnicodePlatform(unittest.TestCase): @patch('sys.getfilesystemencoding') def test_open_unicode(self, mock): mock.return_value = 'ascii' - + _reload() self.failUnlessRaises(FilenameEncodingError, open_unicode, - u'lumière') + u'lumière', 'rb') -class StringUtils: +class StringUtils(ReallyEqualMixin): def setUp(self): # Mock sys.platform because unicode_platform() uses it self.original_platform = sys.platform @@ -139,29 +161,30 @@ class StringUtils: def tearDown(self): sys.platform = self.original_platform + _reload() @patch('sys.stdout') def test_argv_to_unicode(self, mock): if 'argv' not in dir(self): - raise unittest.SkipTest("There's no way to pass non-ASCII arguments in CLI on this (mocked) platform") - - mock.encoding = self.stdoutencoding + return + mock.encoding = self.output_encoding argu = u'lumière' argv = self.argv - - self.failUnlessEqual(argv_to_unicode(argv), argu) + _reload() + self.failUnlessReallyEqual(argv_to_unicode(argv), argu) def test_unicode_to_url(self): - self.failUnless(unicode_to_url(u'lumière'), u'lumière'.encode('utf-8')) + self.failUnless(unicode_to_url(u'lumière'), "lumi\xc3\xa8re") @patch('sys.stdout') - def test_unicode_to_stdout(self, mock): - if 'argv' not in dir(self): - raise unittest.SkipTest("There's no way to pass non-ASCII arguments in CLI on this (mocked) platform") + def test_unicode_to_output(self, mock): + if 'output' not in dir(self): + return - mock.encoding = self.stdoutencoding - self.failUnlessEqual(unicode_to_stdout(u'lumière'), self.argv) + mock.encoding = self.output_encoding + _reload() + self.failUnlessReallyEqual(unicode_to_output(u'lumière'), self.output) def test_unicode_platform(self): matrix = { @@ -171,113 +194,119 @@ class StringUtils: 'darwin': True, } - self.failUnlessEqual(unicode_platform(), matrix[self.platform]) + _reload() + self.failUnlessReallyEqual(unicode_platform(), matrix[self.platform]) @patch('sys.getfilesystemencoding') @patch('os.listdir') def test_listdir_unicode(self, mock_listdir, mock_getfilesystemencoding): if 'dirlist' not in dir(self): - raise unittest.SkipTest("No way to write non-ASCII filenames on this system") + return mock_listdir.return_value = self.dirlist - mock_getfilesystemencoding.return_value = self.filesystemencoding + mock_getfilesystemencoding.return_value = self.filesystem_encoding + _reload() filenames = listdir_unicode(u'/dummy') for fname in TEST_FILENAMES: self.failUnless(isinstance(fname, unicode)) - - if fname not in filenames: - self.fail("Cannot find %r in %r" % (fname, filenames)) + self.failUnlessIn(fname, filenames) @patch('sys.getfilesystemencoding') @patch('__builtin__.open') def test_open_unicode(self, mock_open, mock_getfilesystemencoding): - mock_getfilesystemencoding.return_value = self.filesystemencoding - + mock_getfilesystemencoding.return_value = self.filesystem_encoding fn = u'/dummy_directory/lumière.txt' + _reload() try: - open_unicode(fn) + open_unicode(fn, 'rb') except FilenameEncodingError: - raise unittest.SkipTest("Cannot represent test filename on this (mocked) platform") + return # Pass Unicode string to open() on Unicode platforms if unicode_platform(): - mock_open.assert_called_with(fn, 'r') + mock_open.assert_called_with(fn, 'rb') # Pass correctly encoded bytestrings to open() on non-Unicode platforms else: - fn_bytestring = fn.encode(self.filesystemencoding) - mock_open.assert_called_with(fn_bytestring, 'r') + fn_bytestring = fn.encode(self.filesystem_encoding) + mock_open.assert_called_with(fn_bytestring, 'rb') + class UbuntuKarmicUTF8(StringUtils, unittest.TestCase): uname = 'Linux korn 2.6.31-14-generic #48-Ubuntu SMP Fri Oct 16 14:05:01 UTC 2009 x86_64' + output = 'lumi\xc3\xa8re' argv = 'lumi\xc3\xa8re' platform = 'linux2' - filesystemencoding = 'UTF-8' - stdoutencoding = 'UTF-8' + filesystem_encoding = 'UTF-8' + output_encoding = 'UTF-8' + argv_encoding = 'UTF-8' dirlist = ['test_file', '\xc3\x84rtonwall.mp3', 'Blah blah.txt'] - class UbuntuKarmicLatin1(StringUtils, unittest.TestCase): uname = 'Linux korn 2.6.31-14-generic #48-Ubuntu SMP Fri Oct 16 14:05:01 UTC 2009 x86_64' + output = 'lumi\xe8re' argv = 'lumi\xe8re' platform = 'linux2' - filesystemencoding = 'ISO-8859-1' - stdoutencoding = 'ISO-8859-1' + filesystem_encoding = 'ISO-8859-1' + output_encoding = 'ISO-8859-1' + argv_encoding = 'ISO-8859-1' dirlist = ['test_file', 'Blah blah.txt', '\xc4rtonwall.mp3'] class WindowsXP(StringUtils, unittest.TestCase): uname = 'Windows XP 5.1.2600 x86 x86 Family 15 Model 75 Step ping 2, AuthenticAMD' - argv = 'lumi\xe8re' + output = 'lumi\x8are' + argv = 'lumi\xc3\xa8re' platform = 'win32' - filesystemencoding = 'mbcs' - stdoutencoding = 'cp850' + filesystem_encoding = 'mbcs' + output_encoding = 'cp850' + argv_encoding = 'utf-8' dirlist = [u'Blah blah.txt', u'test_file', u'\xc4rtonwall.mp3'] - todo = "Unicode arguments on the command-line is not yet supported under Windows, see bug #565." - class WindowsXP_UTF8(StringUtils, unittest.TestCase): uname = 'Windows XP 5.1.2600 x86 x86 Family 15 Model 75 Step ping 2, AuthenticAMD' - argv = 'lumi\xe8re' + output = 'lumi\xc3\xa8re' + argv = 'lumi\xc3\xa8re' platform = 'win32' - filesystemencoding = 'mbcs' - stdoutencoding = 'cp65001' + filesystem_encoding = 'mbcs' + output_encoding = 'cp65001' + argv_encoding = 'utf-8' dirlist = [u'Blah blah.txt', u'test_file', u'\xc4rtonwall.mp3'] - todo = "Unicode arguments on the command-line is not yet supported under Windows, see bug #565." - class WindowsVista(StringUtils, unittest.TestCase): uname = 'Windows Vista 6.0.6000 x86 x86 Family 6 Model 15 Stepping 11, GenuineIntel' - argv = 'lumi\xe8re' + output = 'lumi\x8are' + argv = 'lumi\xc3\xa8re' platform = 'win32' - filesystemencoding = 'mbcs' - stdoutencoding = 'cp850' + filesystem_encoding = 'mbcs' + output_encoding = 'cp850' + argv_encoding = 'utf-8' dirlist = [u'Blah blah.txt', u'test_file', u'\xc4rtonwall.mp3'] - todo = "Unicode arguments on the command-line is not yet supported under Windows, see bug #565." - class MacOSXLeopard(StringUtils, unittest.TestCase): uname = 'Darwin g5.local 9.8.0 Darwin Kernel Version 9.8.0: Wed Jul 15 16:57:01 PDT 2009; root:xnu-1228.15.4~1/RELEASE_PPC Power Macintosh powerpc' + output = 'lumi\xc3\xa8re' argv = 'lumi\xc3\xa8re' platform = 'darwin' - filesystemencoding = 'utf-8' - stdoutencoding = 'UTF-8' + filesystem_encoding = 'utf-8' + output_encoding = 'UTF-8' + argv_encoding = 'UTF-8' dirlist = [u'A\u0308rtonwall.mp3', u'Blah blah.txt', u'test_file'] class MacOSXLeopard7bit(StringUtils, unittest.TestCase): uname = 'Darwin g5.local 9.8.0 Darwin Kernel Version 9.8.0: Wed Jul 15 16:57:01 PDT 2009; root:xnu-1228.15.4~1/RELEASE_PPC Power Macintosh powerpc' - #argv = 'lumiere' platform = 'darwin' - filesystemencoding = 'utf-8' - stdoutencoding = 'US-ASCII' + filesystem_encoding = 'utf-8' + output_encoding = 'US-ASCII' + argv_encoding = 'US-ASCII' dirlist = [u'A\u0308rtonwall.mp3', u'Blah blah.txt', u'test_file'] class OpenBSD(StringUtils, unittest.TestCase): uname = 'OpenBSD 4.1 GENERIC#187 i386 Intel(R) Celeron(R) CPU 2.80GHz ("GenuineIntel" 686-class)' - #argv = 'lumiere' platform = 'openbsd4' - filesystemencoding = '646' - stdoutencoding = '646' + filesystem_encoding = '646' + output_encoding = '646' + argv_encoding = '646' # Oops, I cannot write filenames containing non-ascii characters diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index c1b1f7fa..59c8e7f1 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -1416,7 +1416,7 @@ class SystemTest(SystemTestMixin, unittest.TestCase): d.addCallback(run, "list-aliases") def _check_aliases_1((out,err)): self.failUnlessEqual(err, "") - self.failUnlessEqual(out, "tahoe: %s\n" % private_uri) + self.failUnlessEqual(out.strip(" \n"), "tahoe: %s" % private_uri) d.addCallback(_check_aliases_1) # now that that's out of the way, remove root_dir.cap and work with diff --git a/src/allmydata/util/stringutils.py b/src/allmydata/util/stringutils.py index ab06ed6e..eb511914 100644 --- a/src/allmydata/util/stringutils.py +++ b/src/allmydata/util/stringutils.py @@ -5,125 +5,235 @@ unicode and back. import sys import os +import re import unicodedata from allmydata.util.assertutil import precondition from twisted.python import usage import locale -def get_term_encoding(): + +def _canonical_encoding(encoding): + if encoding is None: + encoding = 'utf-8' + encoding = encoding.lower() + if encoding == "cp65001": + encoding = 'utf-8' + elif encoding == "us-ascii" or encoding == "646": + encoding = 'ascii' + + # sometimes Python returns an encoding name that it doesn't support for conversion + # fail early if this happens + try: + u"test".encode(encoding) + except LookupError: + raise AssertionError("The character encoding '%s' is not supported for conversion." % (encoding,)) + + return encoding + +filesystem_encoding = None +output_encoding = None +argv_encoding = None +is_unicode_platform = False + +def _reload(): + global filesystem_encoding, output_encoding, argv_encoding, is_unicode_platform + + filesystem_encoding = _canonical_encoding(sys.getfilesystemencoding()) + output_encoding = _canonical_encoding(sys.stdout.encoding or locale.getpreferredencoding()) + if sys.platform == 'win32': + # arguments are converted to utf-8 in windows/tahoe.py + argv_encoding = 'utf-8' + else: + argv_encoding = output_encoding + is_unicode_platform = sys.platform in ["win32", "darwin"] + +_reload() + + +def get_filesystem_encoding(): """ - Returns expected encoding for writing to the terminal and reading - arguments from the command-line. + Returns expected encoding for local filenames. """ + return filesystem_encoding - if sys.stdout.encoding: - return sys.stdout.encoding - else: - return locale.getpreferredencoding() +def get_output_encoding(): + """ + Returns expected encoding for writing to stdout or stderr. + """ + return output_encoding -def argv_to_unicode(s): +def get_argv_encoding(): """ - Decode given argv element to unicode. + Returns expected encoding for command-line arguments. """ - # Try to decode the command-line argument with the encoding returned by - # get_term_encoding(), if this fails print an error message to the user. + return argv_encoding +def argv_to_unicode(s): + """ + Decode given argv element to unicode. If this fails, raise a UsageError. + """ precondition(isinstance(s, str), s) try: - return unicode(s, get_term_encoding()) + return unicode(s, argv_encoding) except UnicodeDecodeError: - raise usage.UsageError("Argument '%s' cannot be decoded as %s." % - (s, get_term_encoding())) + raise usage.UsageError("Argument %s cannot be decoded as %s." % + (quote_output(s), argv_encoding)) def unicode_to_url(s): """ Encode an unicode object used in an URL. """ - # According to RFC 2718, non-ascii characters in url's must be UTF-8 encoded. + # According to RFC 2718, non-ascii characters in URLs must be UTF-8 encoded. - precondition(isinstance(s, unicode), s) + # FIXME + return to_str(s) + #precondition(isinstance(s, unicode), s) + #return s.encode('utf-8') + +def to_str(s): + if s is None or isinstance(s, str): + return s return s.encode('utf-8') -def unicode_to_stdout(s): +def to_argv(s): + if isinstance(s, str): + return s + return s.encode(argv_encoding) + +PRINTABLE_ASCII = re.compile(r'^[ -~\n\r]*$', re.DOTALL) +PRINTABLE_8BIT = re.compile(r'^[ -&(-~\n\r\x80-\xFF]*$', re.DOTALL) + +def is_printable_ascii(s): + return PRINTABLE_ASCII.search(s) is not None + +def unicode_to_output(s): """ - Encode an unicode object for representation on stdout. + Encode an unicode object for representation on stdout or stderr. """ - precondition(isinstance(s, unicode), s) - return s.encode(get_term_encoding(), 'replace') -def unicode_platform(): + try: + out = s.encode(output_encoding) + except UnicodeEncodeError: + raise UnicodeEncodeError(output_encoding, s, 0, 0, + "A string could not be encoded as %s for output to the terminal:\n%r" % + (output_encoding, repr(s))) + + if PRINTABLE_8BIT.search(out) is None: + raise UnicodeEncodeError(output_encoding, s, 0, 0, + "A string encoded as %s for output to the terminal contained unsafe bytes:\n%r" % + (output_encoding, repr(s))) + return out + +def quote_output(s, quotemarks=True): """ - Does the current platform handle Unicode filenames natively ? + Encode either a Unicode string or a UTF-8-encoded bytestring for representation + on stdout or stderr, tolerating errors. If 'quotemarks' is True, the string is + always surrounded by single quotes; otherwise, it is quoted only if necessary to + avoid ambiguity or control bytes in the output. """ + precondition(isinstance(s, (str, unicode)), s) + + if isinstance(s, str): + try: + s = s.decode('utf-8') + except UnicodeDecodeError: + return 'b' + repr(s) + + try: + out = s.encode(output_encoding) + except UnicodeEncodeError: + return repr(s) + + if PRINTABLE_8BIT.search(out) is None: + return repr(out) - return sys.platform in ('win32', 'darwin') + if quotemarks: + return "'" + out.replace("\\", "\\\\").replace("'", "\'") + "'" + else: + return out + +def quote_path(path, quotemarks=True): + return quote_output("/".join(map(to_str, path)), quotemarks=quotemarks) + + +def unicode_platform(): + """ + Does the current platform handle Unicode filenames natively? + """ + return is_unicode_platform class FilenameEncodingError(Exception): """ Filename cannot be encoded using the current encoding of your filesystem (%s). Please configure your locale correctly or rename this file. """ - pass -def listdir_unicode_unix(path): +def listdir_unicode_fallback(path): """ - This function emulates an Unicode API under Unix similar to one available + This function emulates a fallback Unicode API similar to one available under Windows or MacOS X. If badly encoded filenames are encountered, an exception is raised. """ precondition(isinstance(path, unicode), path) - encoding = sys.getfilesystemencoding() try: - byte_path = path.encode(encoding) + byte_path = path.encode(filesystem_encoding) except UnicodeEncodeError: raise FilenameEncodingError(path) try: - return [unicode(fn, encoding) for fn in os.listdir(byte_path)] + return [unicode(fn, filesystem_encoding) for fn in os.listdir(byte_path)] except UnicodeDecodeError: raise FilenameEncodingError(fn) -def listdir_unicode(path, encoding = None): +def listdir_unicode(path): """ Wrapper around listdir() which provides safe access to the convenient - Unicode API even under Unix. + Unicode API even under platforms that don't provide one natively. """ - precondition(isinstance(path, unicode), path) # On Windows and MacOS X, the Unicode API is used - if unicode_platform(): - dirlist = os.listdir(path) - # On other platforms (ie. Unix systems), the byte-level API is used + + if is_unicode_platform: + dirlist = os.listdir(path) else: - dirlist = listdir_unicode_unix(path) + dirlist = listdir_unicode_fallback(path) # Normalize the resulting unicode filenames # - # This prevents different OS from generating non-equal unicode strings for + # This prevents different OSes from generating non-equal unicode strings for # the same filename representation return [unicodedata.normalize('NFC', fname) for fname in dirlist] -def open_unicode(path, mode='r'): +def open_unicode(path, mode): """ Wrapper around open() which provides safe access to the convenient Unicode API even under Unix. """ - precondition(isinstance(path, unicode), path) - if unicode_platform(): - return open(path, mode) + if is_unicode_platform: + return open(os.path.expanduser(path), mode) else: - encoding = sys.getfilesystemencoding() - try: - return open(path.encode(encoding), mode) + return open(os.path.expanduser(path.encode(filesystem_encoding)), mode) except UnicodeEncodeError: raise FilenameEncodingError(path) + +def abspath_expanduser_unicode(path): + precondition(isinstance(path, unicode), path) + + if is_unicode_platform: + return os.path.abspath(os.path.expanduser(path)) + else: + try: + pathstr = path.encode(filesystem_encoding) + return os.path.abspath(os.path.expanduser(pathstr)).decode(filesystem_encoding) + except (UnicodeEncodeError, UnicodeDecodeError): + raise FilenameEncodingError(path) diff --git a/windows/tahoe.py b/windows/tahoe.py index 38dd1a78..42161f48 100644 --- a/windows/tahoe.py +++ b/windows/tahoe.py @@ -3,5 +3,17 @@ pkgresutil.install() # this is done before nevow is imported by depends import depends # import dependencies so that py2exe finds them _junk = depends # appease pyflakes +import sys +from ctypes import WINFUNCTYPE, POINTER, byref, c_wchar_p, c_int, windll from allmydata.scripts import runner -runner.run(install_node_control=False) + +GetCommandLineW = WINFUNCTYPE(c_wchar_p)(("GetCommandLineW", windll.kernel32)) +CommandLineToArgvW = WINFUNCTYPE(POINTER(c_wchar_p), c_wchar_p, POINTER(c_int)) \ + (("CommandLineToArgvW", windll.shell32)) + +argc = c_int(0) +argv = CommandLineToArgvW(GetCommandLineW(), byref(argc)) +argv_utf8 = [argv[i].encode('utf-8') for i in xrange(1, argc.value)] + +rc = runner(argv_utf8, install_node_control=False) +sys.exit(rc) \ No newline at end of file -- 2.37.2