From: Brian Warner Date: Wed, 4 Mar 2015 02:19:58 +0000 (-0800) Subject: tahoe cp: overhaul target assignment, update tests X-Git-Tag: allmydata-tahoe-1.10.1a1~60^2~8 X-Git-Url: https://git.rkrishnan.org/provisioning?a=commitdiff_plain;h=2a361bc46ffa26b5f6cb25b7130d99870d2fa718;p=tahoe-lafs%2Ftahoe-lafs.git tahoe cp: overhaul target assignment, update tests This substantially changes the internals of "tahoe cp", to behave in accordance with the scheme developed in ticket:2329. test_cli_cp.py got a large new test to exercise all the various combinations. This also changes the set of error messages that "tahoe cp" can produce. This modifies try_copy(), inserts a new implementation of copy_things_to_directory() (and supporting methods), and fixes a few bugs elsewhere. fixes ticket:2329 --- diff --git a/src/allmydata/scripts/tahoe_cp.py b/src/allmydata/scripts/tahoe_cp.py index 0498cee9..850a1b37 100644 --- a/src/allmydata/scripts/tahoe_cp.py +++ b/src/allmydata/scripts/tahoe_cp.py @@ -2,6 +2,7 @@ import os.path import urllib import simplejson +from collections import defaultdict from cStringIO import StringIO from twisted.python.failure import Failure from allmydata.scripts.common import get_alias, escape_path, \ @@ -12,7 +13,7 @@ from allmydata.util import fileutil from allmydata.util.fileutil import abspath_expanduser_unicode, precondition_abspath from allmydata.util.encodingutil import unicode_to_url, listdir_unicode, quote_output, \ quote_local_unicode_path, to_str -from allmydata.util.assertutil import precondition +from allmydata.util.assertutil import precondition, _assert class MissingSourceError(TahoeError): @@ -156,7 +157,9 @@ class LocalDirectoryTarget: return self.children[name] pathname = os.path.join(self.pathname, name) os.makedirs(pathname) - return LocalDirectoryTarget(self.progressfunc, pathname) + child = LocalDirectoryTarget(self.progressfunc, pathname) + self.children[name] = child + return child def put_file(self, name, inf): precondition(isinstance(name, unicode), name) @@ -337,6 +340,7 @@ class TahoeDirectoryTarget: self.children = None def just_created(self, writecap): + # TODO: maybe integrate this with the constructor self.writecap = writecap self.readcap = uri.from_string(writecap).get_readonly().to_string() self.mutable = True @@ -482,57 +486,83 @@ class Copier: return 1 def try_copy(self): + """ + All usage errors are caught here, not in a subroutine. This bottoms + out in copy_file_to_file() or copy_things_to_directory(). + """ source_specs = self.options.sources destination_spec = self.options.destination recursive = self.options["recursive"] target = self.get_target_info(destination_spec) + precondition(isinstance(target, FileTargets + DirectoryTargets + MissingTargets), target) + target_has_trailing_slash = destination_spec.endswith("/") sources = [] # list of source objects for ss in source_specs: - sources.append(self.get_source_info(ss)) + si = self.get_source_info(ss) + precondition(isinstance(si, FileSources + DirectorySources), si) + sources.append(si) + + # if any source is a directory, must use -r + # if target is missing: + # if source is a single file, target will be a file + # else target will be a directory, so mkdir it + # if there are multiple sources, target must be a dir + # if target is a file, source must be a single file + # if target is directory, sources must be named or a dir have_source_dirs = any([isinstance(s, DirectorySources) for s in sources]) - if have_source_dirs and not recursive: + # 'cp dir target' without -r: error self.to_stderr("cannot copy directories without --recursive") return 1 + del recursive # -r is only used for signalling errors if isinstance(target, FileTargets): - # cp STUFF foo.txt, where foo.txt already exists. This limits the - # possibilities considerably. - if len(sources) > 1: - self.to_stderr("target %s is not a directory" % quote_output(destination_spec)) - return 1 - if have_source_dirs: + target_is_file = True + elif isinstance(target, DirectoryTargets): + target_is_file = False + else: # isinstance(target, MissingTargets) + if len(sources) == 1 and isinstance(sources[0], FileSources): + target_is_file = True + else: + target_is_file = False + + if target_is_file and target_has_trailing_slash: + self.to_stderr("target is not a directory, but has a slash") + return 1 + + if len(sources) > 1 and target_is_file: + self.to_stderr("copying multiple things requires target be a directory") + return 1 + + if target_is_file: + _assert(len(sources) == 1, sources) + if not isinstance(sources[0], FileSources): + # 'cp -r dir existingfile': error self.to_stderr("cannot copy directory into a file") return 1 return self.copy_file_to_file(sources[0], target) - if isinstance(target, MissingTargets): - if recursive: - return self.copy_to_directory(sources, target) - if len(sources) > 1: - # if we have -r, we'll auto-create the target directory. Without - # it, we'll only create a file. - self.to_stderr("cannot copy multiple files into a file without -r") + # else target is a directory, so each source must be one of: + # * a named file (copied to a new file under the target) + # * a named directory (causes a new directory of the same name to be + # created under the target, then the contents of the source are + # copied into that directory) + # * an unnamed directory (the contents of the source are copied into + # the target, without a new directory being made) + # + # If any source is an unnamed file, throw an error, since we have no + # way to name the output file. + _assert(isinstance(target, DirectoryTargets + MissingTargets), target) + + for source in sources: + if isinstance(source, FileSources) and not source.basename(): + self.to_stderr("when copying into a directory, all source files must have names, but %s is unnamed" % quote_output(source_specs[0])) return 1 - # cp file1 newfile - return self.copy_file_to_file(sources[0], target) - - if isinstance(target, DirectoryTargets): - # We're copying to an existing directory -- make sure that we - # have target names for everything - for source in sources: - if source.basename() is None and isinstance(source, TahoeFileSource): - self.to_stderr( - "error: you must specify a destination filename") - return 1 - return self.copy_to_directory(sources, target) - - self.to_stderr("unknown target") - return 1 + return self.copy_things_to_directory(sources, target) def to_stderr(self, text): print >>self.stderr, text @@ -585,6 +615,9 @@ class Copier: return t def get_source_info(self, source_spec): + """ + This turns an argv string into a (Local|Tahoe)(File|Directory)Source. + """ precondition(isinstance(source_spec, unicode), source_spec) rootcap, path_utf8 = get_alias(self.aliases, source_spec, None) path = path_utf8.decode("utf-8") @@ -666,6 +699,113 @@ class Copier: target.put_uri(source.bestcap()) return self.announce_success("file linked") + def copy_things_to_directory(self, sources, target): + # step one: if the target is missing, we should mkdir it + target = self.maybe_create_target(target) + target.populate(False) + + # step two: scan any source dirs, recursively, to find children + for s in sources: + if isinstance(s, DirectorySources): + s.populate(True) + if isinstance(s, FileSources): + # each source must have a name, or be a directory + _assert(s.basename() is not None, s) + + # step three: find a target for each source node, creating + # directories as necessary. 'targetmap' is a dictionary that uses + # target Directory instances as keys, and has values of (name: + # sourceobject) dicts for all the files that need to wind up there. + targetmap = self.build_targetmap(sources, target) + + # step four: walk through the list of targets. For each one, copy all + # the files. If the target is a TahoeDirectory, upload and create + # read-caps, then do a set_children to the target directory. + self.copy_to_targetmap(targetmap) + + return self.announce_success("files copied") + + def maybe_create_target(self, target): + if isinstance(target, LocalMissingTarget): + os.makedirs(target.pathname) + target = LocalDirectoryTarget(self.progress, target.pathname) + elif isinstance(target, TahoeMissingTarget): + writecap = mkdir(target.url) + target = TahoeDirectoryTarget(self.nodeurl, self.cache, + self.progress) + target.just_created(writecap) + # afterwards, or otherwise, it will be a directory + precondition(isinstance(target, DirectoryTargets), target) + return target + + def build_targetmap(self, sources, target): + num_source_files = len([s for s in sources + if isinstance(s, FileSources)]) + num_source_dirs = len([s for s in sources + if isinstance(s, DirectorySources)]) + self.progress("attaching sources to targets, " + "%d files / %d dirs in root" % + (num_source_files, num_source_dirs)) + + # this maps each target directory to a list of source files that need + # to be copied into it. All source files have names. + targetmap = defaultdict(list) + + for s in sources: + if isinstance(s, FileSources): + targetmap[target].append(s) + else: + _assert(isinstance(s, DirectorySources), s) + name = s.basename() + if name is not None: + # named sources get a new directory. see #2329 + new_target = target.get_child_target(name) + else: + # unnamed sources have their contents copied directly + new_target = target + self.assign_targets(targetmap, s, new_target) + + self.progress("targets assigned, %s dirs, %s files" % + (len(targetmap), self.count_files_to_copy(targetmap))) + return targetmap + + def assign_targets(self, targetmap, source, target): + # copy everything in the source into the target + precondition(isinstance(source, DirectorySources), source) + for name, child in source.children.items(): + if isinstance(child, DirectorySources): + # we will need a target directory for this one + subtarget = target.get_child_target(name) + self.assign_targets(targetmap, child, subtarget) + else: + precondition(isinstance(child, FileSources), child) + targetmap[target].append(child) + + def copy_to_targetmap(self, targetmap): + files_to_copy = self.count_files_to_copy(targetmap) + self.progress("starting copy, %d files, %d directories" % + (files_to_copy, len(targetmap))) + files_copied = 0 + targets_finished = 0 + + for target, sources in targetmap.items(): + precondition(isinstance(target, DirectoryTargets), target) + for source in sources: + precondition(isinstance(source, FileSources), source) + self.copy_file_into_dir(source, source.basename(), target) + files_copied += 1 + self.progress("%d/%d files, %d/%d directories" % + (files_copied, files_to_copy, + targets_finished, len(targetmap))) + target.set_children() + targets_finished += 1 + self.progress("%d/%d directories" % + (targets_finished, len(targetmap))) + + def count_files_to_copy(self, targetmap): + files_to_copy = sum([len(sources) for sources in targetmap.values()]) + return files_to_copy + def copy_file_into_dir(self, source, name, target): precondition(isinstance(source, FileSources), source) precondition(isinstance(target, DirectoryTargets), target) diff --git a/src/allmydata/test/test_cli_cp.py b/src/allmydata/test/test_cli_cp.py index 032e104b..837f1874 100644 --- a/src/allmydata/test/test_cli_cp.py +++ b/src/allmydata/test/test_cli_cp.py @@ -1,6 +1,7 @@ -import os.path, simplejson +import os.path, simplejson, shutil from twisted.trial import unittest from twisted.python import usage +from twisted.internet import defer from allmydata.scripts import cli from allmydata.util import fileutil @@ -119,7 +120,7 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(lambda ign: self.do_cli("cp", self.filecap, outdir)) def _resp((rc, out, err)): self.failUnlessReallyEqual(rc, 1) - self.failUnlessIn("error: you must specify a destination filename", + self.failUnlessIn("when copying into a directory, all source files must have names, but", err) self.failUnlessReallyEqual(out, "") d.addCallback(_resp) @@ -653,3 +654,329 @@ starting copy, 2 files, 1 directories (rc, out, err) = res self.failUnlessIn("Success: file copied", out, str(res)) return d + +# trailing slash on target *directory* should not matter, test both +# trailing slash on files should cause error + +COPYOUT_TESTCASES = """ +cp $FILECAP to/existing-file : to/existing-file +cp -r $FILECAP to/existing-file : to/existing-file +cp $DIRCAP/file $PARENTCAP/dir2/file2 to/existing-file : E6-MANYONE +cp -r $DIRCAP/file $PARENTCAP/dir2/file2 to/existing-file : E6-MANYONE +cp $DIRCAP to/existing-file : E4-NEED-R +cp -r $DIRCAP to/existing-file : E5-DIRTOFILE +cp $FILECAP $DIRCAP to/existing-file : E4-NEED-R +cp -r $FILECAP $DIRCAP to/existing-file : E6-MANYONE + +cp $FILECAP to/existing-file/ : E7-BADSLASH +cp -r $FILECAP to/existing-file/ : E7-BADSLASH +cp $DIRCAP/file $PARENTCAP/dir2/file2 to/existing-file/ : E7-BADSLASH +cp -r $DIRCAP/file $PARENTCAP/dir2/file2 to/existing-file/ : E7-BADSLASH +cp $DIRCAP to/existing-file/ : E4-NEED-R +cp -r $DIRCAP to/existing-file/ : E7-BADSLASH +cp $FILECAP $DIRCAP to/existing-file/ : E4-NEED-R +cp -r $FILECAP $DIRCAP to/existing-file/ : E7-BADSLASH + + +cp $FILECAP to : E2-DESTNAME +cp -r $FILECAP to : E2-DESTNAME +cp $DIRCAP/file to : to/file +cp -r $DIRCAP/file to : to/file +cp $PARENTCAP/dir to : E4-NEED-R +cp -r $PARENTCAP/dir to : to/dir/file +cp $DIRCAP to : E4-NEED-R +cp -r $DIRCAP to : to/file +cp $ALIAS to : E4-NEED-R +cp -r $ALIAS to : to/file + +cp $FILECAP to/ : E2-DESTNAME +cp -r $FILECAP to/ : E2-DESTNAME +cp $DIRCAP/file to/ : to/file +cp -r $DIRCAP/file to/ : to/file +cp $PARENTCAP/dir to/ : E4-NEED-R +cp -r $PARENTCAP/dir to/ : to/dir/file +cp $DIRCAP to/ : E4-NEED-R +cp -r $DIRCAP to/ : to/file +cp $ALIAS to/ : E4-NEED-R +cp -r $ALIAS to/ : to/file + + +cp $DIRCAP/file $PARENTCAP/dir2/file2 to : to/file,to/file2 +cp $DIRCAP/file $FILECAP to : E2-DESTNAME +cp $DIRCAP $FILECAP to : E4-NEED-R +cp -r $DIRCAP $FILECAP to : E2-DESTNAME + # namedfile, unnameddir, nameddir +cp $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 to : E4-NEED-R +cp -r $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 to : to/file3,to/file,to/dir2/file2 + # namedfile, unnameddir, nameddir, unnamedfile +cp $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 $FILECAP to : E4-NEED-R +cp -r $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 $FILECAP to : E2-DESTNAME + +cp $DIRCAP/file $PARENTCAP/dir2/file2 to/ : to/file,to/file2 +cp $DIRCAP/file $FILECAP to/ : E2-DESTNAME +cp $DIRCAP $FILECAP to/ : E4-NEED-R +cp -r $DIRCAP $FILECAP to/ : E2-DESTNAME + # namedfile, unnameddir, nameddir +cp $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 to/ : E4-NEED-R +cp -r $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 to/ : to/file3,to/file,to/dir2/file2 + # namedfile, unnameddir, nameddir, unnamedfile +cp $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 $FILECAP to/ : E4-NEED-R +cp -r $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 $FILECAP to/ : E2-DESTNAME + +# single sources to a missing target: should mkdir or create a file +cp $FILECAP to/missing : to/missing +cp -r $FILECAP to/missing : to/missing +cp $DIRCAP/file to/missing : to/missing +cp -r $DIRCAP/file to/missing : to/missing +cp $PARENTCAP/dir to/missing : E4-NEED-R +cp -r $PARENTCAP/dir to/missing : to/missing/dir/file +cp $DIRCAP to/missing : E4-NEED-R +cp -r $DIRCAP to/missing : to/missing/file +cp $ALIAS to/missing : E4-NEED-R +cp -r $ALIAS to/missing : to/missing/file + +cp $FILECAP to/missing/ : E7-BADSLASH +cp -r $FILECAP to/missing/ : E7-BADSLASH +cp $DIRCAP/file to/missing/ : E7-BADSLASH +cp -r $DIRCAP/file to/missing/ : E7-BADSLASH +cp $PARENTCAP/dir to/missing/ : E4-NEED-R +cp -r $PARENTCAP/dir to/missing/ : to/missing/dir/file +cp $DIRCAP to/missing/ : E4-NEED-R +cp -r $DIRCAP to/missing/ : to/missing/file +cp $ALIAS to/missing/ : E4-NEED-R +cp -r $ALIAS to/missing/ : to/missing/file + +# multiple files to a missing target: should mkdir +cp $DIRCAP/file $PARENTCAP/dir2/file2 to/missing : to/missing/file,to/missing/file2 +cp -r $DIRCAP/file $PARENTCAP/dir2/file2 to/missing : to/missing/file,to/missing/file2 + +cp $DIRCAP/file $PARENTCAP/dir2/file2 to/missing/ : to/missing/file,to/missing/file2 +cp -r $DIRCAP/file $PARENTCAP/dir2/file2 to/missing/ : to/missing/file,to/missing/file2 + +# make sure empty directories are copied too +cp -r $PARENTCAP/dir4 to : to/dir4/emptydir/ +cp -r $PARENTCAP/dir4 to/ : to/dir4/emptydir/ + +# name collisions: ensure files are copied in order +cp -r $PARENTCAP/dir6/dir $PARENTCAP/dir5/dir to : to/dir/collide=5 +cp -r $PARENTCAP/dir5/dir $PARENTCAP/dir6/dir to : to/dir/collide=6 +cp -r $DIRCAP6 $DIRCAP5 to : to/dir/collide=5 +cp -r $DIRCAP5 $DIRCAP6 to : to/dir/collide=6 + +""" + +class CopyOut(GridTestMixin, CLITestMixin, unittest.TestCase): + FILE_CONTENTS = "file text" + FILE_CONTENTS_5 = "5" + FILE_CONTENTS_6 = "6" + + def do_setup(self): + # first we build a tahoe filesystem that contains: + # $PARENTCAP + # $PARENTCAP/dir == $DIRCAP == alias: + # $PARENTCAP/dir/file == $FILECAP + # $PARENTCAP/dir2 (named directory) + # $PARENTCAP/dir2/file2 + # $PARENTCAP/dir3/file3 (a second named file) + # $PARENTCAP/dir4 + # $PARENTCAP/dir4/emptydir/ (an empty directory) + # $PARENTCAP/dir5 == $DIRCAP5 + # $PARENTCAP/dir5/dir/collide (contents are "5") + # $PARENTCAP/dir6 == $DIRCAP6 + # $PARENTCAP/dir6/dir/collide (contents are "6") + + source_file = os.path.join(self.basedir, "file") + fileutil.write(source_file, self.FILE_CONTENTS) + source_file_5 = os.path.join(self.basedir, "file5") + fileutil.write(source_file_5, self.FILE_CONTENTS_5) + source_file_6 = os.path.join(self.basedir, "file6") + fileutil.write(source_file_6, self.FILE_CONTENTS_6) + + d = self.do_cli("mkdir") + def _stash_parentdircap(res): + (rc, out, err) = res + self.failUnlessEqual(rc, 0, str(res)) + self.failUnlessEqual(err, "", str(res)) + self.PARENTCAP = out.strip() + return self.do_cli("mkdir", "%s/dir" % self.PARENTCAP) + d.addCallback(_stash_parentdircap) + def _stash_dircap(res): + (rc, out, err) = res + self.failUnlessEqual(rc, 0, str(res)) + self.failUnlessEqual(err, "", str(res)) + self.DIRCAP = out.strip() + return self.do_cli("add-alias", "ALIAS", self.DIRCAP) + d.addCallback(_stash_dircap) + d.addCallback(lambda ign: + self.do_cli("put", source_file, "%s/dir/file" % self.PARENTCAP)) + def _stash_filecap(res): + (rc, out, err) = res + self.failUnlessEqual(rc, 0, str(res)) + self.failUnlessEqual(err.strip(), "201 Created", str(res)) + self.FILECAP = out.strip() + assert self.FILECAP.startswith("URI:LIT:") + d.addCallback(_stash_filecap) + d.addCallback(lambda ign: + self.do_cli("mkdir", "%s/dir2" % self.PARENTCAP)) + d.addCallback(lambda ign: + self.do_cli("put", source_file, "%s/dir2/file2" % self.PARENTCAP)) + d.addCallback(lambda ign: + self.do_cli("mkdir", "%s/dir3" % self.PARENTCAP)) + d.addCallback(lambda ign: + self.do_cli("put", source_file, "%s/dir3/file3" % self.PARENTCAP)) + d.addCallback(lambda ign: + self.do_cli("mkdir", "%s/dir4" % self.PARENTCAP)) + d.addCallback(lambda ign: + self.do_cli("mkdir", "%s/dir4/emptydir" % self.PARENTCAP)) + + d.addCallback(lambda ign: + self.do_cli("mkdir", "%s/dir5" % self.PARENTCAP)) + def _stash_dircap_5(res): + (rc, out, err) = res + self.failUnlessEqual(rc, 0, str(res)) + self.failUnlessEqual(err, "", str(res)) + self.DIRCAP5 = out.strip() + d.addCallback(_stash_dircap_5) + d.addCallback(lambda ign: + self.do_cli("mkdir", "%s/dir5/dir" % self.PARENTCAP)) + d.addCallback(lambda ign: + self.do_cli("put", source_file_5, "%s/dir5/dir/collide" % self.PARENTCAP)) + + d.addCallback(lambda ign: + self.do_cli("mkdir", "%s/dir6" % self.PARENTCAP)) + def _stash_dircap_6(res): + (rc, out, err) = res + self.failUnlessEqual(rc, 0, str(res)) + self.failUnlessEqual(err, "", str(res)) + self.DIRCAP6 = out.strip() + d.addCallback(_stash_dircap_6) + d.addCallback(lambda ign: + self.do_cli("mkdir", "%s/dir6/dir" % self.PARENTCAP)) + d.addCallback(lambda ign: + self.do_cli("put", source_file_6, "%s/dir6/dir/collide" % self.PARENTCAP)) + + return d + + def check_output(self): + # locate the files and directories created (if any) under to/ + top = os.path.join(self.basedir, "to") + results = set() + for (dirpath, dirnames, filenames) in os.walk(top): + assert dirpath.startswith(top) + here = "/".join(dirpath.split(os.sep)[len(top.split(os.sep))-1:]) + results.add(here+"/") + for fn in filenames: + f = open(os.path.join(dirpath, fn), "rb") + contents = f.read() + f.close() + if contents == self.FILE_CONTENTS: + results.add("%s/%s" % (here, fn)) + elif contents == self.FILE_CONTENTS_5: + results.add("%s/%s=5" % (here, fn)) + elif contents == self.FILE_CONTENTS_6: + results.add("%s/%s=6" % (here, fn)) + return results + + def run_one_case(self, case): + cmd = (case + .replace("$PARENTCAP", self.PARENTCAP) + .replace("$DIRCAP5", self.DIRCAP5) + .replace("$DIRCAP6", self.DIRCAP6) + .replace("$DIRCAP", self.DIRCAP) + .replace("$ALIAS", "ALIAS:") + .replace("$FILECAP", self.FILECAP) + .split()) + target = cmd[-1] + cmd[-1] = os.path.abspath(os.path.join(self.basedir, cmd[-1])) + + # reset + targetdir = os.path.abspath(os.path.join(self.basedir, "to")) + if os.path.exists(targetdir): + shutil.rmtree(targetdir) + os.mkdir(targetdir) + + if target.rstrip("/") == "to/existing-file": + fileutil.write(cmd[-1], "existing file contents\n") + + # The abspath() for cmd[-1] strips a trailing slash, and we want to + # test what happens when it is present. So put it back. + if target.endswith("/"): + cmd[-1] += "/" + + d = self.do_cli(*cmd) + def _check(res): + (rc, out, err) = res + err = err.strip() + if rc == 0: + return self.check_output() + if rc == 1: + self.failUnlessEqual(out, "", str(res)) + if "when copying into a directory, all source files must have names, but" in err: + return set(["E2-DESTNAME"]) + if err == "cannot copy directories without --recursive": + return set(["E4-NEED-R"]) + if err == "cannot copy directory into a file": + return set(["E5-DIRTOFILE"]) + if err == "copying multiple things requires target be a directory": + return set(["E6-MANYONE"]) + if err == "target is not a directory, but has a slash": + return set(["E7-BADSLASH"]) + self.fail("unrecognized error ('%s') %s" % (case, res)) + d.addCallback(_check) + return d + + def do_one_test(self, case, expected): + expected = expected.copy() + printable_expected = ",".join(sorted(expected)) + #print "---", case, ":", printable_expected + + for f in list(expected): + # f is "dir/file" or "dir/sub/file" or "dir/" or "dir/sub/" + # we want all parent directories in the set, with trailing / + pieces = f.rstrip("/").split("/") + for i in range(1,len(pieces)): + parent = "/".join(pieces[:i]) + expected.add(parent+"/") + + d = self.run_one_case(case) + def _dump(got): + ok = "ok" if got == expected else "FAIL" + printable_got = ",".join(sorted(got)) + print "%-31s: got %-19s, want %-19s %s" % (case, printable_got, + printable_expected, ok) + return got + d.addCallback(_dump) + def _check(got): + self.failUnlessEqual(got, expected, case) + #d.addCallback(_check) + return d + + def do_tests(self): + # then we run various forms of "cp [-r] TAHOETHING to[/missing]" + # and see what happens. + d = defer.succeed(None) + print + + for line in COPYOUT_TESTCASES.splitlines(): + if "#" in line: + line = line[:line.find("#")] + line = line.strip() + if not line: + continue + case, expected = line.split(":") + case = case.strip() + expected = set(expected.strip().split(",")) + + d.addCallback(lambda ign, case=case, expected=expected: + self.do_one_test(case, expected)) + + return d + + def test_cp_out(self): + # test copying all sorts of things out of a tahoe filesystem + self.basedir = "cli_cp/CopyOut/cp_out" + self.set_up_grid(num_servers=1) + + d = self.do_setup() + d.addCallback(lambda ign: self.do_tests()) + return d