src/allmydata/test/test_cli_cp.py

   1 import os.path, simplejson, shutil
   2 from twisted.trial import unittest
   3 from twisted.python import usage
   4 from twisted.internet import defer
   5
   6 from allmydata.scripts import cli
   7 from allmydata.util import fileutil
   8 from allmydata.util.encodingutil import (quote_output, get_io_encoding,
   9                                          unicode_to_output, to_str)
  10 from .no_network import GridTestMixin
  11 from .test_cli import CLITestMixin
  12
  13 timeout = 480 # deep_check takes 360s on Zandr's linksys box, others take > 240s
  14
  15 class Cp(GridTestMixin, CLITestMixin, unittest.TestCase):
  16
  17     def test_not_enough_args(self):
  18         o = cli.CpOptions()
  19         self.failUnlessRaises(usage.UsageError,
  20                               o.parseOptions, ["onearg"])
  21
  22     def test_unicode_filename(self):
  23         self.basedir = "cli/Cp/unicode_filename"
  24
  25         fn1 = os.path.join(unicode(self.basedir), u"\u00C4rtonwall")
  26         try:
  27             fn1_arg = fn1.encode(get_io_encoding())
  28             artonwall_arg = u"\u00C4rtonwall".encode(get_io_encoding())
  29         except UnicodeEncodeError:
  30             raise unittest.SkipTest("A non-ASCII command argument could not be encoded on this platform.")
  31
  32         self.skip_if_cannot_represent_filename(fn1)
  33
  34         self.set_up_grid()
  35
  36         DATA1 = "unicode file content"
  37         fileutil.write(fn1, DATA1)
  38
  39         fn2 = os.path.join(self.basedir, "Metallica")
  40         DATA2 = "non-unicode file content"
  41         fileutil.write(fn2, DATA2)
  42
  43         d = self.do_cli("create-alias", "tahoe")
  44
  45         d.addCallback(lambda res: self.do_cli("cp", fn1_arg, "tahoe:"))
  46
  47         d.addCallback(lambda res: self.do_cli("get", "tahoe:" + artonwall_arg))
  48         d.addCallback(lambda (rc,out,err): self.failUnlessReallyEqual(out, DATA1))
  49
  50         d.addCallback(lambda res: self.do_cli("cp", fn2, "tahoe:"))
  51
  52         d.addCallback(lambda res: self.do_cli("get", "tahoe:Metallica"))
  53         d.addCallback(lambda (rc,out,err): self.failUnlessReallyEqual(out, DATA2))
  54
  55         d.addCallback(lambda res: self.do_cli("ls", "tahoe:"))
  56         def _check((rc, out, err)):
  57             try:
  58                 unicode_to_output(u"\u00C4rtonwall")
  59             except UnicodeEncodeError:
  60                 self.failUnlessReallyEqual(rc, 1)
  61                 self.failUnlessReallyEqual(out, "Metallica\n")
  62                 self.failUnlessIn(quote_output(u"\u00C4rtonwall"), err)
  63                 self.failUnlessIn("files whose names could not be converted", err)
  64             else:
  65                 self.failUnlessReallyEqual(rc, 0)
  66                 self.failUnlessReallyEqual(out.decode(get_io_encoding()), u"Metallica\n\u00C4rtonwall\n")
  67                 self.failUnlessReallyEqual(err, "")
  68         d.addCallback(_check)
  69
  70         return d
  71
  72     def test_dangling_symlink_vs_recursion(self):
  73         if not hasattr(os, 'symlink'):
  74             raise unittest.SkipTest("Symlinks are not supported by Python on this platform.")
  75
  76         # cp -r on a directory containing a dangling symlink shouldn't assert
  77         self.basedir = "cli/Cp/dangling_symlink_vs_recursion"
  78         self.set_up_grid()
  79         dn = os.path.join(self.basedir, "dir")
  80         os.mkdir(dn)
  81         fn = os.path.join(dn, "Fakebandica")
  82         ln = os.path.join(dn, "link")
  83         os.symlink(fn, ln)
  84
  85         d = self.do_cli("create-alias", "tahoe")
  86         d.addCallback(lambda res: self.do_cli("cp", "--recursive",
  87                                               dn, "tahoe:"))
  88         return d
  89
  90     def test_copy_using_filecap(self):
  91         self.basedir = "cli/Cp/test_copy_using_filecap"
  92         self.set_up_grid()
  93         outdir = os.path.join(self.basedir, "outdir")
  94         os.mkdir(outdir)
  95         fn1 = os.path.join(self.basedir, "Metallica")
  96         fn2 = os.path.join(outdir, "Not Metallica")
  97         fn3 = os.path.join(outdir, "test2")
  98         DATA1 = "puppies" * 10000
  99         fileutil.write(fn1, DATA1)
 100
 101         d = self.do_cli("create-alias", "tahoe")
 102         d.addCallback(lambda ign: self.do_cli("put", fn1))
 103         def _put_file((rc, out, err)):
 104             self.failUnlessReallyEqual(rc, 0)
 105             self.failUnlessIn("200 OK", err)
 106             # keep track of the filecap
 107             self.filecap = out.strip()
 108         d.addCallback(_put_file)
 109
 110         # Let's try copying this to the disk using the filecap.
 111         d.addCallback(lambda ign: self.do_cli("cp", self.filecap, fn2))
 112         def _copy_file((rc, out, err)):
 113             self.failUnlessReallyEqual(rc, 0)
 114             results = fileutil.read(fn2)
 115             self.failUnlessReallyEqual(results, DATA1)
 116         d.addCallback(_copy_file)
 117
 118         # Test copying a filecap to local dir, which should fail without a
 119         # destination filename (#761).
 120         d.addCallback(lambda ign: self.do_cli("cp", self.filecap, outdir))
 121         def _resp((rc, out, err)):
 122             self.failUnlessReallyEqual(rc, 1)
 123             self.failUnlessIn("when copying into a directory, all source files must have names, but",
 124                               err)
 125             self.failUnlessReallyEqual(out, "")
 126         d.addCallback(_resp)
 127
 128         # Create a directory, linked at tahoe:test .
 129         d.addCallback(lambda ign: self.do_cli("mkdir", "tahoe:test"))
 130         def _get_dir((rc, out, err)):
 131             self.failUnlessReallyEqual(rc, 0)
 132             self.dircap = out.strip()
 133         d.addCallback(_get_dir)
 134
 135         # Upload a file to the directory.
 136         d.addCallback(lambda ign:
 137                       self.do_cli("put", fn1, "tahoe:test/test_file"))
 138         d.addCallback(lambda (rc, out, err): self.failUnlessReallyEqual(rc, 0))
 139
 140         # Copying DIRCAP/filename to a local dir should work, because the
 141         # destination filename can be inferred.
 142         d.addCallback(lambda ign:
 143                       self.do_cli("cp",  self.dircap + "/test_file", outdir))
 144         def _get_resp((rc, out, err)):
 145             self.failUnlessReallyEqual(rc, 0)
 146             results = fileutil.read(os.path.join(outdir, "test_file"))
 147             self.failUnlessReallyEqual(results, DATA1)
 148         d.addCallback(_get_resp)
 149
 150         # ... and to an explicit filename different from the source filename.
 151         d.addCallback(lambda ign:
 152                       self.do_cli("cp",  self.dircap + "/test_file", fn3))
 153         def _get_resp2((rc, out, err)):
 154             self.failUnlessReallyEqual(rc, 0)
 155             results = fileutil.read(fn3)
 156             self.failUnlessReallyEqual(results, DATA1)
 157         d.addCallback(_get_resp2)
 158
 159         # Test that the --verbose option prints correct indices (#1805).
 160         d.addCallback(lambda ign:
 161                       self.do_cli("cp", "--verbose", fn3, self.dircap))
 162         def _test_for_wrong_indices((rc, out, err)):
 163             lines = err.split('\n')
 164             self.failUnlessIn('examining 1 of 1', lines)
 165             self.failUnlessIn('starting copy, 1 files, 1 directories', lines)
 166             self.failIfIn('examining 0 of', err)
 167         d.addCallback(_test_for_wrong_indices)
 168         return d
 169
 170     def test_cp_with_nonexistent_alias(self):
 171         # when invoked with an alias or aliases that don't exist, 'tahoe cp'
 172         # should output a sensible error message rather than a stack trace.
 173         self.basedir = "cli/Cp/cp_with_nonexistent_alias"
 174         self.set_up_grid()
 175         d = self.do_cli("cp", "fake:file1", "fake:file2")
 176         def _check((rc, out, err)):
 177             self.failUnlessReallyEqual(rc, 1)
 178             self.failUnlessIn("error:", err)
 179         d.addCallback(_check)
 180         # 'tahoe cp' actually processes the target argument first, so we need
 181         # to check to make sure that validation extends to the source
 182         # argument.
 183         d.addCallback(lambda ign: self.do_cli("create-alias", "tahoe"))
 184         d.addCallback(lambda ign: self.do_cli("cp", "fake:file1",
 185                                               "tahoe:file2"))
 186         d.addCallback(_check)
 187         return d
 188
 189     def test_unicode_dirnames(self):
 190         self.basedir = "cli/Cp/unicode_dirnames"
 191
 192         fn1 = os.path.join(unicode(self.basedir), u"\u00C4rtonwall")
 193         try:
 194             fn1_arg = fn1.encode(get_io_encoding())
 195             del fn1_arg # hush pyflakes
 196             artonwall_arg = u"\u00C4rtonwall".encode(get_io_encoding())
 197         except UnicodeEncodeError:
 198             raise unittest.SkipTest("A non-ASCII command argument could not be encoded on this platform.")
 199
 200         self.skip_if_cannot_represent_filename(fn1)
 201
 202         self.set_up_grid()
 203
 204         d = self.do_cli("create-alias", "tahoe")
 205         d.addCallback(lambda res: self.do_cli("mkdir", "tahoe:test/" + artonwall_arg))
 206         d.addCallback(lambda res: self.do_cli("cp", "-r", "tahoe:test", "tahoe:test2"))
 207         d.addCallback(lambda res: self.do_cli("ls", "tahoe:test2/test"))
 208         def _check((rc, out, err)):
 209             try:
 210                 unicode_to_output(u"\u00C4rtonwall")
 211             except UnicodeEncodeError:
 212                 self.failUnlessReallyEqual(rc, 1)
 213                 self.failUnlessReallyEqual(out, "")
 214                 self.failUnlessIn(quote_output(u"\u00C4rtonwall"), err)
 215                 self.failUnlessIn("files whose names could not be converted", err)
 216             else:
 217                 self.failUnlessReallyEqual(rc, 0)
 218                 self.failUnlessReallyEqual(out.decode(get_io_encoding()), u"\u00C4rtonwall\n")
 219                 self.failUnlessReallyEqual(err, "")
 220         d.addCallback(_check)
 221
 222         return d
 223
 224     def test_cp_replaces_mutable_file_contents(self):
 225         self.basedir = "cli/Cp/cp_replaces_mutable_file_contents"
 226         self.set_up_grid()
 227
 228         # Write a test file, which we'll copy to the grid.
 229         test_txt_path = os.path.join(self.basedir, "test.txt")
 230         test_txt_contents = "foo bar baz"
 231         f = open(test_txt_path, "w")
 232         f.write(test_txt_contents)
 233         f.close()
 234
 235         d = self.do_cli("create-alias", "tahoe")
 236         d.addCallback(lambda ignored:
 237             self.do_cli("mkdir", "tahoe:test"))
 238         # We have to use 'tahoe put' here because 'tahoe cp' doesn't
 239         # know how to make mutable files at the destination.
 240         d.addCallback(lambda ignored:
 241             self.do_cli("put", "--mutable", test_txt_path, "tahoe:test/test.txt"))
 242         d.addCallback(lambda ignored:
 243             self.do_cli("get", "tahoe:test/test.txt"))
 244         def _check((rc, out, err)):
 245             self.failUnlessEqual(rc, 0)
 246             self.failUnlessEqual(out, test_txt_contents)
 247         d.addCallback(_check)
 248
 249         # We'll do ls --json to get the read uri and write uri for the
 250         # file we've just uploaded.
 251         d.addCallback(lambda ignored:
 252             self.do_cli("ls", "--json", "tahoe:test/test.txt"))
 253         def _get_test_txt_uris((rc, out, err)):
 254             self.failUnlessEqual(rc, 0)
 255             filetype, data = simplejson.loads(out)
 256
 257             self.failUnlessEqual(filetype, "filenode")
 258             self.failUnless(data['mutable'])
 259
 260             self.failUnlessIn("rw_uri", data)
 261             self.rw_uri = to_str(data["rw_uri"])
 262             self.failUnlessIn("ro_uri", data)
 263             self.ro_uri = to_str(data["ro_uri"])
 264         d.addCallback(_get_test_txt_uris)
 265
 266         # Now make a new file to copy in place of test.txt.
 267         new_txt_path = os.path.join(self.basedir, "new.txt")
 268         new_txt_contents = "baz bar foo" * 100000
 269         f = open(new_txt_path, "w")
 270         f.write(new_txt_contents)
 271         f.close()
 272
 273         # Copy the new file on top of the old file.
 274         d.addCallback(lambda ignored:
 275             self.do_cli("cp", new_txt_path, "tahoe:test/test.txt"))
 276
 277         # If we get test.txt now, we should see the new data.
 278         d.addCallback(lambda ignored:
 279             self.do_cli("get", "tahoe:test/test.txt"))
 280         d.addCallback(lambda (rc, out, err):
 281             self.failUnlessEqual(out, new_txt_contents))
 282         # If we get the json of the new file, we should see that the old
 283         # uri is there
 284         d.addCallback(lambda ignored:
 285             self.do_cli("ls", "--json", "tahoe:test/test.txt"))
 286         def _check_json((rc, out, err)):
 287             self.failUnlessEqual(rc, 0)
 288             filetype, data = simplejson.loads(out)
 289
 290             self.failUnlessEqual(filetype, "filenode")
 291             self.failUnless(data['mutable'])
 292
 293             self.failUnlessIn("ro_uri", data)
 294             self.failUnlessEqual(to_str(data["ro_uri"]), self.ro_uri)
 295             self.failUnlessIn("rw_uri", data)
 296             self.failUnlessEqual(to_str(data["rw_uri"]), self.rw_uri)
 297         d.addCallback(_check_json)
 298
 299         # and, finally, doing a GET directly on one of the old uris
 300         # should give us the new contents.
 301         d.addCallback(lambda ignored:
 302             self.do_cli("get", self.rw_uri))
 303         d.addCallback(lambda (rc, out, err):
 304             self.failUnlessEqual(out, new_txt_contents))
 305         # Now copy the old test.txt without an explicit destination
 306         # file. tahoe cp will match it to the existing file and
 307         # overwrite it appropriately.
 308         d.addCallback(lambda ignored:
 309             self.do_cli("cp", test_txt_path, "tahoe:test"))
 310         d.addCallback(lambda ignored:
 311             self.do_cli("get", "tahoe:test/test.txt"))
 312         d.addCallback(lambda (rc, out, err):
 313             self.failUnlessEqual(out, test_txt_contents))
 314         d.addCallback(lambda ignored:
 315             self.do_cli("ls", "--json", "tahoe:test/test.txt"))
 316         d.addCallback(_check_json)
 317         d.addCallback(lambda ignored:
 318             self.do_cli("get", self.rw_uri))
 319         d.addCallback(lambda (rc, out, err):
 320             self.failUnlessEqual(out, test_txt_contents))
 321
 322         # Now we'll make a more complicated directory structure.
 323         # test2/
 324         # test2/mutable1
 325         # test2/mutable2
 326         # test2/imm1
 327         # test2/imm2
 328         imm_test_txt_path = os.path.join(self.basedir, "imm_test.txt")
 329         imm_test_txt_contents = test_txt_contents * 10000
 330         fileutil.write(imm_test_txt_path, imm_test_txt_contents)
 331         d.addCallback(lambda ignored:
 332             self.do_cli("mkdir", "tahoe:test2"))
 333         d.addCallback(lambda ignored:
 334             self.do_cli("put", "--mutable", new_txt_path,
 335                         "tahoe:test2/mutable1"))
 336         d.addCallback(lambda ignored:
 337             self.do_cli("put", "--mutable", new_txt_path,
 338                         "tahoe:test2/mutable2"))
 339         d.addCallback(lambda ignored:
 340             self.do_cli('put', new_txt_path, "tahoe:test2/imm1"))
 341         d.addCallback(lambda ignored:
 342             self.do_cli("put", imm_test_txt_path, "tahoe:test2/imm2"))
 343         d.addCallback(lambda ignored:
 344             self.do_cli("ls", "--json", "tahoe:test2"))
 345         def _process_directory_json((rc, out, err)):
 346             self.failUnlessEqual(rc, 0)
 347
 348             filetype, data = simplejson.loads(out)
 349             self.failUnlessEqual(filetype, "dirnode")
 350             self.failUnless(data['mutable'])
 351             self.failUnlessIn("children", data)
 352             children = data['children']
 353
 354             # Store the URIs for later use.
 355             self.childuris = {}
 356             for k in ["mutable1", "mutable2", "imm1", "imm2"]:
 357                 self.failUnlessIn(k, children)
 358                 childtype, childdata = children[k]
 359                 self.failUnlessEqual(childtype, "filenode")
 360                 if "mutable" in k:
 361                     self.failUnless(childdata['mutable'])
 362                     self.failUnlessIn("rw_uri", childdata)
 363                     uri_key = "rw_uri"
 364                 else:
 365                     self.failIf(childdata['mutable'])
 366                     self.failUnlessIn("ro_uri", childdata)
 367                     uri_key = "ro_uri"
 368                 self.childuris[k] = to_str(childdata[uri_key])
 369         d.addCallback(_process_directory_json)
 370         # Now build a local directory to copy into place, like the following:
 371         # test2/
 372         # test2/mutable1
 373         # test2/mutable2
 374         # test2/imm1
 375         # test2/imm3
 376         def _build_local_directory(ignored):
 377             test2_path = os.path.join(self.basedir, "test2")
 378             fileutil.make_dirs(test2_path)
 379             for fn in ("mutable1", "mutable2", "imm1", "imm3"):
 380                 fileutil.write(os.path.join(test2_path, fn), fn * 1000)
 381             self.test2_path = test2_path
 382         d.addCallback(_build_local_directory)
 383         d.addCallback(lambda ignored:
 384             self.do_cli("cp", "-r", self.test2_path, "tahoe:"))
 385
 386         # We expect that mutable1 and mutable2 are overwritten in-place,
 387         # so they'll retain their URIs but have different content.
 388         def _process_file_json((rc, out, err), fn):
 389             self.failUnlessEqual(rc, 0)
 390             filetype, data = simplejson.loads(out)
 391             self.failUnlessEqual(filetype, "filenode")
 392
 393             if "mutable" in fn:
 394                 self.failUnless(data['mutable'])
 395                 self.failUnlessIn("rw_uri", data)
 396                 self.failUnlessEqual(to_str(data["rw_uri"]), self.childuris[fn])
 397             else:
 398                 self.failIf(data['mutable'])
 399                 self.failUnlessIn("ro_uri", data)
 400                 self.failIfEqual(to_str(data["ro_uri"]), self.childuris[fn])
 401
 402         for fn in ("mutable1", "mutable2"):
 403             d.addCallback(lambda ignored, fn=fn:
 404                 self.do_cli("get", "tahoe:test2/%s" % fn))
 405             d.addCallback(lambda (rc, out, err), fn=fn:
 406                 self.failUnlessEqual(out, fn * 1000))
 407             d.addCallback(lambda ignored, fn=fn:
 408                 self.do_cli("ls", "--json", "tahoe:test2/%s" % fn))
 409             d.addCallback(_process_file_json, fn=fn)
 410
 411         # imm1 should have been replaced, so both its uri and content
 412         # should be different.
 413         d.addCallback(lambda ignored:
 414             self.do_cli("get", "tahoe:test2/imm1"))
 415         d.addCallback(lambda (rc, out, err):
 416             self.failUnlessEqual(out, "imm1" * 1000))
 417         d.addCallback(lambda ignored:
 418             self.do_cli("ls", "--json", "tahoe:test2/imm1"))
 419         d.addCallback(_process_file_json, fn="imm1")
 420
 421         # imm3 should have been created.
 422         d.addCallback(lambda ignored:
 423             self.do_cli("get", "tahoe:test2/imm3"))
 424         d.addCallback(lambda (rc, out, err):
 425             self.failUnlessEqual(out, "imm3" * 1000))
 426
 427         # imm2 should be exactly as we left it, since our newly-copied
 428         # directory didn't contain an imm2 entry.
 429         d.addCallback(lambda ignored:
 430             self.do_cli("get", "tahoe:test2/imm2"))
 431         d.addCallback(lambda (rc, out, err):
 432             self.failUnlessEqual(out, imm_test_txt_contents))
 433         d.addCallback(lambda ignored:
 434             self.do_cli("ls", "--json", "tahoe:test2/imm2"))
 435         def _process_imm2_json((rc, out, err)):
 436             self.failUnlessEqual(rc, 0)
 437             filetype, data = simplejson.loads(out)
 438             self.failUnlessEqual(filetype, "filenode")
 439             self.failIf(data['mutable'])
 440             self.failUnlessIn("ro_uri", data)
 441             self.failUnlessEqual(to_str(data["ro_uri"]), self.childuris["imm2"])
 442         d.addCallback(_process_imm2_json)
 443         return d
 444
 445     def test_cp_overwrite_readonly_mutable_file(self):
 446         # tahoe cp should print an error when asked to overwrite a
 447         # mutable file that it can't overwrite.
 448         self.basedir = "cli/Cp/overwrite_readonly_mutable_file"
 449         self.set_up_grid()
 450
 451         # This is our initial file. We'll link its readcap into the
 452         # tahoe: alias.
 453         test_file_path = os.path.join(self.basedir, "test_file.txt")
 454         test_file_contents = "This is a test file."
 455         fileutil.write(test_file_path, test_file_contents)
 456
 457         # This is our replacement file. We'll try and fail to upload it
 458         # over the readcap that we linked into the tahoe: alias.
 459         replacement_file_path = os.path.join(self.basedir, "replacement.txt")
 460         replacement_file_contents = "These are new contents."
 461         fileutil.write(replacement_file_path, replacement_file_contents)
 462
 463         d = self.do_cli("create-alias", "tahoe:")
 464         d.addCallback(lambda ignored:
 465             self.do_cli("put", "--mutable", test_file_path))
 466         def _get_test_uri((rc, out, err)):
 467             self.failUnlessEqual(rc, 0)
 468             # this should be a write uri
 469             self._test_write_uri = out
 470         d.addCallback(_get_test_uri)
 471         d.addCallback(lambda ignored:
 472             self.do_cli("ls", "--json", self._test_write_uri))
 473         def _process_test_json((rc, out, err)):
 474             self.failUnlessEqual(rc, 0)
 475             filetype, data = simplejson.loads(out)
 476
 477             self.failUnlessEqual(filetype, "filenode")
 478             self.failUnless(data['mutable'])
 479             self.failUnlessIn("ro_uri", data)
 480             self._test_read_uri = to_str(data["ro_uri"])
 481         d.addCallback(_process_test_json)
 482         # Now we'll link the readonly URI into the tahoe: alias.
 483         d.addCallback(lambda ignored:
 484             self.do_cli("ln", self._test_read_uri, "tahoe:test_file.txt"))
 485         d.addCallback(lambda (rc, out, err):
 486             self.failUnlessEqual(rc, 0))
 487         # Let's grab the json of that to make sure that we did it right.
 488         d.addCallback(lambda ignored:
 489             self.do_cli("ls", "--json", "tahoe:"))
 490         def _process_tahoe_json((rc, out, err)):
 491             self.failUnlessEqual(rc, 0)
 492
 493             filetype, data = simplejson.loads(out)
 494             self.failUnlessEqual(filetype, "dirnode")
 495             self.failUnlessIn("children", data)
 496             kiddata = data['children']
 497
 498             self.failUnlessIn("test_file.txt", kiddata)
 499             testtype, testdata = kiddata['test_file.txt']
 500             self.failUnlessEqual(testtype, "filenode")
 501             self.failUnless(testdata['mutable'])
 502             self.failUnlessIn("ro_uri", testdata)
 503             self.failUnlessEqual(to_str(testdata["ro_uri"]), self._test_read_uri)
 504             self.failIfIn("rw_uri", testdata)
 505         d.addCallback(_process_tahoe_json)
 506         # Okay, now we're going to try uploading another mutable file in
 507         # place of that one. We should get an error.
 508         d.addCallback(lambda ignored:
 509             self.do_cli("cp", replacement_file_path, "tahoe:test_file.txt"))
 510         def _check_error_message((rc, out, err)):
 511             self.failUnlessEqual(rc, 1)
 512             self.failUnlessIn("replace or update requested with read-only cap", err)
 513         d.addCallback(_check_error_message)
 514         # Make extra sure that that didn't work.
 515         d.addCallback(lambda ignored:
 516             self.do_cli("get", "tahoe:test_file.txt"))
 517         d.addCallback(lambda (rc, out, err):
 518             self.failUnlessEqual(out, test_file_contents))
 519         d.addCallback(lambda ignored:
 520             self.do_cli("get", self._test_read_uri))
 521         d.addCallback(lambda (rc, out, err):
 522             self.failUnlessEqual(out, test_file_contents))
 523         # Now we'll do it without an explicit destination.
 524         d.addCallback(lambda ignored:
 525             self.do_cli("cp", test_file_path, "tahoe:"))
 526         d.addCallback(_check_error_message)
 527         d.addCallback(lambda ignored:
 528             self.do_cli("get", "tahoe:test_file.txt"))
 529         d.addCallback(lambda (rc, out, err):
 530             self.failUnlessEqual(out, test_file_contents))
 531         d.addCallback(lambda ignored:
 532             self.do_cli("get", self._test_read_uri))
 533         d.addCallback(lambda (rc, out, err):
 534             self.failUnlessEqual(out, test_file_contents))
 535         # Now we'll link a readonly file into a subdirectory.
 536         d.addCallback(lambda ignored:
 537             self.do_cli("mkdir", "tahoe:testdir"))
 538         d.addCallback(lambda (rc, out, err):
 539             self.failUnlessEqual(rc, 0))
 540         d.addCallback(lambda ignored:
 541             self.do_cli("ln", self._test_read_uri, "tahoe:test/file2.txt"))
 542         d.addCallback(lambda (rc, out, err):
 543             self.failUnlessEqual(rc, 0))
 544
 545         test_dir_path = os.path.join(self.basedir, "test")
 546         fileutil.make_dirs(test_dir_path)
 547         for f in ("file1.txt", "file2.txt"):
 548             fileutil.write(os.path.join(test_dir_path, f), f * 10000)
 549
 550         d.addCallback(lambda ignored:
 551             self.do_cli("cp", "-r", test_dir_path, "tahoe:"))
 552         d.addCallback(_check_error_message)
 553         d.addCallback(lambda ignored:
 554             self.do_cli("ls", "--json", "tahoe:test"))
 555         def _got_testdir_json((rc, out, err)):
 556             self.failUnlessEqual(rc, 0)
 557
 558             filetype, data = simplejson.loads(out)
 559             self.failUnlessEqual(filetype, "dirnode")
 560
 561             self.failUnlessIn("children", data)
 562             childdata = data['children']
 563
 564             self.failUnlessIn("file2.txt", childdata)
 565             file2type, file2data = childdata['file2.txt']
 566             self.failUnlessEqual(file2type, "filenode")
 567             self.failUnless(file2data['mutable'])
 568             self.failUnlessIn("ro_uri", file2data)
 569             self.failUnlessEqual(to_str(file2data["ro_uri"]), self._test_read_uri)
 570             self.failIfIn("rw_uri", file2data)
 571         d.addCallback(_got_testdir_json)
 572         return d
 573
 574     def test_cp_verbose(self):
 575         self.basedir = "cli/Cp/cp_verbose"
 576         self.set_up_grid()
 577
 578         # Write two test files, which we'll copy to the grid.
 579         test1_path = os.path.join(self.basedir, "test1")
 580         test2_path = os.path.join(self.basedir, "test2")
 581         fileutil.write(test1_path, "test1")
 582         fileutil.write(test2_path, "test2")
 583
 584         d = self.do_cli("create-alias", "tahoe")
 585         d.addCallback(lambda ign:
 586             self.do_cli("cp", "--verbose", test1_path, test2_path, "tahoe:"))
 587         def _check(res):
 588             (rc, out, err) = res
 589             self.failUnlessEqual(rc, 0, str(res))
 590             self.failUnlessIn("Success: files copied", out, str(res))
 591             self.failUnlessEqual(err, """\
 592 attaching sources to targets, 2 files / 0 dirs in root
 593 targets assigned, 1 dirs, 2 files
 594 starting copy, 2 files, 1 directories
 595 1/2 files, 0/1 directories
 596 2/2 files, 0/1 directories
 597 1/1 directories
 598 """, str(res))
 599         d.addCallback(_check)
 600         return d
 601
 602     def test_cp_copies_dir(self):
 603         # This test ensures that a directory is copied using
 604         # tahoe cp -r. Refer to ticket #712:
 605         # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/712
 606
 607         self.basedir = "cli/Cp/cp_copies_dir"
 608         self.set_up_grid()
 609         subdir = os.path.join(self.basedir, "foo")
 610         os.mkdir(subdir)
 611         test1_path = os.path.join(subdir, "test1")
 612         fileutil.write(test1_path, "test1")
 613
 614         d = self.do_cli("create-alias", "tahoe")
 615         d.addCallback(lambda ign:
 616             self.do_cli("cp", "-r", subdir, "tahoe:"))
 617         d.addCallback(lambda ign:
 618             self.do_cli("ls", "tahoe:"))
 619         def _check(res, item):
 620             (rc, out, err) = res
 621             self.failUnlessEqual(rc, 0)
 622             self.failUnlessEqual(err, "")
 623             self.failUnlessIn(item, out, str(res))
 624         d.addCallback(_check, "foo")
 625         d.addCallback(lambda ign:
 626             self.do_cli("ls", "tahoe:foo/"))
 627         d.addCallback(_check, "test1")
 628
 629         d.addCallback(lambda ign: fileutil.rm_dir(subdir))
 630         d.addCallback(lambda ign: self.do_cli("cp", "-r", "tahoe:foo", self.basedir))
 631         def _check_local_fs(ign):
 632             self.failUnless(os.path.isdir(self.basedir))
 633             self.failUnless(os.path.isfile(test1_path))
 634         d.addCallback(_check_local_fs)
 635         return d
 636
 637     def test_ticket_2027(self):
 638         # This test ensures that tahoe will copy a file from the grid to
 639         # a local directory without a specified file name.
 640         # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/2027
 641         self.basedir = "cli/Cp/cp_verbose"
 642         self.set_up_grid()
 643
 644         # Write a test file, which we'll copy to the grid.
 645         test1_path = os.path.join(self.basedir, "test1")
 646         fileutil.write(test1_path, "test1")
 647
 648         d = self.do_cli("create-alias", "tahoe")
 649         d.addCallback(lambda ign:
 650             self.do_cli("cp", test1_path, "tahoe:"))
 651         d.addCallback(lambda ign:
 652             self.do_cli("cp", "tahoe:test1", self.basedir))
 653         def _check(res):
 654             (rc, out, err) = res
 655             self.failUnlessIn("Success: file copied", out, str(res))
 656         return d
 657
 658 # trailing slash on target *directory* should not matter, test both
 659 # trailing slash on files should cause error
 660
 661 COPYOUT_TESTCASES = """
 662 cp    $FILECAP          to/existing-file : to/existing-file
 663 cp -r $FILECAP          to/existing-file : to/existing-file
 664 cp    $DIRCAP/file $PARENTCAP/dir2/file2 to/existing-file : E6-MANYONE
 665 cp -r $DIRCAP/file $PARENTCAP/dir2/file2 to/existing-file : E6-MANYONE
 666 cp    $DIRCAP           to/existing-file : E4-NEED-R
 667 cp -r $DIRCAP           to/existing-file : E5-DIRTOFILE
 668 cp    $FILECAP $DIRCAP  to/existing-file : E4-NEED-R
 669 cp -r $FILECAP $DIRCAP  to/existing-file : E6-MANYONE
 670
 671 cp    $FILECAP          to/existing-file/ : E7-BADSLASH
 672 cp -r $FILECAP          to/existing-file/ : E7-BADSLASH
 673 cp    $DIRCAP/file $PARENTCAP/dir2/file2 to/existing-file/ : E7-BADSLASH
 674 cp -r $DIRCAP/file $PARENTCAP/dir2/file2 to/existing-file/ : E7-BADSLASH
 675 cp    $DIRCAP           to/existing-file/ : E4-NEED-R
 676 cp -r $DIRCAP           to/existing-file/ : E7-BADSLASH
 677 cp    $FILECAP $DIRCAP  to/existing-file/ : E4-NEED-R
 678 cp -r $FILECAP $DIRCAP  to/existing-file/ : E7-BADSLASH
 679
 680
 681 cp    $FILECAP       to : E2-DESTNAME
 682 cp -r $FILECAP       to : E2-DESTNAME
 683 cp    $DIRCAP/file   to : to/file
 684 cp -r $DIRCAP/file   to : to/file
 685 cp    $PARENTCAP/dir to : E4-NEED-R
 686 cp -r $PARENTCAP/dir to : to/dir/file
 687 cp    $DIRCAP        to : E4-NEED-R
 688 cp -r $DIRCAP        to : to/file
 689 cp    $ALIAS         to : E4-NEED-R
 690 cp -r $ALIAS         to : to/file
 691
 692 cp    $FILECAP       to/ : E2-DESTNAME
 693 cp -r $FILECAP       to/ : E2-DESTNAME
 694 cp    $DIRCAP/file   to/ : to/file
 695 cp -r $DIRCAP/file   to/ : to/file
 696 cp    $PARENTCAP/dir to/ : E4-NEED-R
 697 cp -r $PARENTCAP/dir to/ : to/dir/file
 698 cp    $DIRCAP        to/ : E4-NEED-R
 699 cp -r $DIRCAP        to/ : to/file
 700 cp    $ALIAS         to/ : E4-NEED-R
 701 cp -r $ALIAS         to/ : to/file
 702
 703
 704 cp $DIRCAP/file $PARENTCAP/dir2/file2 to : to/file,to/file2
 705 cp    $DIRCAP/file $FILECAP           to : E2-DESTNAME
 706 cp    $DIRCAP $FILECAP                to : E4-NEED-R
 707 cp -r $DIRCAP $FILECAP                to : E2-DESTNAME
 708       # namedfile, unnameddir, nameddir
 709 cp    $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2          to : E4-NEED-R
 710 cp -r $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2          to : to/file3,to/file,to/dir2/file2
 711       # namedfile, unnameddir, nameddir, unnamedfile
 712 cp    $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 $FILECAP to : E4-NEED-R
 713 cp -r $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 $FILECAP to : E2-DESTNAME
 714
 715 cp $DIRCAP/file $PARENTCAP/dir2/file2 to/ : to/file,to/file2
 716 cp    $DIRCAP/file $FILECAP           to/ : E2-DESTNAME
 717 cp    $DIRCAP $FILECAP                to/ : E4-NEED-R
 718 cp -r $DIRCAP $FILECAP                to/ : E2-DESTNAME
 719       # namedfile, unnameddir, nameddir
 720 cp    $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2          to/ : E4-NEED-R
 721 cp -r $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2          to/ : to/file3,to/file,to/dir2/file2
 722       # namedfile, unnameddir, nameddir, unnamedfile
 723 cp    $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 $FILECAP to/ : E4-NEED-R
 724 cp -r $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 $FILECAP to/ : E2-DESTNAME
 725
 726 # single sources to a missing target: should mkdir or create a file
 727 cp    $FILECAP       to/missing : to/missing
 728 cp -r $FILECAP       to/missing : to/missing
 729 cp    $DIRCAP/file   to/missing : to/missing
 730 cp -r $DIRCAP/file   to/missing : to/missing
 731 cp    $PARENTCAP/dir to/missing : E4-NEED-R
 732 cp -r $PARENTCAP/dir to/missing : to/missing/dir/file
 733 cp    $DIRCAP        to/missing : E4-NEED-R
 734 cp -r $DIRCAP        to/missing : to/missing/file
 735 cp    $ALIAS         to/missing : E4-NEED-R
 736 cp -r $ALIAS         to/missing : to/missing/file
 737
 738 cp    $FILECAP       to/missing/ : E7-BADSLASH
 739 cp -r $FILECAP       to/missing/ : E7-BADSLASH
 740 cp    $DIRCAP/file   to/missing/ : E7-BADSLASH
 741 cp -r $DIRCAP/file   to/missing/ : E7-BADSLASH
 742 cp    $PARENTCAP/dir to/missing/ : E4-NEED-R
 743 cp -r $PARENTCAP/dir to/missing/ : to/missing/dir/file
 744 cp    $DIRCAP        to/missing/ : E4-NEED-R
 745 cp -r $DIRCAP        to/missing/ : to/missing/file
 746 cp    $ALIAS         to/missing/ : E4-NEED-R
 747 cp -r $ALIAS         to/missing/ : to/missing/file
 748
 749 # multiple files to a missing target: should mkdir
 750 cp    $DIRCAP/file $PARENTCAP/dir2/file2 to/missing : to/missing/file,to/missing/file2
 751 cp -r $DIRCAP/file $PARENTCAP/dir2/file2 to/missing : to/missing/file,to/missing/file2
 752
 753 cp    $DIRCAP/file $PARENTCAP/dir2/file2 to/missing/ : to/missing/file,to/missing/file2
 754 cp -r $DIRCAP/file $PARENTCAP/dir2/file2 to/missing/ : to/missing/file,to/missing/file2
 755
 756 # make sure empty directories are copied too
 757 cp -r $PARENTCAP/dir4 to  : to/dir4/emptydir/
 758 cp -r $PARENTCAP/dir4 to/ : to/dir4/emptydir/
 759
 760 # name collisions: ensure files are copied in order
 761 cp -r $PARENTCAP/dir6/dir $PARENTCAP/dir5/dir to : to/dir/collide=5
 762 cp -r $PARENTCAP/dir5/dir $PARENTCAP/dir6/dir to : to/dir/collide=6
 763 cp -r $DIRCAP6 $DIRCAP5 to : to/dir/collide=5
 764 cp -r $DIRCAP5 $DIRCAP6 to : to/dir/collide=6
 765
 766 """
 767
 768 class CopyOut(GridTestMixin, CLITestMixin, unittest.TestCase):
 769     FILE_CONTENTS = "file text"
 770     FILE_CONTENTS_5 = "5"
 771     FILE_CONTENTS_6 = "6"
 772
 773     def do_setup(self):
 774         # first we build a tahoe filesystem that contains:
 775         #  $PARENTCAP
 776         #  $PARENTCAP/dir  == $DIRCAP == alias:
 777         #  $PARENTCAP/dir/file == $FILECAP
 778         #  $PARENTCAP/dir2        (named directory)
 779         #  $PARENTCAP/dir2/file2
 780         #  $PARENTCAP/dir3/file3  (a second named file)
 781         #  $PARENTCAP/dir4
 782         #  $PARENTCAP/dir4/emptydir/ (an empty directory)
 783         #  $PARENTCAP/dir5 == $DIRCAP5
 784         #  $PARENTCAP/dir5/dir/collide (contents are "5")
 785         #  $PARENTCAP/dir6 == $DIRCAP6
 786         #  $PARENTCAP/dir6/dir/collide (contents are "6")
 787
 788         source_file = os.path.join(self.basedir, "file")
 789         fileutil.write(source_file, self.FILE_CONTENTS)
 790         source_file_5 = os.path.join(self.basedir, "file5")
 791         fileutil.write(source_file_5, self.FILE_CONTENTS_5)
 792         source_file_6 = os.path.join(self.basedir, "file6")
 793         fileutil.write(source_file_6, self.FILE_CONTENTS_6)
 794
 795         d = self.do_cli("mkdir")
 796         def _stash_parentdircap(res):
 797             (rc, out, err) = res
 798             self.failUnlessEqual(rc, 0, str(res))
 799             self.failUnlessEqual(err, "", str(res))
 800             self.PARENTCAP = out.strip()
 801             return self.do_cli("mkdir", "%s/dir" % self.PARENTCAP)
 802         d.addCallback(_stash_parentdircap)
 803         def _stash_dircap(res):
 804             (rc, out, err) = res
 805             self.failUnlessEqual(rc, 0, str(res))
 806             self.failUnlessEqual(err, "", str(res))
 807             self.DIRCAP = out.strip()
 808             return self.do_cli("add-alias", "ALIAS", self.DIRCAP)
 809         d.addCallback(_stash_dircap)
 810         d.addCallback(lambda ign:
 811             self.do_cli("put", source_file, "%s/dir/file" % self.PARENTCAP))
 812         def _stash_filecap(res):
 813             (rc, out, err) = res
 814             self.failUnlessEqual(rc, 0, str(res))
 815             self.failUnlessEqual(err.strip(), "201 Created", str(res))
 816             self.FILECAP = out.strip()
 817             assert self.FILECAP.startswith("URI:LIT:")
 818         d.addCallback(_stash_filecap)
 819         d.addCallback(lambda ign:
 820             self.do_cli("mkdir", "%s/dir2" % self.PARENTCAP))
 821         d.addCallback(lambda ign:
 822             self.do_cli("put", source_file, "%s/dir2/file2" % self.PARENTCAP))
 823         d.addCallback(lambda ign:
 824             self.do_cli("mkdir", "%s/dir3" % self.PARENTCAP))
 825         d.addCallback(lambda ign:
 826             self.do_cli("put", source_file, "%s/dir3/file3" % self.PARENTCAP))
 827         d.addCallback(lambda ign:
 828             self.do_cli("mkdir", "%s/dir4" % self.PARENTCAP))
 829         d.addCallback(lambda ign:
 830             self.do_cli("mkdir", "%s/dir4/emptydir" % self.PARENTCAP))
 831
 832         d.addCallback(lambda ign:
 833             self.do_cli("mkdir", "%s/dir5" % self.PARENTCAP))
 834         def _stash_dircap_5(res):
 835             (rc, out, err) = res
 836             self.failUnlessEqual(rc, 0, str(res))
 837             self.failUnlessEqual(err, "", str(res))
 838             self.DIRCAP5 = out.strip()
 839         d.addCallback(_stash_dircap_5)
 840         d.addCallback(lambda ign:
 841             self.do_cli("mkdir", "%s/dir5/dir" % self.PARENTCAP))
 842         d.addCallback(lambda ign:
 843             self.do_cli("put", source_file_5, "%s/dir5/dir/collide" % self.PARENTCAP))
 844
 845         d.addCallback(lambda ign:
 846             self.do_cli("mkdir", "%s/dir6" % self.PARENTCAP))
 847         def _stash_dircap_6(res):
 848             (rc, out, err) = res
 849             self.failUnlessEqual(rc, 0, str(res))
 850             self.failUnlessEqual(err, "", str(res))
 851             self.DIRCAP6 = out.strip()
 852         d.addCallback(_stash_dircap_6)
 853         d.addCallback(lambda ign:
 854             self.do_cli("mkdir", "%s/dir6/dir" % self.PARENTCAP))
 855         d.addCallback(lambda ign:
 856             self.do_cli("put", source_file_6, "%s/dir6/dir/collide" % self.PARENTCAP))
 857
 858         return d
 859
 860     def check_output(self):
 861         # locate the files and directories created (if any) under to/
 862         top = os.path.join(self.basedir, "to")
 863         results = set()
 864         for (dirpath, dirnames, filenames) in os.walk(top):
 865             assert dirpath.startswith(top)
 866             here = "/".join(dirpath.split(os.sep)[len(top.split(os.sep))-1:])
 867             results.add(here+"/")
 868             for fn in filenames:
 869                 f = open(os.path.join(dirpath, fn), "rb")
 870                 contents = f.read()
 871                 f.close()
 872                 if contents == self.FILE_CONTENTS:
 873                     results.add("%s/%s" % (here, fn))
 874                 elif contents == self.FILE_CONTENTS_5:
 875                     results.add("%s/%s=5" % (here, fn))
 876                 elif contents == self.FILE_CONTENTS_6:
 877                     results.add("%s/%s=6" % (here, fn))
 878         return results
 879
 880     def run_one_case(self, case):
 881         cmd = (case
 882                .replace("$PARENTCAP", self.PARENTCAP)
 883                .replace("$DIRCAP5", self.DIRCAP5)
 884                .replace("$DIRCAP6", self.DIRCAP6)
 885                .replace("$DIRCAP", self.DIRCAP)
 886                .replace("$ALIAS", "ALIAS:")
 887                .replace("$FILECAP", self.FILECAP)
 888                .split())
 889         target = cmd[-1]
 890         cmd[-1] = os.path.abspath(os.path.join(self.basedir, cmd[-1]))
 891
 892         # reset
 893         targetdir = os.path.abspath(os.path.join(self.basedir, "to"))
 894         if os.path.exists(targetdir):
 895             shutil.rmtree(targetdir)
 896         os.mkdir(targetdir)
 897
 898         if target.rstrip("/") == "to/existing-file":
 899             fileutil.write(cmd[-1], "existing file contents\n")
 900
 901         # The abspath() for cmd[-1] strips a trailing slash, and we want to
 902         # test what happens when it is present. So put it back.
 903         if target.endswith("/"):
 904             cmd[-1] += "/"
 905
 906         d = self.do_cli(*cmd)
 907         def _check(res):
 908             (rc, out, err) = res
 909             err = err.strip()
 910             if rc == 0:
 911                 return self.check_output()
 912             if rc == 1:
 913                 self.failUnlessEqual(out, "", str(res))
 914                 if "when copying into a directory, all source files must have names, but" in err:
 915                     return set(["E2-DESTNAME"])
 916                 if err == "cannot copy directories without --recursive":
 917                     return set(["E4-NEED-R"])
 918                 if err == "cannot copy directory into a file":
 919                     return set(["E5-DIRTOFILE"])
 920                 if err == "copying multiple things requires target be a directory":
 921                     return set(["E6-MANYONE"])
 922                 if err == "target is not a directory, but ends with a slash":
 923                     return set(["E7-BADSLASH"])
 924             self.fail("unrecognized error ('%s') %s" % (case, res))
 925         d.addCallback(_check)
 926         return d
 927
 928     def do_one_test(self, case, expected):
 929         expected = expected.copy()
 930         printable_expected = ",".join(sorted(expected))
 931         #print "---", case, ":", printable_expected
 932
 933         for f in list(expected):
 934             # f is "dir/file" or "dir/sub/file" or "dir/" or "dir/sub/"
 935             # we want all parent directories in the set, with trailing /
 936             pieces = f.rstrip("/").split("/")
 937             for i in range(1,len(pieces)):
 938                 parent = "/".join(pieces[:i])
 939                 expected.add(parent+"/")
 940
 941         d = self.run_one_case(case)
 942         def _dump(got):
 943             ok = "ok" if got == expected else "FAIL"
 944             printable_got = ",".join(sorted(got))
 945             print "%-31s: got %-19s, want %-19s %s" % (case, printable_got,
 946                                                        printable_expected, ok)
 947             return got
 948         #d.addCallback(_dump)
 949         def _check(got):
 950             self.failUnlessEqual(got, expected, case)
 951         d.addCallback(_check)
 952         return d
 953
 954     def do_tests(self):
 955         # then we run various forms of "cp [-r] TAHOETHING to[/missing]"
 956         # and see what happens.
 957         d = defer.succeed(None)
 958         #print
 959
 960         for line in COPYOUT_TESTCASES.splitlines():
 961             if "#" in line:
 962                 line = line[:line.find("#")]
 963             line = line.strip()
 964             if not line:
 965                 continue
 966             case, expected = line.split(":")
 967             case = case.strip()
 968             expected = set(expected.strip().split(","))
 969
 970             d.addCallback(lambda ign, case=case, expected=expected:
 971                           self.do_one_test(case, expected))
 972
 973         return d
 974
 975     def test_cp_out(self):
 976         # test copying all sorts of things out of a tahoe filesystem
 977         self.basedir = "cli_cp/CopyOut/cp_out"
 978         self.set_up_grid(num_servers=1)
 979
 980         d = self.do_setup()
 981         d.addCallback(lambda ign: self.do_tests())
 982         return d