1 import os.path, simplejson, shutil
2 from twisted.trial import unittest
3 from twisted.python import usage
4 from twisted.internet import defer
6 from allmydata.scripts import cli
7 from allmydata.util import fileutil
8 from allmydata.util.encodingutil import (quote_output, get_io_encoding,
9 unicode_to_output, to_str)
10 from .no_network import GridTestMixin
11 from .test_cli import CLITestMixin
13 timeout = 480 # deep_check takes 360s on Zandr's linksys box, others take > 240s
15 class Cp(GridTestMixin, CLITestMixin, unittest.TestCase):
17 def test_not_enough_args(self):
19 self.failUnlessRaises(usage.UsageError,
20 o.parseOptions, ["onearg"])
22 def test_unicode_filename(self):
23 self.basedir = "cli/Cp/unicode_filename"
25 fn1 = os.path.join(unicode(self.basedir), u"\u00C4rtonwall")
27 fn1_arg = fn1.encode(get_io_encoding())
28 artonwall_arg = u"\u00C4rtonwall".encode(get_io_encoding())
29 except UnicodeEncodeError:
30 raise unittest.SkipTest("A non-ASCII command argument could not be encoded on this platform.")
32 self.skip_if_cannot_represent_filename(fn1)
36 DATA1 = "unicode file content"
37 fileutil.write(fn1, DATA1)
39 fn2 = os.path.join(self.basedir, "Metallica")
40 DATA2 = "non-unicode file content"
41 fileutil.write(fn2, DATA2)
43 d = self.do_cli("create-alias", "tahoe")
45 d.addCallback(lambda res: self.do_cli("cp", fn1_arg, "tahoe:"))
47 d.addCallback(lambda res: self.do_cli("get", "tahoe:" + artonwall_arg))
48 d.addCallback(lambda (rc,out,err): self.failUnlessReallyEqual(out, DATA1))
50 d.addCallback(lambda res: self.do_cli("cp", fn2, "tahoe:"))
52 d.addCallback(lambda res: self.do_cli("get", "tahoe:Metallica"))
53 d.addCallback(lambda (rc,out,err): self.failUnlessReallyEqual(out, DATA2))
55 d.addCallback(lambda res: self.do_cli("ls", "tahoe:"))
56 def _check((rc, out, err)):
58 unicode_to_output(u"\u00C4rtonwall")
59 except UnicodeEncodeError:
60 self.failUnlessReallyEqual(rc, 1)
61 self.failUnlessReallyEqual(out, "Metallica\n")
62 self.failUnlessIn(quote_output(u"\u00C4rtonwall"), err)
63 self.failUnlessIn("files whose names could not be converted", err)
65 self.failUnlessReallyEqual(rc, 0)
66 self.failUnlessReallyEqual(out.decode(get_io_encoding()), u"Metallica\n\u00C4rtonwall\n")
67 self.failUnlessReallyEqual(err, "")
72 def test_dangling_symlink_vs_recursion(self):
73 if not hasattr(os, 'symlink'):
74 raise unittest.SkipTest("Symlinks are not supported by Python on this platform.")
76 # cp -r on a directory containing a dangling symlink shouldn't assert
77 self.basedir = "cli/Cp/dangling_symlink_vs_recursion"
79 dn = os.path.join(self.basedir, "dir")
81 fn = os.path.join(dn, "Fakebandica")
82 ln = os.path.join(dn, "link")
85 d = self.do_cli("create-alias", "tahoe")
86 d.addCallback(lambda res: self.do_cli("cp", "--recursive",
90 def test_copy_using_filecap(self):
91 self.basedir = "cli/Cp/test_copy_using_filecap"
93 outdir = os.path.join(self.basedir, "outdir")
95 fn1 = os.path.join(self.basedir, "Metallica")
96 fn2 = os.path.join(outdir, "Not Metallica")
97 fn3 = os.path.join(outdir, "test2")
98 DATA1 = "puppies" * 10000
99 fileutil.write(fn1, DATA1)
101 d = self.do_cli("create-alias", "tahoe")
102 d.addCallback(lambda ign: self.do_cli("put", fn1))
103 def _put_file((rc, out, err)):
104 self.failUnlessReallyEqual(rc, 0)
105 self.failUnlessIn("200 OK", err)
106 # keep track of the filecap
107 self.filecap = out.strip()
108 d.addCallback(_put_file)
110 # Let's try copying this to the disk using the filecap.
111 d.addCallback(lambda ign: self.do_cli("cp", self.filecap, fn2))
112 def _copy_file((rc, out, err)):
113 self.failUnlessReallyEqual(rc, 0)
114 results = fileutil.read(fn2)
115 self.failUnlessReallyEqual(results, DATA1)
116 d.addCallback(_copy_file)
118 # Test copying a filecap to local dir, which should fail without a
119 # destination filename (#761).
120 d.addCallback(lambda ign: self.do_cli("cp", self.filecap, outdir))
121 def _resp((rc, out, err)):
122 self.failUnlessReallyEqual(rc, 1)
123 self.failUnlessIn("when copying into a directory, all source files must have names, but",
125 self.failUnlessReallyEqual(out, "")
128 # Create a directory, linked at tahoe:test .
129 d.addCallback(lambda ign: self.do_cli("mkdir", "tahoe:test"))
130 def _get_dir((rc, out, err)):
131 self.failUnlessReallyEqual(rc, 0)
132 self.dircap = out.strip()
133 d.addCallback(_get_dir)
135 # Upload a file to the directory.
136 d.addCallback(lambda ign:
137 self.do_cli("put", fn1, "tahoe:test/test_file"))
138 d.addCallback(lambda (rc, out, err): self.failUnlessReallyEqual(rc, 0))
140 # Copying DIRCAP/filename to a local dir should work, because the
141 # destination filename can be inferred.
142 d.addCallback(lambda ign:
143 self.do_cli("cp", self.dircap + "/test_file", outdir))
144 def _get_resp((rc, out, err)):
145 self.failUnlessReallyEqual(rc, 0)
146 results = fileutil.read(os.path.join(outdir, "test_file"))
147 self.failUnlessReallyEqual(results, DATA1)
148 d.addCallback(_get_resp)
150 # ... and to an explicit filename different from the source filename.
151 d.addCallback(lambda ign:
152 self.do_cli("cp", self.dircap + "/test_file", fn3))
153 def _get_resp2((rc, out, err)):
154 self.failUnlessReallyEqual(rc, 0)
155 results = fileutil.read(fn3)
156 self.failUnlessReallyEqual(results, DATA1)
157 d.addCallback(_get_resp2)
159 # Test that the --verbose option prints correct indices (#1805).
160 d.addCallback(lambda ign:
161 self.do_cli("cp", "--verbose", fn3, self.dircap))
162 def _test_for_wrong_indices((rc, out, err)):
163 lines = err.split('\n')
164 self.failUnlessIn('examining 1 of 1', lines)
165 self.failUnlessIn('starting copy, 1 files, 1 directories', lines)
166 self.failIfIn('examining 0 of', err)
167 d.addCallback(_test_for_wrong_indices)
170 def test_cp_with_nonexistent_alias(self):
171 # when invoked with an alias or aliases that don't exist, 'tahoe cp'
172 # should output a sensible error message rather than a stack trace.
173 self.basedir = "cli/Cp/cp_with_nonexistent_alias"
175 d = self.do_cli("cp", "fake:file1", "fake:file2")
176 def _check((rc, out, err)):
177 self.failUnlessReallyEqual(rc, 1)
178 self.failUnlessIn("error:", err)
179 d.addCallback(_check)
180 # 'tahoe cp' actually processes the target argument first, so we need
181 # to check to make sure that validation extends to the source
183 d.addCallback(lambda ign: self.do_cli("create-alias", "tahoe"))
184 d.addCallback(lambda ign: self.do_cli("cp", "fake:file1",
186 d.addCallback(_check)
189 def test_unicode_dirnames(self):
190 self.basedir = "cli/Cp/unicode_dirnames"
192 fn1 = os.path.join(unicode(self.basedir), u"\u00C4rtonwall")
194 fn1_arg = fn1.encode(get_io_encoding())
195 del fn1_arg # hush pyflakes
196 artonwall_arg = u"\u00C4rtonwall".encode(get_io_encoding())
197 except UnicodeEncodeError:
198 raise unittest.SkipTest("A non-ASCII command argument could not be encoded on this platform.")
200 self.skip_if_cannot_represent_filename(fn1)
204 d = self.do_cli("create-alias", "tahoe")
205 d.addCallback(lambda res: self.do_cli("mkdir", "tahoe:test/" + artonwall_arg))
206 d.addCallback(lambda res: self.do_cli("cp", "-r", "tahoe:test", "tahoe:test2"))
207 d.addCallback(lambda res: self.do_cli("ls", "tahoe:test2/test"))
208 def _check((rc, out, err)):
210 unicode_to_output(u"\u00C4rtonwall")
211 except UnicodeEncodeError:
212 self.failUnlessReallyEqual(rc, 1)
213 self.failUnlessReallyEqual(out, "")
214 self.failUnlessIn(quote_output(u"\u00C4rtonwall"), err)
215 self.failUnlessIn("files whose names could not be converted", err)
217 self.failUnlessReallyEqual(rc, 0)
218 self.failUnlessReallyEqual(out.decode(get_io_encoding()), u"\u00C4rtonwall\n")
219 self.failUnlessReallyEqual(err, "")
220 d.addCallback(_check)
224 def test_cp_replaces_mutable_file_contents(self):
225 self.basedir = "cli/Cp/cp_replaces_mutable_file_contents"
228 # Write a test file, which we'll copy to the grid.
229 test_txt_path = os.path.join(self.basedir, "test.txt")
230 test_txt_contents = "foo bar baz"
231 f = open(test_txt_path, "w")
232 f.write(test_txt_contents)
235 d = self.do_cli("create-alias", "tahoe")
236 d.addCallback(lambda ignored:
237 self.do_cli("mkdir", "tahoe:test"))
238 # We have to use 'tahoe put' here because 'tahoe cp' doesn't
239 # know how to make mutable files at the destination.
240 d.addCallback(lambda ignored:
241 self.do_cli("put", "--mutable", test_txt_path, "tahoe:test/test.txt"))
242 d.addCallback(lambda ignored:
243 self.do_cli("get", "tahoe:test/test.txt"))
244 def _check((rc, out, err)):
245 self.failUnlessEqual(rc, 0)
246 self.failUnlessEqual(out, test_txt_contents)
247 d.addCallback(_check)
249 # We'll do ls --json to get the read uri and write uri for the
250 # file we've just uploaded.
251 d.addCallback(lambda ignored:
252 self.do_cli("ls", "--json", "tahoe:test/test.txt"))
253 def _get_test_txt_uris((rc, out, err)):
254 self.failUnlessEqual(rc, 0)
255 filetype, data = simplejson.loads(out)
257 self.failUnlessEqual(filetype, "filenode")
258 self.failUnless(data['mutable'])
260 self.failUnlessIn("rw_uri", data)
261 self.rw_uri = to_str(data["rw_uri"])
262 self.failUnlessIn("ro_uri", data)
263 self.ro_uri = to_str(data["ro_uri"])
264 d.addCallback(_get_test_txt_uris)
266 # Now make a new file to copy in place of test.txt.
267 new_txt_path = os.path.join(self.basedir, "new.txt")
268 new_txt_contents = "baz bar foo" * 100000
269 f = open(new_txt_path, "w")
270 f.write(new_txt_contents)
273 # Copy the new file on top of the old file.
274 d.addCallback(lambda ignored:
275 self.do_cli("cp", new_txt_path, "tahoe:test/test.txt"))
277 # If we get test.txt now, we should see the new data.
278 d.addCallback(lambda ignored:
279 self.do_cli("get", "tahoe:test/test.txt"))
280 d.addCallback(lambda (rc, out, err):
281 self.failUnlessEqual(out, new_txt_contents))
282 # If we get the json of the new file, we should see that the old
284 d.addCallback(lambda ignored:
285 self.do_cli("ls", "--json", "tahoe:test/test.txt"))
286 def _check_json((rc, out, err)):
287 self.failUnlessEqual(rc, 0)
288 filetype, data = simplejson.loads(out)
290 self.failUnlessEqual(filetype, "filenode")
291 self.failUnless(data['mutable'])
293 self.failUnlessIn("ro_uri", data)
294 self.failUnlessEqual(to_str(data["ro_uri"]), self.ro_uri)
295 self.failUnlessIn("rw_uri", data)
296 self.failUnlessEqual(to_str(data["rw_uri"]), self.rw_uri)
297 d.addCallback(_check_json)
299 # and, finally, doing a GET directly on one of the old uris
300 # should give us the new contents.
301 d.addCallback(lambda ignored:
302 self.do_cli("get", self.rw_uri))
303 d.addCallback(lambda (rc, out, err):
304 self.failUnlessEqual(out, new_txt_contents))
305 # Now copy the old test.txt without an explicit destination
306 # file. tahoe cp will match it to the existing file and
307 # overwrite it appropriately.
308 d.addCallback(lambda ignored:
309 self.do_cli("cp", test_txt_path, "tahoe:test"))
310 d.addCallback(lambda ignored:
311 self.do_cli("get", "tahoe:test/test.txt"))
312 d.addCallback(lambda (rc, out, err):
313 self.failUnlessEqual(out, test_txt_contents))
314 d.addCallback(lambda ignored:
315 self.do_cli("ls", "--json", "tahoe:test/test.txt"))
316 d.addCallback(_check_json)
317 d.addCallback(lambda ignored:
318 self.do_cli("get", self.rw_uri))
319 d.addCallback(lambda (rc, out, err):
320 self.failUnlessEqual(out, test_txt_contents))
322 # Now we'll make a more complicated directory structure.
328 imm_test_txt_path = os.path.join(self.basedir, "imm_test.txt")
329 imm_test_txt_contents = test_txt_contents * 10000
330 fileutil.write(imm_test_txt_path, imm_test_txt_contents)
331 d.addCallback(lambda ignored:
332 self.do_cli("mkdir", "tahoe:test2"))
333 d.addCallback(lambda ignored:
334 self.do_cli("put", "--mutable", new_txt_path,
335 "tahoe:test2/mutable1"))
336 d.addCallback(lambda ignored:
337 self.do_cli("put", "--mutable", new_txt_path,
338 "tahoe:test2/mutable2"))
339 d.addCallback(lambda ignored:
340 self.do_cli('put', new_txt_path, "tahoe:test2/imm1"))
341 d.addCallback(lambda ignored:
342 self.do_cli("put", imm_test_txt_path, "tahoe:test2/imm2"))
343 d.addCallback(lambda ignored:
344 self.do_cli("ls", "--json", "tahoe:test2"))
345 def _process_directory_json((rc, out, err)):
346 self.failUnlessEqual(rc, 0)
348 filetype, data = simplejson.loads(out)
349 self.failUnlessEqual(filetype, "dirnode")
350 self.failUnless(data['mutable'])
351 self.failUnlessIn("children", data)
352 children = data['children']
354 # Store the URIs for later use.
356 for k in ["mutable1", "mutable2", "imm1", "imm2"]:
357 self.failUnlessIn(k, children)
358 childtype, childdata = children[k]
359 self.failUnlessEqual(childtype, "filenode")
361 self.failUnless(childdata['mutable'])
362 self.failUnlessIn("rw_uri", childdata)
365 self.failIf(childdata['mutable'])
366 self.failUnlessIn("ro_uri", childdata)
368 self.childuris[k] = to_str(childdata[uri_key])
369 d.addCallback(_process_directory_json)
370 # Now build a local directory to copy into place, like the following:
376 def _build_local_directory(ignored):
377 test2_path = os.path.join(self.basedir, "test2")
378 fileutil.make_dirs(test2_path)
379 for fn in ("mutable1", "mutable2", "imm1", "imm3"):
380 fileutil.write(os.path.join(test2_path, fn), fn * 1000)
381 self.test2_path = test2_path
382 d.addCallback(_build_local_directory)
383 d.addCallback(lambda ignored:
384 self.do_cli("cp", "-r", self.test2_path, "tahoe:"))
386 # We expect that mutable1 and mutable2 are overwritten in-place,
387 # so they'll retain their URIs but have different content.
388 def _process_file_json((rc, out, err), fn):
389 self.failUnlessEqual(rc, 0)
390 filetype, data = simplejson.loads(out)
391 self.failUnlessEqual(filetype, "filenode")
394 self.failUnless(data['mutable'])
395 self.failUnlessIn("rw_uri", data)
396 self.failUnlessEqual(to_str(data["rw_uri"]), self.childuris[fn])
398 self.failIf(data['mutable'])
399 self.failUnlessIn("ro_uri", data)
400 self.failIfEqual(to_str(data["ro_uri"]), self.childuris[fn])
402 for fn in ("mutable1", "mutable2"):
403 d.addCallback(lambda ignored, fn=fn:
404 self.do_cli("get", "tahoe:test2/%s" % fn))
405 d.addCallback(lambda (rc, out, err), fn=fn:
406 self.failUnlessEqual(out, fn * 1000))
407 d.addCallback(lambda ignored, fn=fn:
408 self.do_cli("ls", "--json", "tahoe:test2/%s" % fn))
409 d.addCallback(_process_file_json, fn=fn)
411 # imm1 should have been replaced, so both its uri and content
412 # should be different.
413 d.addCallback(lambda ignored:
414 self.do_cli("get", "tahoe:test2/imm1"))
415 d.addCallback(lambda (rc, out, err):
416 self.failUnlessEqual(out, "imm1" * 1000))
417 d.addCallback(lambda ignored:
418 self.do_cli("ls", "--json", "tahoe:test2/imm1"))
419 d.addCallback(_process_file_json, fn="imm1")
421 # imm3 should have been created.
422 d.addCallback(lambda ignored:
423 self.do_cli("get", "tahoe:test2/imm3"))
424 d.addCallback(lambda (rc, out, err):
425 self.failUnlessEqual(out, "imm3" * 1000))
427 # imm2 should be exactly as we left it, since our newly-copied
428 # directory didn't contain an imm2 entry.
429 d.addCallback(lambda ignored:
430 self.do_cli("get", "tahoe:test2/imm2"))
431 d.addCallback(lambda (rc, out, err):
432 self.failUnlessEqual(out, imm_test_txt_contents))
433 d.addCallback(lambda ignored:
434 self.do_cli("ls", "--json", "tahoe:test2/imm2"))
435 def _process_imm2_json((rc, out, err)):
436 self.failUnlessEqual(rc, 0)
437 filetype, data = simplejson.loads(out)
438 self.failUnlessEqual(filetype, "filenode")
439 self.failIf(data['mutable'])
440 self.failUnlessIn("ro_uri", data)
441 self.failUnlessEqual(to_str(data["ro_uri"]), self.childuris["imm2"])
442 d.addCallback(_process_imm2_json)
445 def test_cp_overwrite_readonly_mutable_file(self):
446 # tahoe cp should print an error when asked to overwrite a
447 # mutable file that it can't overwrite.
448 self.basedir = "cli/Cp/overwrite_readonly_mutable_file"
451 # This is our initial file. We'll link its readcap into the
453 test_file_path = os.path.join(self.basedir, "test_file.txt")
454 test_file_contents = "This is a test file."
455 fileutil.write(test_file_path, test_file_contents)
457 # This is our replacement file. We'll try and fail to upload it
458 # over the readcap that we linked into the tahoe: alias.
459 replacement_file_path = os.path.join(self.basedir, "replacement.txt")
460 replacement_file_contents = "These are new contents."
461 fileutil.write(replacement_file_path, replacement_file_contents)
463 d = self.do_cli("create-alias", "tahoe:")
464 d.addCallback(lambda ignored:
465 self.do_cli("put", "--mutable", test_file_path))
466 def _get_test_uri((rc, out, err)):
467 self.failUnlessEqual(rc, 0)
468 # this should be a write uri
469 self._test_write_uri = out
470 d.addCallback(_get_test_uri)
471 d.addCallback(lambda ignored:
472 self.do_cli("ls", "--json", self._test_write_uri))
473 def _process_test_json((rc, out, err)):
474 self.failUnlessEqual(rc, 0)
475 filetype, data = simplejson.loads(out)
477 self.failUnlessEqual(filetype, "filenode")
478 self.failUnless(data['mutable'])
479 self.failUnlessIn("ro_uri", data)
480 self._test_read_uri = to_str(data["ro_uri"])
481 d.addCallback(_process_test_json)
482 # Now we'll link the readonly URI into the tahoe: alias.
483 d.addCallback(lambda ignored:
484 self.do_cli("ln", self._test_read_uri, "tahoe:test_file.txt"))
485 d.addCallback(lambda (rc, out, err):
486 self.failUnlessEqual(rc, 0))
487 # Let's grab the json of that to make sure that we did it right.
488 d.addCallback(lambda ignored:
489 self.do_cli("ls", "--json", "tahoe:"))
490 def _process_tahoe_json((rc, out, err)):
491 self.failUnlessEqual(rc, 0)
493 filetype, data = simplejson.loads(out)
494 self.failUnlessEqual(filetype, "dirnode")
495 self.failUnlessIn("children", data)
496 kiddata = data['children']
498 self.failUnlessIn("test_file.txt", kiddata)
499 testtype, testdata = kiddata['test_file.txt']
500 self.failUnlessEqual(testtype, "filenode")
501 self.failUnless(testdata['mutable'])
502 self.failUnlessIn("ro_uri", testdata)
503 self.failUnlessEqual(to_str(testdata["ro_uri"]), self._test_read_uri)
504 self.failIfIn("rw_uri", testdata)
505 d.addCallback(_process_tahoe_json)
506 # Okay, now we're going to try uploading another mutable file in
507 # place of that one. We should get an error.
508 d.addCallback(lambda ignored:
509 self.do_cli("cp", replacement_file_path, "tahoe:test_file.txt"))
510 def _check_error_message((rc, out, err)):
511 self.failUnlessEqual(rc, 1)
512 self.failUnlessIn("replace or update requested with read-only cap", err)
513 d.addCallback(_check_error_message)
514 # Make extra sure that that didn't work.
515 d.addCallback(lambda ignored:
516 self.do_cli("get", "tahoe:test_file.txt"))
517 d.addCallback(lambda (rc, out, err):
518 self.failUnlessEqual(out, test_file_contents))
519 d.addCallback(lambda ignored:
520 self.do_cli("get", self._test_read_uri))
521 d.addCallback(lambda (rc, out, err):
522 self.failUnlessEqual(out, test_file_contents))
523 # Now we'll do it without an explicit destination.
524 d.addCallback(lambda ignored:
525 self.do_cli("cp", test_file_path, "tahoe:"))
526 d.addCallback(_check_error_message)
527 d.addCallback(lambda ignored:
528 self.do_cli("get", "tahoe:test_file.txt"))
529 d.addCallback(lambda (rc, out, err):
530 self.failUnlessEqual(out, test_file_contents))
531 d.addCallback(lambda ignored:
532 self.do_cli("get", self._test_read_uri))
533 d.addCallback(lambda (rc, out, err):
534 self.failUnlessEqual(out, test_file_contents))
535 # Now we'll link a readonly file into a subdirectory.
536 d.addCallback(lambda ignored:
537 self.do_cli("mkdir", "tahoe:testdir"))
538 d.addCallback(lambda (rc, out, err):
539 self.failUnlessEqual(rc, 0))
540 d.addCallback(lambda ignored:
541 self.do_cli("ln", self._test_read_uri, "tahoe:test/file2.txt"))
542 d.addCallback(lambda (rc, out, err):
543 self.failUnlessEqual(rc, 0))
545 test_dir_path = os.path.join(self.basedir, "test")
546 fileutil.make_dirs(test_dir_path)
547 for f in ("file1.txt", "file2.txt"):
548 fileutil.write(os.path.join(test_dir_path, f), f * 10000)
550 d.addCallback(lambda ignored:
551 self.do_cli("cp", "-r", test_dir_path, "tahoe:"))
552 d.addCallback(_check_error_message)
553 d.addCallback(lambda ignored:
554 self.do_cli("ls", "--json", "tahoe:test"))
555 def _got_testdir_json((rc, out, err)):
556 self.failUnlessEqual(rc, 0)
558 filetype, data = simplejson.loads(out)
559 self.failUnlessEqual(filetype, "dirnode")
561 self.failUnlessIn("children", data)
562 childdata = data['children']
564 self.failUnlessIn("file2.txt", childdata)
565 file2type, file2data = childdata['file2.txt']
566 self.failUnlessEqual(file2type, "filenode")
567 self.failUnless(file2data['mutable'])
568 self.failUnlessIn("ro_uri", file2data)
569 self.failUnlessEqual(to_str(file2data["ro_uri"]), self._test_read_uri)
570 self.failIfIn("rw_uri", file2data)
571 d.addCallback(_got_testdir_json)
574 def test_cp_verbose(self):
575 self.basedir = "cli/Cp/cp_verbose"
578 # Write two test files, which we'll copy to the grid.
579 test1_path = os.path.join(self.basedir, "test1")
580 test2_path = os.path.join(self.basedir, "test2")
581 fileutil.write(test1_path, "test1")
582 fileutil.write(test2_path, "test2")
584 d = self.do_cli("create-alias", "tahoe")
585 d.addCallback(lambda ign:
586 self.do_cli("cp", "--verbose", test1_path, test2_path, "tahoe:"))
589 self.failUnlessEqual(rc, 0, str(res))
590 self.failUnlessIn("Success: files copied", out, str(res))
591 self.failUnlessEqual(err, """\
592 attaching sources to targets, 2 files / 0 dirs in root
593 targets assigned, 1 dirs, 2 files
594 starting copy, 2 files, 1 directories
595 1/2 files, 0/1 directories
596 2/2 files, 0/1 directories
599 d.addCallback(_check)
602 def test_cp_copies_dir(self):
603 # This test ensures that a directory is copied using
604 # tahoe cp -r. Refer to ticket #712:
605 # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/712
607 self.basedir = "cli/Cp/cp_copies_dir"
609 subdir = os.path.join(self.basedir, "foo")
611 test1_path = os.path.join(subdir, "test1")
612 fileutil.write(test1_path, "test1")
614 d = self.do_cli("create-alias", "tahoe")
615 d.addCallback(lambda ign:
616 self.do_cli("cp", "-r", subdir, "tahoe:"))
617 d.addCallback(lambda ign:
618 self.do_cli("ls", "tahoe:"))
619 def _check(res, item):
621 self.failUnlessEqual(rc, 0)
622 self.failUnlessEqual(err, "")
623 self.failUnlessIn(item, out, str(res))
624 d.addCallback(_check, "foo")
625 d.addCallback(lambda ign:
626 self.do_cli("ls", "tahoe:foo/"))
627 d.addCallback(_check, "test1")
629 d.addCallback(lambda ign: fileutil.rm_dir(subdir))
630 d.addCallback(lambda ign: self.do_cli("cp", "-r", "tahoe:foo", self.basedir))
631 def _check_local_fs(ign):
632 self.failUnless(os.path.isdir(self.basedir))
633 self.failUnless(os.path.isfile(test1_path))
634 d.addCallback(_check_local_fs)
637 def test_ticket_2027(self):
638 # This test ensures that tahoe will copy a file from the grid to
639 # a local directory without a specified file name.
640 # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/2027
641 self.basedir = "cli/Cp/cp_verbose"
644 # Write a test file, which we'll copy to the grid.
645 test1_path = os.path.join(self.basedir, "test1")
646 fileutil.write(test1_path, "test1")
648 d = self.do_cli("create-alias", "tahoe")
649 d.addCallback(lambda ign:
650 self.do_cli("cp", test1_path, "tahoe:"))
651 d.addCallback(lambda ign:
652 self.do_cli("cp", "tahoe:test1", self.basedir))
655 self.failUnlessIn("Success: file copied", out, str(res))
658 # these test cases come from ticket #2329 comment 40
659 # trailing slash on target *directory* should not matter, test both
660 # trailing slash on files should cause error
662 COPYOUT_TESTCASES = """
663 cp $FILECAP to/existing-file : to/existing-file
664 cp -r $FILECAP to/existing-file : to/existing-file
665 cp $DIRCAP/file $PARENTCAP/dir2/file2 to/existing-file : E6-MANYONE
666 cp -r $DIRCAP/file $PARENTCAP/dir2/file2 to/existing-file : E6-MANYONE
667 cp $DIRCAP to/existing-file : E4-NEED-R
668 cp -r $DIRCAP to/existing-file : E5-DIRTOFILE
669 cp $FILECAP $DIRCAP to/existing-file : E4-NEED-R
670 cp -r $FILECAP $DIRCAP to/existing-file : E6-MANYONE
672 cp $FILECAP to/existing-file/ : E7-BADSLASH
673 cp -r $FILECAP to/existing-file/ : E7-BADSLASH
674 cp $DIRCAP/file $PARENTCAP/dir2/file2 to/existing-file/ : E7-BADSLASH
675 cp -r $DIRCAP/file $PARENTCAP/dir2/file2 to/existing-file/ : E7-BADSLASH
676 cp $DIRCAP to/existing-file/ : E4-NEED-R
677 cp -r $DIRCAP to/existing-file/ : E7-BADSLASH
678 cp $FILECAP $DIRCAP to/existing-file/ : E4-NEED-R
679 cp -r $FILECAP $DIRCAP to/existing-file/ : E7-BADSLASH
681 # single source to a (present) target directory
682 cp $FILECAP to : E2-DESTNAME
683 cp -r $FILECAP to : E2-DESTNAME
684 cp $DIRCAP/file to : to/file
685 cp -r $DIRCAP/file to : to/file
686 cp $PARENTCAP/dir to : E4-NEED-R
687 cp -r $PARENTCAP/dir to : to/dir/file
688 cp $DIRCAP to : E4-NEED-R
689 cp -r $DIRCAP to : to/file
690 cp $DIRALIAS to : E4-NEED-R
691 cp -r $DIRALIAS to : to/file
693 cp $FILECAP to/ : E2-DESTNAME
694 cp -r $FILECAP to/ : E2-DESTNAME
695 cp $DIRCAP/file to/ : to/file
696 cp -r $DIRCAP/file to/ : to/file
697 cp $PARENTCAP/dir to/ : E4-NEED-R
698 cp -r $PARENTCAP/dir to/ : to/dir/file
699 cp $DIRCAP to/ : E4-NEED-R
700 cp -r $DIRCAP to/ : to/file
701 cp $DIRALIAS to/ : E4-NEED-R
702 cp -r $DIRALIAS to/ : to/file
704 # multiple sources to a (present) target directory
705 cp $DIRCAP/file $PARENTCAP/dir2/file2 to : to/file,to/file2
706 cp $DIRCAP/file $FILECAP to : E2-DESTNAME
707 cp $DIRCAP $FILECAP to : E4-NEED-R
708 cp -r $DIRCAP $FILECAP to : E2-DESTNAME
709 # namedfile, unnameddir, nameddir
710 cp $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 to : E4-NEED-R
711 cp -r $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 to : to/file3,to/file,to/dir2/file2
712 # namedfile, unnameddir, nameddir, unnamedfile
713 cp $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 $FILECAP to : E4-NEED-R
714 cp -r $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 $FILECAP to : E2-DESTNAME
716 cp $DIRCAP/file $PARENTCAP/dir2/file2 to/ : to/file,to/file2
717 cp $DIRCAP/file $FILECAP to/ : E2-DESTNAME
718 cp $DIRCAP $FILECAP to/ : E4-NEED-R
719 cp -r $DIRCAP $FILECAP to/ : E2-DESTNAME
720 # namedfile, unnameddir, nameddir
721 cp $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 to/ : E4-NEED-R
722 cp -r $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 to/ : to/file3,to/file,to/dir2/file2
723 # namedfile, unnameddir, nameddir, unnamedfile
724 cp $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 $FILECAP to/ : E4-NEED-R
725 cp -r $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 $FILECAP to/ : E2-DESTNAME
727 # single sources to a missing target: should mkdir or create a file
728 cp $FILECAP to/missing : to/missing
729 cp -r $FILECAP to/missing : to/missing
730 cp $DIRCAP/file to/missing : to/missing
731 cp -r $DIRCAP/file to/missing : to/missing
732 cp $PARENTCAP/dir to/missing : E4-NEED-R
733 cp -r $PARENTCAP/dir to/missing : to/missing/dir/file
734 cp $DIRCAP to/missing : E4-NEED-R
735 cp -r $DIRCAP to/missing : to/missing/file
736 cp $DIRALIAS to/missing : E4-NEED-R
737 cp -r $DIRALIAS to/missing : to/missing/file
739 cp $FILECAP to/missing/ : E7-BADSLASH
740 cp -r $FILECAP to/missing/ : E7-BADSLASH
741 cp $DIRCAP/file to/missing/ : E7-BADSLASH
742 cp -r $DIRCAP/file to/missing/ : E7-BADSLASH
743 cp $PARENTCAP/dir to/missing/ : E4-NEED-R
744 cp -r $PARENTCAP/dir to/missing/ : to/missing/dir/file
745 cp $DIRCAP to/missing/ : E4-NEED-R
746 cp -r $DIRCAP to/missing/ : to/missing/file
747 cp $DIRALIAS to/missing/ : E4-NEED-R
748 cp -r $DIRALIAS to/missing/ : to/missing/file
750 # multiple files to a missing target: should mkdir
751 cp $DIRCAP/file $PARENTCAP/dir2/file2 to/missing : to/missing/file,to/missing/file2
752 cp -r $DIRCAP/file $PARENTCAP/dir2/file2 to/missing : to/missing/file,to/missing/file2
754 cp $DIRCAP/file $PARENTCAP/dir2/file2 to/missing/ : to/missing/file,to/missing/file2
755 cp -r $DIRCAP/file $PARENTCAP/dir2/file2 to/missing/ : to/missing/file,to/missing/file2
757 # multiple things to a missing target: should mkdir
758 cp $DIRCAP/file $PARENTCAP/dir2/file2 to/missing : to/missing/file,to/missing/file2
759 cp $DIRCAP/file $FILECAP to/missing : E2-DESTNAME
760 cp $DIRCAP $FILECAP to/missing : E4-NEED-R
761 cp -r $DIRCAP $FILECAP to/missing : E2-DESTNAME
762 # namedfile, unnameddir, nameddir
763 cp $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 to/missing : E4-NEED-R
764 cp -r $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 to/missing : to/missing/file3,to/missing/file,to/missing/dir2/file2
765 # namedfile, unnameddir, nameddir, unnamedfile
766 cp $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 $FILECAP to/missing : E4-NEED-R
767 cp -r $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 $FILECAP to/missing : E2-DESTNAME
769 cp $DIRCAP/file $PARENTCAP/dir2/file2 to/missing/ : to/missing/file,to/missing/file2
770 cp $DIRCAP/file $FILECAP to/missing/ : E2-DESTNAME
771 cp $DIRCAP $FILECAP to/missing/ : E4-NEED-R
772 cp -r $DIRCAP $FILECAP to/missing/ : E2-DESTNAME
773 # namedfile, unnameddir, nameddir
774 cp $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 to/missing/ : E4-NEED-R
775 cp -r $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 to/missing/ : to/missing/file3,to/missing/file,to/missing/dir2/file2
776 # namedfile, unnameddir, nameddir, unnamedfile
777 cp $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 $FILECAP to/missing/ : E4-NEED-R
778 cp -r $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 $FILECAP to/missing/ : E2-DESTNAME
780 # make sure empty directories are copied too
781 cp -r $PARENTCAP/dir4 to : to/dir4/emptydir/
782 cp -r $PARENTCAP/dir4 to/ : to/dir4/emptydir/
784 # name collisions: ensure files are copied in order
785 cp -r $PARENTCAP/dir6/dir $PARENTCAP/dir5/dir to : to/dir/collide=5
786 cp -r $PARENTCAP/dir5/dir $PARENTCAP/dir6/dir to : to/dir/collide=6
787 cp -r $DIRCAP6 $DIRCAP5 to : to/dir/collide=5
788 cp -r $DIRCAP5 $DIRCAP6 to : to/dir/collide=6
792 class CopyOut(GridTestMixin, CLITestMixin, unittest.TestCase):
793 FILE_CONTENTS = "file text"
794 FILE_CONTENTS_5 = "5"
795 FILE_CONTENTS_6 = "6"
798 # first we build a tahoe filesystem that contains:
800 # $PARENTCAP/dir == $DIRCAP == alias:
801 # $PARENTCAP/dir/file == $FILECAP
802 # $PARENTCAP/dir2 (named directory)
803 # $PARENTCAP/dir2/file2
804 # $PARENTCAP/dir3/file3 (a second named file)
806 # $PARENTCAP/dir4/emptydir/ (an empty directory)
807 # $PARENTCAP/dir5 == $DIRCAP5
808 # $PARENTCAP/dir5/dir/collide (contents are "5")
809 # $PARENTCAP/dir6 == $DIRCAP6
810 # $PARENTCAP/dir6/dir/collide (contents are "6")
812 source_file = os.path.join(self.basedir, "file")
813 fileutil.write(source_file, self.FILE_CONTENTS)
814 source_file_5 = os.path.join(self.basedir, "file5")
815 fileutil.write(source_file_5, self.FILE_CONTENTS_5)
816 source_file_6 = os.path.join(self.basedir, "file6")
817 fileutil.write(source_file_6, self.FILE_CONTENTS_6)
819 d = self.do_cli("mkdir")
820 def _stash_parentdircap(res):
822 self.failUnlessEqual(rc, 0, str(res))
823 self.failUnlessEqual(err, "", str(res))
824 self.PARENTCAP = out.strip()
825 return self.do_cli("mkdir", "%s/dir" % self.PARENTCAP)
826 d.addCallback(_stash_parentdircap)
827 def _stash_dircap(res):
829 self.failUnlessEqual(rc, 0, str(res))
830 self.failUnlessEqual(err, "", str(res))
831 self.DIRCAP = out.strip()
832 return self.do_cli("add-alias", "ALIAS", self.DIRCAP)
833 d.addCallback(_stash_dircap)
834 d.addCallback(lambda ign:
835 self.do_cli("put", source_file, "%s/dir/file" % self.PARENTCAP))
836 def _stash_filecap(res):
838 self.failUnlessEqual(rc, 0, str(res))
839 self.failUnlessEqual(err.strip(), "201 Created", str(res))
840 self.FILECAP = out.strip()
841 assert self.FILECAP.startswith("URI:LIT:")
842 d.addCallback(_stash_filecap)
843 d.addCallback(lambda ign:
844 self.do_cli("mkdir", "%s/dir2" % self.PARENTCAP))
845 d.addCallback(lambda ign:
846 self.do_cli("put", source_file, "%s/dir2/file2" % self.PARENTCAP))
847 d.addCallback(lambda ign:
848 self.do_cli("mkdir", "%s/dir3" % self.PARENTCAP))
849 d.addCallback(lambda ign:
850 self.do_cli("put", source_file, "%s/dir3/file3" % self.PARENTCAP))
851 d.addCallback(lambda ign:
852 self.do_cli("mkdir", "%s/dir4" % self.PARENTCAP))
853 d.addCallback(lambda ign:
854 self.do_cli("mkdir", "%s/dir4/emptydir" % self.PARENTCAP))
856 d.addCallback(lambda ign:
857 self.do_cli("mkdir", "%s/dir5" % self.PARENTCAP))
858 def _stash_dircap_5(res):
860 self.failUnlessEqual(rc, 0, str(res))
861 self.failUnlessEqual(err, "", str(res))
862 self.DIRCAP5 = out.strip()
863 d.addCallback(_stash_dircap_5)
864 d.addCallback(lambda ign:
865 self.do_cli("mkdir", "%s/dir5/dir" % self.PARENTCAP))
866 d.addCallback(lambda ign:
867 self.do_cli("put", source_file_5, "%s/dir5/dir/collide" % self.PARENTCAP))
869 d.addCallback(lambda ign:
870 self.do_cli("mkdir", "%s/dir6" % self.PARENTCAP))
871 def _stash_dircap_6(res):
873 self.failUnlessEqual(rc, 0, str(res))
874 self.failUnlessEqual(err, "", str(res))
875 self.DIRCAP6 = out.strip()
876 d.addCallback(_stash_dircap_6)
877 d.addCallback(lambda ign:
878 self.do_cli("mkdir", "%s/dir6/dir" % self.PARENTCAP))
879 d.addCallback(lambda ign:
880 self.do_cli("put", source_file_6, "%s/dir6/dir/collide" % self.PARENTCAP))
884 def check_output(self):
885 # locate the files and directories created (if any) under to/
886 top = os.path.join(self.basedir, "to")
888 for (dirpath, dirnames, filenames) in os.walk(top):
889 assert dirpath.startswith(top)
890 here = "/".join(dirpath.split(os.sep)[len(top.split(os.sep))-1:])
891 results.add(here+"/")
893 f = open(os.path.join(dirpath, fn), "rb")
896 if contents == self.FILE_CONTENTS:
897 results.add("%s/%s" % (here, fn))
898 elif contents == self.FILE_CONTENTS_5:
899 results.add("%s/%s=5" % (here, fn))
900 elif contents == self.FILE_CONTENTS_6:
901 results.add("%s/%s=6" % (here, fn))
904 def run_one_case(self, case):
906 .replace("$PARENTCAP", self.PARENTCAP)
907 .replace("$DIRCAP5", self.DIRCAP5)
908 .replace("$DIRCAP6", self.DIRCAP6)
909 .replace("$DIRCAP", self.DIRCAP)
910 .replace("$DIRALIAS", "ALIAS:")
911 .replace("$FILECAP", self.FILECAP)
914 cmd[-1] = os.path.abspath(os.path.join(self.basedir, cmd[-1]))
917 targetdir = os.path.abspath(os.path.join(self.basedir, "to"))
918 if os.path.exists(targetdir):
919 shutil.rmtree(targetdir)
922 if target.rstrip("/") == "to/existing-file":
923 fileutil.write(cmd[-1], "existing file contents\n")
925 # The abspath() for cmd[-1] strips a trailing slash, and we want to
926 # test what happens when it is present. So put it back.
927 if target.endswith("/"):
930 d = self.do_cli(*cmd)
935 return self.check_output()
937 self.failUnlessEqual(out, "", str(res))
938 if "when copying into a directory, all source files must have names, but" in err:
939 return set(["E2-DESTNAME"])
940 if err == "cannot copy directories without --recursive":
941 return set(["E4-NEED-R"])
942 if err == "cannot copy directory into a file":
943 return set(["E5-DIRTOFILE"])
944 if err == "copying multiple things requires target be a directory":
945 return set(["E6-MANYONE"])
946 if err == "target is not a directory, but ends with a slash":
947 return set(["E7-BADSLASH"])
948 self.fail("unrecognized error ('%s') %s" % (case, res))
949 d.addCallback(_check)
952 def do_one_test(self, case, expected):
953 expected = expected.copy()
954 printable_expected = ",".join(sorted(expected))
955 #print "---", case, ":", printable_expected
957 for f in list(expected):
958 # f is "dir/file" or "dir/sub/file" or "dir/" or "dir/sub/"
959 # we want all parent directories in the set, with trailing /
960 pieces = f.rstrip("/").split("/")
961 for i in range(1,len(pieces)):
962 parent = "/".join(pieces[:i])
963 expected.add(parent+"/")
965 d = self.run_one_case(case)
967 ok = "ok" if got == expected else "FAIL"
968 printable_got = ",".join(sorted(got))
969 print "%-31s: got %-19s, want %-19s %s" % (case, printable_got,
970 printable_expected, ok)
972 #d.addCallback(_dump)
974 self.failUnlessEqual(got, expected, case)
975 d.addCallback(_check)
979 # then we run various forms of "cp [-r] TAHOETHING to[/missing]"
980 # and see what happens.
981 d = defer.succeed(None)
984 for line in COPYOUT_TESTCASES.splitlines():
986 line = line[:line.find("#")]
990 case, expected = line.split(":")
992 expected = set(expected.strip().split(","))
994 d.addCallback(lambda ign, case=case, expected=expected:
995 self.do_one_test(case, expected))
999 def test_cp_out(self):
1000 # test copying all sorts of things out of a tahoe filesystem
1001 self.basedir = "cli_cp/CopyOut/cp_out"
1002 self.set_up_grid(num_servers=1)
1005 d.addCallback(lambda ign: self.do_tests())