1 import os.path, simplejson
2 from twisted.trial import unittest
3 from twisted.python import usage
4 from twisted.internet import defer
6 from allmydata.scripts import cli
7 from allmydata.util import fileutil
8 from allmydata.util.encodingutil import (quote_output, get_io_encoding,
9 unicode_to_output, to_str)
10 from allmydata.util.assertutil import _assert
11 from .no_network import GridTestMixin
12 from .test_cli import CLITestMixin
14 timeout = 480 # deep_check takes 360s on Zandr's linksys box, others take > 240s
16 class Cp(GridTestMixin, CLITestMixin, unittest.TestCase):
18 def test_not_enough_args(self):
20 self.failUnlessRaises(usage.UsageError,
21 o.parseOptions, ["onearg"])
23 def test_unicode_filename(self):
24 self.basedir = "cli/Cp/unicode_filename"
26 fn1 = os.path.join(unicode(self.basedir), u"\u00C4rtonwall")
28 fn1_arg = fn1.encode(get_io_encoding())
29 artonwall_arg = u"\u00C4rtonwall".encode(get_io_encoding())
30 except UnicodeEncodeError:
31 raise unittest.SkipTest("A non-ASCII command argument could not be encoded on this platform.")
33 self.skip_if_cannot_represent_filename(fn1)
37 DATA1 = "unicode file content"
38 fileutil.write(fn1, DATA1)
40 fn2 = os.path.join(self.basedir, "Metallica")
41 DATA2 = "non-unicode file content"
42 fileutil.write(fn2, DATA2)
44 d = self.do_cli("create-alias", "tahoe")
46 d.addCallback(lambda res: self.do_cli("cp", fn1_arg, "tahoe:"))
48 d.addCallback(lambda res: self.do_cli("get", "tahoe:" + artonwall_arg))
49 d.addCallback(lambda (rc,out,err): self.failUnlessReallyEqual(out, DATA1))
51 d.addCallback(lambda res: self.do_cli("cp", fn2, "tahoe:"))
53 d.addCallback(lambda res: self.do_cli("get", "tahoe:Metallica"))
54 d.addCallback(lambda (rc,out,err): self.failUnlessReallyEqual(out, DATA2))
56 d.addCallback(lambda res: self.do_cli("ls", "tahoe:"))
57 def _check((rc, out, err)):
59 unicode_to_output(u"\u00C4rtonwall")
60 except UnicodeEncodeError:
61 self.failUnlessReallyEqual(rc, 1)
62 self.failUnlessReallyEqual(out, "Metallica\n")
63 self.failUnlessIn(quote_output(u"\u00C4rtonwall"), err)
64 self.failUnlessIn("files whose names could not be converted", err)
66 self.failUnlessReallyEqual(rc, 0)
67 self.failUnlessReallyEqual(out.decode(get_io_encoding()), u"Metallica\n\u00C4rtonwall\n")
68 self.failUnlessReallyEqual(err, "")
73 def test_dangling_symlink_vs_recursion(self):
74 if not hasattr(os, 'symlink'):
75 raise unittest.SkipTest("Symlinks are not supported by Python on this platform.")
77 # cp -r on a directory containing a dangling symlink shouldn't assert
78 self.basedir = "cli/Cp/dangling_symlink_vs_recursion"
80 dn = os.path.join(self.basedir, "dir")
82 fn = os.path.join(dn, "Fakebandica")
83 ln = os.path.join(dn, "link")
86 d = self.do_cli("create-alias", "tahoe")
87 d.addCallback(lambda res: self.do_cli("cp", "--recursive",
91 def test_copy_using_filecap(self):
92 self.basedir = "cli/Cp/test_copy_using_filecap"
94 outdir = os.path.join(self.basedir, "outdir")
96 fn1 = os.path.join(self.basedir, "Metallica")
97 fn2 = os.path.join(outdir, "Not Metallica")
98 fn3 = os.path.join(outdir, "test2")
99 DATA1 = "puppies" * 10000
100 fileutil.write(fn1, DATA1)
102 d = self.do_cli("create-alias", "tahoe")
103 d.addCallback(lambda ign: self.do_cli("put", fn1))
104 def _put_file((rc, out, err)):
105 self.failUnlessReallyEqual(rc, 0)
106 self.failUnlessIn("200 OK", err)
107 # keep track of the filecap
108 self.filecap = out.strip()
109 d.addCallback(_put_file)
111 # Let's try copying this to the disk using the filecap.
112 d.addCallback(lambda ign: self.do_cli("cp", self.filecap, fn2))
113 def _copy_file((rc, out, err)):
114 self.failUnlessReallyEqual(rc, 0)
115 results = fileutil.read(fn2)
116 self.failUnlessReallyEqual(results, DATA1)
117 d.addCallback(_copy_file)
119 # Test copying a filecap to local dir, which should fail without a
120 # destination filename (#761).
121 d.addCallback(lambda ign: self.do_cli("cp", self.filecap, outdir))
122 def _resp((rc, out, err)):
123 self.failUnlessReallyEqual(rc, 1)
124 self.failUnlessIn("when copying into a directory, all source files must have names, but",
126 self.failUnlessReallyEqual(out, "")
129 # Create a directory, linked at tahoe:test .
130 d.addCallback(lambda ign: self.do_cli("mkdir", "tahoe:test"))
131 def _get_dir((rc, out, err)):
132 self.failUnlessReallyEqual(rc, 0)
133 self.dircap = out.strip()
134 d.addCallback(_get_dir)
136 # Upload a file to the directory.
137 d.addCallback(lambda ign:
138 self.do_cli("put", fn1, "tahoe:test/test_file"))
139 d.addCallback(lambda (rc, out, err): self.failUnlessReallyEqual(rc, 0))
141 # Copying DIRCAP/filename to a local dir should work, because the
142 # destination filename can be inferred.
143 d.addCallback(lambda ign:
144 self.do_cli("cp", self.dircap + "/test_file", outdir))
145 def _get_resp((rc, out, err)):
146 self.failUnlessReallyEqual(rc, 0)
147 results = fileutil.read(os.path.join(outdir, "test_file"))
148 self.failUnlessReallyEqual(results, DATA1)
149 d.addCallback(_get_resp)
151 # ... and to an explicit filename different from the source filename.
152 d.addCallback(lambda ign:
153 self.do_cli("cp", self.dircap + "/test_file", fn3))
154 def _get_resp2((rc, out, err)):
155 self.failUnlessReallyEqual(rc, 0)
156 results = fileutil.read(fn3)
157 self.failUnlessReallyEqual(results, DATA1)
158 d.addCallback(_get_resp2)
160 # Test that the --verbose option prints correct indices (#1805).
161 d.addCallback(lambda ign:
162 self.do_cli("cp", "--verbose", fn3, self.dircap))
163 def _test_for_wrong_indices((rc, out, err)):
164 lines = err.split('\n')
165 self.failUnlessIn('examining 1 of 1', lines)
166 self.failUnlessIn('starting copy, 1 files, 1 directories', lines)
167 self.failIfIn('examining 0 of', err)
168 d.addCallback(_test_for_wrong_indices)
171 def test_cp_with_nonexistent_alias(self):
172 # when invoked with an alias or aliases that don't exist, 'tahoe cp'
173 # should output a sensible error message rather than a stack trace.
174 self.basedir = "cli/Cp/cp_with_nonexistent_alias"
176 d = self.do_cli("cp", "fake:file1", "fake:file2")
177 def _check((rc, out, err)):
178 self.failUnlessReallyEqual(rc, 1)
179 self.failUnlessIn("error:", err)
180 d.addCallback(_check)
181 # 'tahoe cp' actually processes the target argument first, so we need
182 # to check to make sure that validation extends to the source
184 d.addCallback(lambda ign: self.do_cli("create-alias", "tahoe"))
185 d.addCallback(lambda ign: self.do_cli("cp", "fake:file1",
187 d.addCallback(_check)
190 def test_unicode_dirnames(self):
191 self.basedir = "cli/Cp/unicode_dirnames"
193 fn1 = os.path.join(unicode(self.basedir), u"\u00C4rtonwall")
195 fn1_arg = fn1.encode(get_io_encoding())
196 del fn1_arg # hush pyflakes
197 artonwall_arg = u"\u00C4rtonwall".encode(get_io_encoding())
198 except UnicodeEncodeError:
199 raise unittest.SkipTest("A non-ASCII command argument could not be encoded on this platform.")
201 self.skip_if_cannot_represent_filename(fn1)
205 d = self.do_cli("create-alias", "tahoe")
206 d.addCallback(lambda res: self.do_cli("mkdir", "tahoe:test/" + artonwall_arg))
207 d.addCallback(lambda res: self.do_cli("cp", "-r", "tahoe:test", "tahoe:test2"))
208 d.addCallback(lambda res: self.do_cli("ls", "tahoe:test2/test"))
209 def _check((rc, out, err)):
211 unicode_to_output(u"\u00C4rtonwall")
212 except UnicodeEncodeError:
213 self.failUnlessReallyEqual(rc, 1)
214 self.failUnlessReallyEqual(out, "")
215 self.failUnlessIn(quote_output(u"\u00C4rtonwall"), err)
216 self.failUnlessIn("files whose names could not be converted", err)
218 self.failUnlessReallyEqual(rc, 0)
219 self.failUnlessReallyEqual(out.decode(get_io_encoding()), u"\u00C4rtonwall\n")
220 self.failUnlessReallyEqual(err, "")
221 d.addCallback(_check)
225 def test_cp_replaces_mutable_file_contents(self):
226 self.basedir = "cli/Cp/cp_replaces_mutable_file_contents"
229 # Write a test file, which we'll copy to the grid.
230 test_txt_path = os.path.join(self.basedir, "test.txt")
231 test_txt_contents = "foo bar baz"
232 f = open(test_txt_path, "w")
233 f.write(test_txt_contents)
236 d = self.do_cli("create-alias", "tahoe")
237 d.addCallback(lambda ignored:
238 self.do_cli("mkdir", "tahoe:test"))
239 # We have to use 'tahoe put' here because 'tahoe cp' doesn't
240 # know how to make mutable files at the destination.
241 d.addCallback(lambda ignored:
242 self.do_cli("put", "--mutable", test_txt_path, "tahoe:test/test.txt"))
243 d.addCallback(lambda ignored:
244 self.do_cli("get", "tahoe:test/test.txt"))
245 def _check((rc, out, err)):
246 self.failUnlessEqual(rc, 0)
247 self.failUnlessEqual(out, test_txt_contents)
248 d.addCallback(_check)
250 # We'll do ls --json to get the read uri and write uri for the
251 # file we've just uploaded.
252 d.addCallback(lambda ignored:
253 self.do_cli("ls", "--json", "tahoe:test/test.txt"))
254 def _get_test_txt_uris((rc, out, err)):
255 self.failUnlessEqual(rc, 0)
256 filetype, data = simplejson.loads(out)
258 self.failUnlessEqual(filetype, "filenode")
259 self.failUnless(data['mutable'])
261 self.failUnlessIn("rw_uri", data)
262 self.rw_uri = to_str(data["rw_uri"])
263 self.failUnlessIn("ro_uri", data)
264 self.ro_uri = to_str(data["ro_uri"])
265 d.addCallback(_get_test_txt_uris)
267 # Now make a new file to copy in place of test.txt.
268 new_txt_path = os.path.join(self.basedir, "new.txt")
269 new_txt_contents = "baz bar foo" * 100000
270 f = open(new_txt_path, "w")
271 f.write(new_txt_contents)
274 # Copy the new file on top of the old file.
275 d.addCallback(lambda ignored:
276 self.do_cli("cp", new_txt_path, "tahoe:test/test.txt"))
278 # If we get test.txt now, we should see the new data.
279 d.addCallback(lambda ignored:
280 self.do_cli("get", "tahoe:test/test.txt"))
281 d.addCallback(lambda (rc, out, err):
282 self.failUnlessEqual(out, new_txt_contents))
283 # If we get the json of the new file, we should see that the old
285 d.addCallback(lambda ignored:
286 self.do_cli("ls", "--json", "tahoe:test/test.txt"))
287 def _check_json((rc, out, err)):
288 self.failUnlessEqual(rc, 0)
289 filetype, data = simplejson.loads(out)
291 self.failUnlessEqual(filetype, "filenode")
292 self.failUnless(data['mutable'])
294 self.failUnlessIn("ro_uri", data)
295 self.failUnlessEqual(to_str(data["ro_uri"]), self.ro_uri)
296 self.failUnlessIn("rw_uri", data)
297 self.failUnlessEqual(to_str(data["rw_uri"]), self.rw_uri)
298 d.addCallback(_check_json)
300 # and, finally, doing a GET directly on one of the old uris
301 # should give us the new contents.
302 d.addCallback(lambda ignored:
303 self.do_cli("get", self.rw_uri))
304 d.addCallback(lambda (rc, out, err):
305 self.failUnlessEqual(out, new_txt_contents))
306 # Now copy the old test.txt without an explicit destination
307 # file. tahoe cp will match it to the existing file and
308 # overwrite it appropriately.
309 d.addCallback(lambda ignored:
310 self.do_cli("cp", test_txt_path, "tahoe:test"))
311 d.addCallback(lambda ignored:
312 self.do_cli("get", "tahoe:test/test.txt"))
313 d.addCallback(lambda (rc, out, err):
314 self.failUnlessEqual(out, test_txt_contents))
315 d.addCallback(lambda ignored:
316 self.do_cli("ls", "--json", "tahoe:test/test.txt"))
317 d.addCallback(_check_json)
318 d.addCallback(lambda ignored:
319 self.do_cli("get", self.rw_uri))
320 d.addCallback(lambda (rc, out, err):
321 self.failUnlessEqual(out, test_txt_contents))
323 # Now we'll make a more complicated directory structure.
329 imm_test_txt_path = os.path.join(self.basedir, "imm_test.txt")
330 imm_test_txt_contents = test_txt_contents * 10000
331 fileutil.write(imm_test_txt_path, imm_test_txt_contents)
332 d.addCallback(lambda ignored:
333 self.do_cli("mkdir", "tahoe:test2"))
334 d.addCallback(lambda ignored:
335 self.do_cli("put", "--mutable", new_txt_path,
336 "tahoe:test2/mutable1"))
337 d.addCallback(lambda ignored:
338 self.do_cli("put", "--mutable", new_txt_path,
339 "tahoe:test2/mutable2"))
340 d.addCallback(lambda ignored:
341 self.do_cli('put', new_txt_path, "tahoe:test2/imm1"))
342 d.addCallback(lambda ignored:
343 self.do_cli("put", imm_test_txt_path, "tahoe:test2/imm2"))
344 d.addCallback(lambda ignored:
345 self.do_cli("ls", "--json", "tahoe:test2"))
346 def _process_directory_json((rc, out, err)):
347 self.failUnlessEqual(rc, 0)
349 filetype, data = simplejson.loads(out)
350 self.failUnlessEqual(filetype, "dirnode")
351 self.failUnless(data['mutable'])
352 self.failUnlessIn("children", data)
353 children = data['children']
355 # Store the URIs for later use.
357 for k in ["mutable1", "mutable2", "imm1", "imm2"]:
358 self.failUnlessIn(k, children)
359 childtype, childdata = children[k]
360 self.failUnlessEqual(childtype, "filenode")
362 self.failUnless(childdata['mutable'])
363 self.failUnlessIn("rw_uri", childdata)
366 self.failIf(childdata['mutable'])
367 self.failUnlessIn("ro_uri", childdata)
369 self.childuris[k] = to_str(childdata[uri_key])
370 d.addCallback(_process_directory_json)
371 # Now build a local directory to copy into place, like the following:
377 def _build_local_directory(ignored):
378 test2_path = os.path.join(self.basedir, "test2")
379 fileutil.make_dirs(test2_path)
380 for fn in ("mutable1", "mutable2", "imm1", "imm3"):
381 fileutil.write(os.path.join(test2_path, fn), fn * 1000)
382 self.test2_path = test2_path
383 d.addCallback(_build_local_directory)
384 d.addCallback(lambda ignored:
385 self.do_cli("cp", "-r", self.test2_path, "tahoe:"))
387 # We expect that mutable1 and mutable2 are overwritten in-place,
388 # so they'll retain their URIs but have different content.
389 def _process_file_json((rc, out, err), fn):
390 self.failUnlessEqual(rc, 0)
391 filetype, data = simplejson.loads(out)
392 self.failUnlessEqual(filetype, "filenode")
395 self.failUnless(data['mutable'])
396 self.failUnlessIn("rw_uri", data)
397 self.failUnlessEqual(to_str(data["rw_uri"]), self.childuris[fn])
399 self.failIf(data['mutable'])
400 self.failUnlessIn("ro_uri", data)
401 self.failIfEqual(to_str(data["ro_uri"]), self.childuris[fn])
403 for fn in ("mutable1", "mutable2"):
404 d.addCallback(lambda ignored, fn=fn:
405 self.do_cli("get", "tahoe:test2/%s" % fn))
406 d.addCallback(lambda (rc, out, err), fn=fn:
407 self.failUnlessEqual(out, fn * 1000))
408 d.addCallback(lambda ignored, fn=fn:
409 self.do_cli("ls", "--json", "tahoe:test2/%s" % fn))
410 d.addCallback(_process_file_json, fn=fn)
412 # imm1 should have been replaced, so both its uri and content
413 # should be different.
414 d.addCallback(lambda ignored:
415 self.do_cli("get", "tahoe:test2/imm1"))
416 d.addCallback(lambda (rc, out, err):
417 self.failUnlessEqual(out, "imm1" * 1000))
418 d.addCallback(lambda ignored:
419 self.do_cli("ls", "--json", "tahoe:test2/imm1"))
420 d.addCallback(_process_file_json, fn="imm1")
422 # imm3 should have been created.
423 d.addCallback(lambda ignored:
424 self.do_cli("get", "tahoe:test2/imm3"))
425 d.addCallback(lambda (rc, out, err):
426 self.failUnlessEqual(out, "imm3" * 1000))
428 # imm2 should be exactly as we left it, since our newly-copied
429 # directory didn't contain an imm2 entry.
430 d.addCallback(lambda ignored:
431 self.do_cli("get", "tahoe:test2/imm2"))
432 d.addCallback(lambda (rc, out, err):
433 self.failUnlessEqual(out, imm_test_txt_contents))
434 d.addCallback(lambda ignored:
435 self.do_cli("ls", "--json", "tahoe:test2/imm2"))
436 def _process_imm2_json((rc, out, err)):
437 self.failUnlessEqual(rc, 0)
438 filetype, data = simplejson.loads(out)
439 self.failUnlessEqual(filetype, "filenode")
440 self.failIf(data['mutable'])
441 self.failUnlessIn("ro_uri", data)
442 self.failUnlessEqual(to_str(data["ro_uri"]), self.childuris["imm2"])
443 d.addCallback(_process_imm2_json)
446 def test_cp_overwrite_readonly_mutable_file(self):
447 # tahoe cp should print an error when asked to overwrite a
448 # mutable file that it can't overwrite.
449 self.basedir = "cli/Cp/overwrite_readonly_mutable_file"
452 # This is our initial file. We'll link its readcap into the
454 test_file_path = os.path.join(self.basedir, "test_file.txt")
455 test_file_contents = "This is a test file."
456 fileutil.write(test_file_path, test_file_contents)
458 # This is our replacement file. We'll try and fail to upload it
459 # over the readcap that we linked into the tahoe: alias.
460 replacement_file_path = os.path.join(self.basedir, "replacement.txt")
461 replacement_file_contents = "These are new contents."
462 fileutil.write(replacement_file_path, replacement_file_contents)
464 d = self.do_cli("create-alias", "tahoe:")
465 d.addCallback(lambda ignored:
466 self.do_cli("put", "--mutable", test_file_path))
467 def _get_test_uri((rc, out, err)):
468 self.failUnlessEqual(rc, 0)
469 # this should be a write uri
470 self._test_write_uri = out
471 d.addCallback(_get_test_uri)
472 d.addCallback(lambda ignored:
473 self.do_cli("ls", "--json", self._test_write_uri))
474 def _process_test_json((rc, out, err)):
475 self.failUnlessEqual(rc, 0)
476 filetype, data = simplejson.loads(out)
478 self.failUnlessEqual(filetype, "filenode")
479 self.failUnless(data['mutable'])
480 self.failUnlessIn("ro_uri", data)
481 self._test_read_uri = to_str(data["ro_uri"])
482 d.addCallback(_process_test_json)
483 # Now we'll link the readonly URI into the tahoe: alias.
484 d.addCallback(lambda ignored:
485 self.do_cli("ln", self._test_read_uri, "tahoe:test_file.txt"))
486 d.addCallback(lambda (rc, out, err):
487 self.failUnlessEqual(rc, 0))
488 # Let's grab the json of that to make sure that we did it right.
489 d.addCallback(lambda ignored:
490 self.do_cli("ls", "--json", "tahoe:"))
491 def _process_tahoe_json((rc, out, err)):
492 self.failUnlessEqual(rc, 0)
494 filetype, data = simplejson.loads(out)
495 self.failUnlessEqual(filetype, "dirnode")
496 self.failUnlessIn("children", data)
497 kiddata = data['children']
499 self.failUnlessIn("test_file.txt", kiddata)
500 testtype, testdata = kiddata['test_file.txt']
501 self.failUnlessEqual(testtype, "filenode")
502 self.failUnless(testdata['mutable'])
503 self.failUnlessIn("ro_uri", testdata)
504 self.failUnlessEqual(to_str(testdata["ro_uri"]), self._test_read_uri)
505 self.failIfIn("rw_uri", testdata)
506 d.addCallback(_process_tahoe_json)
507 # Okay, now we're going to try uploading another mutable file in
508 # place of that one. We should get an error.
509 d.addCallback(lambda ignored:
510 self.do_cli("cp", replacement_file_path, "tahoe:test_file.txt"))
511 def _check_error_message((rc, out, err)):
512 self.failUnlessEqual(rc, 1)
513 self.failUnlessIn("replace or update requested with read-only cap", err)
514 d.addCallback(_check_error_message)
515 # Make extra sure that that didn't work.
516 d.addCallback(lambda ignored:
517 self.do_cli("get", "tahoe:test_file.txt"))
518 d.addCallback(lambda (rc, out, err):
519 self.failUnlessEqual(out, test_file_contents))
520 d.addCallback(lambda ignored:
521 self.do_cli("get", self._test_read_uri))
522 d.addCallback(lambda (rc, out, err):
523 self.failUnlessEqual(out, test_file_contents))
524 # Now we'll do it without an explicit destination.
525 d.addCallback(lambda ignored:
526 self.do_cli("cp", test_file_path, "tahoe:"))
527 d.addCallback(_check_error_message)
528 d.addCallback(lambda ignored:
529 self.do_cli("get", "tahoe:test_file.txt"))
530 d.addCallback(lambda (rc, out, err):
531 self.failUnlessEqual(out, test_file_contents))
532 d.addCallback(lambda ignored:
533 self.do_cli("get", self._test_read_uri))
534 d.addCallback(lambda (rc, out, err):
535 self.failUnlessEqual(out, test_file_contents))
536 # Now we'll link a readonly file into a subdirectory.
537 d.addCallback(lambda ignored:
538 self.do_cli("mkdir", "tahoe:testdir"))
539 d.addCallback(lambda (rc, out, err):
540 self.failUnlessEqual(rc, 0))
541 d.addCallback(lambda ignored:
542 self.do_cli("ln", self._test_read_uri, "tahoe:test/file2.txt"))
543 d.addCallback(lambda (rc, out, err):
544 self.failUnlessEqual(rc, 0))
546 test_dir_path = os.path.join(self.basedir, "test")
547 fileutil.make_dirs(test_dir_path)
548 for f in ("file1.txt", "file2.txt"):
549 fileutil.write(os.path.join(test_dir_path, f), f * 10000)
551 d.addCallback(lambda ignored:
552 self.do_cli("cp", "-r", test_dir_path, "tahoe:"))
553 d.addCallback(_check_error_message)
554 d.addCallback(lambda ignored:
555 self.do_cli("ls", "--json", "tahoe:test"))
556 def _got_testdir_json((rc, out, err)):
557 self.failUnlessEqual(rc, 0)
559 filetype, data = simplejson.loads(out)
560 self.failUnlessEqual(filetype, "dirnode")
562 self.failUnlessIn("children", data)
563 childdata = data['children']
565 self.failUnlessIn("file2.txt", childdata)
566 file2type, file2data = childdata['file2.txt']
567 self.failUnlessEqual(file2type, "filenode")
568 self.failUnless(file2data['mutable'])
569 self.failUnlessIn("ro_uri", file2data)
570 self.failUnlessEqual(to_str(file2data["ro_uri"]), self._test_read_uri)
571 self.failIfIn("rw_uri", file2data)
572 d.addCallback(_got_testdir_json)
575 def test_cp_verbose(self):
576 self.basedir = "cli/Cp/cp_verbose"
579 # Write two test files, which we'll copy to the grid.
580 test1_path = os.path.join(self.basedir, "test1")
581 test2_path = os.path.join(self.basedir, "test2")
582 fileutil.write(test1_path, "test1")
583 fileutil.write(test2_path, "test2")
585 d = self.do_cli("create-alias", "tahoe")
586 d.addCallback(lambda ign:
587 self.do_cli("cp", "--verbose", test1_path, test2_path, "tahoe:"))
590 self.failUnlessEqual(rc, 0, str(res))
591 self.failUnlessIn("Success: files copied", out, str(res))
592 self.failUnlessEqual(err, """\
593 attaching sources to targets, 2 files / 0 dirs in root
594 targets assigned, 1 dirs, 2 files
595 starting copy, 2 files, 1 directories
596 1/2 files, 0/1 directories
597 2/2 files, 0/1 directories
600 d.addCallback(_check)
603 def test_cp_copies_dir(self):
604 # This test ensures that a directory is copied using
605 # tahoe cp -r. Refer to ticket #712:
606 # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/712
608 self.basedir = "cli/Cp/cp_copies_dir"
610 subdir = os.path.join(self.basedir, "foo")
612 test1_path = os.path.join(subdir, "test1")
613 fileutil.write(test1_path, "test1")
615 d = self.do_cli("create-alias", "tahoe")
616 d.addCallback(lambda ign:
617 self.do_cli("cp", "-r", subdir, "tahoe:"))
618 d.addCallback(lambda ign:
619 self.do_cli("ls", "tahoe:"))
620 def _check(res, item):
622 self.failUnlessEqual(rc, 0)
623 self.failUnlessEqual(err, "")
624 self.failUnlessIn(item, out, str(res))
625 d.addCallback(_check, "foo")
626 d.addCallback(lambda ign:
627 self.do_cli("ls", "tahoe:foo/"))
628 d.addCallback(_check, "test1")
630 d.addCallback(lambda ign: fileutil.rm_dir(subdir))
631 d.addCallback(lambda ign: self.do_cli("cp", "-r", "tahoe:foo", self.basedir))
632 def _check_local_fs(ign):
633 self.failUnless(os.path.isdir(self.basedir))
634 self.failUnless(os.path.isfile(test1_path))
635 d.addCallback(_check_local_fs)
638 def test_ticket_2027(self):
639 # This test ensures that tahoe will copy a file from the grid to
640 # a local directory without a specified file name.
641 # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/2027
642 self.basedir = "cli/Cp/cp_verbose"
645 # Write a test file, which we'll copy to the grid.
646 test1_path = os.path.join(self.basedir, "test1")
647 fileutil.write(test1_path, "test1")
649 d = self.do_cli("create-alias", "tahoe")
650 d.addCallback(lambda ign:
651 self.do_cli("cp", test1_path, "tahoe:"))
652 d.addCallback(lambda ign:
653 self.do_cli("cp", "tahoe:test1", self.basedir))
656 self.failUnlessIn("Success: file copied", out, str(res))
659 # these test cases come from ticket #2329 comment 40
660 # trailing slash on target *directory* should not matter, test both
661 # trailing slash on target files should cause error
662 # trailing slash on source directory should not matter, test a few
663 # trailing slash on source files should cause error
665 COPYOUT_TESTCASES = """
666 cp $FILECAP to/existing-file : to/existing-file
667 cp -r $FILECAP to/existing-file : to/existing-file
668 cp $DIRCAP/file $PARENTCAP/dir2/file2 to/existing-file : E6-MANYONE
669 cp -r $DIRCAP/file $PARENTCAP/dir2/file2 to/existing-file : E6-MANYONE
670 cp $DIRCAP to/existing-file : E4-NEED-R
671 cp -r $DIRCAP to/existing-file : E5-DIRTOFILE
672 cp $FILECAP $DIRCAP to/existing-file : E4-NEED-R
673 cp -r $FILECAP $DIRCAP to/existing-file : E6-MANYONE
675 cp $FILECAP to/existing-file/ : E7-BADSLASH
676 cp -r $FILECAP to/existing-file/ : E7-BADSLASH
677 cp $DIRCAP/file $PARENTCAP/dir2/file2 to/existing-file/ : E7-BADSLASH
678 cp -r $DIRCAP/file $PARENTCAP/dir2/file2 to/existing-file/ : E7-BADSLASH
679 cp $DIRCAP to/existing-file/ : E4-NEED-R
680 cp -r $DIRCAP to/existing-file/ : E7-BADSLASH
681 cp $FILECAP $DIRCAP to/existing-file/ : E4-NEED-R
682 cp -r $FILECAP $DIRCAP to/existing-file/ : E7-BADSLASH
684 # single source to a (present) target directory
685 cp $FILECAP to : E2-DESTNAME
686 cp -r $FILECAP to : E2-DESTNAME
687 cp $DIRCAP/file to : to/file
688 cp -r $DIRCAP/file to : to/file
689 # these two are errors
690 cp $DIRCAP/file/ to : E8-BADSLASH
691 cp -r $DIRCAP/file/ to : E8-BADSLASH
692 cp $PARENTCAP/dir to : E4-NEED-R
693 cp -r $PARENTCAP/dir to : to/dir/file
694 # but these two should ignore the trailing source slash
695 cp $PARENTCAP/dir/ to : E4-NEED-R
696 cp -r $PARENTCAP/dir/ to : to/dir/file
697 cp $DIRCAP to : E4-NEED-R
698 cp -r $DIRCAP to : to/file
699 cp $DIRALIAS to : E4-NEED-R
700 cp -r $DIRALIAS to : to/file
702 cp $FILECAP to/ : E2-DESTNAME
703 cp -r $FILECAP to/ : E2-DESTNAME
704 cp $DIRCAP/file to/ : to/file
705 cp -r $DIRCAP/file to/ : to/file
706 cp $PARENTCAP/dir to/ : E4-NEED-R
707 cp -r $PARENTCAP/dir to/ : to/dir/file
708 cp $DIRCAP to/ : E4-NEED-R
709 cp -r $DIRCAP to/ : to/file
710 cp $DIRALIAS to/ : E4-NEED-R
711 cp -r $DIRALIAS to/ : to/file
713 # multiple sources to a (present) target directory
714 cp $DIRCAP/file $PARENTCAP/dir2/file2 to : to/file,to/file2
715 cp $DIRCAP/file $FILECAP to : E2-DESTNAME
716 cp $DIRCAP $FILECAP to : E4-NEED-R
717 cp -r $DIRCAP $FILECAP to : E2-DESTNAME
718 # namedfile, unnameddir, nameddir
719 cp $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 to : E4-NEED-R
720 cp -r $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 to : to/file3,to/file,to/dir2/file2
721 # namedfile, unnameddir, nameddir, unnamedfile
722 cp $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 $FILECAP to : E4-NEED-R
723 cp -r $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 $FILECAP to : E2-DESTNAME
725 cp $DIRCAP/file $PARENTCAP/dir2/file2 to/ : to/file,to/file2
726 cp $DIRCAP/file $FILECAP to/ : E2-DESTNAME
727 cp $DIRCAP $FILECAP to/ : E4-NEED-R
728 cp -r $DIRCAP $FILECAP to/ : E2-DESTNAME
729 # namedfile, unnameddir, nameddir
730 cp $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 to/ : E4-NEED-R
731 cp -r $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 to/ : to/file3,to/file,to/dir2/file2
732 # namedfile, unnameddir, nameddir, unnamedfile
733 cp $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 $FILECAP to/ : E4-NEED-R
734 cp -r $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 $FILECAP to/ : E2-DESTNAME
736 # single sources to a missing target: should mkdir or create a file
737 cp $FILECAP to/missing : to/missing
738 cp -r $FILECAP to/missing : to/missing
739 cp $DIRCAP/file to/missing : to/missing
740 cp -r $DIRCAP/file to/missing : to/missing
741 cp $PARENTCAP/dir to/missing : E4-NEED-R
742 cp -r $PARENTCAP/dir to/missing : to/missing/dir/file
743 cp $DIRCAP to/missing : E4-NEED-R
744 cp -r $DIRCAP to/missing : to/missing/file
745 cp $DIRALIAS to/missing : E4-NEED-R
746 cp -r $DIRALIAS to/missing : to/missing/file
748 cp $FILECAP to/missing/ : E7-BADSLASH
749 cp -r $FILECAP to/missing/ : E7-BADSLASH
750 cp $DIRCAP/file to/missing/ : E7-BADSLASH
751 cp -r $DIRCAP/file to/missing/ : E7-BADSLASH
752 cp $PARENTCAP/dir to/missing/ : E4-NEED-R
753 cp -r $PARENTCAP/dir to/missing/ : to/missing/dir/file
754 cp $DIRCAP to/missing/ : E4-NEED-R
755 cp -r $DIRCAP to/missing/ : to/missing/file
756 cp $DIRALIAS to/missing/ : E4-NEED-R
757 cp -r $DIRALIAS to/missing/ : to/missing/file
759 # multiple things to a missing target: should mkdir
760 cp $DIRCAP/file $PARENTCAP/dir2/file2 to/missing : to/missing/file,to/missing/file2
761 cp -r $DIRCAP/file $PARENTCAP/dir2/file2 to/missing : to/missing/file,to/missing/file2
762 cp $DIRCAP/file $FILECAP to/missing : E2-DESTNAME
763 cp -r $DIRCAP/file $FILECAP to/missing : E2-DESTNAME
764 cp $DIRCAP $FILECAP to/missing : E4-NEED-R
765 cp -r $DIRCAP $FILECAP to/missing : E2-DESTNAME
766 # namedfile, unnameddir, nameddir
767 cp $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 to/missing : E4-NEED-R
768 cp -r $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 to/missing : to/missing/file3,to/missing/file,to/missing/dir2/file2
769 # namedfile, unnameddir, nameddir, unnamedfile
770 cp $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 $FILECAP to/missing : E4-NEED-R
771 cp -r $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 $FILECAP to/missing : E2-DESTNAME
773 cp $DIRCAP/file $PARENTCAP/dir2/file2 to/missing/ : to/missing/file,to/missing/file2
774 cp -r $DIRCAP/file $PARENTCAP/dir2/file2 to/missing/ : to/missing/file,to/missing/file2
775 cp $DIRCAP/file $FILECAP to/missing/ : E2-DESTNAME
776 cp -r $DIRCAP/file $FILECAP to/missing/ : E2-DESTNAME
777 cp $DIRCAP $FILECAP to/missing/ : E4-NEED-R
778 cp -r $DIRCAP $FILECAP to/missing/ : E2-DESTNAME
779 # namedfile, unnameddir, nameddir
780 cp $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 to/missing/ : E4-NEED-R
781 cp -r $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 to/missing/ : to/missing/file3,to/missing/file,to/missing/dir2/file2
782 # namedfile, unnameddir, nameddir, unnamedfile
783 cp $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 $FILECAP to/missing/ : E4-NEED-R
784 cp -r $PARENTCAP/dir3/file3 $DIRCAP $PARENTCAP/dir2 $FILECAP to/missing/ : E2-DESTNAME
786 # make sure empty directories are copied too
787 cp -r $PARENTCAP/dir4 to : to/dir4/emptydir/
788 cp -r $PARENTCAP/dir4 to/ : to/dir4/emptydir/
790 # name collisions should cause errors, not overwrites
791 cp -r $PARENTCAP/dir6/dir $PARENTCAP/dir5/dir to : E9-COLLIDING-TARGETS
792 cp -r $PARENTCAP/dir5/dir $PARENTCAP/dir6/dir to : E9-COLLIDING-TARGETS
793 cp -r $DIRCAP6 $DIRCAP5 to : E9-COLLIDING-TARGETS
794 cp -r $DIRCAP5 $DIRCAP6 to : E9-COLLIDING-TARGETS
798 class CopyOut(GridTestMixin, CLITestMixin, unittest.TestCase):
799 FILE_CONTENTS = "file text"
800 FILE_CONTENTS_5 = "5"
801 FILE_CONTENTS_6 = "6"
804 # first we build a tahoe filesystem that contains:
806 # $PARENTCAP/dir == $DIRCAP == alias:
807 # $PARENTCAP/dir/file == $FILECAP
808 # $PARENTCAP/dir2 (named directory)
809 # $PARENTCAP/dir2/file2
810 # $PARENTCAP/dir3/file3 (a second named file)
812 # $PARENTCAP/dir4/emptydir/ (an empty directory)
813 # $PARENTCAP/dir5 == $DIRCAP5
814 # $PARENTCAP/dir5/dir/collide (contents are "5")
815 # $PARENTCAP/dir6 == $DIRCAP6
816 # $PARENTCAP/dir6/dir/collide (contents are "6")
818 source_file = os.path.join(self.basedir, "file")
819 fileutil.write(source_file, self.FILE_CONTENTS)
820 source_file_5 = os.path.join(self.basedir, "file5")
821 fileutil.write(source_file_5, self.FILE_CONTENTS_5)
822 source_file_6 = os.path.join(self.basedir, "file6")
823 fileutil.write(source_file_6, self.FILE_CONTENTS_6)
825 d = self.do_cli("mkdir")
826 def _stash_parentdircap(res):
828 self.failUnlessEqual(rc, 0, str(res))
829 self.failUnlessEqual(err, "", str(res))
830 self.PARENTCAP = out.strip()
831 return self.do_cli("mkdir", "%s/dir" % self.PARENTCAP)
832 d.addCallback(_stash_parentdircap)
833 def _stash_dircap(res):
835 self.failUnlessEqual(rc, 0, str(res))
836 self.failUnlessEqual(err, "", str(res))
837 self.DIRCAP = out.strip()
838 return self.do_cli("add-alias", "ALIAS", self.DIRCAP)
839 d.addCallback(_stash_dircap)
840 d.addCallback(lambda ign:
841 self.do_cli("put", source_file, "%s/dir/file" % self.PARENTCAP))
842 def _stash_filecap(res):
844 self.failUnlessEqual(rc, 0, str(res))
845 self.failUnlessEqual(err.strip(), "201 Created", str(res))
846 self.FILECAP = out.strip()
847 assert self.FILECAP.startswith("URI:LIT:")
848 d.addCallback(_stash_filecap)
849 d.addCallback(lambda ign:
850 self.do_cli("mkdir", "%s/dir2" % self.PARENTCAP))
851 d.addCallback(lambda ign:
852 self.do_cli("put", source_file, "%s/dir2/file2" % self.PARENTCAP))
853 d.addCallback(lambda ign:
854 self.do_cli("mkdir", "%s/dir3" % self.PARENTCAP))
855 d.addCallback(lambda ign:
856 self.do_cli("put", source_file, "%s/dir3/file3" % self.PARENTCAP))
857 d.addCallback(lambda ign:
858 self.do_cli("mkdir", "%s/dir4" % self.PARENTCAP))
859 d.addCallback(lambda ign:
860 self.do_cli("mkdir", "%s/dir4/emptydir" % self.PARENTCAP))
862 d.addCallback(lambda ign:
863 self.do_cli("mkdir", "%s/dir5" % self.PARENTCAP))
864 def _stash_dircap_5(res):
866 self.failUnlessEqual(rc, 0, str(res))
867 self.failUnlessEqual(err, "", str(res))
868 self.DIRCAP5 = out.strip()
869 d.addCallback(_stash_dircap_5)
870 d.addCallback(lambda ign:
871 self.do_cli("mkdir", "%s/dir5/dir" % self.PARENTCAP))
872 d.addCallback(lambda ign:
873 self.do_cli("put", source_file_5, "%s/dir5/dir/collide" % self.PARENTCAP))
875 d.addCallback(lambda ign:
876 self.do_cli("mkdir", "%s/dir6" % self.PARENTCAP))
877 def _stash_dircap_6(res):
879 self.failUnlessEqual(rc, 0, str(res))
880 self.failUnlessEqual(err, "", str(res))
881 self.DIRCAP6 = out.strip()
882 d.addCallback(_stash_dircap_6)
883 d.addCallback(lambda ign:
884 self.do_cli("mkdir", "%s/dir6/dir" % self.PARENTCAP))
885 d.addCallback(lambda ign:
886 self.do_cli("put", source_file_6, "%s/dir6/dir/collide" % self.PARENTCAP))
890 def check_output(self):
891 # locate the files and directories created (if any) under to/
892 top = os.path.join(self.basedir, "to")
894 for (dirpath, dirnames, filenames) in os.walk(top):
895 assert dirpath.startswith(top)
896 here = "/".join(dirpath.split(os.sep)[len(top.split(os.sep))-1:])
897 results.add(here+"/")
899 contents = fileutil.read(os.path.join(dirpath, fn))
900 if contents == self.FILE_CONTENTS:
901 results.add("%s/%s" % (here, fn))
902 elif contents == self.FILE_CONTENTS_5:
903 results.add("%s/%s=5" % (here, fn))
904 elif contents == self.FILE_CONTENTS_6:
905 results.add("%s/%s=6" % (here, fn))
908 def run_one_case(self, case):
910 .replace("$PARENTCAP", self.PARENTCAP)
911 .replace("$DIRCAP5", self.DIRCAP5)
912 .replace("$DIRCAP6", self.DIRCAP6)
913 .replace("$DIRCAP", self.DIRCAP)
914 .replace("$DIRALIAS", "ALIAS:")
915 .replace("$FILECAP", self.FILECAP)
918 _assert(target == "to" or target.startswith("to/"), target)
919 cmd[-1] = os.path.abspath(os.path.join(self.basedir, cmd[-1]))
922 targetdir = os.path.abspath(os.path.join(self.basedir, "to"))
923 fileutil.rm_dir(targetdir)
926 if target.rstrip("/") == "to/existing-file":
927 fileutil.write(cmd[-1], "existing file contents\n")
929 # The abspath() for cmd[-1] strips a trailing slash, and we want to
930 # test what happens when it is present. So put it back.
931 if target.endswith("/"):
934 d = self.do_cli(*cmd)
939 return self.check_output()
941 self.failUnlessEqual(out, "", str(res))
942 if "when copying into a directory, all source files must have names, but" in err:
943 return set(["E2-DESTNAME"])
944 if err == "cannot copy directories without --recursive":
945 return set(["E4-NEED-R"])
946 if err == "cannot copy directory into a file":
947 return set(["E5-DIRTOFILE"])
948 if err == "copying multiple things requires target be a directory":
949 return set(["E6-MANYONE"])
950 if err == "target is not a directory, but ends with a slash":
951 return set(["E7-BADSLASH"])
952 if err == "cannot copy multiple files with the same name into the same target directory":
953 return set(["E9-COLLIDING-TARGETS"])
954 if (err.startswith("source ") and
955 "is not a directory, but ends with a slash" in err):
956 return set(["E8-BADSLASH"])
957 self.fail("unrecognized error ('%s') %s" % (case, res))
958 d.addCallback(_check)
961 def do_one_test(self, case, orig_expected):
962 expected = set(orig_expected)
963 printable_expected = ",".join(sorted(expected))
964 #print "---", case, ":", printable_expected
966 for f in orig_expected:
967 # f is "dir/file" or "dir/sub/file" or "dir/" or "dir/sub/"
968 # we want all parent directories in the set, with trailing /
969 pieces = f.rstrip("/").split("/")
970 for i in range(1,len(pieces)):
971 parent = "/".join(pieces[:i])
972 expected.add(parent+"/")
974 d = self.run_one_case(case)
976 ok = "ok" if got == expected else "FAIL"
977 printable_got = ",".join(sorted(got))
978 print "%-31s: got %-19s, want %-19s %s" % (case, printable_got,
979 printable_expected, ok)
981 #d.addCallback(_dump)
983 self.failUnlessEqual(got, expected, case)
984 d.addCallback(_check)
988 # then we run various forms of "cp [-r] TAHOETHING to[/missing]"
989 # and see what happens.
990 d = defer.succeed(None)
993 for line in COPYOUT_TESTCASES.splitlines():
995 line = line[:line.find("#")]
999 case, expected = line.split(":")
1001 expected = frozenset(expected.strip().split(","))
1003 d.addCallback(lambda ign, case=case, expected=expected:
1004 self.do_one_test(case, expected))
1008 def test_cp_out(self):
1009 # test copying all sorts of things out of a tahoe filesystem
1010 self.basedir = "cli_cp/CopyOut/cp_out"
1011 self.set_up_grid(num_servers=1)
1014 d.addCallback(lambda ign: self.do_tests())