3 from zope.interface import implements
4 from twisted.trial import unittest
5 from twisted.internet import defer
6 from allmydata import uri, dirnode
7 from allmydata.client import Client
8 from allmydata.immutable import upload
9 from allmydata.interfaces import IFileNode, IMutableFileNode, \
10 ExistingChildError, NoSuchChildError, NotDeepImmutableError, \
11 IDeepCheckResults, IDeepCheckAndRepairResults, CannotPackUnknownNodeError
12 from allmydata.mutable.filenode import MutableFileNode
13 from allmydata.mutable.common import UncoordinatedWriteError
14 from allmydata.util import hashutil, base32
15 from allmydata.monitor import Monitor
16 from allmydata.test.common import make_chk_file_uri, make_mutable_file_uri, \
18 from allmydata.test.no_network import GridTestMixin
19 from allmydata.unknown import UnknownNode
20 from allmydata.nodemaker import NodeMaker
21 from base64 import b32decode
22 import common_util as testutil
24 class Dirnode(GridTestMixin, unittest.TestCase,
25 testutil.ShouldFailMixin, testutil.StallMixin, ErrorMixin):
26 timeout = 240 # It takes longer than 120 seconds on Francois's arm box.
29 self.basedir = "dirnode/Dirnode/test_basic"
32 d = c.create_dirnode()
34 self.failUnless(isinstance(res, dirnode.DirectoryNode))
36 self.failUnless("RW-MUT" in rep)
40 def test_initial_children(self):
41 self.basedir = "dirnode/Dirnode/test_initial_children"
45 setup_py_uri = "URI:CHK:n7r3m6wmomelk4sep3kw5cvduq:os7ijw5c3maek7pg65e5254k2fzjflavtpejjyhshpsxuqzhcwwq:3:20:14861"
46 one_uri = "URI:LIT:n5xgk" # LIT for "one"
47 kids = {u"one": (nm.create_from_cap(one_uri), {}),
48 u"two": (nm.create_from_cap(setup_py_uri),
49 {"metakey": "metavalue"}),
51 d = c.create_dirnode(kids)
53 self.failUnless(isinstance(dn, dirnode.DirectoryNode))
55 self.failUnless("RW-MUT" in rep)
57 d.addCallback(_created)
58 def _check_kids(children):
59 self.failUnlessEqual(sorted(children.keys()), [u"one", u"two"])
60 one_node, one_metadata = children[u"one"]
61 two_node, two_metadata = children[u"two"]
62 self.failUnlessEqual(one_node.get_size(), 3)
63 self.failUnlessEqual(two_node.get_size(), 14861)
64 self.failUnless(isinstance(one_metadata, dict), one_metadata)
65 self.failUnlessEqual(two_metadata["metakey"], "metavalue")
66 d.addCallback(_check_kids)
67 d.addCallback(lambda ign: nm.create_new_mutable_directory(kids))
68 d.addCallback(lambda dn: dn.list())
69 d.addCallback(_check_kids)
70 future_writecap = "x-tahoe-crazy://I_am_from_the_future."
71 future_readcap = "x-tahoe-crazy-readonly://I_am_from_the_future."
72 future_node = UnknownNode(future_writecap, future_readcap)
73 bad_kids1 = {u"one": (future_node, {})}
74 d.addCallback(lambda ign:
75 self.shouldFail(AssertionError, "bad_kids1",
76 "does not accept UnknownNode",
77 nm.create_new_mutable_directory,
79 bad_kids2 = {u"one": (nm.create_from_cap(one_uri), None)}
80 d.addCallback(lambda ign:
81 self.shouldFail(AssertionError, "bad_kids2",
82 "requires metadata to be a dict",
83 nm.create_new_mutable_directory,
87 def test_immutable(self):
88 self.basedir = "dirnode/Dirnode/test_immutable"
92 setup_py_uri = "URI:CHK:n7r3m6wmomelk4sep3kw5cvduq:os7ijw5c3maek7pg65e5254k2fzjflavtpejjyhshpsxuqzhcwwq:3:20:14861"
93 one_uri = "URI:LIT:n5xgk" # LIT for "one"
94 mut_readcap = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
95 mut_writecap = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
96 kids = {u"one": (nm.create_from_cap(one_uri), {}),
97 u"two": (nm.create_from_cap(setup_py_uri),
98 {"metakey": "metavalue"}),
100 d = c.create_immutable_dirnode(kids)
102 self.failUnless(isinstance(dn, dirnode.DirectoryNode))
103 self.failIf(dn.is_mutable())
104 self.failUnless(dn.is_readonly())
106 self.failUnless("RO-IMM" in rep)
108 self.failUnlessIn("CHK", cap.to_string())
111 d.addCallback(_created)
112 def _check_kids(children):
113 self.failUnlessEqual(sorted(children.keys()), [u"one", u"two"])
114 one_node, one_metadata = children[u"one"]
115 two_node, two_metadata = children[u"two"]
116 self.failUnlessEqual(one_node.get_size(), 3)
117 self.failUnlessEqual(two_node.get_size(), 14861)
118 self.failUnless(isinstance(one_metadata, dict), one_metadata)
119 self.failUnlessEqual(two_metadata["metakey"], "metavalue")
120 d.addCallback(_check_kids)
121 d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
122 d.addCallback(lambda dn: dn.list())
123 d.addCallback(_check_kids)
124 future_writecap = "x-tahoe-crazy://I_am_from_the_future."
125 future_readcap = "x-tahoe-crazy-readonly://I_am_from_the_future."
126 future_node = UnknownNode(future_writecap, future_readcap)
127 bad_kids1 = {u"one": (future_node, {})}
128 d.addCallback(lambda ign:
129 self.shouldFail(AssertionError, "bad_kids1",
130 "does not accept UnknownNode",
131 c.create_immutable_dirnode,
133 bad_kids2 = {u"one": (nm.create_from_cap(one_uri), None)}
134 d.addCallback(lambda ign:
135 self.shouldFail(AssertionError, "bad_kids2",
136 "requires metadata to be a dict",
137 c.create_immutable_dirnode,
139 bad_kids3 = {u"one": (nm.create_from_cap(mut_writecap), {})}
140 d.addCallback(lambda ign:
141 self.shouldFail(NotDeepImmutableError, "bad_kids3",
143 c.create_immutable_dirnode,
145 bad_kids4 = {u"one": (nm.create_from_cap(mut_readcap), {})}
146 d.addCallback(lambda ign:
147 self.shouldFail(NotDeepImmutableError, "bad_kids4",
149 c.create_immutable_dirnode,
151 d.addCallback(lambda ign: c.create_immutable_dirnode({}))
152 def _created_empty(dn):
153 self.failUnless(isinstance(dn, dirnode.DirectoryNode))
154 self.failIf(dn.is_mutable())
155 self.failUnless(dn.is_readonly())
157 self.failUnless("RO-IMM" in rep)
159 self.failUnlessIn("LIT", cap.to_string())
160 self.failUnlessEqual(cap.to_string(), "URI:DIR2-LIT:")
163 d.addCallback(_created_empty)
164 d.addCallback(lambda kids: self.failUnlessEqual(kids, {}))
165 smallkids = {u"o": (nm.create_from_cap(one_uri), {})}
166 d.addCallback(lambda ign: c.create_immutable_dirnode(smallkids))
167 def _created_small(dn):
168 self.failUnless(isinstance(dn, dirnode.DirectoryNode))
169 self.failIf(dn.is_mutable())
170 self.failUnless(dn.is_readonly())
172 self.failUnless("RO-IMM" in rep)
174 self.failUnlessIn("LIT", cap.to_string())
175 self.failUnlessEqual(cap.to_string(),
176 "URI:DIR2-LIT:gi4tumj2n4wdcmz2kvjesosmjfkdu3rvpbtwwlbqhiwdeot3puwcy")
179 d.addCallback(_created_small)
180 d.addCallback(lambda kids: self.failUnlessEqual(kids.keys(), [u"o"]))
182 # now test n.create_subdirectory(mutable=False)
183 d.addCallback(lambda ign: c.create_dirnode())
185 d = n.create_subdirectory(u"subdir", kids, mutable=False)
186 d.addCallback(lambda sd: sd.list())
187 d.addCallback(_check_kids)
188 d.addCallback(lambda ign: n.list())
189 d.addCallback(lambda children:
190 self.failUnlessEqual(children.keys(), [u"subdir"]))
191 d.addCallback(lambda ign: n.get(u"subdir"))
192 d.addCallback(lambda sd: sd.list())
193 d.addCallback(_check_kids)
194 d.addCallback(lambda ign: n.get(u"subdir"))
195 d.addCallback(lambda sd: self.failIf(sd.is_mutable()))
196 bad_kids = {u"one": (nm.create_from_cap(mut_writecap), {})}
197 d.addCallback(lambda ign:
198 self.shouldFail(NotDeepImmutableError, "YZ",
200 n.create_subdirectory,
201 u"sub2", bad_kids, mutable=False))
203 d.addCallback(_made_parent)
207 def test_check(self):
208 self.basedir = "dirnode/Dirnode/test_check"
210 c = self.g.clients[0]
211 d = c.create_dirnode()
212 d.addCallback(lambda dn: dn.check(Monitor()))
214 self.failUnless(res.is_healthy())
218 def _test_deepcheck_create(self):
219 # create a small tree with a loop, and some non-directories
223 # root/subdir/link -> root
225 c = self.g.clients[0]
226 d = c.create_dirnode()
227 def _created_root(rootnode):
228 self._rootnode = rootnode
229 return rootnode.create_subdirectory(u"subdir")
230 d.addCallback(_created_root)
231 def _created_subdir(subdir):
232 self._subdir = subdir
233 d = subdir.add_file(u"file1", upload.Data("data"*100, None))
234 d.addCallback(lambda res: subdir.set_node(u"link", self._rootnode))
235 d.addCallback(lambda res: c.create_dirnode())
236 d.addCallback(lambda dn:
237 self._rootnode.set_uri(u"rodir",
239 dn.get_readonly_uri()))
241 d.addCallback(_created_subdir)
243 return self._rootnode
247 def test_deepcheck(self):
248 self.basedir = "dirnode/Dirnode/test_deepcheck"
250 d = self._test_deepcheck_create()
251 d.addCallback(lambda rootnode: rootnode.start_deep_check().when_done())
252 def _check_results(r):
253 self.failUnless(IDeepCheckResults.providedBy(r))
255 self.failUnlessEqual(c,
256 {"count-objects-checked": 4,
257 "count-objects-healthy": 4,
258 "count-objects-unhealthy": 0,
259 "count-objects-unrecoverable": 0,
260 "count-corrupt-shares": 0,
262 self.failIf(r.get_corrupt_shares())
263 self.failUnlessEqual(len(r.get_all_results()), 4)
264 d.addCallback(_check_results)
267 def test_deepcheck_and_repair(self):
268 self.basedir = "dirnode/Dirnode/test_deepcheck_and_repair"
270 d = self._test_deepcheck_create()
271 d.addCallback(lambda rootnode:
272 rootnode.start_deep_check_and_repair().when_done())
273 def _check_results(r):
274 self.failUnless(IDeepCheckAndRepairResults.providedBy(r))
276 self.failUnlessEqual(c,
277 {"count-objects-checked": 4,
278 "count-objects-healthy-pre-repair": 4,
279 "count-objects-unhealthy-pre-repair": 0,
280 "count-objects-unrecoverable-pre-repair": 0,
281 "count-corrupt-shares-pre-repair": 0,
282 "count-objects-healthy-post-repair": 4,
283 "count-objects-unhealthy-post-repair": 0,
284 "count-objects-unrecoverable-post-repair": 0,
285 "count-corrupt-shares-post-repair": 0,
286 "count-repairs-attempted": 0,
287 "count-repairs-successful": 0,
288 "count-repairs-unsuccessful": 0,
290 self.failIf(r.get_corrupt_shares())
291 self.failIf(r.get_remaining_corrupt_shares())
292 self.failUnlessEqual(len(r.get_all_results()), 4)
293 d.addCallback(_check_results)
296 def _mark_file_bad(self, rootnode):
297 si = rootnode.get_storage_index()
298 self.delete_shares_numbered(rootnode.get_uri(), [0])
301 def test_deepcheck_problems(self):
302 self.basedir = "dirnode/Dirnode/test_deepcheck_problems"
304 d = self._test_deepcheck_create()
305 d.addCallback(lambda rootnode: self._mark_file_bad(rootnode))
306 d.addCallback(lambda rootnode: rootnode.start_deep_check().when_done())
307 def _check_results(r):
309 self.failUnlessEqual(c,
310 {"count-objects-checked": 4,
311 "count-objects-healthy": 3,
312 "count-objects-unhealthy": 1,
313 "count-objects-unrecoverable": 0,
314 "count-corrupt-shares": 0,
316 #self.failUnlessEqual(len(r.get_problems()), 1) # TODO
317 d.addCallback(_check_results)
320 def test_readonly(self):
321 self.basedir = "dirnode/Dirnode/test_readonly"
323 c = self.g.clients[0]
325 filecap = make_chk_file_uri(1234)
326 filenode = nm.create_from_cap(filecap)
327 uploadable = upload.Data("some data", convergence="some convergence string")
329 d = c.create_dirnode()
331 d2 = rw_dn.set_uri(u"child", filecap, filecap)
332 d2.addCallback(lambda res: rw_dn)
334 d.addCallback(_created)
337 ro_uri = rw_dn.get_readonly_uri()
338 ro_dn = c.create_node_from_uri(ro_uri)
339 self.failUnless(ro_dn.is_readonly())
340 self.failUnless(ro_dn.is_mutable())
342 self.shouldFail(dirnode.NotMutableError, "set_uri ro", None,
343 ro_dn.set_uri, u"newchild", filecap, filecap)
344 self.shouldFail(dirnode.NotMutableError, "set_uri ro", None,
345 ro_dn.set_node, u"newchild", filenode)
346 self.shouldFail(dirnode.NotMutableError, "set_nodes ro", None,
347 ro_dn.set_nodes, { u"newchild": (filenode, None) })
348 self.shouldFail(dirnode.NotMutableError, "set_uri ro", None,
349 ro_dn.add_file, u"newchild", uploadable)
350 self.shouldFail(dirnode.NotMutableError, "set_uri ro", None,
351 ro_dn.delete, u"child")
352 self.shouldFail(dirnode.NotMutableError, "set_uri ro", None,
353 ro_dn.create_subdirectory, u"newchild")
354 self.shouldFail(dirnode.NotMutableError, "set_metadata_for ro", None,
355 ro_dn.set_metadata_for, u"child", {})
356 self.shouldFail(dirnode.NotMutableError, "set_uri ro", None,
357 ro_dn.move_child_to, u"child", rw_dn)
358 self.shouldFail(dirnode.NotMutableError, "set_uri ro", None,
359 rw_dn.move_child_to, u"child", ro_dn)
361 d.addCallback(_ready)
362 def _listed(children):
363 self.failUnless(u"child" in children)
364 d.addCallback(_listed)
367 def failUnlessGreaterThan(self, a, b):
368 self.failUnless(a > b, "%r should be > %r" % (a, b))
370 def failUnlessGreaterOrEqualThan(self, a, b):
371 self.failUnless(a >= b, "%r should be >= %r" % (a, b))
373 def test_create(self):
374 self.basedir = "dirnode/Dirnode/test_create"
376 c = self.g.clients[0]
378 self.expected_manifest = []
379 self.expected_verifycaps = set()
380 self.expected_storage_indexes = set()
382 d = c.create_dirnode()
386 self.failUnless(n.is_mutable())
389 self.failUnless(u.startswith("URI:DIR2:"), u)
390 u_ro = n.get_readonly_uri()
391 self.failUnless(u_ro.startswith("URI:DIR2-RO:"), u_ro)
392 u_v = n.get_verify_cap().to_string()
393 self.failUnless(u_v.startswith("URI:DIR2-Verifier:"), u_v)
394 u_r = n.get_repair_cap().to_string()
395 self.failUnlessEqual(u_r, u)
396 self.expected_manifest.append( ((), u) )
397 self.expected_verifycaps.add(u_v)
398 si = n.get_storage_index()
399 self.expected_storage_indexes.add(base32.b2a(si))
400 expected_si = n._uri._filenode_uri.storage_index
401 self.failUnlessEqual(si, expected_si)
404 d.addCallback(lambda res: self.failUnlessEqual(res, {}))
405 d.addCallback(lambda res: n.has_child(u"missing"))
406 d.addCallback(lambda res: self.failIf(res))
408 fake_file_uri = make_mutable_file_uri()
409 other_file_uri = make_mutable_file_uri()
410 m = c.nodemaker.create_from_cap(fake_file_uri)
411 ffu_v = m.get_verify_cap().to_string()
412 self.expected_manifest.append( ((u"child",) , m.get_uri()) )
413 self.expected_verifycaps.add(ffu_v)
414 self.expected_storage_indexes.add(base32.b2a(m.get_storage_index()))
415 d.addCallback(lambda res: n.set_uri(u"child",
416 fake_file_uri, fake_file_uri))
417 d.addCallback(lambda res:
418 self.shouldFail(ExistingChildError, "set_uri-no",
419 "child 'child' already exists",
421 other_file_uri, other_file_uri,
426 d.addCallback(lambda res: n.create_subdirectory(u"subdir"))
430 # /subdir = directory
431 def _created(subdir):
432 self.failUnless(isinstance(subdir, dirnode.DirectoryNode))
434 new_v = subdir.get_verify_cap().to_string()
435 assert isinstance(new_v, str)
436 self.expected_manifest.append( ((u"subdir",), subdir.get_uri()) )
437 self.expected_verifycaps.add(new_v)
438 si = subdir.get_storage_index()
439 self.expected_storage_indexes.add(base32.b2a(si))
440 d.addCallback(_created)
442 d.addCallback(lambda res:
443 self.shouldFail(ExistingChildError, "mkdir-no",
444 "child 'subdir' already exists",
445 n.create_subdirectory, u"subdir",
448 d.addCallback(lambda res: n.list())
449 d.addCallback(lambda children:
450 self.failUnlessEqual(sorted(children.keys()),
451 sorted([u"child", u"subdir"])))
453 d.addCallback(lambda res: n.start_deep_stats().when_done())
454 def _check_deepstats(stats):
455 self.failUnless(isinstance(stats, dict))
456 expected = {"count-immutable-files": 0,
457 "count-mutable-files": 1,
458 "count-literal-files": 0,
460 "count-directories": 2,
461 "size-immutable-files": 0,
462 "size-literal-files": 0,
463 #"size-directories": 616, # varies
464 #"largest-directory": 616,
465 "largest-directory-children": 2,
466 "largest-immutable-file": 0,
468 for k,v in expected.iteritems():
469 self.failUnlessEqual(stats[k], v,
470 "stats[%s] was %s, not %s" %
472 self.failUnless(stats["size-directories"] > 500,
473 stats["size-directories"])
474 self.failUnless(stats["largest-directory"] > 500,
475 stats["largest-directory"])
476 self.failUnlessEqual(stats["size-files-histogram"], [])
477 d.addCallback(_check_deepstats)
479 d.addCallback(lambda res: n.build_manifest().when_done())
480 def _check_manifest(res):
481 manifest = res["manifest"]
482 self.failUnlessEqual(sorted(manifest),
483 sorted(self.expected_manifest))
485 _check_deepstats(stats)
486 self.failUnlessEqual(self.expected_verifycaps,
488 self.failUnlessEqual(self.expected_storage_indexes,
489 res["storage-index"])
490 d.addCallback(_check_manifest)
492 def _add_subsubdir(res):
493 return self.subdir.create_subdirectory(u"subsubdir")
494 d.addCallback(_add_subsubdir)
497 # /subdir = directory
498 # /subdir/subsubdir = directory
499 d.addCallback(lambda res: n.get_child_at_path(u"subdir/subsubdir"))
500 d.addCallback(lambda subsubdir:
501 self.failUnless(isinstance(subsubdir,
502 dirnode.DirectoryNode)))
503 d.addCallback(lambda res: n.get_child_at_path(u""))
504 d.addCallback(lambda res: self.failUnlessEqual(res.get_uri(),
507 d.addCallback(lambda res: n.get_metadata_for(u"child"))
508 d.addCallback(lambda metadata:
509 self.failUnlessEqual(set(metadata.keys()),
510 set(["tahoe", "ctime", "mtime"])))
512 d.addCallback(lambda res:
513 self.shouldFail(NoSuchChildError, "gcamap-no",
515 n.get_child_and_metadata_at_path,
517 d.addCallback(lambda res:
518 n.get_child_and_metadata_at_path(u""))
519 def _check_child_and_metadata1(res):
520 child, metadata = res
521 self.failUnless(isinstance(child, dirnode.DirectoryNode))
522 # edge-metadata needs at least one path segment
523 self.failUnlessEqual(sorted(metadata.keys()), [])
524 d.addCallback(_check_child_and_metadata1)
525 d.addCallback(lambda res:
526 n.get_child_and_metadata_at_path(u"child"))
528 def _check_child_and_metadata2(res):
529 child, metadata = res
530 self.failUnlessEqual(child.get_uri(),
532 self.failUnlessEqual(set(metadata.keys()),
533 set(["tahoe", "ctime", "mtime"]))
534 d.addCallback(_check_child_and_metadata2)
536 d.addCallback(lambda res:
537 n.get_child_and_metadata_at_path(u"subdir/subsubdir"))
538 def _check_child_and_metadata3(res):
539 child, metadata = res
540 self.failUnless(isinstance(child, dirnode.DirectoryNode))
541 self.failUnlessEqual(set(metadata.keys()),
542 set(["tahoe", "ctime", "mtime"]))
543 d.addCallback(_check_child_and_metadata3)
546 # it should be possible to add a child without any metadata
547 d.addCallback(lambda res: n.set_uri(u"c2",
548 fake_file_uri, fake_file_uri,
550 d.addCallback(lambda res: n.get_metadata_for(u"c2"))
551 d.addCallback(lambda metadata: self.failUnlessEqual(metadata.keys(), ['tahoe']))
553 # You can't override the link timestamps.
554 d.addCallback(lambda res: n.set_uri(u"c2",
555 fake_file_uri, fake_file_uri,
556 { 'tahoe': {'linkcrtime': "bogus"}}))
557 d.addCallback(lambda res: n.get_metadata_for(u"c2"))
558 def _has_good_linkcrtime(metadata):
559 self.failUnless(metadata.has_key('tahoe'))
560 self.failUnless(metadata['tahoe'].has_key('linkcrtime'))
561 self.failIfEqual(metadata['tahoe']['linkcrtime'], 'bogus')
562 d.addCallback(_has_good_linkcrtime)
564 # if we don't set any defaults, the child should get timestamps
565 d.addCallback(lambda res: n.set_uri(u"c3",
566 fake_file_uri, fake_file_uri))
567 d.addCallback(lambda res: n.get_metadata_for(u"c3"))
568 d.addCallback(lambda metadata:
569 self.failUnlessEqual(set(metadata.keys()),
570 set(["tahoe", "ctime", "mtime"])))
572 # or we can add specific metadata at set_uri() time, which
573 # overrides the timestamps
574 d.addCallback(lambda res: n.set_uri(u"c4",
575 fake_file_uri, fake_file_uri,
577 d.addCallback(lambda res: n.get_metadata_for(u"c4"))
578 d.addCallback(lambda metadata:
579 self.failUnless((set(metadata.keys()) == set(["key", "tahoe"])) and
580 (metadata['key'] == "value"), metadata))
582 d.addCallback(lambda res: n.delete(u"c2"))
583 d.addCallback(lambda res: n.delete(u"c3"))
584 d.addCallback(lambda res: n.delete(u"c4"))
586 # set_node + metadata
587 # it should be possible to add a child without any metadata
588 d.addCallback(lambda res: n.set_node(u"d2", n, {}))
589 d.addCallback(lambda res: c.create_dirnode())
590 d.addCallback(lambda n2:
591 self.shouldFail(ExistingChildError, "set_node-no",
592 "child 'd2' already exists",
593 n.set_node, u"d2", n2,
595 d.addCallback(lambda res: n.get_metadata_for(u"d2"))
596 d.addCallback(lambda metadata: self.failUnlessEqual(metadata.keys(), ['tahoe']))
598 # if we don't set any defaults, the child should get timestamps
599 d.addCallback(lambda res: n.set_node(u"d3", n))
600 d.addCallback(lambda res: n.get_metadata_for(u"d3"))
601 d.addCallback(lambda metadata:
602 self.failUnlessEqual(set(metadata.keys()),
603 set(["tahoe", "ctime", "mtime"])))
605 # or we can add specific metadata at set_node() time, which
606 # overrides the timestamps
607 d.addCallback(lambda res: n.set_node(u"d4", n,
609 d.addCallback(lambda res: n.get_metadata_for(u"d4"))
610 d.addCallback(lambda metadata:
611 self.failUnless((set(metadata.keys()) == set(["key", "tahoe"])) and
612 (metadata['key'] == "value"), metadata))
614 d.addCallback(lambda res: n.delete(u"d2"))
615 d.addCallback(lambda res: n.delete(u"d3"))
616 d.addCallback(lambda res: n.delete(u"d4"))
618 # metadata through set_children()
619 d.addCallback(lambda res:
621 u"e1": (fake_file_uri, fake_file_uri),
622 u"e2": (fake_file_uri, fake_file_uri, {}),
623 u"e3": (fake_file_uri, fake_file_uri,
626 d.addCallback(lambda n2: self.failUnlessIdentical(n2, n))
627 d.addCallback(lambda res:
628 self.shouldFail(ExistingChildError, "set_children-no",
629 "child 'e1' already exists",
631 { u"e1": (other_file_uri,
633 u"new": (other_file_uri,
637 # and 'new' should not have been created
638 d.addCallback(lambda res: n.list())
639 d.addCallback(lambda children: self.failIf(u"new" in children))
640 d.addCallback(lambda res: n.get_metadata_for(u"e1"))
641 d.addCallback(lambda metadata:
642 self.failUnlessEqual(set(metadata.keys()),
643 set(["tahoe", "ctime", "mtime"])))
644 d.addCallback(lambda res: n.get_metadata_for(u"e2"))
645 d.addCallback(lambda metadata:
646 self.failUnlessEqual(set(metadata.keys()), set(['tahoe'])))
647 d.addCallback(lambda res: n.get_metadata_for(u"e3"))
648 d.addCallback(lambda metadata:
649 self.failUnless((set(metadata.keys()) == set(["key", "tahoe"]))
650 and (metadata['key'] == "value"), metadata))
652 d.addCallback(lambda res: n.delete(u"e1"))
653 d.addCallback(lambda res: n.delete(u"e2"))
654 d.addCallback(lambda res: n.delete(u"e3"))
656 # metadata through set_nodes()
657 d.addCallback(lambda res:
658 n.set_nodes({ u"f1": (n, None),
660 u"f3": (n, {"key": "value"}),
662 d.addCallback(lambda n2: self.failUnlessIdentical(n2, n))
663 d.addCallback(lambda res:
664 self.shouldFail(ExistingChildError, "set_nodes-no",
665 "child 'f1' already exists",
666 n.set_nodes, { u"f1": (n, None),
667 u"new": (n, None), },
669 # and 'new' should not have been created
670 d.addCallback(lambda res: n.list())
671 d.addCallback(lambda children: self.failIf(u"new" in children))
672 d.addCallback(lambda res: n.get_metadata_for(u"f1"))
673 d.addCallback(lambda metadata:
674 self.failUnlessEqual(set(metadata.keys()),
675 set(["tahoe", "ctime", "mtime"])))
676 d.addCallback(lambda res: n.get_metadata_for(u"f2"))
678 lambda metadata: self.failUnlessEqual(set(metadata.keys()), set(['tahoe'])))
679 d.addCallback(lambda res: n.get_metadata_for(u"f3"))
680 d.addCallback(lambda metadata:
681 self.failUnless((set(metadata.keys()) == set(["key", "tahoe"])) and
682 (metadata['key'] == "value"), metadata))
684 d.addCallback(lambda res: n.delete(u"f1"))
685 d.addCallback(lambda res: n.delete(u"f2"))
686 d.addCallback(lambda res: n.delete(u"f3"))
689 d.addCallback(lambda res:
690 n.set_metadata_for(u"child",
691 {"tags": ["web2.0-compatible"]}))
692 d.addCallback(lambda n1: n1.get_metadata_for(u"child"))
693 d.addCallback(lambda metadata:
694 self.failUnlessEqual(metadata,
695 {"tags": ["web2.0-compatible"]}))
698 self._start_timestamp = time.time()
699 d.addCallback(_start)
700 # simplejson-1.7.1 (as shipped on Ubuntu 'gutsy') rounds all
701 # floats to hundredeths (it uses str(num) instead of repr(num)).
702 # simplejson-1.7.3 does not have this bug. To prevent this bug
703 # from causing the test to fail, stall for more than a few
704 # hundrededths of a second.
705 d.addCallback(self.stall, 0.1)
706 d.addCallback(lambda res: n.add_file(u"timestamps",
707 upload.Data("stamp me", convergence="some convergence string")))
708 d.addCallback(self.stall, 0.1)
710 self._stop_timestamp = time.time()
713 d.addCallback(lambda res: n.get_metadata_for(u"timestamps"))
714 def _check_timestamp1(metadata):
715 self.failUnless("ctime" in metadata)
716 self.failUnless("mtime" in metadata)
717 self.failUnlessGreaterOrEqualThan(metadata["ctime"],
718 self._start_timestamp)
719 self.failUnlessGreaterOrEqualThan(self._stop_timestamp,
721 self.failUnlessGreaterOrEqualThan(metadata["mtime"],
722 self._start_timestamp)
723 self.failUnlessGreaterOrEqualThan(self._stop_timestamp,
725 # Our current timestamp rules say that replacing an existing
726 # child should preserve the 'ctime' but update the mtime
727 self._old_ctime = metadata["ctime"]
728 self._old_mtime = metadata["mtime"]
729 d.addCallback(_check_timestamp1)
730 d.addCallback(self.stall, 2.0) # accomodate low-res timestamps
731 d.addCallback(lambda res: n.set_node(u"timestamps", n))
732 d.addCallback(lambda res: n.get_metadata_for(u"timestamps"))
733 def _check_timestamp2(metadata):
734 self.failUnlessEqual(metadata["ctime"], self._old_ctime,
735 "%s != %s" % (metadata["ctime"],
737 self.failUnlessGreaterThan(metadata["mtime"], self._old_mtime)
738 return n.delete(u"timestamps")
739 d.addCallback(_check_timestamp2)
741 # also make sure we can add/update timestamps on a
742 # previously-existing child that didn't have any, since there are
743 # a lot of 0.7.0-generated edges around out there
744 d.addCallback(lambda res: n.set_node(u"no_timestamps", n, {}))
745 d.addCallback(lambda res: n.set_node(u"no_timestamps", n))
746 d.addCallback(lambda res: n.get_metadata_for(u"no_timestamps"))
747 d.addCallback(lambda metadata:
748 self.failUnlessEqual(set(metadata.keys()),
749 set(["tahoe", "ctime", "mtime"])))
750 d.addCallback(lambda res: n.delete(u"no_timestamps"))
752 d.addCallback(lambda res: n.delete(u"subdir"))
753 d.addCallback(lambda old_child:
754 self.failUnlessEqual(old_child.get_uri(),
755 self.subdir.get_uri()))
757 d.addCallback(lambda res: n.list())
758 d.addCallback(lambda children:
759 self.failUnlessEqual(sorted(children.keys()),
762 uploadable1 = upload.Data("some data", convergence="converge")
763 d.addCallback(lambda res: n.add_file(u"newfile", uploadable1))
764 d.addCallback(lambda newnode:
765 self.failUnless(IFileNode.providedBy(newnode)))
766 uploadable2 = upload.Data("some data", convergence="stuff")
767 d.addCallback(lambda res:
768 self.shouldFail(ExistingChildError, "add_file-no",
769 "child 'newfile' already exists",
770 n.add_file, u"newfile",
773 d.addCallback(lambda res: n.list())
774 d.addCallback(lambda children:
775 self.failUnlessEqual(sorted(children.keys()),
776 sorted([u"child", u"newfile"])))
777 d.addCallback(lambda res: n.get_metadata_for(u"newfile"))
778 d.addCallback(lambda metadata:
779 self.failUnlessEqual(set(metadata.keys()),
780 set(["tahoe", "ctime", "mtime"])))
782 uploadable3 = upload.Data("some data", convergence="converge")
783 d.addCallback(lambda res: n.add_file(u"newfile-metadata",
786 d.addCallback(lambda newnode:
787 self.failUnless(IFileNode.providedBy(newnode)))
788 d.addCallback(lambda res: n.get_metadata_for(u"newfile-metadata"))
789 d.addCallback(lambda metadata:
790 self.failUnless((set(metadata.keys()) == set(["key", "tahoe"])) and
791 (metadata['key'] == "value"), metadata))
792 d.addCallback(lambda res: n.delete(u"newfile-metadata"))
794 d.addCallback(lambda res: n.create_subdirectory(u"subdir2"))
795 def _created2(subdir2):
796 self.subdir2 = subdir2
797 # put something in the way, to make sure it gets overwritten
798 return subdir2.add_file(u"child", upload.Data("overwrite me",
800 d.addCallback(_created2)
802 d.addCallback(lambda res:
803 n.move_child_to(u"child", self.subdir2))
804 d.addCallback(lambda res: n.list())
805 d.addCallback(lambda children:
806 self.failUnlessEqual(sorted(children.keys()),
807 sorted([u"newfile", u"subdir2"])))
808 d.addCallback(lambda res: self.subdir2.list())
809 d.addCallback(lambda children:
810 self.failUnlessEqual(sorted(children.keys()),
812 d.addCallback(lambda res: self.subdir2.get(u"child"))
813 d.addCallback(lambda child:
814 self.failUnlessEqual(child.get_uri(),
817 # move it back, using new_child_name=
818 d.addCallback(lambda res:
819 self.subdir2.move_child_to(u"child", n, u"newchild"))
820 d.addCallback(lambda res: n.list())
821 d.addCallback(lambda children:
822 self.failUnlessEqual(sorted(children.keys()),
823 sorted([u"newchild", u"newfile",
825 d.addCallback(lambda res: self.subdir2.list())
826 d.addCallback(lambda children:
827 self.failUnlessEqual(sorted(children.keys()), []))
829 # now make sure that we honor overwrite=False
830 d.addCallback(lambda res:
831 self.subdir2.set_uri(u"newchild",
832 other_file_uri, other_file_uri))
834 d.addCallback(lambda res:
835 self.shouldFail(ExistingChildError, "move_child_to-no",
836 "child 'newchild' already exists",
837 n.move_child_to, u"newchild",
840 d.addCallback(lambda res: self.subdir2.get(u"newchild"))
841 d.addCallback(lambda child:
842 self.failUnlessEqual(child.get_uri(),
849 d.addErrback(self.explain_error)
852 def test_create_subdirectory(self):
853 self.basedir = "dirnode/Dirnode/test_create_subdirectory"
855 c = self.g.clients[0]
858 d = c.create_dirnode()
862 fake_file_uri = make_mutable_file_uri()
863 other_file_uri = make_mutable_file_uri()
864 md = {"metakey": "metavalue"}
865 kids = {u"kid1": (nm.create_from_cap(fake_file_uri), {}),
866 u"kid2": (nm.create_from_cap(other_file_uri), md),
868 d = n.create_subdirectory(u"subdir", kids)
870 d = n.get_child_at_path(u"subdir")
871 d.addCallback(lambda sub2: self.failUnlessEqual(sub2.get_uri(),
873 d.addCallback(lambda ign: sub.list())
875 d.addCallback(_check)
876 def _check_kids(kids2):
877 self.failUnlessEqual(sorted(kids.keys()), sorted(kids2.keys()))
878 self.failUnlessEqual(kids2[u"kid2"][1]["metakey"], "metavalue")
879 d.addCallback(_check_kids)
884 class MinimalFakeMutableFile:
885 def get_writekey(self):
888 class Packing(unittest.TestCase):
889 # This is a base32-encoded representation of the directory tree
893 # as represented after being fed to _pack_contents.
894 # We have it here so we can decode it, feed it to
895 # _unpack_contents, and verify that _unpack_contents
898 known_tree = "GM4TOORVHJTGS3DFGEWDSNJ2KVJESOSDJBFTU33MPB2GS3LZNVYG6N3GGI3WU5TIORTXC3DOMJ2G4NB2MVWXUZDONBVTE5LNGRZWK2LYN55GY23XGNYXQMTOMZUWU5TENN4DG23ZG5UTO2L2NQ2DO6LFMRWDMZJWGRQTUMZ2GEYDUMJQFQYTIMZ22XZKZORX5XS7CAQCSK3URR6QOHISHRCMGER5LRFSZRNAS5ZSALCS6TWFQAE754IVOIKJVK73WZPP3VUUEDTX3WHTBBZ5YX3CEKHCPG3ZWQLYA4QM6LDRCF7TJQYWLIZHKGN5ROA3AUZPXESBNLQQ6JTC2DBJU2D47IZJTLR3PKZ4RVF57XLPWY7FX7SZV3T6IJ3ORFW37FXUPGOE3ROPFNUX5DCGMAQJ3PGGULBRGM3TU6ZCMN2GS3LFEI5CAMJSGQ3DMNRTHA4TOLRUGI3TKNRWGEWCAITUMFUG6ZJCHIQHWITMNFXGW3LPORUW2ZJCHIQDCMRUGY3DMMZYHE3S4NBSG42TMNRRFQQCE3DJNZVWG4TUNFWWKIR2EAYTENBWGY3DGOBZG4XDIMRXGU3DMML5FQQCE3LUNFWWKIR2EAYTENBWGY3DGOBZG4XDIMRXGU3DMML5FQWDGOJRHI2TUZTJNRSTELBZGQ5FKUSJHJBUQSZ2MFYGKZ3SOBSWQ43IO52WO23CNAZWU3DUGVSWSNTIOE5DK33POVTW4ZLNMNWDK6DHPA2GS2THNF2W25DEN5VGY2LQNFRGG5DKNNRHO5TZPFTWI6LNMRYGQ2LCGJTHM4J2GM5DCMB2GQWDCNBSHKVVQBGRYMACKJ27CVQ6O6B4QPR72RFVTGOZUI76XUSWAX73JRV5PYRHMIFYZIA25MXDPGUGML6M2NMRSG4YD4W4K37ZDYSXHMJ3IUVT4F64YTQQVBJFFFOUC7J7LAB2VFCL5UKKGMR2D3F4EPOYC7UYWQZNR5KXHBSNXLCNBX2SNF22DCXJIHSMEKWEWOG5XCJEVVZ7UW5IB6I64XXQSJ34B5CAYZGZIIMR6LBRGMZTU6ZCMN2GS3LFEI5CAMJSGQ3DMNRTHA4TOLRUGMYDEMJYFQQCE5DBNBXWKIR2EB5SE3DJNZVW233UNFWWKIR2EAYTENBWGY3DGOBZG4XDIMZQGIYTQLBAEJWGS3TLMNZHI2LNMURDUIBRGI2DMNRWGM4DSNZOGQZTAMRRHB6SYIBCNV2GS3LFEI5CAMJSGQ3DMNRTHA4TOLRUGMYDEMJYPUWCYMZZGU5DKOTGNFWGKMZMHE2DUVKSJE5EGSCLHJRW25DDPBYTO2DXPB3GM6DBNYZTI6LJMV3DM2LWNB4TU4LWMNSWW3LKORXWK5DEMN3TI23NNE3WEM3SORRGY5THPA3TKNBUMNZG453BOF2GSZLXMVWWI3DJOFZW623RHIZTUMJQHI2SYMJUGI5BOSHWDPG3WKPAVXCF3XMKA7QVIWPRMWJHDTQHD27AHDCPJWDQENQ5H5ZZILTXQNIXXCIW4LKQABU2GCFRG5FHQN7CHD7HF4EKNRZFIV2ZYQIBM7IQU7F4RGB3XCX3FREPBKQ7UCICHVWPCYFGA6OLH3J45LXQ6GWWICJ3PGWJNLZ7PCRNLAPNYUGU6BENS7OXMBEOOFRIZV3PF2FFWZ5WHDPKXERYP7GNHKRMGEZTOOT3EJRXI2LNMURDUIBRGI2DMNRWGM4DSNZOGQZTGNRSGY4SYIBCORQWQ33FEI5CA6ZCNRUW423NN52GS3LFEI5CAMJSGQ3DMNRTHA4TOLRUGMZTMMRWHEWCAITMNFXGWY3SORUW2ZJCHIQDCMRUGY3DMMZYHE3S4NBTGM3DENRZPUWCAITNORUW2ZJCHIQDCMRUGY3DMMZYHE3S4NBTGM3DENRZPUWCY==="
900 def test_unpack_and_pack_behavior(self):
901 known_tree = b32decode(self.known_tree)
902 nodemaker = NodeMaker(None, None, None,
904 {"k": 3, "n": 10}, None)
905 writecap = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
906 filenode = nodemaker.create_from_cap(writecap)
907 node = dirnode.DirectoryNode(filenode, nodemaker, None)
908 children = node._unpack_contents(known_tree)
909 self._check_children(children)
911 packed_children = node._pack_contents(children)
912 children = node._unpack_contents(packed_children)
913 self._check_children(children)
915 def _check_children(self, children):
916 # Are all the expected child nodes there?
917 self.failUnless(children.has_key(u'file1'))
918 self.failUnless(children.has_key(u'file2'))
919 self.failUnless(children.has_key(u'file3'))
921 # Are the metadata for child 3 right?
922 file3_rocap = "URI:CHK:cmtcxq7hwxvfxan34yiev6ivhy:qvcekmjtoetdcw4kmi7b3rtblvgx7544crnwaqtiewemdliqsokq:3:10:5"
923 file3_rwcap = "URI:CHK:cmtcxq7hwxvfxan34yiev6ivhy:qvcekmjtoetdcw4kmi7b3rtblvgx7544crnwaqtiewemdliqsokq:3:10:5"
924 file3_metadata = {'ctime': 1246663897.4336269, 'tahoe': {'linkmotime': 1246663897.4336269, 'linkcrtime': 1246663897.4336269}, 'mtime': 1246663897.4336269}
925 self.failUnlessEqual(file3_metadata, children[u'file3'][1])
926 self.failUnlessEqual(file3_rocap,
927 children[u'file3'][0].get_readonly_uri())
928 self.failUnlessEqual(file3_rwcap,
929 children[u'file3'][0].get_uri())
931 # Are the metadata for child 2 right?
932 file2_rocap = "URI:CHK:apegrpehshwugkbh3jlt5ei6hq:5oougnemcl5xgx4ijgiumtdojlipibctjkbwvyygdymdphib2fvq:3:10:4"
933 file2_rwcap = "URI:CHK:apegrpehshwugkbh3jlt5ei6hq:5oougnemcl5xgx4ijgiumtdojlipibctjkbwvyygdymdphib2fvq:3:10:4"
934 file2_metadata = {'ctime': 1246663897.430218, 'tahoe': {'linkmotime': 1246663897.430218, 'linkcrtime': 1246663897.430218}, 'mtime': 1246663897.430218}
935 self.failUnlessEqual(file2_metadata, children[u'file2'][1])
936 self.failUnlessEqual(file2_rocap,
937 children[u'file2'][0].get_readonly_uri())
938 self.failUnlessEqual(file2_rwcap,
939 children[u'file2'][0].get_uri())
941 # Are the metadata for child 1 right?
942 file1_rocap = "URI:CHK:olxtimympo7f27jvhtgqlnbtn4:emzdnhk2um4seixozlkw3qx2nfijvdkx3ky7i7izl47yedl6e64a:3:10:10"
943 file1_rwcap = "URI:CHK:olxtimympo7f27jvhtgqlnbtn4:emzdnhk2um4seixozlkw3qx2nfijvdkx3ky7i7izl47yedl6e64a:3:10:10"
944 file1_metadata = {'ctime': 1246663897.4275661, 'tahoe': {'linkmotime': 1246663897.4275661, 'linkcrtime': 1246663897.4275661}, 'mtime': 1246663897.4275661}
945 self.failUnlessEqual(file1_metadata, children[u'file1'][1])
946 self.failUnlessEqual(file1_rocap,
947 children[u'file1'][0].get_readonly_uri())
948 self.failUnlessEqual(file1_rwcap,
949 children[u'file1'][0].get_uri())
951 def _make_kids(self, nm, which):
952 caps = {"imm": "URI:CHK:n7r3m6wmomelk4sep3kw5cvduq:os7ijw5c3maek7pg65e5254k2fzjflavtpejjyhshpsxuqzhcwwq:3:20:14861",
953 "lit": "URI:LIT:n5xgk", # LIT for "one"
954 "write": "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq",
955 "read": "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q",
956 "dirwrite": "URI:DIR2:n6x24zd3seu725yluj75q5boaa:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq",
957 "dirread": "URI:DIR2-RO:b7sr5qsifnicca7cbk3rhrhbvq:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq",
961 kids[unicode(name)] = (nm.create_from_cap(caps[name]), {})
964 def test_deep_immutable(self):
965 nm = NodeMaker(None, None, None, None, None, None, {"k": 3, "n": 10},
967 fn = MinimalFakeMutableFile()
969 kids = self._make_kids(nm, ["imm", "lit", "write", "read",
970 "dirwrite", "dirread"])
971 packed = dirnode.pack_children(fn, kids, deep_immutable=False)
972 self.failUnlessIn("lit", packed)
974 kids = self._make_kids(nm, ["imm", "lit"])
975 packed = dirnode.pack_children(fn, kids, deep_immutable=True)
976 self.failUnlessIn("lit", packed)
978 kids = self._make_kids(nm, ["imm", "lit", "write"])
979 e = self.failUnlessRaises(dirnode.MustBeDeepImmutable,
980 dirnode.pack_children,
981 fn, kids, deep_immutable=True)
983 # read-only is not enough: all children must be immutable
984 kids = self._make_kids(nm, ["imm", "lit", "read"])
985 e = self.failUnlessRaises(dirnode.MustBeDeepImmutable,
986 dirnode.pack_children,
987 fn, kids, deep_immutable=True)
989 kids = self._make_kids(nm, ["imm", "lit", "dirwrite"])
990 e = self.failUnlessRaises(dirnode.MustBeDeepImmutable,
991 dirnode.pack_children,
992 fn, kids, deep_immutable=True)
994 kids = self._make_kids(nm, ["imm", "lit", "dirread"])
995 e = self.failUnlessRaises(dirnode.MustBeDeepImmutable,
996 dirnode.pack_children,
997 fn, kids, deep_immutable=True)
999 class FakeMutableFile:
1000 implements(IMutableFileNode)
1002 def __init__(self, initial_contents=""):
1003 self.data = self._get_initial_contents(initial_contents)
1004 counter = FakeMutableFile.counter
1005 FakeMutableFile.counter += 1
1006 writekey = hashutil.ssk_writekey_hash(str(counter))
1007 fingerprint = hashutil.ssk_pubkey_fingerprint_hash(str(counter))
1008 self.uri = uri.WriteableSSKFileURI(writekey, fingerprint)
1010 def _get_initial_contents(self, contents):
1011 if isinstance(contents, str):
1013 if contents is None:
1015 assert callable(contents), "%s should be callable, not %s" % \
1016 (contents, type(contents))
1017 return contents(self)
1022 return self.uri.to_string()
1023 def download_best_version(self):
1024 return defer.succeed(self.data)
1025 def get_writekey(self):
1027 def is_readonly(self):
1029 def is_mutable(self):
1031 def modify(self, modifier):
1032 self.data = modifier(self.data, None, True)
1033 return defer.succeed(None)
1035 class FakeNodeMaker(NodeMaker):
1036 def create_mutable_file(self, contents="", keysize=None):
1037 return defer.succeed(FakeMutableFile(contents))
1039 class FakeClient2(Client):
1041 self.nodemaker = FakeNodeMaker(None, None, None,
1043 {"k":3,"n":10}, None)
1044 def create_node_from_uri(self, rwcap, rocap):
1045 return self.nodemaker.create_from_cap(rwcap, rocap)
1047 class Dirnode2(unittest.TestCase, testutil.ShouldFailMixin):
1049 self.client = FakeClient2()
1050 self.nodemaker = self.client.nodemaker
1052 def test_from_future(self):
1053 # create a dirnode that contains unknown URI types, and make sure we
1054 # tolerate them properly. Since dirnodes aren't allowed to add
1055 # unknown node types, we have to be tricky.
1056 d = self.nodemaker.create_new_mutable_directory()
1057 future_writecap = "x-tahoe-crazy://I_am_from_the_future."
1058 future_readcap = "x-tahoe-crazy-readonly://I_am_from_the_future."
1059 future_node = UnknownNode(future_writecap, future_readcap)
1062 return n.set_node(u"future", future_node)
1063 d.addCallback(_then)
1065 # we should be prohibited from adding an unknown URI to a directory,
1066 # since we don't know how to diminish the cap to a readcap (for the
1067 # dirnode's rocap slot), and we don't want to accidentally grant
1068 # write access to a holder of the dirnode's readcap.
1069 d.addCallback(lambda ign:
1070 self.shouldFail(CannotPackUnknownNodeError,
1072 "cannot pack unknown node as child add",
1073 self._node.set_uri, u"add",
1074 future_writecap, future_readcap))
1075 d.addCallback(lambda ign: self._node.list())
1076 def _check(children):
1077 self.failUnlessEqual(len(children), 1)
1078 (fn, metadata) = children[u"future"]
1079 self.failUnless(isinstance(fn, UnknownNode), fn)
1080 self.failUnlessEqual(fn.get_uri(), future_writecap)
1081 self.failUnlessEqual(fn.get_readonly_uri(), future_readcap)
1082 # but we *should* be allowed to copy this node, because the
1083 # UnknownNode contains all the information that was in the
1084 # original directory (readcap and writecap), so we're preserving
1086 return self._node.set_node(u"copy", fn)
1087 d.addCallback(_check)
1088 d.addCallback(lambda ign: self._node.list())
1089 def _check2(children):
1090 self.failUnlessEqual(len(children), 2)
1091 (fn, metadata) = children[u"copy"]
1092 self.failUnless(isinstance(fn, UnknownNode), fn)
1093 self.failUnlessEqual(fn.get_uri(), future_writecap)
1094 self.failUnlessEqual(fn.get_readonly_uri(), future_readcap)
1097 class DeepStats(unittest.TestCase):
1098 timeout = 240 # It takes longer than 120 seconds on Francois's arm box.
1099 def test_stats(self):
1100 ds = dirnode.DeepStats(None)
1101 ds.add("count-files")
1102 ds.add("size-immutable-files", 123)
1103 ds.histogram("size-files-histogram", 123)
1104 ds.max("largest-directory", 444)
1106 s = ds.get_results()
1107 self.failUnlessEqual(s["count-files"], 1)
1108 self.failUnlessEqual(s["size-immutable-files"], 123)
1109 self.failUnlessEqual(s["largest-directory"], 444)
1110 self.failUnlessEqual(s["count-literal-files"], 0)
1112 ds.add("count-files")
1113 ds.add("size-immutable-files", 321)
1114 ds.histogram("size-files-histogram", 321)
1115 ds.max("largest-directory", 2)
1117 s = ds.get_results()
1118 self.failUnlessEqual(s["count-files"], 2)
1119 self.failUnlessEqual(s["size-immutable-files"], 444)
1120 self.failUnlessEqual(s["largest-directory"], 444)
1121 self.failUnlessEqual(s["count-literal-files"], 0)
1122 self.failUnlessEqual(s["size-files-histogram"],
1123 [ (101, 316, 1), (317, 1000, 1) ])
1125 ds = dirnode.DeepStats(None)
1126 for i in range(1, 1100):
1127 ds.histogram("size-files-histogram", i)
1128 ds.histogram("size-files-histogram", 4*1000*1000*1000*1000) # 4TB
1129 s = ds.get_results()
1130 self.failUnlessEqual(s["size-files-histogram"],
1138 (3162277660169L, 10000000000000L, 1),
1141 class UCWEingMutableFileNode(MutableFileNode):
1142 please_ucwe_after_next_upload = False
1144 def _upload(self, new_contents, servermap):
1145 d = MutableFileNode._upload(self, new_contents, servermap)
1147 if self.please_ucwe_after_next_upload:
1148 self.please_ucwe_after_next_upload = False
1149 raise UncoordinatedWriteError()
1151 d.addCallback(_ucwe)
1154 class UCWEingNodeMaker(NodeMaker):
1155 def _create_mutable(self, cap):
1156 n = UCWEingMutableFileNode(self.storage_broker, self.secret_holder,
1157 self.default_encoding_parameters,
1159 return n.init_from_cap(cap)
1162 class Deleter(GridTestMixin, unittest.TestCase):
1163 timeout = 3600 # It takes longer than 433 seconds on Zandr's ARM box.
1164 def test_retry(self):
1165 # ticket #550, a dirnode.delete which experiences an
1166 # UncoordinatedWriteError will fail with an incorrect "you're
1167 # deleting something which isn't there" NoSuchChildError exception.
1169 # to trigger this, we start by creating a directory with a single
1170 # file in it. Then we create a special dirnode that uses a modified
1171 # MutableFileNode which will raise UncoordinatedWriteError once on
1172 # demand. We then call dirnode.delete, which ought to retry and
1175 self.basedir = self.mktemp()
1177 c0 = self.g.clients[0]
1178 d = c0.create_dirnode()
1179 small = upload.Data("Small enough for a LIT", None)
1180 def _created_dir(dn):
1182 self.root_uri = dn.get_uri()
1183 return dn.add_file(u"file", small)
1184 d.addCallback(_created_dir)
1185 def _do_delete(ignored):
1186 nm = UCWEingNodeMaker(c0.storage_broker, c0._secret_holder,
1187 c0.get_history(), c0.getServiceNamed("uploader"),
1189 c0.download_cache_dirman,
1190 c0.get_encoding_parameters(),
1192 n = nm.create_from_cap(self.root_uri)
1193 assert n._node.please_ucwe_after_next_upload == False
1194 n._node.please_ucwe_after_next_upload = True
1195 # This should succeed, not raise an exception
1196 return n.delete(u"file")
1197 d.addCallback(_do_delete)
1201 class Adder(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin):
1203 def test_overwrite(self):
1204 # note: This functionality could be tested without actually creating
1205 # several RSA keys. It would be faster without the GridTestMixin: use
1206 # dn.set_node(nodemaker.create_from_cap(make_chk_file_uri())) instead
1207 # of dn.add_file, and use a special NodeMaker that creates fake
1209 self.basedir = "dirnode/Adder/test_overwrite"
1211 c = self.g.clients[0]
1212 fileuri = make_chk_file_uri(1234)
1213 filenode = c.nodemaker.create_from_cap(fileuri)
1214 d = c.create_dirnode()
1216 def _create_directory_tree(root_node):
1221 d = root_node.add_file(u'file1', upload.Data("Important Things",
1223 d.addCallback(lambda res:
1224 root_node.add_file(u'file2', upload.Data("Sekrit Codes", None)))
1225 d.addCallback(lambda res:
1226 root_node.create_subdirectory(u"dir1"))
1227 d.addCallback(lambda res: root_node)
1230 d.addCallback(_create_directory_tree)
1232 def _test_adder(root_node):
1233 d = root_node.set_node(u'file1', filenode)
1234 # We've overwritten file1. Let's try it with a directory
1235 d.addCallback(lambda res:
1236 root_node.create_subdirectory(u'dir2'))
1237 d.addCallback(lambda res:
1238 root_node.set_node(u'dir2', filenode))
1239 # We try overwriting a file with a child while also specifying
1240 # overwrite=False. We should receive an ExistingChildError
1242 d.addCallback(lambda res:
1243 self.shouldFail(ExistingChildError, "set_node",
1244 "child 'file1' already exists",
1245 root_node.set_node, u"file1",
1246 filenode, overwrite=False))
1247 # If we try with a directory, we should see the same thing
1248 d.addCallback(lambda res:
1249 self.shouldFail(ExistingChildError, "set_node",
1250 "child 'dir1' already exists",
1251 root_node.set_node, u'dir1', filenode,
1253 d.addCallback(lambda res:
1254 root_node.set_node(u'file1', filenode,
1255 overwrite="only-files"))
1256 d.addCallback(lambda res:
1257 self.shouldFail(ExistingChildError, "set_node",
1258 "child 'dir1' already exists",
1259 root_node.set_node, u'dir1', filenode,
1260 overwrite="only-files"))
1263 d.addCallback(_test_adder)