4 from zope.interface import implements
5 from twisted.trial import unittest
6 from twisted.internet import defer
7 from twisted.internet.interfaces import IConsumer
8 from allmydata import uri, dirnode
9 from allmydata.client import Client
10 from allmydata.immutable import upload
11 from allmydata.interfaces import IImmutableFileNode, IMutableFileNode, \
12 ExistingChildError, NoSuchChildError, MustNotBeUnknownRWError, \
13 MustBeDeepImmutableError, MustBeReadonlyError, \
14 IDeepCheckResults, IDeepCheckAndRepairResults
15 from allmydata.mutable.filenode import MutableFileNode
16 from allmydata.mutable.common import UncoordinatedWriteError
17 from allmydata.util import hashutil, base32
18 from allmydata.util.netstring import split_netstring
19 from allmydata.monitor import Monitor
20 from allmydata.test.common import make_chk_file_uri, make_mutable_file_uri, \
22 from allmydata.test.no_network import GridTestMixin
23 from allmydata.unknown import UnknownNode, strip_prefix_for_ro
24 from allmydata.nodemaker import NodeMaker
25 from base64 import b32decode
26 import allmydata.test.common_util as testutil
30 def registerProducer(self, producer, streaming):
31 self.producer = producer
32 self.producer.resumeProducing()
34 def unregisterProducer(self):
36 def write(self, data):
37 assert not hasattr(self, 'data')
39 self.producer.resumeProducing()
41 setup_py_uri = "URI:CHK:n7r3m6wmomelk4sep3kw5cvduq:os7ijw5c3maek7pg65e5254k2fzjflavtpejjyhshpsxuqzhcwwq:3:20:14861"
42 one_uri = "URI:LIT:n5xgk" # LIT for "one"
43 mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
44 empty_litdir_uri = "URI:DIR2-LIT:"
45 tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
46 mut_read_uri = "URI:SSK-RO:jf6wkflosyvntwxqcdo7a54jvm:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
47 future_write_uri = "x-tahoe-crazy://I_am_from_the_future."
48 future_read_uri = "x-tahoe-crazy-readonly://I_am_from_the_future."
49 future_nonascii_write_uri = u"x-tahoe-even-more-crazy://I_am_from_the_future_rw_\u263A".encode('utf-8')
50 future_nonascii_read_uri = u"x-tahoe-even-more-crazy-readonly://I_am_from_the_future_ro_\u263A".encode('utf-8')
54 one_nfd = u"one\u0304"
56 class Dirnode(GridTestMixin, unittest.TestCase,
57 testutil.ReallyEqualMixin, testutil.ShouldFailMixin, testutil.StallMixin, ErrorMixin):
58 timeout = 240 # It takes longer than 120 seconds on Francois's arm box.
61 self.basedir = "dirnode/Dirnode/test_basic"
64 d = c.create_dirnode()
66 self.failUnless(isinstance(res, dirnode.DirectoryNode))
67 self.failUnless(res.is_mutable())
68 self.failIf(res.is_readonly())
69 self.failIf(res.is_unknown())
70 self.failIf(res.is_allowed_in_immutable_directory())
73 self.failUnless("RW-MUT" in rep)
77 def test_initial_children(self):
78 self.basedir = "dirnode/Dirnode/test_initial_children"
83 kids = {one_nfd: (nm.create_from_cap(one_uri), {}),
84 u"two": (nm.create_from_cap(setup_py_uri),
85 {"metakey": "metavalue"}),
86 u"mut": (nm.create_from_cap(mut_write_uri, mut_read_uri), {}),
87 u"fut": (nm.create_from_cap(future_write_uri, future_read_uri), {}),
88 u"fro": (nm.create_from_cap(None, future_read_uri), {}),
89 u"fut-unic": (nm.create_from_cap(future_nonascii_write_uri, future_nonascii_read_uri), {}),
90 u"fro-unic": (nm.create_from_cap(None, future_nonascii_read_uri), {}),
91 u"empty_litdir": (nm.create_from_cap(empty_litdir_uri), {}),
92 u"tiny_litdir": (nm.create_from_cap(tiny_litdir_uri), {}),
94 d = c.create_dirnode(kids)
97 self.failUnless(isinstance(dn, dirnode.DirectoryNode))
98 self.failUnless(dn.is_mutable())
99 self.failIf(dn.is_readonly())
100 self.failIf(dn.is_unknown())
101 self.failIf(dn.is_allowed_in_immutable_directory())
104 self.failUnless("RW-MUT" in rep)
106 d.addCallback(_created)
108 def _check_kids(children):
109 self.failUnlessReallyEqual(set(children.keys()),
110 set([one_nfc, u"two", u"mut", u"fut", u"fro",
111 u"fut-unic", u"fro-unic", u"empty_litdir", u"tiny_litdir"]))
112 one_node, one_metadata = children[one_nfc]
113 two_node, two_metadata = children[u"two"]
114 mut_node, mut_metadata = children[u"mut"]
115 fut_node, fut_metadata = children[u"fut"]
116 fro_node, fro_metadata = children[u"fro"]
117 futna_node, futna_metadata = children[u"fut-unic"]
118 frona_node, frona_metadata = children[u"fro-unic"]
119 emptylit_node, emptylit_metadata = children[u"empty_litdir"]
120 tinylit_node, tinylit_metadata = children[u"tiny_litdir"]
122 self.failUnlessReallyEqual(one_node.get_size(), 3)
123 self.failUnlessReallyEqual(one_node.get_uri(), one_uri)
124 self.failUnlessReallyEqual(one_node.get_readonly_uri(), one_uri)
125 self.failUnless(isinstance(one_metadata, dict), one_metadata)
127 self.failUnlessReallyEqual(two_node.get_size(), 14861)
128 self.failUnlessReallyEqual(two_node.get_uri(), setup_py_uri)
129 self.failUnlessReallyEqual(two_node.get_readonly_uri(), setup_py_uri)
130 self.failUnlessEqual(two_metadata["metakey"], "metavalue")
132 self.failUnlessReallyEqual(mut_node.get_uri(), mut_write_uri)
133 self.failUnlessReallyEqual(mut_node.get_readonly_uri(), mut_read_uri)
134 self.failUnless(isinstance(mut_metadata, dict), mut_metadata)
136 self.failUnless(fut_node.is_unknown())
137 self.failUnlessReallyEqual(fut_node.get_uri(), future_write_uri)
138 self.failUnlessReallyEqual(fut_node.get_readonly_uri(), "ro." + future_read_uri)
139 self.failUnless(isinstance(fut_metadata, dict), fut_metadata)
141 self.failUnless(futna_node.is_unknown())
142 self.failUnlessReallyEqual(futna_node.get_uri(), future_nonascii_write_uri)
143 self.failUnlessReallyEqual(futna_node.get_readonly_uri(), "ro." + future_nonascii_read_uri)
144 self.failUnless(isinstance(futna_metadata, dict), futna_metadata)
146 self.failUnless(fro_node.is_unknown())
147 self.failUnlessReallyEqual(fro_node.get_uri(), "ro." + future_read_uri)
148 self.failUnlessReallyEqual(fut_node.get_readonly_uri(), "ro." + future_read_uri)
149 self.failUnless(isinstance(fro_metadata, dict), fro_metadata)
151 self.failUnless(frona_node.is_unknown())
152 self.failUnlessReallyEqual(frona_node.get_uri(), "ro." + future_nonascii_read_uri)
153 self.failUnlessReallyEqual(futna_node.get_readonly_uri(), "ro." + future_nonascii_read_uri)
154 self.failUnless(isinstance(frona_metadata, dict), frona_metadata)
156 self.failIf(emptylit_node.is_unknown())
157 self.failUnlessReallyEqual(emptylit_node.get_storage_index(), None)
158 self.failIf(tinylit_node.is_unknown())
159 self.failUnlessReallyEqual(tinylit_node.get_storage_index(), None)
161 d2 = defer.succeed(None)
162 d2.addCallback(lambda ignored: emptylit_node.list())
163 d2.addCallback(lambda children: self.failUnlessEqual(children, {}))
164 d2.addCallback(lambda ignored: tinylit_node.list())
165 d2.addCallback(lambda children: self.failUnlessReallyEqual(set(children.keys()),
167 d2.addCallback(lambda ignored: tinylit_node.list())
168 d2.addCallback(lambda children: children[u"short"][0].read(MemAccum()))
169 d2.addCallback(lambda accum: self.failUnlessReallyEqual(accum.data, "The end."))
172 d.addCallback(_check_kids)
174 d.addCallback(lambda ign: nm.create_new_mutable_directory(kids))
175 d.addCallback(lambda dn: dn.list())
176 d.addCallback(_check_kids)
178 bad_future_node = UnknownNode(future_write_uri, None)
179 bad_kids1 = {one_nfd: (bad_future_node, {})}
180 # This should fail because we don't know how to diminish the future_write_uri
181 # cap (given in a write slot and not prefixed with "ro." or "imm.") to a readcap.
182 d.addCallback(lambda ign:
183 self.shouldFail(MustNotBeUnknownRWError, "bad_kids1",
184 "cannot attach unknown",
185 nm.create_new_mutable_directory,
187 bad_kids2 = {one_nfd: (nm.create_from_cap(one_uri), None)}
188 d.addCallback(lambda ign:
189 self.shouldFail(AssertionError, "bad_kids2",
190 "requires metadata to be a dict",
191 nm.create_new_mutable_directory,
195 def test_immutable(self):
196 self.basedir = "dirnode/Dirnode/test_immutable"
198 c = self.g.clients[0]
201 kids = {one_nfd: (nm.create_from_cap(one_uri), {}),
202 u"two": (nm.create_from_cap(setup_py_uri),
203 {"metakey": "metavalue"}),
204 u"fut": (nm.create_from_cap(None, future_read_uri), {}),
205 u"futna": (nm.create_from_cap(None, future_nonascii_read_uri), {}),
206 u"empty_litdir": (nm.create_from_cap(empty_litdir_uri), {}),
207 u"tiny_litdir": (nm.create_from_cap(tiny_litdir_uri), {}),
209 d = c.create_immutable_dirnode(kids)
212 self.failUnless(isinstance(dn, dirnode.DirectoryNode))
213 self.failIf(dn.is_mutable())
214 self.failUnless(dn.is_readonly())
215 self.failIf(dn.is_unknown())
216 self.failUnless(dn.is_allowed_in_immutable_directory())
219 self.failUnless("RO-IMM" in rep)
221 self.failUnlessIn("CHK", cap.to_string())
224 d.addCallback(_created)
226 def _check_kids(children):
227 self.failUnlessReallyEqual(set(children.keys()),
228 set([one_nfc, u"two", u"fut", u"futna", u"empty_litdir", u"tiny_litdir"]))
229 one_node, one_metadata = children[one_nfc]
230 two_node, two_metadata = children[u"two"]
231 fut_node, fut_metadata = children[u"fut"]
232 futna_node, futna_metadata = children[u"futna"]
233 emptylit_node, emptylit_metadata = children[u"empty_litdir"]
234 tinylit_node, tinylit_metadata = children[u"tiny_litdir"]
236 self.failUnlessReallyEqual(one_node.get_size(), 3)
237 self.failUnlessReallyEqual(one_node.get_uri(), one_uri)
238 self.failUnlessReallyEqual(one_node.get_readonly_uri(), one_uri)
239 self.failUnless(isinstance(one_metadata, dict), one_metadata)
241 self.failUnlessReallyEqual(two_node.get_size(), 14861)
242 self.failUnlessReallyEqual(two_node.get_uri(), setup_py_uri)
243 self.failUnlessReallyEqual(two_node.get_readonly_uri(), setup_py_uri)
244 self.failUnlessEqual(two_metadata["metakey"], "metavalue")
246 self.failUnless(fut_node.is_unknown())
247 self.failUnlessReallyEqual(fut_node.get_uri(), "imm." + future_read_uri)
248 self.failUnlessReallyEqual(fut_node.get_readonly_uri(), "imm." + future_read_uri)
249 self.failUnless(isinstance(fut_metadata, dict), fut_metadata)
251 self.failUnless(futna_node.is_unknown())
252 self.failUnlessReallyEqual(futna_node.get_uri(), "imm." + future_nonascii_read_uri)
253 self.failUnlessReallyEqual(futna_node.get_readonly_uri(), "imm." + future_nonascii_read_uri)
254 self.failUnless(isinstance(futna_metadata, dict), futna_metadata)
256 self.failIf(emptylit_node.is_unknown())
257 self.failUnlessReallyEqual(emptylit_node.get_storage_index(), None)
258 self.failIf(tinylit_node.is_unknown())
259 self.failUnlessReallyEqual(tinylit_node.get_storage_index(), None)
261 d2 = defer.succeed(None)
262 d2.addCallback(lambda ignored: emptylit_node.list())
263 d2.addCallback(lambda children: self.failUnlessEqual(children, {}))
264 d2.addCallback(lambda ignored: tinylit_node.list())
265 d2.addCallback(lambda children: self.failUnlessReallyEqual(set(children.keys()),
267 d2.addCallback(lambda ignored: tinylit_node.list())
268 d2.addCallback(lambda children: children[u"short"][0].read(MemAccum()))
269 d2.addCallback(lambda accum: self.failUnlessReallyEqual(accum.data, "The end."))
272 d.addCallback(_check_kids)
274 d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
275 d.addCallback(lambda dn: dn.list())
276 d.addCallback(_check_kids)
278 bad_future_node1 = UnknownNode(future_write_uri, None)
279 bad_kids1 = {one_nfd: (bad_future_node1, {})}
280 d.addCallback(lambda ign:
281 self.shouldFail(MustNotBeUnknownRWError, "bad_kids1",
282 "cannot attach unknown",
283 c.create_immutable_dirnode,
285 bad_future_node2 = UnknownNode(future_write_uri, future_read_uri)
286 bad_kids2 = {one_nfd: (bad_future_node2, {})}
287 d.addCallback(lambda ign:
288 self.shouldFail(MustBeDeepImmutableError, "bad_kids2",
289 "is not allowed in an immutable directory",
290 c.create_immutable_dirnode,
292 bad_kids3 = {one_nfd: (nm.create_from_cap(one_uri), None)}
293 d.addCallback(lambda ign:
294 self.shouldFail(AssertionError, "bad_kids3",
295 "requires metadata to be a dict",
296 c.create_immutable_dirnode,
298 bad_kids4 = {one_nfd: (nm.create_from_cap(mut_write_uri), {})}
299 d.addCallback(lambda ign:
300 self.shouldFail(MustBeDeepImmutableError, "bad_kids4",
301 "is not allowed in an immutable directory",
302 c.create_immutable_dirnode,
304 bad_kids5 = {one_nfd: (nm.create_from_cap(mut_read_uri), {})}
305 d.addCallback(lambda ign:
306 self.shouldFail(MustBeDeepImmutableError, "bad_kids5",
307 "is not allowed in an immutable directory",
308 c.create_immutable_dirnode,
310 d.addCallback(lambda ign: c.create_immutable_dirnode({}))
311 def _created_empty(dn):
312 self.failUnless(isinstance(dn, dirnode.DirectoryNode))
313 self.failIf(dn.is_mutable())
314 self.failUnless(dn.is_readonly())
315 self.failIf(dn.is_unknown())
316 self.failUnless(dn.is_allowed_in_immutable_directory())
319 self.failUnless("RO-IMM" in rep)
321 self.failUnlessIn("LIT", cap.to_string())
322 self.failUnlessReallyEqual(cap.to_string(), "URI:DIR2-LIT:")
325 d.addCallback(_created_empty)
326 d.addCallback(lambda kids: self.failUnlessEqual(kids, {}))
327 smallkids = {u"o": (nm.create_from_cap(one_uri), {})}
328 d.addCallback(lambda ign: c.create_immutable_dirnode(smallkids))
329 def _created_small(dn):
330 self.failUnless(isinstance(dn, dirnode.DirectoryNode))
331 self.failIf(dn.is_mutable())
332 self.failUnless(dn.is_readonly())
333 self.failIf(dn.is_unknown())
334 self.failUnless(dn.is_allowed_in_immutable_directory())
337 self.failUnless("RO-IMM" in rep)
339 self.failUnlessIn("LIT", cap.to_string())
340 self.failUnlessReallyEqual(cap.to_string(),
341 "URI:DIR2-LIT:gi4tumj2n4wdcmz2kvjesosmjfkdu3rvpbtwwlbqhiwdeot3puwcy")
344 d.addCallback(_created_small)
345 d.addCallback(lambda kids: self.failUnlessReallyEqual(kids.keys(), [u"o"]))
347 # now test n.create_subdirectory(mutable=False)
348 d.addCallback(lambda ign: c.create_dirnode())
350 d = n.create_subdirectory(u"subdir", kids, mutable=False)
351 d.addCallback(lambda sd: sd.list())
352 d.addCallback(_check_kids)
353 d.addCallback(lambda ign: n.list())
354 d.addCallback(lambda children:
355 self.failUnlessReallyEqual(children.keys(), [u"subdir"]))
356 d.addCallback(lambda ign: n.get(u"subdir"))
357 d.addCallback(lambda sd: sd.list())
358 d.addCallback(_check_kids)
359 d.addCallback(lambda ign: n.get(u"subdir"))
360 d.addCallback(lambda sd: self.failIf(sd.is_mutable()))
361 bad_kids = {one_nfd: (nm.create_from_cap(mut_write_uri), {})}
362 d.addCallback(lambda ign:
363 self.shouldFail(MustBeDeepImmutableError, "YZ",
364 "is not allowed in an immutable directory",
365 n.create_subdirectory,
366 u"sub2", bad_kids, mutable=False))
368 d.addCallback(_made_parent)
371 def test_directory_representation(self):
372 self.basedir = "dirnode/Dirnode/test_directory_representation"
374 c = self.g.clients[0]
377 # This test checks that any trailing spaces in URIs are retained in the
378 # encoded directory, but stripped when we get them out of the directory.
379 # See ticket #925 for why we want that.
380 # It also tests that we store child names as UTF-8 NFC, and normalize
381 # them again when retrieving them.
383 stripped_write_uri = "lafs://from_the_future\t"
384 stripped_read_uri = "lafs://readonly_from_the_future\t"
385 spacedout_write_uri = stripped_write_uri + " "
386 spacedout_read_uri = stripped_read_uri + " "
388 child = nm.create_from_cap(spacedout_write_uri, spacedout_read_uri)
389 self.failUnlessReallyEqual(child.get_write_uri(), spacedout_write_uri)
390 self.failUnlessReallyEqual(child.get_readonly_uri(), "ro." + spacedout_read_uri)
392 child_dottedi = u"ch\u0131\u0307ld"
394 kids_in = {child_dottedi: (child, {}), one_nfd: (child, {})}
395 kids_out = {child_dottedi: (child, {}), one_nfc: (child, {})}
396 kids_norm = {u"child": (child, {}), one_nfc: (child, {})}
397 d = c.create_dirnode(kids_in)
400 self.failUnless(isinstance(dn, dirnode.DirectoryNode))
401 self.failUnless(dn.is_mutable())
402 self.failIf(dn.is_readonly())
404 self.cap = dn.get_cap()
406 return dn._node.download_best_version()
407 d.addCallback(_created)
409 def _check_data(data):
410 # Decode the netstring representation of the directory to check that the
411 # spaces are retained when the URIs are stored, and that the names are stored
415 while position < len(data):
416 entries, position = split_netstring(data, 1, position)
418 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
419 name = name_utf8.decode("utf-8")
420 rw_uri = self.rootnode._decrypt_rwcapdata(rwcapdata)
421 self.failUnlessIn(name, kids_out)
422 (expected_child, ign) = kids_out[name]
423 self.failUnlessReallyEqual(rw_uri, expected_child.get_write_uri())
424 self.failUnlessReallyEqual("ro." + ro_uri, expected_child.get_readonly_uri())
427 self.failUnlessReallyEqual(numkids, len(kids_out))
429 d.addCallback(_check_data)
431 # Mock up a hypothetical future version of Unicode that adds a canonical equivalence
432 # between dotless-i + dot-above, and 'i'. That would actually be prohibited by the
433 # stability rules, but similar additions involving currently-unassigned characters
435 old_normalize = unicodedata.normalize
436 def future_normalize(form, s):
437 assert form == 'NFC', form
438 return old_normalize(form, s).replace(u"\u0131\u0307", u"i")
441 unicodedata.normalize = future_normalize
444 unicodedata.normalize = old_normalize
446 d2.addBoth(_undo_mock)
450 def _check_kids(children):
451 # Now when we use the real directory listing code, the trailing spaces
452 # should have been stripped (and "ro." should have been prepended to the
453 # ro_uri, since it's unknown). Also the dotless-i + dot-above should have been
456 self.failUnlessReallyEqual(set(children.keys()), set(kids_norm.keys()))
457 child_node, child_metadata = children[u"child"]
459 self.failUnlessReallyEqual(child_node.get_write_uri(), stripped_write_uri)
460 self.failUnlessReallyEqual(child_node.get_readonly_uri(), "ro." + stripped_read_uri)
461 d.addCallback(_check_kids)
463 d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
465 d.addCallback(_check_kids) # again with dirnode recreated from cap
468 def test_check(self):
469 self.basedir = "dirnode/Dirnode/test_check"
471 c = self.g.clients[0]
472 d = c.create_dirnode()
473 d.addCallback(lambda dn: dn.check(Monitor()))
475 self.failUnless(res.is_healthy())
479 def _test_deepcheck_create(self):
480 # create a small tree with a loop, and some non-directories
484 # root/subdir/link -> root
486 c = self.g.clients[0]
487 d = c.create_dirnode()
488 def _created_root(rootnode):
489 self._rootnode = rootnode
490 return rootnode.create_subdirectory(u"subdir")
491 d.addCallback(_created_root)
492 def _created_subdir(subdir):
493 self._subdir = subdir
494 d = subdir.add_file(u"file1", upload.Data("data"*100, None))
495 d.addCallback(lambda res: subdir.set_node(u"link", self._rootnode))
496 d.addCallback(lambda res: c.create_dirnode())
497 d.addCallback(lambda dn:
498 self._rootnode.set_uri(u"rodir",
500 dn.get_readonly_uri()))
502 d.addCallback(_created_subdir)
504 return self._rootnode
508 def test_deepcheck(self):
509 self.basedir = "dirnode/Dirnode/test_deepcheck"
511 d = self._test_deepcheck_create()
512 d.addCallback(lambda rootnode: rootnode.start_deep_check().when_done())
513 def _check_results(r):
514 self.failUnless(IDeepCheckResults.providedBy(r))
516 self.failUnlessReallyEqual(c,
517 {"count-objects-checked": 4,
518 "count-objects-healthy": 4,
519 "count-objects-unhealthy": 0,
520 "count-objects-unrecoverable": 0,
521 "count-corrupt-shares": 0,
523 self.failIf(r.get_corrupt_shares())
524 self.failUnlessReallyEqual(len(r.get_all_results()), 4)
525 d.addCallback(_check_results)
528 def test_deepcheck_and_repair(self):
529 self.basedir = "dirnode/Dirnode/test_deepcheck_and_repair"
531 d = self._test_deepcheck_create()
532 d.addCallback(lambda rootnode:
533 rootnode.start_deep_check_and_repair().when_done())
534 def _check_results(r):
535 self.failUnless(IDeepCheckAndRepairResults.providedBy(r))
537 self.failUnlessReallyEqual(c,
538 {"count-objects-checked": 4,
539 "count-objects-healthy-pre-repair": 4,
540 "count-objects-unhealthy-pre-repair": 0,
541 "count-objects-unrecoverable-pre-repair": 0,
542 "count-corrupt-shares-pre-repair": 0,
543 "count-objects-healthy-post-repair": 4,
544 "count-objects-unhealthy-post-repair": 0,
545 "count-objects-unrecoverable-post-repair": 0,
546 "count-corrupt-shares-post-repair": 0,
547 "count-repairs-attempted": 0,
548 "count-repairs-successful": 0,
549 "count-repairs-unsuccessful": 0,
551 self.failIf(r.get_corrupt_shares())
552 self.failIf(r.get_remaining_corrupt_shares())
553 self.failUnlessReallyEqual(len(r.get_all_results()), 4)
554 d.addCallback(_check_results)
557 def _mark_file_bad(self, rootnode):
558 self.delete_shares_numbered(rootnode.get_uri(), [0])
561 def test_deepcheck_problems(self):
562 self.basedir = "dirnode/Dirnode/test_deepcheck_problems"
564 d = self._test_deepcheck_create()
565 d.addCallback(lambda rootnode: self._mark_file_bad(rootnode))
566 d.addCallback(lambda rootnode: rootnode.start_deep_check().when_done())
567 def _check_results(r):
569 self.failUnlessReallyEqual(c,
570 {"count-objects-checked": 4,
571 "count-objects-healthy": 3,
572 "count-objects-unhealthy": 1,
573 "count-objects-unrecoverable": 0,
574 "count-corrupt-shares": 0,
576 #self.failUnlessReallyEqual(len(r.get_problems()), 1) # TODO
577 d.addCallback(_check_results)
580 def test_readonly(self):
581 self.basedir = "dirnode/Dirnode/test_readonly"
583 c = self.g.clients[0]
585 filecap = make_chk_file_uri(1234)
586 filenode = nm.create_from_cap(filecap)
587 uploadable = upload.Data("some data", convergence="some convergence string")
589 d = c.create_dirnode()
591 d2 = rw_dn.set_uri(u"child", filecap, filecap)
592 d2.addCallback(lambda res: rw_dn)
594 d.addCallback(_created)
597 ro_uri = rw_dn.get_readonly_uri()
598 ro_dn = c.create_node_from_uri(ro_uri)
599 self.failUnless(ro_dn.is_readonly())
600 self.failUnless(ro_dn.is_mutable())
601 self.failIf(ro_dn.is_unknown())
602 self.failIf(ro_dn.is_allowed_in_immutable_directory())
605 self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
606 ro_dn.set_uri, u"newchild", filecap, filecap)
607 self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
608 ro_dn.set_node, u"newchild", filenode)
609 self.shouldFail(dirnode.NotWriteableError, "set_nodes ro", None,
610 ro_dn.set_nodes, { u"newchild": (filenode, None) })
611 self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
612 ro_dn.add_file, u"newchild", uploadable)
613 self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
614 ro_dn.delete, u"child")
615 self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
616 ro_dn.create_subdirectory, u"newchild")
617 self.shouldFail(dirnode.NotWriteableError, "set_metadata_for ro", None,
618 ro_dn.set_metadata_for, u"child", {})
619 self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
620 ro_dn.move_child_to, u"child", rw_dn)
621 self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
622 rw_dn.move_child_to, u"child", ro_dn)
624 d.addCallback(_ready)
625 def _listed(children):
626 self.failUnless(u"child" in children)
627 d.addCallback(_listed)
630 def failUnlessGreaterThan(self, a, b):
631 self.failUnless(a > b, "%r should be > %r" % (a, b))
633 def failUnlessGreaterOrEqualThan(self, a, b):
634 self.failUnless(a >= b, "%r should be >= %r" % (a, b))
636 def test_create(self):
637 self.basedir = "dirnode/Dirnode/test_create"
639 c = self.g.clients[0]
641 self.expected_manifest = []
642 self.expected_verifycaps = set()
643 self.expected_storage_indexes = set()
645 d = c.create_dirnode()
649 self.failUnless(n.is_mutable())
652 self.failUnless(u.startswith("URI:DIR2:"), u)
653 u_ro = n.get_readonly_uri()
654 self.failUnless(u_ro.startswith("URI:DIR2-RO:"), u_ro)
655 u_v = n.get_verify_cap().to_string()
656 self.failUnless(u_v.startswith("URI:DIR2-Verifier:"), u_v)
657 u_r = n.get_repair_cap().to_string()
658 self.failUnlessReallyEqual(u_r, u)
659 self.expected_manifest.append( ((), u) )
660 self.expected_verifycaps.add(u_v)
661 si = n.get_storage_index()
662 self.expected_storage_indexes.add(base32.b2a(si))
663 expected_si = n._uri.get_storage_index()
664 self.failUnlessReallyEqual(si, expected_si)
667 d.addCallback(lambda res: self.failUnlessEqual(res, {}))
668 d.addCallback(lambda res: n.has_child(u"missing"))
669 d.addCallback(lambda res: self.failIf(res))
671 fake_file_uri = make_mutable_file_uri()
672 other_file_uri = make_mutable_file_uri()
673 m = c.nodemaker.create_from_cap(fake_file_uri)
674 ffu_v = m.get_verify_cap().to_string()
675 self.expected_manifest.append( ((u"child",) , m.get_uri()) )
676 self.expected_verifycaps.add(ffu_v)
677 self.expected_storage_indexes.add(base32.b2a(m.get_storage_index()))
678 d.addCallback(lambda res: n.set_uri(u"child",
679 fake_file_uri, fake_file_uri))
680 d.addCallback(lambda res:
681 self.shouldFail(ExistingChildError, "set_uri-no",
682 "child 'child' already exists",
684 other_file_uri, other_file_uri,
689 d.addCallback(lambda res: n.create_subdirectory(u"subdir"))
693 # /subdir = directory
694 def _created(subdir):
695 self.failUnless(isinstance(subdir, dirnode.DirectoryNode))
697 new_v = subdir.get_verify_cap().to_string()
698 assert isinstance(new_v, str)
699 self.expected_manifest.append( ((u"subdir",), subdir.get_uri()) )
700 self.expected_verifycaps.add(new_v)
701 si = subdir.get_storage_index()
702 self.expected_storage_indexes.add(base32.b2a(si))
703 d.addCallback(_created)
705 d.addCallback(lambda res:
706 self.shouldFail(ExistingChildError, "mkdir-no",
707 "child 'subdir' already exists",
708 n.create_subdirectory, u"subdir",
711 d.addCallback(lambda res: n.list())
712 d.addCallback(lambda children:
713 self.failUnlessReallyEqual(set(children.keys()),
714 set([u"child", u"subdir"])))
716 d.addCallback(lambda res: n.start_deep_stats().when_done())
717 def _check_deepstats(stats):
718 self.failUnless(isinstance(stats, dict))
719 expected = {"count-immutable-files": 0,
720 "count-mutable-files": 1,
721 "count-literal-files": 0,
723 "count-directories": 2,
724 "size-immutable-files": 0,
725 "size-literal-files": 0,
726 #"size-directories": 616, # varies
727 #"largest-directory": 616,
728 "largest-directory-children": 2,
729 "largest-immutable-file": 0,
731 for k,v in expected.iteritems():
732 self.failUnlessReallyEqual(stats[k], v,
733 "stats[%s] was %s, not %s" %
735 self.failUnless(stats["size-directories"] > 500,
736 stats["size-directories"])
737 self.failUnless(stats["largest-directory"] > 500,
738 stats["largest-directory"])
739 self.failUnlessReallyEqual(stats["size-files-histogram"], [])
740 d.addCallback(_check_deepstats)
742 d.addCallback(lambda res: n.build_manifest().when_done())
743 def _check_manifest(res):
744 manifest = res["manifest"]
745 self.failUnlessReallyEqual(sorted(manifest),
746 sorted(self.expected_manifest))
748 _check_deepstats(stats)
749 self.failUnlessReallyEqual(self.expected_verifycaps,
751 self.failUnlessReallyEqual(self.expected_storage_indexes,
752 res["storage-index"])
753 d.addCallback(_check_manifest)
755 def _add_subsubdir(res):
756 return self.subdir.create_subdirectory(u"subsubdir")
757 d.addCallback(_add_subsubdir)
760 # /subdir = directory
761 # /subdir/subsubdir = directory
762 d.addCallback(lambda res: n.get_child_at_path(u"subdir/subsubdir"))
763 d.addCallback(lambda subsubdir:
764 self.failUnless(isinstance(subsubdir,
765 dirnode.DirectoryNode)))
766 d.addCallback(lambda res: n.get_child_at_path(u""))
767 d.addCallback(lambda res: self.failUnlessReallyEqual(res.get_uri(),
770 d.addCallback(lambda res: n.get_metadata_for(u"child"))
771 d.addCallback(lambda metadata:
772 self.failUnlessEqual(set(metadata.keys()),
775 d.addCallback(lambda res:
776 self.shouldFail(NoSuchChildError, "gcamap-no",
778 n.get_child_and_metadata_at_path,
780 d.addCallback(lambda res:
781 n.get_child_and_metadata_at_path(u""))
782 def _check_child_and_metadata1(res):
783 child, metadata = res
784 self.failUnless(isinstance(child, dirnode.DirectoryNode))
785 # edge-metadata needs at least one path segment
786 self.failUnlessEqual(set(metadata.keys()), set([]))
787 d.addCallback(_check_child_and_metadata1)
788 d.addCallback(lambda res:
789 n.get_child_and_metadata_at_path(u"child"))
791 def _check_child_and_metadata2(res):
792 child, metadata = res
793 self.failUnlessReallyEqual(child.get_uri(),
795 self.failUnlessEqual(set(metadata.keys()), set(["tahoe"]))
796 d.addCallback(_check_child_and_metadata2)
798 d.addCallback(lambda res:
799 n.get_child_and_metadata_at_path(u"subdir/subsubdir"))
800 def _check_child_and_metadata3(res):
801 child, metadata = res
802 self.failUnless(isinstance(child, dirnode.DirectoryNode))
803 self.failUnlessEqual(set(metadata.keys()), set(["tahoe"]))
804 d.addCallback(_check_child_and_metadata3)
807 # it should be possible to add a child without any metadata
808 d.addCallback(lambda res: n.set_uri(u"c2",
809 fake_file_uri, fake_file_uri,
811 d.addCallback(lambda res: n.get_metadata_for(u"c2"))
812 d.addCallback(lambda metadata:
813 self.failUnlessEqual(set(metadata.keys()), set(["tahoe"])))
815 # You can't override the link timestamps.
816 d.addCallback(lambda res: n.set_uri(u"c2",
817 fake_file_uri, fake_file_uri,
818 { 'tahoe': {'linkcrtime': "bogus"}}))
819 d.addCallback(lambda res: n.get_metadata_for(u"c2"))
820 def _has_good_linkcrtime(metadata):
821 self.failUnless(metadata.has_key('tahoe'))
822 self.failUnless(metadata['tahoe'].has_key('linkcrtime'))
823 self.failIfEqual(metadata['tahoe']['linkcrtime'], 'bogus')
824 d.addCallback(_has_good_linkcrtime)
826 # if we don't set any defaults, the child should get timestamps
827 d.addCallback(lambda res: n.set_uri(u"c3",
828 fake_file_uri, fake_file_uri))
829 d.addCallback(lambda res: n.get_metadata_for(u"c3"))
830 d.addCallback(lambda metadata:
831 self.failUnlessEqual(set(metadata.keys()), set(["tahoe"])))
833 # we can also add specific metadata at set_uri() time
834 d.addCallback(lambda res: n.set_uri(u"c4",
835 fake_file_uri, fake_file_uri,
837 d.addCallback(lambda res: n.get_metadata_for(u"c4"))
838 d.addCallback(lambda metadata:
839 self.failUnless((set(metadata.keys()) == set(["key", "tahoe"])) and
840 (metadata['key'] == "value"), metadata))
842 d.addCallback(lambda res: n.delete(u"c2"))
843 d.addCallback(lambda res: n.delete(u"c3"))
844 d.addCallback(lambda res: n.delete(u"c4"))
846 # set_node + metadata
847 # it should be possible to add a child without any metadata except for timestamps
848 d.addCallback(lambda res: n.set_node(u"d2", n, {}))
849 d.addCallback(lambda res: c.create_dirnode())
850 d.addCallback(lambda n2:
851 self.shouldFail(ExistingChildError, "set_node-no",
852 "child 'd2' already exists",
853 n.set_node, u"d2", n2,
855 d.addCallback(lambda res: n.get_metadata_for(u"d2"))
856 d.addCallback(lambda metadata:
857 self.failUnlessEqual(set(metadata.keys()), set(["tahoe"])))
859 # if we don't set any defaults, the child should get timestamps
860 d.addCallback(lambda res: n.set_node(u"d3", n))
861 d.addCallback(lambda res: n.get_metadata_for(u"d3"))
862 d.addCallback(lambda metadata:
863 self.failUnlessEqual(set(metadata.keys()), set(["tahoe"])))
865 # we can also add specific metadata at set_node() time
866 d.addCallback(lambda res: n.set_node(u"d4", n,
868 d.addCallback(lambda res: n.get_metadata_for(u"d4"))
869 d.addCallback(lambda metadata:
870 self.failUnless((set(metadata.keys()) == set(["key", "tahoe"])) and
871 (metadata["key"] == "value"), metadata))
873 d.addCallback(lambda res: n.delete(u"d2"))
874 d.addCallback(lambda res: n.delete(u"d3"))
875 d.addCallback(lambda res: n.delete(u"d4"))
877 # metadata through set_children()
878 d.addCallback(lambda res:
880 u"e1": (fake_file_uri, fake_file_uri),
881 u"e2": (fake_file_uri, fake_file_uri, {}),
882 u"e3": (fake_file_uri, fake_file_uri,
885 d.addCallback(lambda n2: self.failUnlessIdentical(n2, n))
886 d.addCallback(lambda res:
887 self.shouldFail(ExistingChildError, "set_children-no",
888 "child 'e1' already exists",
890 { u"e1": (other_file_uri,
892 u"new": (other_file_uri,
896 # and 'new' should not have been created
897 d.addCallback(lambda res: n.list())
898 d.addCallback(lambda children: self.failIf(u"new" in children))
899 d.addCallback(lambda res: n.get_metadata_for(u"e1"))
900 d.addCallback(lambda metadata:
901 self.failUnlessEqual(set(metadata.keys()), set(["tahoe"])))
902 d.addCallback(lambda res: n.get_metadata_for(u"e2"))
903 d.addCallback(lambda metadata:
904 self.failUnlessEqual(set(metadata.keys()), set(["tahoe"])))
905 d.addCallback(lambda res: n.get_metadata_for(u"e3"))
906 d.addCallback(lambda metadata:
907 self.failUnless((set(metadata.keys()) == set(["key", "tahoe"])) and
908 (metadata["key"] == "value"), metadata))
910 d.addCallback(lambda res: n.delete(u"e1"))
911 d.addCallback(lambda res: n.delete(u"e2"))
912 d.addCallback(lambda res: n.delete(u"e3"))
914 # metadata through set_nodes()
915 d.addCallback(lambda res:
916 n.set_nodes({ u"f1": (n, None),
918 u"f3": (n, {"key": "value"}),
920 d.addCallback(lambda n2: self.failUnlessIdentical(n2, n))
921 d.addCallback(lambda res:
922 self.shouldFail(ExistingChildError, "set_nodes-no",
923 "child 'f1' already exists",
924 n.set_nodes, { u"f1": (n, None),
925 u"new": (n, None), },
927 # and 'new' should not have been created
928 d.addCallback(lambda res: n.list())
929 d.addCallback(lambda children: self.failIf(u"new" in children))
930 d.addCallback(lambda res: n.get_metadata_for(u"f1"))
931 d.addCallback(lambda metadata:
932 self.failUnlessEqual(set(metadata.keys()), set(["tahoe"])))
933 d.addCallback(lambda res: n.get_metadata_for(u"f2"))
934 d.addCallback(lambda metadata:
935 self.failUnlessEqual(set(metadata.keys()), set(["tahoe"])))
936 d.addCallback(lambda res: n.get_metadata_for(u"f3"))
937 d.addCallback(lambda metadata:
938 self.failUnless((set(metadata.keys()) == set(["key", "tahoe"])) and
939 (metadata["key"] == "value"), metadata))
941 d.addCallback(lambda res: n.delete(u"f1"))
942 d.addCallback(lambda res: n.delete(u"f2"))
943 d.addCallback(lambda res: n.delete(u"f3"))
946 d.addCallback(lambda res:
947 n.set_metadata_for(u"child",
948 {"tags": ["web2.0-compatible"], "tahoe": {"bad": "mojo"}}))
949 d.addCallback(lambda n1: n1.get_metadata_for(u"child"))
950 d.addCallback(lambda metadata:
951 self.failUnless((set(metadata.keys()) == set(["tags", "tahoe"])) and
952 metadata["tags"] == ["web2.0-compatible"] and
953 "bad" not in metadata["tahoe"], metadata))
955 d.addCallback(lambda res:
956 self.shouldFail(NoSuchChildError, "set_metadata_for-nosuch", "",
957 n.set_metadata_for, u"nosuch", {}))
961 self._start_timestamp = time.time()
962 d.addCallback(_start)
963 # simplejson-1.7.1 (as shipped on Ubuntu 'gutsy') rounds all
964 # floats to hundredeths (it uses str(num) instead of repr(num)).
965 # simplejson-1.7.3 does not have this bug. To prevent this bug
966 # from causing the test to fail, stall for more than a few
967 # hundrededths of a second.
968 d.addCallback(self.stall, 0.1)
969 d.addCallback(lambda res: n.add_file(u"timestamps",
970 upload.Data("stamp me", convergence="some convergence string")))
971 d.addCallback(self.stall, 0.1)
973 self._stop_timestamp = time.time()
976 d.addCallback(lambda res: n.get_metadata_for(u"timestamps"))
977 def _check_timestamp1(metadata):
978 self.failUnlessEqual(set(metadata.keys()), set(["tahoe"]))
979 tahoe_md = metadata["tahoe"]
980 self.failUnlessEqual(set(tahoe_md.keys()), set(["linkcrtime", "linkmotime"]))
982 self.failUnlessGreaterOrEqualThan(tahoe_md["linkcrtime"],
983 self._start_timestamp)
984 self.failUnlessGreaterOrEqualThan(self._stop_timestamp,
985 tahoe_md["linkcrtime"])
986 self.failUnlessGreaterOrEqualThan(tahoe_md["linkmotime"],
987 self._start_timestamp)
988 self.failUnlessGreaterOrEqualThan(self._stop_timestamp,
989 tahoe_md["linkmotime"])
990 # Our current timestamp rules say that replacing an existing
991 # child should preserve the 'linkcrtime' but update the
993 self._old_linkcrtime = tahoe_md["linkcrtime"]
994 self._old_linkmotime = tahoe_md["linkmotime"]
995 d.addCallback(_check_timestamp1)
996 d.addCallback(self.stall, 2.0) # accomodate low-res timestamps
997 d.addCallback(lambda res: n.set_node(u"timestamps", n))
998 d.addCallback(lambda res: n.get_metadata_for(u"timestamps"))
999 def _check_timestamp2(metadata):
1000 self.failUnlessIn("tahoe", metadata)
1001 tahoe_md = metadata["tahoe"]
1002 self.failUnlessEqual(set(tahoe_md.keys()), set(["linkcrtime", "linkmotime"]))
1004 self.failUnlessReallyEqual(tahoe_md["linkcrtime"], self._old_linkcrtime)
1005 self.failUnlessGreaterThan(tahoe_md["linkmotime"], self._old_linkmotime)
1006 return n.delete(u"timestamps")
1007 d.addCallback(_check_timestamp2)
1009 d.addCallback(lambda res: n.delete(u"subdir"))
1010 d.addCallback(lambda old_child:
1011 self.failUnlessReallyEqual(old_child.get_uri(),
1012 self.subdir.get_uri()))
1014 d.addCallback(lambda res: n.list())
1015 d.addCallback(lambda children:
1016 self.failUnlessReallyEqual(set(children.keys()),
1019 uploadable1 = upload.Data("some data", convergence="converge")
1020 d.addCallback(lambda res: n.add_file(u"newfile", uploadable1))
1021 d.addCallback(lambda newnode:
1022 self.failUnless(IImmutableFileNode.providedBy(newnode)))
1023 uploadable2 = upload.Data("some data", convergence="stuff")
1024 d.addCallback(lambda res:
1025 self.shouldFail(ExistingChildError, "add_file-no",
1026 "child 'newfile' already exists",
1027 n.add_file, u"newfile",
1030 d.addCallback(lambda res: n.list())
1031 d.addCallback(lambda children:
1032 self.failUnlessReallyEqual(set(children.keys()),
1033 set([u"child", u"newfile"])))
1034 d.addCallback(lambda res: n.get_metadata_for(u"newfile"))
1035 d.addCallback(lambda metadata:
1036 self.failUnlessEqual(set(metadata.keys()), set(["tahoe"])))
1038 uploadable3 = upload.Data("some data", convergence="converge")
1039 d.addCallback(lambda res: n.add_file(u"newfile-metadata",
1042 d.addCallback(lambda newnode:
1043 self.failUnless(IImmutableFileNode.providedBy(newnode)))
1044 d.addCallback(lambda res: n.get_metadata_for(u"newfile-metadata"))
1045 d.addCallback(lambda metadata:
1046 self.failUnless((set(metadata.keys()) == set(["key", "tahoe"])) and
1047 (metadata['key'] == "value"), metadata))
1048 d.addCallback(lambda res: n.delete(u"newfile-metadata"))
1050 d.addCallback(lambda res: n.create_subdirectory(u"subdir2"))
1051 def _created2(subdir2):
1052 self.subdir2 = subdir2
1053 # put something in the way, to make sure it gets overwritten
1054 return subdir2.add_file(u"child", upload.Data("overwrite me",
1056 d.addCallback(_created2)
1058 d.addCallback(lambda res:
1059 n.move_child_to(u"child", self.subdir2))
1060 d.addCallback(lambda res: n.list())
1061 d.addCallback(lambda children:
1062 self.failUnlessReallyEqual(set(children.keys()),
1063 set([u"newfile", u"subdir2"])))
1064 d.addCallback(lambda res: self.subdir2.list())
1065 d.addCallback(lambda children:
1066 self.failUnlessReallyEqual(set(children.keys()),
1068 d.addCallback(lambda res: self.subdir2.get(u"child"))
1069 d.addCallback(lambda child:
1070 self.failUnlessReallyEqual(child.get_uri(),
1073 # move it back, using new_child_name=
1074 d.addCallback(lambda res:
1075 self.subdir2.move_child_to(u"child", n, u"newchild"))
1076 d.addCallback(lambda res: n.list())
1077 d.addCallback(lambda children:
1078 self.failUnlessReallyEqual(set(children.keys()),
1079 set([u"newchild", u"newfile",
1081 d.addCallback(lambda res: self.subdir2.list())
1082 d.addCallback(lambda children:
1083 self.failUnlessReallyEqual(set(children.keys()), set([])))
1085 # now make sure that we honor overwrite=False
1086 d.addCallback(lambda res:
1087 self.subdir2.set_uri(u"newchild",
1088 other_file_uri, other_file_uri))
1090 d.addCallback(lambda res:
1091 self.shouldFail(ExistingChildError, "move_child_to-no",
1092 "child 'newchild' already exists",
1093 n.move_child_to, u"newchild",
1096 d.addCallback(lambda res: self.subdir2.get(u"newchild"))
1097 d.addCallback(lambda child:
1098 self.failUnlessReallyEqual(child.get_uri(),
1102 # Setting the no-write field should diminish a mutable cap to read-only
1103 # (for both files and directories).
1105 d.addCallback(lambda ign: n.set_uri(u"mutable", other_file_uri, other_file_uri))
1106 d.addCallback(lambda ign: n.get(u"mutable"))
1107 d.addCallback(lambda mutable: self.failIf(mutable.is_readonly(), mutable))
1108 d.addCallback(lambda ign: n.set_metadata_for(u"mutable", {"no-write": True}))
1109 d.addCallback(lambda ign: n.get(u"mutable"))
1110 d.addCallback(lambda mutable: self.failUnless(mutable.is_readonly(), mutable))
1111 d.addCallback(lambda ign: n.set_metadata_for(u"mutable", {"no-write": True}))
1112 d.addCallback(lambda ign: n.get(u"mutable"))
1113 d.addCallback(lambda mutable: self.failUnless(mutable.is_readonly(), mutable))
1115 d.addCallback(lambda ign: n.get(u"subdir2"))
1116 d.addCallback(lambda subdir2: self.failIf(subdir2.is_readonly()))
1117 d.addCallback(lambda ign: n.set_metadata_for(u"subdir2", {"no-write": True}))
1118 d.addCallback(lambda ign: n.get(u"subdir2"))
1119 d.addCallback(lambda subdir2: self.failUnless(subdir2.is_readonly(), subdir2))
1121 d.addCallback(lambda ign: n.set_uri(u"mutable_ro", other_file_uri, other_file_uri,
1122 metadata={"no-write": True}))
1123 d.addCallback(lambda ign: n.get(u"mutable_ro"))
1124 d.addCallback(lambda mutable_ro: self.failUnless(mutable_ro.is_readonly(), mutable_ro))
1126 d.addCallback(lambda ign: n.create_subdirectory(u"subdir_ro", metadata={"no-write": True}))
1127 d.addCallback(lambda ign: n.get(u"subdir_ro"))
1128 d.addCallback(lambda subdir_ro: self.failUnless(subdir_ro.is_readonly(), subdir_ro))
1132 d.addCallback(_then)
1134 d.addErrback(self.explain_error)
1137 def test_update_metadata(self):
1138 (t1, t2, t3) = (626644800.0, 634745640.0, 892226160.0)
1140 md1 = dirnode.update_metadata({"ctime": t1}, {}, t2)
1141 self.failUnlessEqual(md1, {"tahoe":{"linkcrtime": t1, "linkmotime": t2}})
1143 md2 = dirnode.update_metadata(md1, {"key": "value", "tahoe": {"bad": "mojo"}}, t3)
1144 self.failUnlessEqual(md2, {"key": "value",
1145 "tahoe":{"linkcrtime": t1, "linkmotime": t3}})
1147 md3 = dirnode.update_metadata({}, None, t3)
1148 self.failUnlessEqual(md3, {"tahoe":{"linkcrtime": t3, "linkmotime": t3}})
1150 md4 = dirnode.update_metadata({}, {"bool": True, "number": 42}, t1)
1151 self.failUnlessEqual(md4, {"bool": True, "number": 42,
1152 "tahoe":{"linkcrtime": t1, "linkmotime": t1}})
1154 def test_create_subdirectory(self):
1155 self.basedir = "dirnode/Dirnode/test_create_subdirectory"
1157 c = self.g.clients[0]
1160 d = c.create_dirnode()
1164 fake_file_uri = make_mutable_file_uri()
1165 other_file_uri = make_mutable_file_uri()
1166 md = {"metakey": "metavalue"}
1167 kids = {u"kid1": (nm.create_from_cap(fake_file_uri), {}),
1168 u"kid2": (nm.create_from_cap(other_file_uri), md),
1170 d = n.create_subdirectory(u"subdir", kids)
1172 d = n.get_child_at_path(u"subdir")
1173 d.addCallback(lambda sub2: self.failUnlessReallyEqual(sub2.get_uri(),
1175 d.addCallback(lambda ign: sub.list())
1177 d.addCallback(_check)
1178 def _check_kids(kids2):
1179 self.failUnlessEqual(set(kids.keys()), set(kids2.keys()))
1180 self.failUnlessEqual(kids2[u"kid2"][1]["metakey"], "metavalue")
1181 d.addCallback(_check_kids)
1183 d.addCallback(_then)
1186 class MinimalFakeMutableFile:
1187 def get_writekey(self):
1190 class Packing(testutil.ReallyEqualMixin, unittest.TestCase):
1191 # This is a base32-encoded representation of the directory tree
1195 # as represented after being fed to _pack_contents.
1196 # We have it here so we can decode it, feed it to
1197 # _unpack_contents, and verify that _unpack_contents
1200 known_tree = "GM4TOORVHJTGS3DFGEWDSNJ2KVJESOSDJBFTU33MPB2GS3LZNVYG6N3GGI3WU5TIORTXC3DOMJ2G4NB2MVWXUZDONBVTE5LNGRZWK2LYN55GY23XGNYXQMTOMZUWU5TENN4DG23ZG5UTO2L2NQ2DO6LFMRWDMZJWGRQTUMZ2GEYDUMJQFQYTIMZ22XZKZORX5XS7CAQCSK3URR6QOHISHRCMGER5LRFSZRNAS5ZSALCS6TWFQAE754IVOIKJVK73WZPP3VUUEDTX3WHTBBZ5YX3CEKHCPG3ZWQLYA4QM6LDRCF7TJQYWLIZHKGN5ROA3AUZPXESBNLQQ6JTC2DBJU2D47IZJTLR3PKZ4RVF57XLPWY7FX7SZV3T6IJ3ORFW37FXUPGOE3ROPFNUX5DCGMAQJ3PGGULBRGM3TU6ZCMN2GS3LFEI5CAMJSGQ3DMNRTHA4TOLRUGI3TKNRWGEWCAITUMFUG6ZJCHIQHWITMNFXGW3LPORUW2ZJCHIQDCMRUGY3DMMZYHE3S4NBSG42TMNRRFQQCE3DJNZVWG4TUNFWWKIR2EAYTENBWGY3DGOBZG4XDIMRXGU3DMML5FQQCE3LUNFWWKIR2EAYTENBWGY3DGOBZG4XDIMRXGU3DMML5FQWDGOJRHI2TUZTJNRSTELBZGQ5FKUSJHJBUQSZ2MFYGKZ3SOBSWQ43IO52WO23CNAZWU3DUGVSWSNTIOE5DK33POVTW4ZLNMNWDK6DHPA2GS2THNF2W25DEN5VGY2LQNFRGG5DKNNRHO5TZPFTWI6LNMRYGQ2LCGJTHM4J2GM5DCMB2GQWDCNBSHKVVQBGRYMACKJ27CVQ6O6B4QPR72RFVTGOZUI76XUSWAX73JRV5PYRHMIFYZIA25MXDPGUGML6M2NMRSG4YD4W4K37ZDYSXHMJ3IUVT4F64YTQQVBJFFFOUC7J7LAB2VFCL5UKKGMR2D3F4EPOYC7UYWQZNR5KXHBSNXLCNBX2SNF22DCXJIHSMEKWEWOG5XCJEVVZ7UW5IB6I64XXQSJ34B5CAYZGZIIMR6LBRGMZTU6ZCMN2GS3LFEI5CAMJSGQ3DMNRTHA4TOLRUGMYDEMJYFQQCE5DBNBXWKIR2EB5SE3DJNZVW233UNFWWKIR2EAYTENBWGY3DGOBZG4XDIMZQGIYTQLBAEJWGS3TLMNZHI2LNMURDUIBRGI2DMNRWGM4DSNZOGQZTAMRRHB6SYIBCNV2GS3LFEI5CAMJSGQ3DMNRTHA4TOLRUGMYDEMJYPUWCYMZZGU5DKOTGNFWGKMZMHE2DUVKSJE5EGSCLHJRW25DDPBYTO2DXPB3GM6DBNYZTI6LJMV3DM2LWNB4TU4LWMNSWW3LKORXWK5DEMN3TI23NNE3WEM3SORRGY5THPA3TKNBUMNZG453BOF2GSZLXMVWWI3DJOFZW623RHIZTUMJQHI2SYMJUGI5BOSHWDPG3WKPAVXCF3XMKA7QVIWPRMWJHDTQHD27AHDCPJWDQENQ5H5ZZILTXQNIXXCIW4LKQABU2GCFRG5FHQN7CHD7HF4EKNRZFIV2ZYQIBM7IQU7F4RGB3XCX3FREPBKQ7UCICHVWPCYFGA6OLH3J45LXQ6GWWICJ3PGWJNLZ7PCRNLAPNYUGU6BENS7OXMBEOOFRIZV3PF2FFWZ5WHDPKXERYP7GNHKRMGEZTOOT3EJRXI2LNMURDUIBRGI2DMNRWGM4DSNZOGQZTGNRSGY4SYIBCORQWQ33FEI5CA6ZCNRUW423NN52GS3LFEI5CAMJSGQ3DMNRTHA4TOLRUGMZTMMRWHEWCAITMNFXGWY3SORUW2ZJCHIQDCMRUGY3DMMZYHE3S4NBTGM3DENRZPUWCAITNORUW2ZJCHIQDCMRUGY3DMMZYHE3S4NBTGM3DENRZPUWCY==="
1202 def test_unpack_and_pack_behavior(self):
1203 known_tree = b32decode(self.known_tree)
1204 nodemaker = NodeMaker(None, None, None,
1206 {"k": 3, "n": 10}, None)
1207 write_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
1208 filenode = nodemaker.create_from_cap(write_uri)
1209 node = dirnode.DirectoryNode(filenode, nodemaker, None)
1210 children = node._unpack_contents(known_tree)
1211 self._check_children(children)
1213 packed_children = node._pack_contents(children)
1214 children = node._unpack_contents(packed_children)
1215 self._check_children(children)
1217 def _check_children(self, children):
1218 # Are all the expected child nodes there?
1219 self.failUnless(children.has_key(u'file1'))
1220 self.failUnless(children.has_key(u'file2'))
1221 self.failUnless(children.has_key(u'file3'))
1223 # Are the metadata for child 3 right?
1224 file3_rocap = "URI:CHK:cmtcxq7hwxvfxan34yiev6ivhy:qvcekmjtoetdcw4kmi7b3rtblvgx7544crnwaqtiewemdliqsokq:3:10:5"
1225 file3_rwcap = "URI:CHK:cmtcxq7hwxvfxan34yiev6ivhy:qvcekmjtoetdcw4kmi7b3rtblvgx7544crnwaqtiewemdliqsokq:3:10:5"
1226 file3_metadata = {'ctime': 1246663897.4336269, 'tahoe': {'linkmotime': 1246663897.4336269, 'linkcrtime': 1246663897.4336269}, 'mtime': 1246663897.4336269}
1227 self.failUnlessEqual(file3_metadata, children[u'file3'][1])
1228 self.failUnlessReallyEqual(file3_rocap,
1229 children[u'file3'][0].get_readonly_uri())
1230 self.failUnlessReallyEqual(file3_rwcap,
1231 children[u'file3'][0].get_uri())
1233 # Are the metadata for child 2 right?
1234 file2_rocap = "URI:CHK:apegrpehshwugkbh3jlt5ei6hq:5oougnemcl5xgx4ijgiumtdojlipibctjkbwvyygdymdphib2fvq:3:10:4"
1235 file2_rwcap = "URI:CHK:apegrpehshwugkbh3jlt5ei6hq:5oougnemcl5xgx4ijgiumtdojlipibctjkbwvyygdymdphib2fvq:3:10:4"
1236 file2_metadata = {'ctime': 1246663897.430218, 'tahoe': {'linkmotime': 1246663897.430218, 'linkcrtime': 1246663897.430218}, 'mtime': 1246663897.430218}
1237 self.failUnlessEqual(file2_metadata, children[u'file2'][1])
1238 self.failUnlessReallyEqual(file2_rocap,
1239 children[u'file2'][0].get_readonly_uri())
1240 self.failUnlessReallyEqual(file2_rwcap,
1241 children[u'file2'][0].get_uri())
1243 # Are the metadata for child 1 right?
1244 file1_rocap = "URI:CHK:olxtimympo7f27jvhtgqlnbtn4:emzdnhk2um4seixozlkw3qx2nfijvdkx3ky7i7izl47yedl6e64a:3:10:10"
1245 file1_rwcap = "URI:CHK:olxtimympo7f27jvhtgqlnbtn4:emzdnhk2um4seixozlkw3qx2nfijvdkx3ky7i7izl47yedl6e64a:3:10:10"
1246 file1_metadata = {'ctime': 1246663897.4275661, 'tahoe': {'linkmotime': 1246663897.4275661, 'linkcrtime': 1246663897.4275661}, 'mtime': 1246663897.4275661}
1247 self.failUnlessEqual(file1_metadata, children[u'file1'][1])
1248 self.failUnlessReallyEqual(file1_rocap,
1249 children[u'file1'][0].get_readonly_uri())
1250 self.failUnlessReallyEqual(file1_rwcap,
1251 children[u'file1'][0].get_uri())
1253 def _make_kids(self, nm, which):
1254 caps = {"imm": "URI:CHK:n7r3m6wmomelk4sep3kw5cvduq:os7ijw5c3maek7pg65e5254k2fzjflavtpejjyhshpsxuqzhcwwq:3:20:14861",
1255 "lit": "URI:LIT:n5xgk", # LIT for "one"
1256 "write": "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq",
1257 "read": "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q",
1258 "dirwrite": "URI:DIR2:n6x24zd3seu725yluj75q5boaa:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq",
1259 "dirread": "URI:DIR2-RO:b7sr5qsifnicca7cbk3rhrhbvq:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq",
1263 kids[unicode(name)] = (nm.create_from_cap(caps[name]), {})
1266 def test_deep_immutable(self):
1267 nm = NodeMaker(None, None, None, None, None, {"k": 3, "n": 10}, None)
1268 fn = MinimalFakeMutableFile()
1270 kids = self._make_kids(nm, ["imm", "lit", "write", "read",
1271 "dirwrite", "dirread"])
1272 packed = dirnode.pack_children(kids, fn.get_writekey(), deep_immutable=False)
1273 self.failUnlessIn("lit", packed)
1275 kids = self._make_kids(nm, ["imm", "lit"])
1276 packed = dirnode.pack_children(kids, fn.get_writekey(), deep_immutable=True)
1277 self.failUnlessIn("lit", packed)
1279 kids = self._make_kids(nm, ["imm", "lit", "write"])
1280 self.failUnlessRaises(dirnode.MustBeDeepImmutableError,
1281 dirnode.pack_children,
1282 kids, fn.get_writekey(), deep_immutable=True)
1284 # read-only is not enough: all children must be immutable
1285 kids = self._make_kids(nm, ["imm", "lit", "read"])
1286 self.failUnlessRaises(dirnode.MustBeDeepImmutableError,
1287 dirnode.pack_children,
1288 kids, fn.get_writekey(), deep_immutable=True)
1290 kids = self._make_kids(nm, ["imm", "lit", "dirwrite"])
1291 self.failUnlessRaises(dirnode.MustBeDeepImmutableError,
1292 dirnode.pack_children,
1293 kids, fn.get_writekey(), deep_immutable=True)
1295 kids = self._make_kids(nm, ["imm", "lit", "dirread"])
1296 self.failUnlessRaises(dirnode.MustBeDeepImmutableError,
1297 dirnode.pack_children,
1298 kids, fn.get_writekey(), deep_immutable=True)
1300 class FakeMutableFile:
1301 implements(IMutableFileNode)
1303 def __init__(self, initial_contents=""):
1304 self.data = self._get_initial_contents(initial_contents)
1305 counter = FakeMutableFile.counter
1306 FakeMutableFile.counter += 1
1307 writekey = hashutil.ssk_writekey_hash(str(counter))
1308 fingerprint = hashutil.ssk_pubkey_fingerprint_hash(str(counter))
1309 self.uri = uri.WriteableSSKFileURI(writekey, fingerprint)
1311 def _get_initial_contents(self, contents):
1312 if isinstance(contents, str):
1314 if contents is None:
1316 assert callable(contents), "%s should be callable, not %s" % \
1317 (contents, type(contents))
1318 return contents(self)
1324 return self.uri.to_string()
1326 def get_write_uri(self):
1327 return self.uri.to_string()
1329 def download_best_version(self):
1330 return defer.succeed(self.data)
1332 def get_writekey(self):
1335 def is_readonly(self):
1338 def is_mutable(self):
1341 def is_unknown(self):
1344 def is_allowed_in_immutable_directory(self):
1347 def raise_error(self):
1350 def modify(self, modifier):
1351 self.data = modifier(self.data, None, True)
1352 return defer.succeed(None)
1354 class FakeNodeMaker(NodeMaker):
1355 def create_mutable_file(self, contents="", keysize=None):
1356 return defer.succeed(FakeMutableFile(contents))
1358 class FakeClient2(Client):
1360 self.nodemaker = FakeNodeMaker(None, None, None,
1362 {"k":3,"n":10}, None)
1363 def create_node_from_uri(self, rwcap, rocap):
1364 return self.nodemaker.create_from_cap(rwcap, rocap)
1366 class Dirnode2(testutil.ReallyEqualMixin, testutil.ShouldFailMixin, unittest.TestCase):
1368 client = FakeClient2()
1369 self.nodemaker = client.nodemaker
1371 def test_from_future(self):
1372 # Create a mutable directory that contains unknown URI types, and make sure
1373 # we tolerate them properly.
1374 d = self.nodemaker.create_new_mutable_directory()
1375 future_write_uri = u"x-tahoe-crazy://I_am_from_the_future_rw_\u263A".encode('utf-8')
1376 future_read_uri = u"x-tahoe-crazy-readonly://I_am_from_the_future_ro_\u263A".encode('utf-8')
1377 future_imm_uri = u"x-tahoe-crazy-immutable://I_am_from_the_future_imm_\u263A".encode('utf-8')
1378 future_node = UnknownNode(future_write_uri, future_read_uri)
1381 return n.set_node(u"future", future_node)
1382 d.addCallback(_then)
1384 # We should be prohibited from adding an unknown URI to a directory
1385 # just in the rw_uri slot, since we don't know how to diminish the cap
1386 # to a readcap (for the ro_uri slot).
1387 d.addCallback(lambda ign:
1388 self.shouldFail(MustNotBeUnknownRWError,
1390 "cannot attach unknown rw cap as child",
1391 self._node.set_uri, u"add",
1392 future_write_uri, None))
1394 # However, we should be able to add both rw_uri and ro_uri as a pair of
1396 d.addCallback(lambda ign: self._node.set_uri(u"add-pair",
1397 future_write_uri, future_read_uri))
1399 # and to add an URI prefixed with "ro." or "imm." when it is given in a
1400 # write slot (or URL parameter).
1401 d.addCallback(lambda ign: self._node.set_uri(u"add-ro",
1402 "ro." + future_read_uri, None))
1403 d.addCallback(lambda ign: self._node.set_uri(u"add-imm",
1404 "imm." + future_imm_uri, None))
1406 d.addCallback(lambda ign: self._node.list())
1407 def _check(children):
1408 self.failUnlessReallyEqual(len(children), 4)
1409 (fn, metadata) = children[u"future"]
1410 self.failUnless(isinstance(fn, UnknownNode), fn)
1411 self.failUnlessReallyEqual(fn.get_uri(), future_write_uri)
1412 self.failUnlessReallyEqual(fn.get_write_uri(), future_write_uri)
1413 self.failUnlessReallyEqual(fn.get_readonly_uri(), "ro." + future_read_uri)
1415 (fn2, metadata2) = children[u"add-pair"]
1416 self.failUnless(isinstance(fn2, UnknownNode), fn2)
1417 self.failUnlessReallyEqual(fn2.get_uri(), future_write_uri)
1418 self.failUnlessReallyEqual(fn2.get_write_uri(), future_write_uri)
1419 self.failUnlessReallyEqual(fn2.get_readonly_uri(), "ro." + future_read_uri)
1421 (fn3, metadata3) = children[u"add-ro"]
1422 self.failUnless(isinstance(fn3, UnknownNode), fn3)
1423 self.failUnlessReallyEqual(fn3.get_uri(), "ro." + future_read_uri)
1424 self.failUnlessReallyEqual(fn3.get_write_uri(), None)
1425 self.failUnlessReallyEqual(fn3.get_readonly_uri(), "ro." + future_read_uri)
1427 (fn4, metadata4) = children[u"add-imm"]
1428 self.failUnless(isinstance(fn4, UnknownNode), fn4)
1429 self.failUnlessReallyEqual(fn4.get_uri(), "imm." + future_imm_uri)
1430 self.failUnlessReallyEqual(fn4.get_write_uri(), None)
1431 self.failUnlessReallyEqual(fn4.get_readonly_uri(), "imm." + future_imm_uri)
1433 # We should also be allowed to copy the "future" UnknownNode, because
1434 # it contains all the information that was in the original directory
1435 # (readcap and writecap), so we're preserving everything.
1436 return self._node.set_node(u"copy", fn)
1437 d.addCallback(_check)
1439 d.addCallback(lambda ign: self._node.list())
1440 def _check2(children):
1441 self.failUnlessReallyEqual(len(children), 5)
1442 (fn, metadata) = children[u"copy"]
1443 self.failUnless(isinstance(fn, UnknownNode), fn)
1444 self.failUnlessReallyEqual(fn.get_uri(), future_write_uri)
1445 self.failUnlessReallyEqual(fn.get_write_uri(), future_write_uri)
1446 self.failUnlessReallyEqual(fn.get_readonly_uri(), "ro." + future_read_uri)
1447 d.addCallback(_check2)
1450 def test_unknown_strip_prefix_for_ro(self):
1451 self.failUnlessReallyEqual(strip_prefix_for_ro("foo", False), "foo")
1452 self.failUnlessReallyEqual(strip_prefix_for_ro("ro.foo", False), "foo")
1453 self.failUnlessReallyEqual(strip_prefix_for_ro("imm.foo", False), "imm.foo")
1454 self.failUnlessReallyEqual(strip_prefix_for_ro("foo", True), "foo")
1455 self.failUnlessReallyEqual(strip_prefix_for_ro("ro.foo", True), "foo")
1456 self.failUnlessReallyEqual(strip_prefix_for_ro("imm.foo", True), "foo")
1458 def test_unknownnode(self):
1459 mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
1460 mut_read_uri = "URI:SSK-RO:jf6wkflosyvntwxqcdo7a54jvm:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
1461 lit_uri = "URI:LIT:n5xgk"
1463 # This does not attempt to be exhaustive.
1464 no_no = [# Opaque node, but not an error.
1465 ( 0, UnknownNode(None, None)),
1466 ( 1, UnknownNode(None, None, deep_immutable=True)),
1468 unknown_rw = [# These are errors because we're only given a rw_uri, and we can't
1470 ( 2, UnknownNode("foo", None)),
1471 ( 3, UnknownNode("foo", None, deep_immutable=True)),
1472 ( 4, UnknownNode("ro.foo", None, deep_immutable=True)),
1473 ( 5, UnknownNode("ro." + mut_read_uri, None, deep_immutable=True)),
1474 ( 6, UnknownNode("URI:SSK-RO:foo", None, deep_immutable=True)),
1475 ( 7, UnknownNode("URI:SSK:foo", None)),
1477 must_be_ro = [# These are errors because a readonly constraint is not met.
1478 ( 8, UnknownNode("ro." + mut_write_uri, None)),
1479 ( 9, UnknownNode(None, "ro." + mut_write_uri)),
1481 must_be_imm = [# These are errors because an immutable constraint is not met.
1482 (10, UnknownNode(None, "ro.URI:SSK-RO:foo", deep_immutable=True)),
1483 (11, UnknownNode(None, "imm.URI:SSK:foo")),
1484 (12, UnknownNode(None, "imm.URI:SSK-RO:foo")),
1485 (13, UnknownNode("bar", "ro.foo", deep_immutable=True)),
1486 (14, UnknownNode("bar", "imm.foo", deep_immutable=True)),
1487 (15, UnknownNode("bar", "imm." + lit_uri, deep_immutable=True)),
1488 (16, UnknownNode("imm." + mut_write_uri, None)),
1489 (17, UnknownNode("imm." + mut_read_uri, None)),
1490 (18, UnknownNode("bar", "imm.foo")),
1492 bad_uri = [# These are errors because the URI is bad once we've stripped the prefix.
1493 (19, UnknownNode("ro.URI:SSK-RO:foo", None)),
1494 (20, UnknownNode("imm.URI:CHK:foo", None, deep_immutable=True)),
1495 (21, UnknownNode(None, "URI:CHK:foo")),
1496 (22, UnknownNode(None, "URI:CHK:foo", deep_immutable=True)),
1498 ro_prefixed = [# These are valid, and the readcap should end up with a ro. prefix.
1499 (23, UnknownNode(None, "foo")),
1500 (24, UnknownNode(None, "ro.foo")),
1501 (25, UnknownNode(None, "ro." + lit_uri)),
1502 (26, UnknownNode("bar", "foo")),
1503 (27, UnknownNode("bar", "ro.foo")),
1504 (28, UnknownNode("bar", "ro." + lit_uri)),
1505 (29, UnknownNode("ro.foo", None)),
1506 (30, UnknownNode("ro." + lit_uri, None)),
1508 imm_prefixed = [# These are valid, and the readcap should end up with an imm. prefix.
1509 (31, UnknownNode(None, "foo", deep_immutable=True)),
1510 (32, UnknownNode(None, "ro.foo", deep_immutable=True)),
1511 (33, UnknownNode(None, "imm.foo")),
1512 (34, UnknownNode(None, "imm.foo", deep_immutable=True)),
1513 (35, UnknownNode("imm." + lit_uri, None)),
1514 (36, UnknownNode("imm." + lit_uri, None, deep_immutable=True)),
1515 (37, UnknownNode(None, "imm." + lit_uri)),
1516 (38, UnknownNode(None, "imm." + lit_uri, deep_immutable=True)),
1518 error = unknown_rw + must_be_ro + must_be_imm + bad_uri
1519 ok = ro_prefixed + imm_prefixed
1521 for (i, n) in no_no + error + ok:
1522 self.failUnless(n.is_unknown(), i)
1524 for (i, n) in no_no + error:
1525 self.failUnless(n.get_uri() is None, i)
1526 self.failUnless(n.get_write_uri() is None, i)
1527 self.failUnless(n.get_readonly_uri() is None, i)
1529 for (i, n) in no_no + ok:
1532 for (i, n) in unknown_rw:
1533 self.failUnlessRaises(MustNotBeUnknownRWError, lambda: n.raise_error())
1535 for (i, n) in must_be_ro:
1536 self.failUnlessRaises(MustBeReadonlyError, lambda: n.raise_error())
1538 for (i, n) in must_be_imm:
1539 self.failUnlessRaises(MustBeDeepImmutableError, lambda: n.raise_error())
1541 for (i, n) in bad_uri:
1542 self.failUnlessRaises(uri.BadURIError, lambda: n.raise_error())
1545 self.failIf(n.get_readonly_uri() is None, i)
1547 for (i, n) in ro_prefixed:
1548 self.failUnless(n.get_readonly_uri().startswith("ro."), i)
1550 for (i, n) in imm_prefixed:
1551 self.failUnless(n.get_readonly_uri().startswith("imm."), i)
1554 class DeepStats(testutil.ReallyEqualMixin, unittest.TestCase):
1555 timeout = 240 # It takes longer than 120 seconds on Francois's arm box.
1556 def test_stats(self):
1557 ds = dirnode.DeepStats(None)
1558 ds.add("count-files")
1559 ds.add("size-immutable-files", 123)
1560 ds.histogram("size-files-histogram", 123)
1561 ds.max("largest-directory", 444)
1563 s = ds.get_results()
1564 self.failUnlessReallyEqual(s["count-files"], 1)
1565 self.failUnlessReallyEqual(s["size-immutable-files"], 123)
1566 self.failUnlessReallyEqual(s["largest-directory"], 444)
1567 self.failUnlessReallyEqual(s["count-literal-files"], 0)
1569 ds.add("count-files")
1570 ds.add("size-immutable-files", 321)
1571 ds.histogram("size-files-histogram", 321)
1572 ds.max("largest-directory", 2)
1574 s = ds.get_results()
1575 self.failUnlessReallyEqual(s["count-files"], 2)
1576 self.failUnlessReallyEqual(s["size-immutable-files"], 444)
1577 self.failUnlessReallyEqual(s["largest-directory"], 444)
1578 self.failUnlessReallyEqual(s["count-literal-files"], 0)
1579 self.failUnlessReallyEqual(s["size-files-histogram"],
1580 [ (101, 316, 1), (317, 1000, 1) ])
1582 ds = dirnode.DeepStats(None)
1583 for i in range(1, 1100):
1584 ds.histogram("size-files-histogram", i)
1585 ds.histogram("size-files-histogram", 4*1000*1000*1000*1000) # 4TB
1586 s = ds.get_results()
1587 self.failUnlessReallyEqual(s["size-files-histogram"],
1595 (3162277660169L, 10000000000000L, 1),
1598 class UCWEingMutableFileNode(MutableFileNode):
1599 please_ucwe_after_next_upload = False
1601 def _upload(self, new_contents, servermap):
1602 d = MutableFileNode._upload(self, new_contents, servermap)
1604 if self.please_ucwe_after_next_upload:
1605 self.please_ucwe_after_next_upload = False
1606 raise UncoordinatedWriteError()
1608 d.addCallback(_ucwe)
1611 class UCWEingNodeMaker(NodeMaker):
1612 def _create_mutable(self, cap):
1613 n = UCWEingMutableFileNode(self.storage_broker, self.secret_holder,
1614 self.default_encoding_parameters,
1616 return n.init_from_cap(cap)
1619 class Deleter(GridTestMixin, testutil.ReallyEqualMixin, unittest.TestCase):
1620 timeout = 3600 # It takes longer than 433 seconds on Zandr's ARM box.
1621 def test_retry(self):
1622 # ticket #550, a dirnode.delete which experiences an
1623 # UncoordinatedWriteError will fail with an incorrect "you're
1624 # deleting something which isn't there" NoSuchChildError exception.
1626 # to trigger this, we start by creating a directory with a single
1627 # file in it. Then we create a special dirnode that uses a modified
1628 # MutableFileNode which will raise UncoordinatedWriteError once on
1629 # demand. We then call dirnode.delete, which ought to retry and
1632 self.basedir = self.mktemp()
1634 c0 = self.g.clients[0]
1635 d = c0.create_dirnode()
1636 small = upload.Data("Small enough for a LIT", None)
1637 def _created_dir(dn):
1639 self.root_uri = dn.get_uri()
1640 return dn.add_file(u"file", small)
1641 d.addCallback(_created_dir)
1642 def _do_delete(ignored):
1643 nm = UCWEingNodeMaker(c0.storage_broker, c0._secret_holder,
1644 c0.get_history(), c0.getServiceNamed("uploader"),
1646 c0.get_encoding_parameters(),
1648 n = nm.create_from_cap(self.root_uri)
1649 assert n._node.please_ucwe_after_next_upload == False
1650 n._node.please_ucwe_after_next_upload = True
1651 # This should succeed, not raise an exception
1652 return n.delete(u"file")
1653 d.addCallback(_do_delete)
1657 class Adder(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin):
1659 def test_overwrite(self):
1660 # note: This functionality could be tested without actually creating
1661 # several RSA keys. It would be faster without the GridTestMixin: use
1662 # dn.set_node(nodemaker.create_from_cap(make_chk_file_uri())) instead
1663 # of dn.add_file, and use a special NodeMaker that creates fake
1665 self.basedir = "dirnode/Adder/test_overwrite"
1667 c = self.g.clients[0]
1668 fileuri = make_chk_file_uri(1234)
1669 filenode = c.nodemaker.create_from_cap(fileuri)
1670 d = c.create_dirnode()
1672 def _create_directory_tree(root_node):
1677 d = root_node.add_file(u'file1', upload.Data("Important Things",
1679 d.addCallback(lambda res:
1680 root_node.add_file(u'file2', upload.Data("Sekrit Codes", None)))
1681 d.addCallback(lambda res:
1682 root_node.create_subdirectory(u"dir1"))
1683 d.addCallback(lambda res: root_node)
1686 d.addCallback(_create_directory_tree)
1688 def _test_adder(root_node):
1689 d = root_node.set_node(u'file1', filenode)
1690 # We've overwritten file1. Let's try it with a directory
1691 d.addCallback(lambda res:
1692 root_node.create_subdirectory(u'dir2'))
1693 d.addCallback(lambda res:
1694 root_node.set_node(u'dir2', filenode))
1695 # We try overwriting a file with a child while also specifying
1696 # overwrite=False. We should receive an ExistingChildError
1698 d.addCallback(lambda res:
1699 self.shouldFail(ExistingChildError, "set_node",
1700 "child 'file1' already exists",
1701 root_node.set_node, u"file1",
1702 filenode, overwrite=False))
1703 # If we try with a directory, we should see the same thing
1704 d.addCallback(lambda res:
1705 self.shouldFail(ExistingChildError, "set_node",
1706 "child 'dir1' already exists",
1707 root_node.set_node, u'dir1', filenode,
1709 d.addCallback(lambda res:
1710 root_node.set_node(u'file1', filenode,
1711 overwrite="only-files"))
1712 d.addCallback(lambda res:
1713 self.shouldFail(ExistingChildError, "set_node",
1714 "child 'dir1' already exists",
1715 root_node.set_node, u'dir1', filenode,
1716 overwrite="only-files"))
1719 d.addCallback(_test_adder)