src/allmydata/test/test_dirnode.py

   1
   2 import time
   3 from zope.interface import implements
   4 from twisted.trial import unittest
   5 from twisted.internet import defer
   6 from allmydata import uri, dirnode
   7 from allmydata.client import Client
   8 from allmydata.immutable import upload
   9 from allmydata.interfaces import IImmutableFileNode, IMutableFileNode, \
  10      ExistingChildError, NoSuchChildError, MustNotBeUnknownRWError, \
  11      MustBeDeepImmutableError, MustBeReadonlyError, \
  12      IDeepCheckResults, IDeepCheckAndRepairResults
  13 from allmydata.mutable.filenode import MutableFileNode
  14 from allmydata.mutable.common import UncoordinatedWriteError
  15 from allmydata.util import hashutil, base32
  16 from allmydata.util.netstring import split_netstring
  17 from allmydata.monitor import Monitor
  18 from allmydata.test.common import make_chk_file_uri, make_mutable_file_uri, \
  19      ErrorMixin
  20 from allmydata.test.no_network import GridTestMixin
  21 from allmydata.unknown import UnknownNode, strip_prefix_for_ro
  22 from allmydata.nodemaker import NodeMaker
  23 from base64 import b32decode
  24 import common_util as testutil
  25
  26 class Dirnode(GridTestMixin, unittest.TestCase,
  27               testutil.ShouldFailMixin, testutil.StallMixin, ErrorMixin):
  28     timeout = 240 # It takes longer than 120 seconds on Francois's arm box.
  29
  30     def test_basic(self):
  31         self.basedir = "dirnode/Dirnode/test_basic"
  32         self.set_up_grid()
  33         c = self.g.clients[0]
  34         d = c.create_dirnode()
  35         def _done(res):
  36             self.failUnless(isinstance(res, dirnode.DirectoryNode))
  37             self.failUnless(res.is_mutable())
  38             self.failIf(res.is_readonly())
  39             self.failIf(res.is_unknown())
  40             self.failIf(res.is_allowed_in_immutable_directory())
  41             res.raise_error()
  42             rep = str(res)
  43             self.failUnless("RW-MUT" in rep)
  44         d.addCallback(_done)
  45         return d
  46
  47     def test_initial_children(self):
  48         self.basedir = "dirnode/Dirnode/test_initial_children"
  49         self.set_up_grid()
  50         c = self.g.clients[0]
  51         nm = c.nodemaker
  52
  53         setup_py_uri = "URI:CHK:n7r3m6wmomelk4sep3kw5cvduq:os7ijw5c3maek7pg65e5254k2fzjflavtpejjyhshpsxuqzhcwwq:3:20:14861"
  54         one_uri = "URI:LIT:n5xgk" # LIT for "one"
  55         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
  56         mut_read_uri = "URI:SSK-RO:jf6wkflosyvntwxqcdo7a54jvm:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
  57         future_write_uri = "x-tahoe-crazy://I_am_from_the_future."
  58         future_read_uri = "x-tahoe-crazy-readonly://I_am_from_the_future."
  59         kids = {u"one": (nm.create_from_cap(one_uri), {}),
  60                 u"two": (nm.create_from_cap(setup_py_uri),
  61                          {"metakey": "metavalue"}),
  62                 u"mut": (nm.create_from_cap(mut_write_uri, mut_read_uri), {}),
  63                 u"fut": (nm.create_from_cap(future_write_uri, future_read_uri), {}),
  64                 u"fro": (nm.create_from_cap(None, future_read_uri), {}),
  65                 }
  66         d = c.create_dirnode(kids)
  67
  68         def _created(dn):
  69             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
  70             self.failUnless(dn.is_mutable())
  71             self.failIf(dn.is_readonly())
  72             self.failIf(dn.is_unknown())
  73             self.failIf(dn.is_allowed_in_immutable_directory())
  74             dn.raise_error()
  75             rep = str(dn)
  76             self.failUnless("RW-MUT" in rep)
  77             return dn.list()
  78         d.addCallback(_created)
  79
  80         def _check_kids(children):
  81             self.failUnlessEqual(sorted(children.keys()),
  82                                  [u"fro", u"fut", u"mut", u"one", u"two"])
  83             one_node, one_metadata = children[u"one"]
  84             two_node, two_metadata = children[u"two"]
  85             mut_node, mut_metadata = children[u"mut"]
  86             fut_node, fut_metadata = children[u"fut"]
  87             fro_node, fro_metadata = children[u"fro"]
  88
  89             self.failUnlessEqual(one_node.get_size(), 3)
  90             self.failUnlessEqual(one_node.get_uri(), one_uri)
  91             self.failUnlessEqual(one_node.get_readonly_uri(), one_uri)
  92             self.failUnless(isinstance(one_metadata, dict), one_metadata)
  93
  94             self.failUnlessEqual(two_node.get_size(), 14861)
  95             self.failUnlessEqual(two_node.get_uri(), setup_py_uri)
  96             self.failUnlessEqual(two_node.get_readonly_uri(), setup_py_uri)
  97             self.failUnlessEqual(two_metadata["metakey"], "metavalue")
  98
  99             self.failUnlessEqual(mut_node.get_uri(), mut_write_uri)
 100             self.failUnlessEqual(mut_node.get_readonly_uri(), mut_read_uri)
 101             self.failUnless(isinstance(mut_metadata, dict), mut_metadata)
 102
 103             self.failUnless(fut_node.is_unknown())
 104             self.failUnlessEqual(fut_node.get_uri(), future_write_uri)
 105             self.failUnlessEqual(fut_node.get_readonly_uri(), "ro." + future_read_uri)
 106             self.failUnless(isinstance(fut_metadata, dict), fut_metadata)
 107
 108             self.failUnless(fro_node.is_unknown())
 109             self.failUnlessEqual(fro_node.get_uri(), "ro." + future_read_uri)
 110             self.failUnlessEqual(fut_node.get_readonly_uri(), "ro." + future_read_uri)
 111             self.failUnless(isinstance(fro_metadata, dict), fro_metadata)
 112         d.addCallback(_check_kids)
 113
 114         d.addCallback(lambda ign: nm.create_new_mutable_directory(kids))
 115         d.addCallback(lambda dn: dn.list())
 116         d.addCallback(_check_kids)
 117
 118         bad_future_node = UnknownNode(future_write_uri, None)
 119         bad_kids1 = {u"one": (bad_future_node, {})}
 120         # This should fail because we don't know how to diminish the future_write_uri
 121         # cap (given in a write slot and not prefixed with "ro." or "imm.") to a readcap.
 122         d.addCallback(lambda ign:
 123                       self.shouldFail(MustNotBeUnknownRWError, "bad_kids1",
 124                                       "cannot attach unknown",
 125                                       nm.create_new_mutable_directory,
 126                                       bad_kids1))
 127         bad_kids2 = {u"one": (nm.create_from_cap(one_uri), None)}
 128         d.addCallback(lambda ign:
 129                       self.shouldFail(AssertionError, "bad_kids2",
 130                                       "requires metadata to be a dict",
 131                                       nm.create_new_mutable_directory,
 132                                       bad_kids2))
 133         return d
 134
 135     def test_immutable(self):
 136         self.basedir = "dirnode/Dirnode/test_immutable"
 137         self.set_up_grid()
 138         c = self.g.clients[0]
 139         nm = c.nodemaker
 140
 141         setup_py_uri = "URI:CHK:n7r3m6wmomelk4sep3kw5cvduq:os7ijw5c3maek7pg65e5254k2fzjflavtpejjyhshpsxuqzhcwwq:3:20:14861"
 142         one_uri = "URI:LIT:n5xgk" # LIT for "one"
 143         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
 144         mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
 145         future_write_uri = "x-tahoe-crazy://I_am_from_the_future."
 146         future_read_uri = "x-tahoe-crazy-readonly://I_am_from_the_future."
 147         kids = {u"one": (nm.create_from_cap(one_uri), {}),
 148                 u"two": (nm.create_from_cap(setup_py_uri),
 149                          {"metakey": "metavalue"}),
 150                 u"fut": (nm.create_from_cap(None, future_read_uri), {}),
 151                 }
 152         d = c.create_immutable_dirnode(kids)
 153
 154         def _created(dn):
 155             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
 156             self.failIf(dn.is_mutable())
 157             self.failUnless(dn.is_readonly())
 158             self.failIf(dn.is_unknown())
 159             self.failUnless(dn.is_allowed_in_immutable_directory())
 160             dn.raise_error()
 161             rep = str(dn)
 162             self.failUnless("RO-IMM" in rep)
 163             cap = dn.get_cap()
 164             self.failUnlessIn("CHK", cap.to_string())
 165             self.cap = cap
 166             return dn.list()
 167         d.addCallback(_created)
 168
 169         def _check_kids(children):
 170             self.failUnlessEqual(sorted(children.keys()), [u"fut", u"one", u"two"])
 171             one_node, one_metadata = children[u"one"]
 172             two_node, two_metadata = children[u"two"]
 173             fut_node, fut_metadata = children[u"fut"]
 174
 175             self.failUnlessEqual(one_node.get_size(), 3)
 176             self.failUnlessEqual(one_node.get_uri(), one_uri)
 177             self.failUnlessEqual(one_node.get_readonly_uri(), one_uri)
 178             self.failUnless(isinstance(one_metadata, dict), one_metadata)
 179
 180             self.failUnlessEqual(two_node.get_size(), 14861)
 181             self.failUnlessEqual(two_node.get_uri(), setup_py_uri)
 182             self.failUnlessEqual(two_node.get_readonly_uri(), setup_py_uri)
 183             self.failUnlessEqual(two_metadata["metakey"], "metavalue")
 184
 185             self.failUnless(fut_node.is_unknown())
 186             self.failUnlessEqual(fut_node.get_uri(), "imm." + future_read_uri)
 187             self.failUnlessEqual(fut_node.get_readonly_uri(), "imm." + future_read_uri)
 188             self.failUnless(isinstance(fut_metadata, dict), fut_metadata)
 189         d.addCallback(_check_kids)
 190
 191         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
 192         d.addCallback(lambda dn: dn.list())
 193         d.addCallback(_check_kids)
 194
 195         bad_future_node1 = UnknownNode(future_write_uri, None)
 196         bad_kids1 = {u"one": (bad_future_node1, {})}
 197         d.addCallback(lambda ign:
 198                       self.shouldFail(MustNotBeUnknownRWError, "bad_kids1",
 199                                       "cannot attach unknown",
 200                                       c.create_immutable_dirnode,
 201                                       bad_kids1))
 202         bad_future_node2 = UnknownNode(future_write_uri, future_read_uri)
 203         bad_kids2 = {u"one": (bad_future_node2, {})}
 204         d.addCallback(lambda ign:
 205                       self.shouldFail(MustBeDeepImmutableError, "bad_kids2",
 206                                       "is not immutable",
 207                                       c.create_immutable_dirnode,
 208                                       bad_kids2))
 209         bad_kids3 = {u"one": (nm.create_from_cap(one_uri), None)}
 210         d.addCallback(lambda ign:
 211                       self.shouldFail(AssertionError, "bad_kids3",
 212                                       "requires metadata to be a dict",
 213                                       c.create_immutable_dirnode,
 214                                       bad_kids3))
 215         bad_kids4 = {u"one": (nm.create_from_cap(mut_write_uri), {})}
 216         d.addCallback(lambda ign:
 217                       self.shouldFail(MustBeDeepImmutableError, "bad_kids4",
 218                                       "is not immutable",
 219                                       c.create_immutable_dirnode,
 220                                       bad_kids4))
 221         bad_kids5 = {u"one": (nm.create_from_cap(mut_read_uri), {})}
 222         d.addCallback(lambda ign:
 223                       self.shouldFail(MustBeDeepImmutableError, "bad_kids5",
 224                                       "is not immutable",
 225                                       c.create_immutable_dirnode,
 226                                       bad_kids5))
 227         d.addCallback(lambda ign: c.create_immutable_dirnode({}))
 228         def _created_empty(dn):
 229             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
 230             self.failIf(dn.is_mutable())
 231             self.failUnless(dn.is_readonly())
 232             self.failIf(dn.is_unknown())
 233             self.failUnless(dn.is_allowed_in_immutable_directory())
 234             dn.raise_error()
 235             rep = str(dn)
 236             self.failUnless("RO-IMM" in rep)
 237             cap = dn.get_cap()
 238             self.failUnlessIn("LIT", cap.to_string())
 239             self.failUnlessEqual(cap.to_string(), "URI:DIR2-LIT:")
 240             self.cap = cap
 241             return dn.list()
 242         d.addCallback(_created_empty)
 243         d.addCallback(lambda kids: self.failUnlessEqual(kids, {}))
 244         smallkids = {u"o": (nm.create_from_cap(one_uri), {})}
 245         d.addCallback(lambda ign: c.create_immutable_dirnode(smallkids))
 246         def _created_small(dn):
 247             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
 248             self.failIf(dn.is_mutable())
 249             self.failUnless(dn.is_readonly())
 250             self.failIf(dn.is_unknown())
 251             self.failUnless(dn.is_allowed_in_immutable_directory())
 252             dn.raise_error()
 253             rep = str(dn)
 254             self.failUnless("RO-IMM" in rep)
 255             cap = dn.get_cap()
 256             self.failUnlessIn("LIT", cap.to_string())
 257             self.failUnlessEqual(cap.to_string(),
 258                                  "URI:DIR2-LIT:gi4tumj2n4wdcmz2kvjesosmjfkdu3rvpbtwwlbqhiwdeot3puwcy")
 259             self.cap = cap
 260             return dn.list()
 261         d.addCallback(_created_small)
 262         d.addCallback(lambda kids: self.failUnlessEqual(kids.keys(), [u"o"]))
 263
 264         # now test n.create_subdirectory(mutable=False)
 265         d.addCallback(lambda ign: c.create_dirnode())
 266         def _made_parent(n):
 267             d = n.create_subdirectory(u"subdir", kids, mutable=False)
 268             d.addCallback(lambda sd: sd.list())
 269             d.addCallback(_check_kids)
 270             d.addCallback(lambda ign: n.list())
 271             d.addCallback(lambda children:
 272                           self.failUnlessEqual(children.keys(), [u"subdir"]))
 273             d.addCallback(lambda ign: n.get(u"subdir"))
 274             d.addCallback(lambda sd: sd.list())
 275             d.addCallback(_check_kids)
 276             d.addCallback(lambda ign: n.get(u"subdir"))
 277             d.addCallback(lambda sd: self.failIf(sd.is_mutable()))
 278             bad_kids = {u"one": (nm.create_from_cap(mut_write_uri), {})}
 279             d.addCallback(lambda ign:
 280                           self.shouldFail(MustBeDeepImmutableError, "YZ",
 281                                           "is not immutable",
 282                                           n.create_subdirectory,
 283                                           u"sub2", bad_kids, mutable=False))
 284             return d
 285         d.addCallback(_made_parent)
 286         return d
 287
 288     def test_spaces_are_stripped_on_the_way_out(self):
 289         self.basedir = "dirnode/Dirnode/test_spaces_are_stripped_on_the_way_out"
 290         self.set_up_grid()
 291         c = self.g.clients[0]
 292         nm = c.nodemaker
 293
 294         # This test checks that any trailing spaces in URIs are retained in the
 295         # encoded directory, but stripped when we get them out of the directory.
 296         # See ticket #925 for why we want that.
 297
 298         stripped_write_uri = "lafs://from_the_future\t"
 299         stripped_read_uri = "lafs://readonly_from_the_future\t"
 300         spacedout_write_uri = stripped_write_uri + "  "
 301         spacedout_read_uri = stripped_read_uri + "  "
 302
 303         child = nm.create_from_cap(spacedout_write_uri, spacedout_read_uri)
 304         self.failUnlessEqual(child.get_write_uri(), spacedout_write_uri)
 305         self.failUnlessEqual(child.get_readonly_uri(), "ro." + spacedout_read_uri)
 306
 307         kids = {u"child": (child, {})}
 308         d = c.create_dirnode(kids)
 309
 310         def _created(dn):
 311             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
 312             self.failUnless(dn.is_mutable())
 313             self.failIf(dn.is_readonly())
 314             dn.raise_error()
 315             self.cap = dn.get_cap()
 316             self.rootnode = dn
 317             return dn._node.download_best_version()
 318         d.addCallback(_created)
 319
 320         def _check_data(data):
 321             # Decode the netstring representation of the directory to check that the
 322             # spaces are retained when the URIs are stored.
 323             position = 0
 324             numkids = 0
 325             while position < len(data):
 326                 entries, position = split_netstring(data, 1, position)
 327                 entry = entries[0]
 328                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
 329                 name = name_utf8.decode("utf-8")
 330                 rw_uri = self.rootnode._decrypt_rwcapdata(rwcapdata)
 331                 self.failUnless(name in kids)
 332                 (expected_child, ign) = kids[name]
 333                 self.failUnlessEqual(rw_uri, expected_child.get_write_uri())
 334                 self.failUnlessEqual("ro." + ro_uri, expected_child.get_readonly_uri())
 335                 numkids += 1
 336
 337             self.failUnlessEqual(numkids, 1)
 338             return self.rootnode.list()
 339         d.addCallback(_check_data)
 340
 341         # Now when we use the real directory listing code, the trailing spaces
 342         # should have been stripped (and "ro." should have been prepended to the
 343         # ro_uri, since it's unknown).
 344         def _check_kids(children):
 345             self.failUnlessEqual(sorted(children.keys()), [u"child"])
 346             child_node, child_metadata = children[u"child"]
 347
 348             self.failUnlessEqual(child_node.get_write_uri(), stripped_write_uri)
 349             self.failUnlessEqual(child_node.get_readonly_uri(), "ro." + stripped_read_uri)
 350         d.addCallback(_check_kids)
 351
 352         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
 353         d.addCallback(lambda n: n.list())
 354         d.addCallback(_check_kids)  # again with dirnode recreated from cap
 355         return d
 356
 357     def test_check(self):
 358         self.basedir = "dirnode/Dirnode/test_check"
 359         self.set_up_grid()
 360         c = self.g.clients[0]
 361         d = c.create_dirnode()
 362         d.addCallback(lambda dn: dn.check(Monitor()))
 363         def _done(res):
 364             self.failUnless(res.is_healthy())
 365         d.addCallback(_done)
 366         return d
 367
 368     def _test_deepcheck_create(self):
 369         # create a small tree with a loop, and some non-directories
 370         #  root/
 371         #  root/subdir/
 372         #  root/subdir/file1
 373         #  root/subdir/link -> root
 374         #  root/rodir
 375         c = self.g.clients[0]
 376         d = c.create_dirnode()
 377         def _created_root(rootnode):
 378             self._rootnode = rootnode
 379             return rootnode.create_subdirectory(u"subdir")
 380         d.addCallback(_created_root)
 381         def _created_subdir(subdir):
 382             self._subdir = subdir
 383             d = subdir.add_file(u"file1", upload.Data("data"*100, None))
 384             d.addCallback(lambda res: subdir.set_node(u"link", self._rootnode))
 385             d.addCallback(lambda res: c.create_dirnode())
 386             d.addCallback(lambda dn:
 387                           self._rootnode.set_uri(u"rodir",
 388                                                  dn.get_uri(),
 389                                                  dn.get_readonly_uri()))
 390             return d
 391         d.addCallback(_created_subdir)
 392         def _done(res):
 393             return self._rootnode
 394         d.addCallback(_done)
 395         return d
 396
 397     def test_deepcheck(self):
 398         self.basedir = "dirnode/Dirnode/test_deepcheck"
 399         self.set_up_grid()
 400         d = self._test_deepcheck_create()
 401         d.addCallback(lambda rootnode: rootnode.start_deep_check().when_done())
 402         def _check_results(r):
 403             self.failUnless(IDeepCheckResults.providedBy(r))
 404             c = r.get_counters()
 405             self.failUnlessEqual(c,
 406                                  {"count-objects-checked": 4,
 407                                   "count-objects-healthy": 4,
 408                                   "count-objects-unhealthy": 0,
 409                                   "count-objects-unrecoverable": 0,
 410                                   "count-corrupt-shares": 0,
 411                                   })
 412             self.failIf(r.get_corrupt_shares())
 413             self.failUnlessEqual(len(r.get_all_results()), 4)
 414         d.addCallback(_check_results)
 415         return d
 416
 417     def test_deepcheck_and_repair(self):
 418         self.basedir = "dirnode/Dirnode/test_deepcheck_and_repair"
 419         self.set_up_grid()
 420         d = self._test_deepcheck_create()
 421         d.addCallback(lambda rootnode:
 422                       rootnode.start_deep_check_and_repair().when_done())
 423         def _check_results(r):
 424             self.failUnless(IDeepCheckAndRepairResults.providedBy(r))
 425             c = r.get_counters()
 426             self.failUnlessEqual(c,
 427                                  {"count-objects-checked": 4,
 428                                   "count-objects-healthy-pre-repair": 4,
 429                                   "count-objects-unhealthy-pre-repair": 0,
 430                                   "count-objects-unrecoverable-pre-repair": 0,
 431                                   "count-corrupt-shares-pre-repair": 0,
 432                                   "count-objects-healthy-post-repair": 4,
 433                                   "count-objects-unhealthy-post-repair": 0,
 434                                   "count-objects-unrecoverable-post-repair": 0,
 435                                   "count-corrupt-shares-post-repair": 0,
 436                                   "count-repairs-attempted": 0,
 437                                   "count-repairs-successful": 0,
 438                                   "count-repairs-unsuccessful": 0,
 439                                   })
 440             self.failIf(r.get_corrupt_shares())
 441             self.failIf(r.get_remaining_corrupt_shares())
 442             self.failUnlessEqual(len(r.get_all_results()), 4)
 443         d.addCallback(_check_results)
 444         return d
 445
 446     def _mark_file_bad(self, rootnode):
 447         self.delete_shares_numbered(rootnode.get_uri(), [0])
 448         return rootnode
 449
 450     def test_deepcheck_problems(self):
 451         self.basedir = "dirnode/Dirnode/test_deepcheck_problems"
 452         self.set_up_grid()
 453         d = self._test_deepcheck_create()
 454         d.addCallback(lambda rootnode: self._mark_file_bad(rootnode))
 455         d.addCallback(lambda rootnode: rootnode.start_deep_check().when_done())
 456         def _check_results(r):
 457             c = r.get_counters()
 458             self.failUnlessEqual(c,
 459                                  {"count-objects-checked": 4,
 460                                   "count-objects-healthy": 3,
 461                                   "count-objects-unhealthy": 1,
 462                                   "count-objects-unrecoverable": 0,
 463                                   "count-corrupt-shares": 0,
 464                                   })
 465             #self.failUnlessEqual(len(r.get_problems()), 1) # TODO
 466         d.addCallback(_check_results)
 467         return d
 468
 469     def test_readonly(self):
 470         self.basedir = "dirnode/Dirnode/test_readonly"
 471         self.set_up_grid()
 472         c = self.g.clients[0]
 473         nm = c.nodemaker
 474         filecap = make_chk_file_uri(1234)
 475         filenode = nm.create_from_cap(filecap)
 476         uploadable = upload.Data("some data", convergence="some convergence string")
 477
 478         d = c.create_dirnode()
 479         def _created(rw_dn):
 480             d2 = rw_dn.set_uri(u"child", filecap, filecap)
 481             d2.addCallback(lambda res: rw_dn)
 482             return d2
 483         d.addCallback(_created)
 484
 485         def _ready(rw_dn):
 486             ro_uri = rw_dn.get_readonly_uri()
 487             ro_dn = c.create_node_from_uri(ro_uri)
 488             self.failUnless(ro_dn.is_readonly())
 489             self.failUnless(ro_dn.is_mutable())
 490             self.failIf(ro_dn.is_unknown())
 491             self.failIf(ro_dn.is_allowed_in_immutable_directory())
 492             ro_dn.raise_error()
 493
 494             self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
 495                             ro_dn.set_uri, u"newchild", filecap, filecap)
 496             self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
 497                             ro_dn.set_node, u"newchild", filenode)
 498             self.shouldFail(dirnode.NotWriteableError, "set_nodes ro", None,
 499                             ro_dn.set_nodes, { u"newchild": (filenode, None) })
 500             self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
 501                             ro_dn.add_file, u"newchild", uploadable)
 502             self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
 503                             ro_dn.delete, u"child")
 504             self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
 505                             ro_dn.create_subdirectory, u"newchild")
 506             self.shouldFail(dirnode.NotWriteableError, "set_metadata_for ro", None,
 507                             ro_dn.set_metadata_for, u"child", {})
 508             self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
 509                             ro_dn.move_child_to, u"child", rw_dn)
 510             self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
 511                             rw_dn.move_child_to, u"child", ro_dn)
 512             return ro_dn.list()
 513         d.addCallback(_ready)
 514         def _listed(children):
 515             self.failUnless(u"child" in children)
 516         d.addCallback(_listed)
 517         return d
 518
 519     def failUnlessGreaterThan(self, a, b):
 520         self.failUnless(a > b, "%r should be > %r" % (a, b))
 521
 522     def failUnlessGreaterOrEqualThan(self, a, b):
 523         self.failUnless(a >= b, "%r should be >= %r" % (a, b))
 524
 525     def test_create(self):
 526         self.basedir = "dirnode/Dirnode/test_create"
 527         self.set_up_grid()
 528         c = self.g.clients[0]
 529
 530         self.expected_manifest = []
 531         self.expected_verifycaps = set()
 532         self.expected_storage_indexes = set()
 533
 534         d = c.create_dirnode()
 535         def _then(n):
 536             # /
 537             self.rootnode = n
 538             self.failUnless(n.is_mutable())
 539             u = n.get_uri()
 540             self.failUnless(u)
 541             self.failUnless(u.startswith("URI:DIR2:"), u)
 542             u_ro = n.get_readonly_uri()
 543             self.failUnless(u_ro.startswith("URI:DIR2-RO:"), u_ro)
 544             u_v = n.get_verify_cap().to_string()
 545             self.failUnless(u_v.startswith("URI:DIR2-Verifier:"), u_v)
 546             u_r = n.get_repair_cap().to_string()
 547             self.failUnlessEqual(u_r, u)
 548             self.expected_manifest.append( ((), u) )
 549             self.expected_verifycaps.add(u_v)
 550             si = n.get_storage_index()
 551             self.expected_storage_indexes.add(base32.b2a(si))
 552             expected_si = n._uri.get_storage_index()
 553             self.failUnlessEqual(si, expected_si)
 554
 555             d = n.list()
 556             d.addCallback(lambda res: self.failUnlessEqual(res, {}))
 557             d.addCallback(lambda res: n.has_child(u"missing"))
 558             d.addCallback(lambda res: self.failIf(res))
 559
 560             fake_file_uri = make_mutable_file_uri()
 561             other_file_uri = make_mutable_file_uri()
 562             m = c.nodemaker.create_from_cap(fake_file_uri)
 563             ffu_v = m.get_verify_cap().to_string()
 564             self.expected_manifest.append( ((u"child",) , m.get_uri()) )
 565             self.expected_verifycaps.add(ffu_v)
 566             self.expected_storage_indexes.add(base32.b2a(m.get_storage_index()))
 567             d.addCallback(lambda res: n.set_uri(u"child",
 568                                                 fake_file_uri, fake_file_uri))
 569             d.addCallback(lambda res:
 570                           self.shouldFail(ExistingChildError, "set_uri-no",
 571                                           "child 'child' already exists",
 572                                           n.set_uri, u"child",
 573                                           other_file_uri, other_file_uri,
 574                                           overwrite=False))
 575             # /
 576             # /child = mutable
 577
 578             d.addCallback(lambda res: n.create_subdirectory(u"subdir"))
 579
 580             # /
 581             # /child = mutable
 582             # /subdir = directory
 583             def _created(subdir):
 584                 self.failUnless(isinstance(subdir, dirnode.DirectoryNode))
 585                 self.subdir = subdir
 586                 new_v = subdir.get_verify_cap().to_string()
 587                 assert isinstance(new_v, str)
 588                 self.expected_manifest.append( ((u"subdir",), subdir.get_uri()) )
 589                 self.expected_verifycaps.add(new_v)
 590                 si = subdir.get_storage_index()
 591                 self.expected_storage_indexes.add(base32.b2a(si))
 592             d.addCallback(_created)
 593
 594             d.addCallback(lambda res:
 595                           self.shouldFail(ExistingChildError, "mkdir-no",
 596                                           "child 'subdir' already exists",
 597                                           n.create_subdirectory, u"subdir",
 598                                           overwrite=False))
 599
 600             d.addCallback(lambda res: n.list())
 601             d.addCallback(lambda children:
 602                           self.failUnlessEqual(sorted(children.keys()),
 603                                                sorted([u"child", u"subdir"])))
 604
 605             d.addCallback(lambda res: n.start_deep_stats().when_done())
 606             def _check_deepstats(stats):
 607                 self.failUnless(isinstance(stats, dict))
 608                 expected = {"count-immutable-files": 0,
 609                             "count-mutable-files": 1,
 610                             "count-literal-files": 0,
 611                             "count-files": 1,
 612                             "count-directories": 2,
 613                             "size-immutable-files": 0,
 614                             "size-literal-files": 0,
 615                             #"size-directories": 616, # varies
 616                             #"largest-directory": 616,
 617                             "largest-directory-children": 2,
 618                             "largest-immutable-file": 0,
 619                             }
 620                 for k,v in expected.iteritems():
 621                     self.failUnlessEqual(stats[k], v,
 622                                          "stats[%s] was %s, not %s" %
 623                                          (k, stats[k], v))
 624                 self.failUnless(stats["size-directories"] > 500,
 625                                 stats["size-directories"])
 626                 self.failUnless(stats["largest-directory"] > 500,
 627                                 stats["largest-directory"])
 628                 self.failUnlessEqual(stats["size-files-histogram"], [])
 629             d.addCallback(_check_deepstats)
 630
 631             d.addCallback(lambda res: n.build_manifest().when_done())
 632             def _check_manifest(res):
 633                 manifest = res["manifest"]
 634                 self.failUnlessEqual(sorted(manifest),
 635                                      sorted(self.expected_manifest))
 636                 stats = res["stats"]
 637                 _check_deepstats(stats)
 638                 self.failUnlessEqual(self.expected_verifycaps,
 639                                      res["verifycaps"])
 640                 self.failUnlessEqual(self.expected_storage_indexes,
 641                                      res["storage-index"])
 642             d.addCallback(_check_manifest)
 643
 644             def _add_subsubdir(res):
 645                 return self.subdir.create_subdirectory(u"subsubdir")
 646             d.addCallback(_add_subsubdir)
 647             # /
 648             # /child = mutable
 649             # /subdir = directory
 650             # /subdir/subsubdir = directory
 651             d.addCallback(lambda res: n.get_child_at_path(u"subdir/subsubdir"))
 652             d.addCallback(lambda subsubdir:
 653                           self.failUnless(isinstance(subsubdir,
 654                                                      dirnode.DirectoryNode)))
 655             d.addCallback(lambda res: n.get_child_at_path(u""))
 656             d.addCallback(lambda res: self.failUnlessEqual(res.get_uri(),
 657                                                            n.get_uri()))
 658
 659             d.addCallback(lambda res: n.get_metadata_for(u"child"))
 660             d.addCallback(lambda metadata:
 661                           self.failUnlessEqual(set(metadata.keys()),
 662                                                set(["tahoe", "ctime", "mtime"])))
 663
 664             d.addCallback(lambda res:
 665                           self.shouldFail(NoSuchChildError, "gcamap-no",
 666                                           "nope",
 667                                           n.get_child_and_metadata_at_path,
 668                                           u"subdir/nope"))
 669             d.addCallback(lambda res:
 670                           n.get_child_and_metadata_at_path(u""))
 671             def _check_child_and_metadata1(res):
 672                 child, metadata = res
 673                 self.failUnless(isinstance(child, dirnode.DirectoryNode))
 674                 # edge-metadata needs at least one path segment
 675                 self.failUnlessEqual(sorted(metadata.keys()), [])
 676             d.addCallback(_check_child_and_metadata1)
 677             d.addCallback(lambda res:
 678                           n.get_child_and_metadata_at_path(u"child"))
 679
 680             def _check_child_and_metadata2(res):
 681                 child, metadata = res
 682                 self.failUnlessEqual(child.get_uri(),
 683                                      fake_file_uri)
 684                 self.failUnlessEqual(set(metadata.keys()),
 685                                      set(["tahoe", "ctime", "mtime"]))
 686             d.addCallback(_check_child_and_metadata2)
 687
 688             d.addCallback(lambda res:
 689                           n.get_child_and_metadata_at_path(u"subdir/subsubdir"))
 690             def _check_child_and_metadata3(res):
 691                 child, metadata = res
 692                 self.failUnless(isinstance(child, dirnode.DirectoryNode))
 693                 self.failUnlessEqual(set(metadata.keys()),
 694                                      set(["tahoe", "ctime", "mtime"]))
 695             d.addCallback(_check_child_and_metadata3)
 696
 697             # set_uri + metadata
 698             # it should be possible to add a child without any metadata
 699             d.addCallback(lambda res: n.set_uri(u"c2",
 700                                                 fake_file_uri, fake_file_uri,
 701                                                 {}))
 702             d.addCallback(lambda res: n.get_metadata_for(u"c2"))
 703             d.addCallback(lambda metadata: self.failUnlessEqual(metadata.keys(), ['tahoe']))
 704
 705             # You can't override the link timestamps.
 706             d.addCallback(lambda res: n.set_uri(u"c2",
 707                                                 fake_file_uri, fake_file_uri,
 708                                                 { 'tahoe': {'linkcrtime': "bogus"}}))
 709             d.addCallback(lambda res: n.get_metadata_for(u"c2"))
 710             def _has_good_linkcrtime(metadata):
 711                 self.failUnless(metadata.has_key('tahoe'))
 712                 self.failUnless(metadata['tahoe'].has_key('linkcrtime'))
 713                 self.failIfEqual(metadata['tahoe']['linkcrtime'], 'bogus')
 714             d.addCallback(_has_good_linkcrtime)
 715
 716             # if we don't set any defaults, the child should get timestamps
 717             d.addCallback(lambda res: n.set_uri(u"c3",
 718                                                 fake_file_uri, fake_file_uri))
 719             d.addCallback(lambda res: n.get_metadata_for(u"c3"))
 720             d.addCallback(lambda metadata:
 721                           self.failUnlessEqual(set(metadata.keys()),
 722                                                set(["tahoe", "ctime", "mtime"])))
 723
 724             # or we can add specific metadata at set_uri() time, which
 725             # overrides the timestamps
 726             d.addCallback(lambda res: n.set_uri(u"c4",
 727                                                 fake_file_uri, fake_file_uri,
 728                                                 {"key": "value"}))
 729             d.addCallback(lambda res: n.get_metadata_for(u"c4"))
 730             d.addCallback(lambda metadata:
 731                               self.failUnless((set(metadata.keys()) == set(["key", "tahoe"])) and
 732                                               (metadata['key'] == "value"), metadata))
 733
 734             d.addCallback(lambda res: n.delete(u"c2"))
 735             d.addCallback(lambda res: n.delete(u"c3"))
 736             d.addCallback(lambda res: n.delete(u"c4"))
 737
 738             # set_node + metadata
 739             # it should be possible to add a child without any metadata
 740             d.addCallback(lambda res: n.set_node(u"d2", n, {}))
 741             d.addCallback(lambda res: c.create_dirnode())
 742             d.addCallback(lambda n2:
 743                           self.shouldFail(ExistingChildError, "set_node-no",
 744                                           "child 'd2' already exists",
 745                                           n.set_node, u"d2", n2,
 746                                           overwrite=False))
 747             d.addCallback(lambda res: n.get_metadata_for(u"d2"))
 748             d.addCallback(lambda metadata: self.failUnlessEqual(metadata.keys(), ['tahoe']))
 749
 750             # if we don't set any defaults, the child should get timestamps
 751             d.addCallback(lambda res: n.set_node(u"d3", n))
 752             d.addCallback(lambda res: n.get_metadata_for(u"d3"))
 753             d.addCallback(lambda metadata:
 754                           self.failUnlessEqual(set(metadata.keys()),
 755                                                set(["tahoe", "ctime", "mtime"])))
 756
 757             # or we can add specific metadata at set_node() time, which
 758             # overrides the timestamps
 759             d.addCallback(lambda res: n.set_node(u"d4", n,
 760                                                 {"key": "value"}))
 761             d.addCallback(lambda res: n.get_metadata_for(u"d4"))
 762             d.addCallback(lambda metadata:
 763                           self.failUnless((set(metadata.keys()) == set(["key", "tahoe"])) and
 764                                           (metadata['key'] == "value"), metadata))
 765
 766             d.addCallback(lambda res: n.delete(u"d2"))
 767             d.addCallback(lambda res: n.delete(u"d3"))
 768             d.addCallback(lambda res: n.delete(u"d4"))
 769
 770             # metadata through set_children()
 771             d.addCallback(lambda res:
 772                           n.set_children({
 773                               u"e1": (fake_file_uri, fake_file_uri),
 774                               u"e2": (fake_file_uri, fake_file_uri, {}),
 775                               u"e3": (fake_file_uri, fake_file_uri,
 776                                       {"key": "value"}),
 777                               }))
 778             d.addCallback(lambda n2: self.failUnlessIdentical(n2, n))
 779             d.addCallback(lambda res:
 780                           self.shouldFail(ExistingChildError, "set_children-no",
 781                                           "child 'e1' already exists",
 782                                           n.set_children,
 783                                           { u"e1": (other_file_uri,
 784                                                     other_file_uri),
 785                                             u"new": (other_file_uri,
 786                                                      other_file_uri),
 787                                             },
 788                                           overwrite=False))
 789             # and 'new' should not have been created
 790             d.addCallback(lambda res: n.list())
 791             d.addCallback(lambda children: self.failIf(u"new" in children))
 792             d.addCallback(lambda res: n.get_metadata_for(u"e1"))
 793             d.addCallback(lambda metadata:
 794                           self.failUnlessEqual(set(metadata.keys()),
 795                                                set(["tahoe", "ctime", "mtime"])))
 796             d.addCallback(lambda res: n.get_metadata_for(u"e2"))
 797             d.addCallback(lambda metadata:
 798                           self.failUnlessEqual(set(metadata.keys()), set(['tahoe'])))
 799             d.addCallback(lambda res: n.get_metadata_for(u"e3"))
 800             d.addCallback(lambda metadata:
 801                               self.failUnless((set(metadata.keys()) == set(["key", "tahoe"]))
 802                                               and (metadata['key'] == "value"), metadata))
 803
 804             d.addCallback(lambda res: n.delete(u"e1"))
 805             d.addCallback(lambda res: n.delete(u"e2"))
 806             d.addCallback(lambda res: n.delete(u"e3"))
 807
 808             # metadata through set_nodes()
 809             d.addCallback(lambda res:
 810                           n.set_nodes({ u"f1": (n, None),
 811                                         u"f2": (n, {}),
 812                                         u"f3": (n, {"key": "value"}),
 813                                         }))
 814             d.addCallback(lambda n2: self.failUnlessIdentical(n2, n))
 815             d.addCallback(lambda res:
 816                           self.shouldFail(ExistingChildError, "set_nodes-no",
 817                                           "child 'f1' already exists",
 818                                           n.set_nodes, { u"f1": (n, None),
 819                                                          u"new": (n, None), },
 820                                           overwrite=False))
 821             # and 'new' should not have been created
 822             d.addCallback(lambda res: n.list())
 823             d.addCallback(lambda children: self.failIf(u"new" in children))
 824             d.addCallback(lambda res: n.get_metadata_for(u"f1"))
 825             d.addCallback(lambda metadata:
 826                           self.failUnlessEqual(set(metadata.keys()),
 827                                                set(["tahoe", "ctime", "mtime"])))
 828             d.addCallback(lambda res: n.get_metadata_for(u"f2"))
 829             d.addCallback(
 830                 lambda metadata: self.failUnlessEqual(set(metadata.keys()), set(['tahoe'])))
 831             d.addCallback(lambda res: n.get_metadata_for(u"f3"))
 832             d.addCallback(lambda metadata:
 833                               self.failUnless((set(metadata.keys()) == set(["key", "tahoe"])) and
 834                                               (metadata['key'] == "value"), metadata))
 835
 836             d.addCallback(lambda res: n.delete(u"f1"))
 837             d.addCallback(lambda res: n.delete(u"f2"))
 838             d.addCallback(lambda res: n.delete(u"f3"))
 839
 840
 841             d.addCallback(lambda res:
 842                           n.set_metadata_for(u"child",
 843                                              {"tags": ["web2.0-compatible"]}))
 844             d.addCallback(lambda n1: n1.get_metadata_for(u"child"))
 845             d.addCallback(lambda metadata:
 846                           self.failUnlessEqual(metadata,
 847                                                {"tags": ["web2.0-compatible"]}))
 848
 849             def _start(res):
 850                 self._start_timestamp = time.time()
 851             d.addCallback(_start)
 852             # simplejson-1.7.1 (as shipped on Ubuntu 'gutsy') rounds all
 853             # floats to hundredeths (it uses str(num) instead of repr(num)).
 854             # simplejson-1.7.3 does not have this bug. To prevent this bug
 855             # from causing the test to fail, stall for more than a few
 856             # hundrededths of a second.
 857             d.addCallback(self.stall, 0.1)
 858             d.addCallback(lambda res: n.add_file(u"timestamps",
 859                                                  upload.Data("stamp me", convergence="some convergence string")))
 860             d.addCallback(self.stall, 0.1)
 861             def _stop(res):
 862                 self._stop_timestamp = time.time()
 863             d.addCallback(_stop)
 864
 865             d.addCallback(lambda res: n.get_metadata_for(u"timestamps"))
 866             def _check_timestamp1(metadata):
 867                 self.failUnless("ctime" in metadata)
 868                 self.failUnless("mtime" in metadata)
 869                 self.failUnlessGreaterOrEqualThan(metadata["ctime"],
 870                                                   self._start_timestamp)
 871                 self.failUnlessGreaterOrEqualThan(self._stop_timestamp,
 872                                                   metadata["ctime"])
 873                 self.failUnlessGreaterOrEqualThan(metadata["mtime"],
 874                                                   self._start_timestamp)
 875                 self.failUnlessGreaterOrEqualThan(self._stop_timestamp,
 876                                                   metadata["mtime"])
 877                 # Our current timestamp rules say that replacing an existing
 878                 # child should preserve the 'ctime' but update the mtime
 879                 self._old_ctime = metadata["ctime"]
 880                 self._old_mtime = metadata["mtime"]
 881             d.addCallback(_check_timestamp1)
 882             d.addCallback(self.stall, 2.0) # accomodate low-res timestamps
 883             d.addCallback(lambda res: n.set_node(u"timestamps", n))
 884             d.addCallback(lambda res: n.get_metadata_for(u"timestamps"))
 885             def _check_timestamp2(metadata):
 886                 self.failUnlessEqual(metadata["ctime"], self._old_ctime,
 887                                      "%s != %s" % (metadata["ctime"],
 888                                                    self._old_ctime))
 889                 self.failUnlessGreaterThan(metadata["mtime"], self._old_mtime)
 890                 return n.delete(u"timestamps")
 891             d.addCallback(_check_timestamp2)
 892
 893             # also make sure we can add/update timestamps on a
 894             # previously-existing child that didn't have any, since there are
 895             # a lot of 0.7.0-generated edges around out there
 896             d.addCallback(lambda res: n.set_node(u"no_timestamps", n, {}))
 897             d.addCallback(lambda res: n.set_node(u"no_timestamps", n))
 898             d.addCallback(lambda res: n.get_metadata_for(u"no_timestamps"))
 899             d.addCallback(lambda metadata:
 900                           self.failUnlessEqual(set(metadata.keys()),
 901                                                set(["tahoe", "ctime", "mtime"])))
 902             d.addCallback(lambda res: n.delete(u"no_timestamps"))
 903
 904             d.addCallback(lambda res: n.delete(u"subdir"))
 905             d.addCallback(lambda old_child:
 906                           self.failUnlessEqual(old_child.get_uri(),
 907                                                self.subdir.get_uri()))
 908
 909             d.addCallback(lambda res: n.list())
 910             d.addCallback(lambda children:
 911                           self.failUnlessEqual(sorted(children.keys()),
 912                                                sorted([u"child"])))
 913
 914             uploadable1 = upload.Data("some data", convergence="converge")
 915             d.addCallback(lambda res: n.add_file(u"newfile", uploadable1))
 916             d.addCallback(lambda newnode:
 917                           self.failUnless(IImmutableFileNode.providedBy(newnode)))
 918             uploadable2 = upload.Data("some data", convergence="stuff")
 919             d.addCallback(lambda res:
 920                           self.shouldFail(ExistingChildError, "add_file-no",
 921                                           "child 'newfile' already exists",
 922                                           n.add_file, u"newfile",
 923                                           uploadable2,
 924                                           overwrite=False))
 925             d.addCallback(lambda res: n.list())
 926             d.addCallback(lambda children:
 927                           self.failUnlessEqual(sorted(children.keys()),
 928                                                sorted([u"child", u"newfile"])))
 929             d.addCallback(lambda res: n.get_metadata_for(u"newfile"))
 930             d.addCallback(lambda metadata:
 931                           self.failUnlessEqual(set(metadata.keys()),
 932                                                set(["tahoe", "ctime", "mtime"])))
 933
 934             uploadable3 = upload.Data("some data", convergence="converge")
 935             d.addCallback(lambda res: n.add_file(u"newfile-metadata",
 936                                                  uploadable3,
 937                                                  {"key": "value"}))
 938             d.addCallback(lambda newnode:
 939                           self.failUnless(IImmutableFileNode.providedBy(newnode)))
 940             d.addCallback(lambda res: n.get_metadata_for(u"newfile-metadata"))
 941             d.addCallback(lambda metadata:
 942                               self.failUnless((set(metadata.keys()) == set(["key", "tahoe"])) and
 943                                               (metadata['key'] == "value"), metadata))
 944             d.addCallback(lambda res: n.delete(u"newfile-metadata"))
 945
 946             d.addCallback(lambda res: n.create_subdirectory(u"subdir2"))
 947             def _created2(subdir2):
 948                 self.subdir2 = subdir2
 949                 # put something in the way, to make sure it gets overwritten
 950                 return subdir2.add_file(u"child", upload.Data("overwrite me",
 951                                                               "converge"))
 952             d.addCallback(_created2)
 953
 954             d.addCallback(lambda res:
 955                           n.move_child_to(u"child", self.subdir2))
 956             d.addCallback(lambda res: n.list())
 957             d.addCallback(lambda children:
 958                           self.failUnlessEqual(sorted(children.keys()),
 959                                                sorted([u"newfile", u"subdir2"])))
 960             d.addCallback(lambda res: self.subdir2.list())
 961             d.addCallback(lambda children:
 962                           self.failUnlessEqual(sorted(children.keys()),
 963                                                sorted([u"child"])))
 964             d.addCallback(lambda res: self.subdir2.get(u"child"))
 965             d.addCallback(lambda child:
 966                           self.failUnlessEqual(child.get_uri(),
 967                                                fake_file_uri))
 968
 969             # move it back, using new_child_name=
 970             d.addCallback(lambda res:
 971                           self.subdir2.move_child_to(u"child", n, u"newchild"))
 972             d.addCallback(lambda res: n.list())
 973             d.addCallback(lambda children:
 974                           self.failUnlessEqual(sorted(children.keys()),
 975                                                sorted([u"newchild", u"newfile",
 976                                                        u"subdir2"])))
 977             d.addCallback(lambda res: self.subdir2.list())
 978             d.addCallback(lambda children:
 979                           self.failUnlessEqual(sorted(children.keys()), []))
 980
 981             # now make sure that we honor overwrite=False
 982             d.addCallback(lambda res:
 983                           self.subdir2.set_uri(u"newchild",
 984                                                other_file_uri, other_file_uri))
 985
 986             d.addCallback(lambda res:
 987                           self.shouldFail(ExistingChildError, "move_child_to-no",
 988                                           "child 'newchild' already exists",
 989                                           n.move_child_to, u"newchild",
 990                                           self.subdir2,
 991                                           overwrite=False))
 992             d.addCallback(lambda res: self.subdir2.get(u"newchild"))
 993             d.addCallback(lambda child:
 994                           self.failUnlessEqual(child.get_uri(),
 995                                                other_file_uri))
 996
 997             return d
 998
 999         d.addCallback(_then)
1000
1001         d.addErrback(self.explain_error)
1002         return d
1003
1004     def test_create_subdirectory(self):
1005         self.basedir = "dirnode/Dirnode/test_create_subdirectory"
1006         self.set_up_grid()
1007         c = self.g.clients[0]
1008         nm = c.nodemaker
1009
1010         d = c.create_dirnode()
1011         def _then(n):
1012             # /
1013             self.rootnode = n
1014             fake_file_uri = make_mutable_file_uri()
1015             other_file_uri = make_mutable_file_uri()
1016             md = {"metakey": "metavalue"}
1017             kids = {u"kid1": (nm.create_from_cap(fake_file_uri), {}),
1018                     u"kid2": (nm.create_from_cap(other_file_uri), md),
1019                     }
1020             d = n.create_subdirectory(u"subdir", kids)
1021             def _check(sub):
1022                 d = n.get_child_at_path(u"subdir")
1023                 d.addCallback(lambda sub2: self.failUnlessEqual(sub2.get_uri(),
1024                                                                 sub.get_uri()))
1025                 d.addCallback(lambda ign: sub.list())
1026                 return d
1027             d.addCallback(_check)
1028             def _check_kids(kids2):
1029                 self.failUnlessEqual(sorted(kids.keys()), sorted(kids2.keys()))
1030                 self.failUnlessEqual(kids2[u"kid2"][1]["metakey"], "metavalue")
1031             d.addCallback(_check_kids)
1032             return d
1033         d.addCallback(_then)
1034         return d
1035
1036 class MinimalFakeMutableFile:
1037     def get_writekey(self):
1038         return "writekey"
1039
1040 class Packing(unittest.TestCase):
1041     # This is a base32-encoded representation of the directory tree
1042     # root/file1
1043     # root/file2
1044     # root/file3
1045     # as represented after being fed to _pack_contents.
1046     # We have it here so we can decode it, feed it to
1047     # _unpack_contents, and verify that _unpack_contents
1048     # works correctly.
1049
1050     known_tree = "GM4TOORVHJTGS3DFGEWDSNJ2KVJESOSDJBFTU33MPB2GS3LZNVYG6N3GGI3WU5TIORTXC3DOMJ2G4NB2MVWXUZDONBVTE5LNGRZWK2LYN55GY23XGNYXQMTOMZUWU5TENN4DG23ZG5UTO2L2NQ2DO6LFMRWDMZJWGRQTUMZ2GEYDUMJQFQYTIMZ22XZKZORX5XS7CAQCSK3URR6QOHISHRCMGER5LRFSZRNAS5ZSALCS6TWFQAE754IVOIKJVK73WZPP3VUUEDTX3WHTBBZ5YX3CEKHCPG3ZWQLYA4QM6LDRCF7TJQYWLIZHKGN5ROA3AUZPXESBNLQQ6JTC2DBJU2D47IZJTLR3PKZ4RVF57XLPWY7FX7SZV3T6IJ3ORFW37FXUPGOE3ROPFNUX5DCGMAQJ3PGGULBRGM3TU6ZCMN2GS3LFEI5CAMJSGQ3DMNRTHA4TOLRUGI3TKNRWGEWCAITUMFUG6ZJCHIQHWITMNFXGW3LPORUW2ZJCHIQDCMRUGY3DMMZYHE3S4NBSG42TMNRRFQQCE3DJNZVWG4TUNFWWKIR2EAYTENBWGY3DGOBZG4XDIMRXGU3DMML5FQQCE3LUNFWWKIR2EAYTENBWGY3DGOBZG4XDIMRXGU3DMML5FQWDGOJRHI2TUZTJNRSTELBZGQ5FKUSJHJBUQSZ2MFYGKZ3SOBSWQ43IO52WO23CNAZWU3DUGVSWSNTIOE5DK33POVTW4ZLNMNWDK6DHPA2GS2THNF2W25DEN5VGY2LQNFRGG5DKNNRHO5TZPFTWI6LNMRYGQ2LCGJTHM4J2GM5DCMB2GQWDCNBSHKVVQBGRYMACKJ27CVQ6O6B4QPR72RFVTGOZUI76XUSWAX73JRV5PYRHMIFYZIA25MXDPGUGML6M2NMRSG4YD4W4K37ZDYSXHMJ3IUVT4F64YTQQVBJFFFOUC7J7LAB2VFCL5UKKGMR2D3F4EPOYC7UYWQZNR5KXHBSNXLCNBX2SNF22DCXJIHSMEKWEWOG5XCJEVVZ7UW5IB6I64XXQSJ34B5CAYZGZIIMR6LBRGMZTU6ZCMN2GS3LFEI5CAMJSGQ3DMNRTHA4TOLRUGMYDEMJYFQQCE5DBNBXWKIR2EB5SE3DJNZVW233UNFWWKIR2EAYTENBWGY3DGOBZG4XDIMZQGIYTQLBAEJWGS3TLMNZHI2LNMURDUIBRGI2DMNRWGM4DSNZOGQZTAMRRHB6SYIBCNV2GS3LFEI5CAMJSGQ3DMNRTHA4TOLRUGMYDEMJYPUWCYMZZGU5DKOTGNFWGKMZMHE2DUVKSJE5EGSCLHJRW25DDPBYTO2DXPB3GM6DBNYZTI6LJMV3DM2LWNB4TU4LWMNSWW3LKORXWK5DEMN3TI23NNE3WEM3SORRGY5THPA3TKNBUMNZG453BOF2GSZLXMVWWI3DJOFZW623RHIZTUMJQHI2SYMJUGI5BOSHWDPG3WKPAVXCF3XMKA7QVIWPRMWJHDTQHD27AHDCPJWDQENQ5H5ZZILTXQNIXXCIW4LKQABU2GCFRG5FHQN7CHD7HF4EKNRZFIV2ZYQIBM7IQU7F4RGB3XCX3FREPBKQ7UCICHVWPCYFGA6OLH3J45LXQ6GWWICJ3PGWJNLZ7PCRNLAPNYUGU6BENS7OXMBEOOFRIZV3PF2FFWZ5WHDPKXERYP7GNHKRMGEZTOOT3EJRXI2LNMURDUIBRGI2DMNRWGM4DSNZOGQZTGNRSGY4SYIBCORQWQ33FEI5CA6ZCNRUW423NN52GS3LFEI5CAMJSGQ3DMNRTHA4TOLRUGMZTMMRWHEWCAITMNFXGWY3SORUW2ZJCHIQDCMRUGY3DMMZYHE3S4NBTGM3DENRZPUWCAITNORUW2ZJCHIQDCMRUGY3DMMZYHE3S4NBTGM3DENRZPUWCY==="
1051
1052     def test_unpack_and_pack_behavior(self):
1053         known_tree = b32decode(self.known_tree)
1054         nodemaker = NodeMaker(None, None, None,
1055                               None, None, None,
1056                               {"k": 3, "n": 10}, None)
1057         write_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
1058         filenode = nodemaker.create_from_cap(write_uri)
1059         node = dirnode.DirectoryNode(filenode, nodemaker, None)
1060         children = node._unpack_contents(known_tree)
1061         self._check_children(children)
1062
1063         packed_children = node._pack_contents(children)
1064         children = node._unpack_contents(packed_children)
1065         self._check_children(children)
1066
1067     def _check_children(self, children):
1068         # Are all the expected child nodes there?
1069         self.failUnless(children.has_key(u'file1'))
1070         self.failUnless(children.has_key(u'file2'))
1071         self.failUnless(children.has_key(u'file3'))
1072
1073         # Are the metadata for child 3 right?
1074         file3_rocap = "URI:CHK:cmtcxq7hwxvfxan34yiev6ivhy:qvcekmjtoetdcw4kmi7b3rtblvgx7544crnwaqtiewemdliqsokq:3:10:5"
1075         file3_rwcap = "URI:CHK:cmtcxq7hwxvfxan34yiev6ivhy:qvcekmjtoetdcw4kmi7b3rtblvgx7544crnwaqtiewemdliqsokq:3:10:5"
1076         file3_metadata = {'ctime': 1246663897.4336269, 'tahoe': {'linkmotime': 1246663897.4336269, 'linkcrtime': 1246663897.4336269}, 'mtime': 1246663897.4336269}
1077         self.failUnlessEqual(file3_metadata, children[u'file3'][1])
1078         self.failUnlessEqual(file3_rocap,
1079                              children[u'file3'][0].get_readonly_uri())
1080         self.failUnlessEqual(file3_rwcap,
1081                              children[u'file3'][0].get_uri())
1082
1083         # Are the metadata for child 2 right?
1084         file2_rocap = "URI:CHK:apegrpehshwugkbh3jlt5ei6hq:5oougnemcl5xgx4ijgiumtdojlipibctjkbwvyygdymdphib2fvq:3:10:4"
1085         file2_rwcap = "URI:CHK:apegrpehshwugkbh3jlt5ei6hq:5oougnemcl5xgx4ijgiumtdojlipibctjkbwvyygdymdphib2fvq:3:10:4"
1086         file2_metadata = {'ctime': 1246663897.430218, 'tahoe': {'linkmotime': 1246663897.430218, 'linkcrtime': 1246663897.430218}, 'mtime': 1246663897.430218}
1087         self.failUnlessEqual(file2_metadata, children[u'file2'][1])
1088         self.failUnlessEqual(file2_rocap,
1089                              children[u'file2'][0].get_readonly_uri())
1090         self.failUnlessEqual(file2_rwcap,
1091                              children[u'file2'][0].get_uri())
1092
1093         # Are the metadata for child 1 right?
1094         file1_rocap = "URI:CHK:olxtimympo7f27jvhtgqlnbtn4:emzdnhk2um4seixozlkw3qx2nfijvdkx3ky7i7izl47yedl6e64a:3:10:10"
1095         file1_rwcap = "URI:CHK:olxtimympo7f27jvhtgqlnbtn4:emzdnhk2um4seixozlkw3qx2nfijvdkx3ky7i7izl47yedl6e64a:3:10:10"
1096         file1_metadata = {'ctime': 1246663897.4275661, 'tahoe': {'linkmotime': 1246663897.4275661, 'linkcrtime': 1246663897.4275661}, 'mtime': 1246663897.4275661}
1097         self.failUnlessEqual(file1_metadata, children[u'file1'][1])
1098         self.failUnlessEqual(file1_rocap,
1099                              children[u'file1'][0].get_readonly_uri())
1100         self.failUnlessEqual(file1_rwcap,
1101                              children[u'file1'][0].get_uri())
1102
1103     def _make_kids(self, nm, which):
1104         caps = {"imm": "URI:CHK:n7r3m6wmomelk4sep3kw5cvduq:os7ijw5c3maek7pg65e5254k2fzjflavtpejjyhshpsxuqzhcwwq:3:20:14861",
1105                 "lit": "URI:LIT:n5xgk", # LIT for "one"
1106                 "write": "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq",
1107                 "read": "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q",
1108                 "dirwrite": "URI:DIR2:n6x24zd3seu725yluj75q5boaa:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq",
1109                 "dirread":  "URI:DIR2-RO:b7sr5qsifnicca7cbk3rhrhbvq:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq",
1110                 }
1111         kids = {}
1112         for name in which:
1113             kids[unicode(name)] = (nm.create_from_cap(caps[name]), {})
1114         return kids
1115
1116     def test_deep_immutable(self):
1117         nm = NodeMaker(None, None, None, None, None, None, {"k": 3, "n": 10},
1118                        None)
1119         fn = MinimalFakeMutableFile()
1120
1121         kids = self._make_kids(nm, ["imm", "lit", "write", "read",
1122                                     "dirwrite", "dirread"])
1123         packed = dirnode.pack_children(fn, kids, deep_immutable=False)
1124         self.failUnlessIn("lit", packed)
1125
1126         kids = self._make_kids(nm, ["imm", "lit"])
1127         packed = dirnode.pack_children(fn, kids, deep_immutable=True)
1128         self.failUnlessIn("lit", packed)
1129
1130         kids = self._make_kids(nm, ["imm", "lit", "write"])
1131         self.failUnlessRaises(dirnode.MustBeDeepImmutableError,
1132                               dirnode.pack_children,
1133                               fn, kids, deep_immutable=True)
1134
1135         # read-only is not enough: all children must be immutable
1136         kids = self._make_kids(nm, ["imm", "lit", "read"])
1137         self.failUnlessRaises(dirnode.MustBeDeepImmutableError,
1138                               dirnode.pack_children,
1139                               fn, kids, deep_immutable=True)
1140
1141         kids = self._make_kids(nm, ["imm", "lit", "dirwrite"])
1142         self.failUnlessRaises(dirnode.MustBeDeepImmutableError,
1143                               dirnode.pack_children,
1144                               fn, kids, deep_immutable=True)
1145
1146         kids = self._make_kids(nm, ["imm", "lit", "dirread"])
1147         self.failUnlessRaises(dirnode.MustBeDeepImmutableError,
1148                               dirnode.pack_children,
1149                               fn, kids, deep_immutable=True)
1150
1151 class FakeMutableFile:
1152     implements(IMutableFileNode)
1153     counter = 0
1154     def __init__(self, initial_contents=""):
1155         self.data = self._get_initial_contents(initial_contents)
1156         counter = FakeMutableFile.counter
1157         FakeMutableFile.counter += 1
1158         writekey = hashutil.ssk_writekey_hash(str(counter))
1159         fingerprint = hashutil.ssk_pubkey_fingerprint_hash(str(counter))
1160         self.uri = uri.WriteableSSKFileURI(writekey, fingerprint)
1161
1162     def _get_initial_contents(self, contents):
1163         if isinstance(contents, str):
1164             return contents
1165         if contents is None:
1166             return ""
1167         assert callable(contents), "%s should be callable, not %s" % \
1168                (contents, type(contents))
1169         return contents(self)
1170
1171     def get_cap(self):
1172         return self.uri
1173
1174     def get_uri(self):
1175         return self.uri.to_string()
1176
1177     def get_write_uri(self):
1178         return self.uri.to_string()
1179
1180     def download_best_version(self):
1181         return defer.succeed(self.data)
1182
1183     def get_writekey(self):
1184         return "writekey"
1185
1186     def is_readonly(self):
1187         return False
1188
1189     def is_mutable(self):
1190         return True
1191
1192     def is_unknown(self):
1193         return False
1194
1195     def is_allowed_in_immutable_directory(self):
1196         return False
1197
1198     def raise_error(self):
1199         pass
1200
1201     def modify(self, modifier):
1202         self.data = modifier(self.data, None, True)
1203         return defer.succeed(None)
1204
1205 class FakeNodeMaker(NodeMaker):
1206     def create_mutable_file(self, contents="", keysize=None):
1207         return defer.succeed(FakeMutableFile(contents))
1208
1209 class FakeClient2(Client):
1210     def __init__(self):
1211         self.nodemaker = FakeNodeMaker(None, None, None,
1212                                        None, None, None,
1213                                        {"k":3,"n":10}, None)
1214     def create_node_from_uri(self, rwcap, rocap):
1215         return self.nodemaker.create_from_cap(rwcap, rocap)
1216
1217 class Dirnode2(unittest.TestCase, testutil.ShouldFailMixin):
1218     def setUp(self):
1219         client = FakeClient2()
1220         self.nodemaker = client.nodemaker
1221
1222     def test_from_future(self):
1223         # Create a mutable directory that contains unknown URI types, and make sure
1224         # we tolerate them properly.
1225         d = self.nodemaker.create_new_mutable_directory()
1226         future_write_uri = "x-tahoe-crazy://I_am_from_the_future."
1227         future_read_uri = "x-tahoe-crazy-readonly://I_am_from_the_future."
1228         future_imm_uri = "x-tahoe-crazy-immutable://I_am_from_the_future."
1229         future_node = UnknownNode(future_write_uri, future_read_uri)
1230         def _then(n):
1231             self._node = n
1232             return n.set_node(u"future", future_node)
1233         d.addCallback(_then)
1234
1235         # We should be prohibited from adding an unknown URI to a directory
1236         # just in the rw_uri slot, since we don't know how to diminish the cap
1237         # to a readcap (for the ro_uri slot).
1238         d.addCallback(lambda ign:
1239              self.shouldFail(MustNotBeUnknownRWError,
1240                              "copy unknown",
1241                              "cannot attach unknown rw cap as child",
1242                              self._node.set_uri, u"add",
1243                              future_write_uri, None))
1244
1245         # However, we should be able to add both rw_uri and ro_uri as a pair of
1246         # unknown URIs.
1247         d.addCallback(lambda ign: self._node.set_uri(u"add-pair",
1248                                                      future_write_uri, future_read_uri))
1249
1250         # and to add an URI prefixed with "ro." or "imm." when it is given in a
1251         # write slot (or URL parameter).
1252         d.addCallback(lambda ign: self._node.set_uri(u"add-ro",
1253                                                      "ro." + future_read_uri, None))
1254         d.addCallback(lambda ign: self._node.set_uri(u"add-imm",
1255                                                      "imm." + future_imm_uri, None))
1256
1257         d.addCallback(lambda ign: self._node.list())
1258         def _check(children):
1259             self.failUnlessEqual(len(children), 4)
1260             (fn, metadata) = children[u"future"]
1261             self.failUnless(isinstance(fn, UnknownNode), fn)
1262             self.failUnlessEqual(fn.get_uri(), future_write_uri)
1263             self.failUnlessEqual(fn.get_write_uri(), future_write_uri)
1264             self.failUnlessEqual(fn.get_readonly_uri(), "ro." + future_read_uri)
1265
1266             (fn2, metadata2) = children[u"add-pair"]
1267             self.failUnless(isinstance(fn2, UnknownNode), fn2)
1268             self.failUnlessEqual(fn2.get_uri(), future_write_uri)
1269             self.failUnlessEqual(fn2.get_write_uri(), future_write_uri)
1270             self.failUnlessEqual(fn2.get_readonly_uri(), "ro." + future_read_uri)
1271
1272             (fn3, metadata3) = children[u"add-ro"]
1273             self.failUnless(isinstance(fn3, UnknownNode), fn3)
1274             self.failUnlessEqual(fn3.get_uri(), "ro." + future_read_uri)
1275             self.failUnlessEqual(fn3.get_write_uri(), None)
1276             self.failUnlessEqual(fn3.get_readonly_uri(), "ro." + future_read_uri)
1277
1278             (fn4, metadata4) = children[u"add-imm"]
1279             self.failUnless(isinstance(fn4, UnknownNode), fn4)
1280             self.failUnlessEqual(fn4.get_uri(), "imm." + future_imm_uri)
1281             self.failUnlessEqual(fn4.get_write_uri(), None)
1282             self.failUnlessEqual(fn4.get_readonly_uri(), "imm." + future_imm_uri)
1283
1284             # We should also be allowed to copy the "future" UnknownNode, because
1285             # it contains all the information that was in the original directory
1286             # (readcap and writecap), so we're preserving everything.
1287             return self._node.set_node(u"copy", fn)
1288         d.addCallback(_check)
1289
1290         d.addCallback(lambda ign: self._node.list())
1291         def _check2(children):
1292             self.failUnlessEqual(len(children), 5)
1293             (fn, metadata) = children[u"copy"]
1294             self.failUnless(isinstance(fn, UnknownNode), fn)
1295             self.failUnlessEqual(fn.get_uri(), future_write_uri)
1296             self.failUnlessEqual(fn.get_write_uri(), future_write_uri)
1297             self.failUnlessEqual(fn.get_readonly_uri(), "ro." + future_read_uri)
1298         d.addCallback(_check2)
1299         return d
1300
1301     def test_unknown_strip_prefix_for_ro(self):
1302         self.failUnlessEqual(strip_prefix_for_ro("foo",     False), "foo")
1303         self.failUnlessEqual(strip_prefix_for_ro("ro.foo",  False), "foo")
1304         self.failUnlessEqual(strip_prefix_for_ro("imm.foo", False), "imm.foo")
1305         self.failUnlessEqual(strip_prefix_for_ro("foo",     True),  "foo")
1306         self.failUnlessEqual(strip_prefix_for_ro("ro.foo",  True),  "foo")
1307         self.failUnlessEqual(strip_prefix_for_ro("imm.foo", True),  "foo")
1308
1309     def test_unknownnode(self):
1310         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
1311         mut_read_uri = "URI:SSK-RO:jf6wkflosyvntwxqcdo7a54jvm:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
1312         lit_uri = "URI:LIT:n5xgk"
1313
1314         # This does not attempt to be exhaustive.
1315         no_no        = [# Opaque node, but not an error.
1316                         ( 0, UnknownNode(None, None)),
1317                         ( 1, UnknownNode(None, None, deep_immutable=True)),
1318                        ]
1319         unknown_rw   = [# These are errors because we're only given a rw_uri, and we can't
1320                         # diminish it.
1321                         ( 2, UnknownNode("foo", None)),
1322                         ( 3, UnknownNode("foo", None, deep_immutable=True)),
1323                         ( 4, UnknownNode("ro.foo", None, deep_immutable=True)),
1324                         ( 5, UnknownNode("ro." + mut_read_uri, None, deep_immutable=True)),
1325                         ( 6, UnknownNode("URI:SSK-RO:foo", None, deep_immutable=True)),
1326                         ( 7, UnknownNode("URI:SSK:foo", None)),
1327                        ]
1328         must_be_ro   = [# These are errors because a readonly constraint is not met.
1329                         ( 8, UnknownNode("ro." + mut_write_uri, None)),
1330                         ( 9, UnknownNode(None, "ro." + mut_write_uri)),
1331                        ]
1332         must_be_imm  = [# These are errors because an immutable constraint is not met.
1333                         (10, UnknownNode(None, "ro.URI:SSK-RO:foo", deep_immutable=True)),
1334                         (11, UnknownNode(None, "imm.URI:SSK:foo")),
1335                         (12, UnknownNode(None, "imm.URI:SSK-RO:foo")),
1336                         (13, UnknownNode("bar", "ro.foo", deep_immutable=True)),
1337                         (14, UnknownNode("bar", "imm.foo", deep_immutable=True)),
1338                         (15, UnknownNode("bar", "imm." + lit_uri, deep_immutable=True)),
1339                         (16, UnknownNode("imm." + mut_write_uri, None)),
1340                         (17, UnknownNode("imm." + mut_read_uri, None)),
1341                         (18, UnknownNode("bar", "imm.foo")),
1342                        ]
1343         bad_uri      = [# These are errors because the URI is bad once we've stripped the prefix.
1344                         (19, UnknownNode("ro.URI:SSK-RO:foo", None)),
1345                         (20, UnknownNode("imm.URI:CHK:foo", None, deep_immutable=True)),
1346                         (21, UnknownNode(None, "URI:CHK:foo")),
1347                         (22, UnknownNode(None, "URI:CHK:foo", deep_immutable=True)),
1348                        ]
1349         ro_prefixed  = [# These are valid, and the readcap should end up with a ro. prefix.
1350                         (23, UnknownNode(None, "foo")),
1351                         (24, UnknownNode(None, "ro.foo")),
1352                         (25, UnknownNode(None, "ro." + lit_uri)),
1353                         (26, UnknownNode("bar", "foo")),
1354                         (27, UnknownNode("bar", "ro.foo")),
1355                         (28, UnknownNode("bar", "ro." + lit_uri)),
1356                         (29, UnknownNode("ro.foo", None)),
1357                         (30, UnknownNode("ro." + lit_uri, None)),
1358                        ]
1359         imm_prefixed = [# These are valid, and the readcap should end up with an imm. prefix.
1360                         (31, UnknownNode(None, "foo", deep_immutable=True)),
1361                         (32, UnknownNode(None, "ro.foo", deep_immutable=True)),
1362                         (33, UnknownNode(None, "imm.foo")),
1363                         (34, UnknownNode(None, "imm.foo", deep_immutable=True)),
1364                         (35, UnknownNode("imm." + lit_uri, None)),
1365                         (36, UnknownNode("imm." + lit_uri, None, deep_immutable=True)),
1366                         (37, UnknownNode(None, "imm." + lit_uri)),
1367                         (38, UnknownNode(None, "imm." + lit_uri, deep_immutable=True)),
1368                        ]
1369         error = unknown_rw + must_be_ro + must_be_imm + bad_uri
1370         ok = ro_prefixed + imm_prefixed
1371
1372         for (i, n) in no_no + error + ok:
1373             self.failUnless(n.is_unknown(), i)
1374
1375         for (i, n) in no_no + error:
1376             self.failUnless(n.get_uri() is None, i)
1377             self.failUnless(n.get_write_uri() is None, i)
1378             self.failUnless(n.get_readonly_uri() is None, i)
1379
1380         for (i, n) in no_no + ok:
1381             n.raise_error()
1382
1383         for (i, n) in unknown_rw:
1384             self.failUnlessRaises(MustNotBeUnknownRWError, lambda: n.raise_error())
1385
1386         for (i, n) in must_be_ro:
1387             self.failUnlessRaises(MustBeReadonlyError, lambda: n.raise_error())
1388
1389         for (i, n) in must_be_imm:
1390             self.failUnlessRaises(MustBeDeepImmutableError, lambda: n.raise_error())
1391
1392         for (i, n) in bad_uri:
1393             self.failUnlessRaises(uri.BadURIError, lambda: n.raise_error())
1394
1395         for (i, n) in ok:
1396             self.failIf(n.get_readonly_uri() is None, i)
1397
1398         for (i, n) in ro_prefixed:
1399             self.failUnless(n.get_readonly_uri().startswith("ro."), i)
1400
1401         for (i, n) in imm_prefixed:
1402             self.failUnless(n.get_readonly_uri().startswith("imm."), i)
1403
1404
1405 class DeepStats(unittest.TestCase):
1406     timeout = 240 # It takes longer than 120 seconds on Francois's arm box.
1407     def test_stats(self):
1408         ds = dirnode.DeepStats(None)
1409         ds.add("count-files")
1410         ds.add("size-immutable-files", 123)
1411         ds.histogram("size-files-histogram", 123)
1412         ds.max("largest-directory", 444)
1413
1414         s = ds.get_results()
1415         self.failUnlessEqual(s["count-files"], 1)
1416         self.failUnlessEqual(s["size-immutable-files"], 123)
1417         self.failUnlessEqual(s["largest-directory"], 444)
1418         self.failUnlessEqual(s["count-literal-files"], 0)
1419
1420         ds.add("count-files")
1421         ds.add("size-immutable-files", 321)
1422         ds.histogram("size-files-histogram", 321)
1423         ds.max("largest-directory", 2)
1424
1425         s = ds.get_results()
1426         self.failUnlessEqual(s["count-files"], 2)
1427         self.failUnlessEqual(s["size-immutable-files"], 444)
1428         self.failUnlessEqual(s["largest-directory"], 444)
1429         self.failUnlessEqual(s["count-literal-files"], 0)
1430         self.failUnlessEqual(s["size-files-histogram"],
1431                              [ (101, 316, 1), (317, 1000, 1) ])
1432
1433         ds = dirnode.DeepStats(None)
1434         for i in range(1, 1100):
1435             ds.histogram("size-files-histogram", i)
1436         ds.histogram("size-files-histogram", 4*1000*1000*1000*1000) # 4TB
1437         s = ds.get_results()
1438         self.failUnlessEqual(s["size-files-histogram"],
1439                              [ (1, 3, 3),
1440                                (4, 10, 7),
1441                                (11, 31, 21),
1442                                (32, 100, 69),
1443                                (101, 316, 216),
1444                                (317, 1000, 684),
1445                                (1001, 3162, 99),
1446                                (3162277660169L, 10000000000000L, 1),
1447                                ])
1448
1449 class UCWEingMutableFileNode(MutableFileNode):
1450     please_ucwe_after_next_upload = False
1451
1452     def _upload(self, new_contents, servermap):
1453         d = MutableFileNode._upload(self, new_contents, servermap)
1454         def _ucwe(res):
1455             if self.please_ucwe_after_next_upload:
1456                 self.please_ucwe_after_next_upload = False
1457                 raise UncoordinatedWriteError()
1458             return res
1459         d.addCallback(_ucwe)
1460         return d
1461
1462 class UCWEingNodeMaker(NodeMaker):
1463     def _create_mutable(self, cap):
1464         n = UCWEingMutableFileNode(self.storage_broker, self.secret_holder,
1465                                    self.default_encoding_parameters,
1466                                    self.history)
1467         return n.init_from_cap(cap)
1468
1469
1470 class Deleter(GridTestMixin, unittest.TestCase):
1471     timeout = 3600 # It takes longer than 433 seconds on Zandr's ARM box.
1472     def test_retry(self):
1473         # ticket #550, a dirnode.delete which experiences an
1474         # UncoordinatedWriteError will fail with an incorrect "you're
1475         # deleting something which isn't there" NoSuchChildError exception.
1476
1477         # to trigger this, we start by creating a directory with a single
1478         # file in it. Then we create a special dirnode that uses a modified
1479         # MutableFileNode which will raise UncoordinatedWriteError once on
1480         # demand. We then call dirnode.delete, which ought to retry and
1481         # succeed.
1482
1483         self.basedir = self.mktemp()
1484         self.set_up_grid()
1485         c0 = self.g.clients[0]
1486         d = c0.create_dirnode()
1487         small = upload.Data("Small enough for a LIT", None)
1488         def _created_dir(dn):
1489             self.root = dn
1490             self.root_uri = dn.get_uri()
1491             return dn.add_file(u"file", small)
1492         d.addCallback(_created_dir)
1493         def _do_delete(ignored):
1494             nm = UCWEingNodeMaker(c0.storage_broker, c0._secret_holder,
1495                                   c0.get_history(), c0.getServiceNamed("uploader"),
1496                                   c0.downloader,
1497                                   c0.download_cache_dirman,
1498                                   c0.get_encoding_parameters(),
1499                                   c0._key_generator)
1500             n = nm.create_from_cap(self.root_uri)
1501             assert n._node.please_ucwe_after_next_upload == False
1502             n._node.please_ucwe_after_next_upload = True
1503             # This should succeed, not raise an exception
1504             return n.delete(u"file")
1505         d.addCallback(_do_delete)
1506
1507         return d
1508
1509 class Adder(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin):
1510
1511     def test_overwrite(self):
1512         # note: This functionality could be tested without actually creating
1513         # several RSA keys. It would be faster without the GridTestMixin: use
1514         # dn.set_node(nodemaker.create_from_cap(make_chk_file_uri())) instead
1515         # of dn.add_file, and use a special NodeMaker that creates fake
1516         # mutable files.
1517         self.basedir = "dirnode/Adder/test_overwrite"
1518         self.set_up_grid()
1519         c = self.g.clients[0]
1520         fileuri = make_chk_file_uri(1234)
1521         filenode = c.nodemaker.create_from_cap(fileuri)
1522         d = c.create_dirnode()
1523
1524         def _create_directory_tree(root_node):
1525             # Build
1526             # root/file1
1527             # root/file2
1528             # root/dir1
1529             d = root_node.add_file(u'file1', upload.Data("Important Things",
1530                 None))
1531             d.addCallback(lambda res:
1532                 root_node.add_file(u'file2', upload.Data("Sekrit Codes", None)))
1533             d.addCallback(lambda res:
1534                 root_node.create_subdirectory(u"dir1"))
1535             d.addCallback(lambda res: root_node)
1536             return d
1537
1538         d.addCallback(_create_directory_tree)
1539
1540         def _test_adder(root_node):
1541             d = root_node.set_node(u'file1', filenode)
1542             # We've overwritten file1. Let's try it with a directory
1543             d.addCallback(lambda res:
1544                 root_node.create_subdirectory(u'dir2'))
1545             d.addCallback(lambda res:
1546                 root_node.set_node(u'dir2', filenode))
1547             # We try overwriting a file with a child while also specifying
1548             # overwrite=False. We should receive an ExistingChildError
1549             # when we do this.
1550             d.addCallback(lambda res:
1551                 self.shouldFail(ExistingChildError, "set_node",
1552                                 "child 'file1' already exists",
1553                                root_node.set_node, u"file1",
1554                                filenode, overwrite=False))
1555             # If we try with a directory, we should see the same thing
1556             d.addCallback(lambda res:
1557                 self.shouldFail(ExistingChildError, "set_node",
1558                                 "child 'dir1' already exists",
1559                                 root_node.set_node, u'dir1', filenode,
1560                                 overwrite=False))
1561             d.addCallback(lambda res:
1562                  root_node.set_node(u'file1', filenode,
1563                                     overwrite="only-files"))
1564             d.addCallback(lambda res:
1565                  self.shouldFail(ExistingChildError, "set_node",
1566                                 "child 'dir1' already exists",
1567                                 root_node.set_node, u'dir1', filenode,
1568                                 overwrite="only-files"))
1569             return d
1570
1571         d.addCallback(_test_adder)
1572         return d