src/allmydata/test/test_dirnode.py

   1
   2 import time
   3 from zope.interface import implements
   4 from twisted.trial import unittest
   5 from twisted.internet import defer
   6 from allmydata import uri, dirnode
   7 from allmydata.client import Client
   8 from allmydata.immutable import upload
   9 from allmydata.interfaces import IImmutableFileNode, IMutableFileNode, \
  10      ExistingChildError, NoSuchChildError, NotDeepImmutableError, \
  11      IDeepCheckResults, IDeepCheckAndRepairResults, CannotPackUnknownNodeError
  12 from allmydata.mutable.filenode import MutableFileNode
  13 from allmydata.mutable.common import UncoordinatedWriteError
  14 from allmydata.util import hashutil, base32
  15 from allmydata.monitor import Monitor
  16 from allmydata.test.common import make_chk_file_uri, make_mutable_file_uri, \
  17      ErrorMixin
  18 from allmydata.test.no_network import GridTestMixin
  19 from allmydata.unknown import UnknownNode
  20 from allmydata.nodemaker import NodeMaker
  21 from base64 import b32decode
  22 import common_util as testutil
  23
  24 class Dirnode(GridTestMixin, unittest.TestCase,
  25               testutil.ShouldFailMixin, testutil.StallMixin, ErrorMixin):
  26     timeout = 240 # It takes longer than 120 seconds on Francois's arm box.
  27
  28     def test_basic(self):
  29         self.basedir = "dirnode/Dirnode/test_basic"
  30         self.set_up_grid()
  31         c = self.g.clients[0]
  32         d = c.create_dirnode()
  33         def _done(res):
  34             self.failUnless(isinstance(res, dirnode.DirectoryNode))
  35             rep = str(res)
  36             self.failUnless("RW-MUT" in rep)
  37         d.addCallback(_done)
  38         return d
  39
  40     def test_initial_children(self):
  41         self.basedir = "dirnode/Dirnode/test_initial_children"
  42         self.set_up_grid()
  43         c = self.g.clients[0]
  44         nm = c.nodemaker
  45         setup_py_uri = "URI:CHK:n7r3m6wmomelk4sep3kw5cvduq:os7ijw5c3maek7pg65e5254k2fzjflavtpejjyhshpsxuqzhcwwq:3:20:14861"
  46         one_uri = "URI:LIT:n5xgk" # LIT for "one"
  47         kids = {u"one": (nm.create_from_cap(one_uri), {}),
  48                 u"two": (nm.create_from_cap(setup_py_uri),
  49                          {"metakey": "metavalue"}),
  50                 }
  51         d = c.create_dirnode(kids)
  52         def _created(dn):
  53             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
  54             rep = str(dn)
  55             self.failUnless("RW-MUT" in rep)
  56             return dn.list()
  57         d.addCallback(_created)
  58         def _check_kids(children):
  59             self.failUnlessEqual(sorted(children.keys()), [u"one", u"two"])
  60             one_node, one_metadata = children[u"one"]
  61             two_node, two_metadata = children[u"two"]
  62             self.failUnlessEqual(one_node.get_size(), 3)
  63             self.failUnlessEqual(two_node.get_size(), 14861)
  64             self.failUnless(isinstance(one_metadata, dict), one_metadata)
  65             self.failUnlessEqual(two_metadata["metakey"], "metavalue")
  66         d.addCallback(_check_kids)
  67         d.addCallback(lambda ign: nm.create_new_mutable_directory(kids))
  68         d.addCallback(lambda dn: dn.list())
  69         d.addCallback(_check_kids)
  70         future_writecap = "x-tahoe-crazy://I_am_from_the_future."
  71         future_readcap = "x-tahoe-crazy-readonly://I_am_from_the_future."
  72         future_node = UnknownNode(future_writecap, future_readcap)
  73         bad_kids1 = {u"one": (future_node, {})}
  74         d.addCallback(lambda ign:
  75                       self.shouldFail(AssertionError, "bad_kids1",
  76                                       "does not accept UnknownNode",
  77                                       nm.create_new_mutable_directory,
  78                                       bad_kids1))
  79         bad_kids2 = {u"one": (nm.create_from_cap(one_uri), None)}
  80         d.addCallback(lambda ign:
  81                       self.shouldFail(AssertionError, "bad_kids2",
  82                                       "requires metadata to be a dict",
  83                                       nm.create_new_mutable_directory,
  84                                       bad_kids2))
  85         return d
  86
  87     def test_immutable(self):
  88         self.basedir = "dirnode/Dirnode/test_immutable"
  89         self.set_up_grid()
  90         c = self.g.clients[0]
  91         nm = c.nodemaker
  92         setup_py_uri = "URI:CHK:n7r3m6wmomelk4sep3kw5cvduq:os7ijw5c3maek7pg65e5254k2fzjflavtpejjyhshpsxuqzhcwwq:3:20:14861"
  93         one_uri = "URI:LIT:n5xgk" # LIT for "one"
  94         mut_readcap = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
  95         mut_writecap = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
  96         kids = {u"one": (nm.create_from_cap(one_uri), {}),
  97                 u"two": (nm.create_from_cap(setup_py_uri),
  98                          {"metakey": "metavalue"}),
  99                 }
 100         d = c.create_immutable_dirnode(kids)
 101         def _created(dn):
 102             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
 103             self.failIf(dn.is_mutable())
 104             self.failUnless(dn.is_readonly())
 105             rep = str(dn)
 106             self.failUnless("RO-IMM" in rep)
 107             cap = dn.get_cap()
 108             self.failUnlessIn("CHK", cap.to_string())
 109             self.cap = cap
 110             return dn.list()
 111         d.addCallback(_created)
 112         def _check_kids(children):
 113             self.failUnlessEqual(sorted(children.keys()), [u"one", u"two"])
 114             one_node, one_metadata = children[u"one"]
 115             two_node, two_metadata = children[u"two"]
 116             self.failUnlessEqual(one_node.get_size(), 3)
 117             self.failUnlessEqual(two_node.get_size(), 14861)
 118             self.failUnless(isinstance(one_metadata, dict), one_metadata)
 119             self.failUnlessEqual(two_metadata["metakey"], "metavalue")
 120         d.addCallback(_check_kids)
 121         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
 122         d.addCallback(lambda dn: dn.list())
 123         d.addCallback(_check_kids)
 124         future_writecap = "x-tahoe-crazy://I_am_from_the_future."
 125         future_readcap = "x-tahoe-crazy-readonly://I_am_from_the_future."
 126         future_node = UnknownNode(future_writecap, future_readcap)
 127         bad_kids1 = {u"one": (future_node, {})}
 128         d.addCallback(lambda ign:
 129                       self.shouldFail(AssertionError, "bad_kids1",
 130                                       "does not accept UnknownNode",
 131                                       c.create_immutable_dirnode,
 132                                       bad_kids1))
 133         bad_kids2 = {u"one": (nm.create_from_cap(one_uri), None)}
 134         d.addCallback(lambda ign:
 135                       self.shouldFail(AssertionError, "bad_kids2",
 136                                       "requires metadata to be a dict",
 137                                       c.create_immutable_dirnode,
 138                                       bad_kids2))
 139         bad_kids3 = {u"one": (nm.create_from_cap(mut_writecap), {})}
 140         d.addCallback(lambda ign:
 141                       self.shouldFail(NotDeepImmutableError, "bad_kids3",
 142                                       "is not immutable",
 143                                       c.create_immutable_dirnode,
 144                                       bad_kids3))
 145         bad_kids4 = {u"one": (nm.create_from_cap(mut_readcap), {})}
 146         d.addCallback(lambda ign:
 147                       self.shouldFail(NotDeepImmutableError, "bad_kids4",
 148                                       "is not immutable",
 149                                       c.create_immutable_dirnode,
 150                                       bad_kids4))
 151         d.addCallback(lambda ign: c.create_immutable_dirnode({}))
 152         def _created_empty(dn):
 153             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
 154             self.failIf(dn.is_mutable())
 155             self.failUnless(dn.is_readonly())
 156             rep = str(dn)
 157             self.failUnless("RO-IMM" in rep)
 158             cap = dn.get_cap()
 159             self.failUnlessIn("LIT", cap.to_string())
 160             self.failUnlessEqual(cap.to_string(), "URI:DIR2-LIT:")
 161             self.cap = cap
 162             return dn.list()
 163         d.addCallback(_created_empty)
 164         d.addCallback(lambda kids: self.failUnlessEqual(kids, {}))
 165         smallkids = {u"o": (nm.create_from_cap(one_uri), {})}
 166         d.addCallback(lambda ign: c.create_immutable_dirnode(smallkids))
 167         def _created_small(dn):
 168             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
 169             self.failIf(dn.is_mutable())
 170             self.failUnless(dn.is_readonly())
 171             rep = str(dn)
 172             self.failUnless("RO-IMM" in rep)
 173             cap = dn.get_cap()
 174             self.failUnlessIn("LIT", cap.to_string())
 175             self.failUnlessEqual(cap.to_string(),
 176                                  "URI:DIR2-LIT:gi4tumj2n4wdcmz2kvjesosmjfkdu3rvpbtwwlbqhiwdeot3puwcy")
 177             self.cap = cap
 178             return dn.list()
 179         d.addCallback(_created_small)
 180         d.addCallback(lambda kids: self.failUnlessEqual(kids.keys(), [u"o"]))
 181
 182         # now test n.create_subdirectory(mutable=False)
 183         d.addCallback(lambda ign: c.create_dirnode())
 184         def _made_parent(n):
 185             d = n.create_subdirectory(u"subdir", kids, mutable=False)
 186             d.addCallback(lambda sd: sd.list())
 187             d.addCallback(_check_kids)
 188             d.addCallback(lambda ign: n.list())
 189             d.addCallback(lambda children:
 190                           self.failUnlessEqual(children.keys(), [u"subdir"]))
 191             d.addCallback(lambda ign: n.get(u"subdir"))
 192             d.addCallback(lambda sd: sd.list())
 193             d.addCallback(_check_kids)
 194             d.addCallback(lambda ign: n.get(u"subdir"))
 195             d.addCallback(lambda sd: self.failIf(sd.is_mutable()))
 196             bad_kids = {u"one": (nm.create_from_cap(mut_writecap), {})}
 197             d.addCallback(lambda ign:
 198                           self.shouldFail(NotDeepImmutableError, "YZ",
 199                                           "is not immutable",
 200                                           n.create_subdirectory,
 201                                           u"sub2", bad_kids, mutable=False))
 202             return d
 203         d.addCallback(_made_parent)
 204         return d
 205
 206
 207     def test_check(self):
 208         self.basedir = "dirnode/Dirnode/test_check"
 209         self.set_up_grid()
 210         c = self.g.clients[0]
 211         d = c.create_dirnode()
 212         d.addCallback(lambda dn: dn.check(Monitor()))
 213         def _done(res):
 214             self.failUnless(res.is_healthy())
 215         d.addCallback(_done)
 216         return d
 217
 218     def _test_deepcheck_create(self):
 219         # create a small tree with a loop, and some non-directories
 220         #  root/
 221         #  root/subdir/
 222         #  root/subdir/file1
 223         #  root/subdir/link -> root
 224         #  root/rodir
 225         c = self.g.clients[0]
 226         d = c.create_dirnode()
 227         def _created_root(rootnode):
 228             self._rootnode = rootnode
 229             return rootnode.create_subdirectory(u"subdir")
 230         d.addCallback(_created_root)
 231         def _created_subdir(subdir):
 232             self._subdir = subdir
 233             d = subdir.add_file(u"file1", upload.Data("data"*100, None))
 234             d.addCallback(lambda res: subdir.set_node(u"link", self._rootnode))
 235             d.addCallback(lambda res: c.create_dirnode())
 236             d.addCallback(lambda dn:
 237                           self._rootnode.set_uri(u"rodir",
 238                                                  dn.get_uri(),
 239                                                  dn.get_readonly_uri()))
 240             return d
 241         d.addCallback(_created_subdir)
 242         def _done(res):
 243             return self._rootnode
 244         d.addCallback(_done)
 245         return d
 246
 247     def test_deepcheck(self):
 248         self.basedir = "dirnode/Dirnode/test_deepcheck"
 249         self.set_up_grid()
 250         d = self._test_deepcheck_create()
 251         d.addCallback(lambda rootnode: rootnode.start_deep_check().when_done())
 252         def _check_results(r):
 253             self.failUnless(IDeepCheckResults.providedBy(r))
 254             c = r.get_counters()
 255             self.failUnlessEqual(c,
 256                                  {"count-objects-checked": 4,
 257                                   "count-objects-healthy": 4,
 258                                   "count-objects-unhealthy": 0,
 259                                   "count-objects-unrecoverable": 0,
 260                                   "count-corrupt-shares": 0,
 261                                   })
 262             self.failIf(r.get_corrupt_shares())
 263             self.failUnlessEqual(len(r.get_all_results()), 4)
 264         d.addCallback(_check_results)
 265         return d
 266
 267     def test_deepcheck_and_repair(self):
 268         self.basedir = "dirnode/Dirnode/test_deepcheck_and_repair"
 269         self.set_up_grid()
 270         d = self._test_deepcheck_create()
 271         d.addCallback(lambda rootnode:
 272                       rootnode.start_deep_check_and_repair().when_done())
 273         def _check_results(r):
 274             self.failUnless(IDeepCheckAndRepairResults.providedBy(r))
 275             c = r.get_counters()
 276             self.failUnlessEqual(c,
 277                                  {"count-objects-checked": 4,
 278                                   "count-objects-healthy-pre-repair": 4,
 279                                   "count-objects-unhealthy-pre-repair": 0,
 280                                   "count-objects-unrecoverable-pre-repair": 0,
 281                                   "count-corrupt-shares-pre-repair": 0,
 282                                   "count-objects-healthy-post-repair": 4,
 283                                   "count-objects-unhealthy-post-repair": 0,
 284                                   "count-objects-unrecoverable-post-repair": 0,
 285                                   "count-corrupt-shares-post-repair": 0,
 286                                   "count-repairs-attempted": 0,
 287                                   "count-repairs-successful": 0,
 288                                   "count-repairs-unsuccessful": 0,
 289                                   })
 290             self.failIf(r.get_corrupt_shares())
 291             self.failIf(r.get_remaining_corrupt_shares())
 292             self.failUnlessEqual(len(r.get_all_results()), 4)
 293         d.addCallback(_check_results)
 294         return d
 295
 296     def _mark_file_bad(self, rootnode):
 297         si = rootnode.get_storage_index()
 298         self.delete_shares_numbered(rootnode.get_uri(), [0])
 299         return rootnode
 300
 301     def test_deepcheck_problems(self):
 302         self.basedir = "dirnode/Dirnode/test_deepcheck_problems"
 303         self.set_up_grid()
 304         d = self._test_deepcheck_create()
 305         d.addCallback(lambda rootnode: self._mark_file_bad(rootnode))
 306         d.addCallback(lambda rootnode: rootnode.start_deep_check().when_done())
 307         def _check_results(r):
 308             c = r.get_counters()
 309             self.failUnlessEqual(c,
 310                                  {"count-objects-checked": 4,
 311                                   "count-objects-healthy": 3,
 312                                   "count-objects-unhealthy": 1,
 313                                   "count-objects-unrecoverable": 0,
 314                                   "count-corrupt-shares": 0,
 315                                   })
 316             #self.failUnlessEqual(len(r.get_problems()), 1) # TODO
 317         d.addCallback(_check_results)
 318         return d
 319
 320     def test_readonly(self):
 321         self.basedir = "dirnode/Dirnode/test_readonly"
 322         self.set_up_grid()
 323         c = self.g.clients[0]
 324         nm = c.nodemaker
 325         filecap = make_chk_file_uri(1234)
 326         filenode = nm.create_from_cap(filecap)
 327         uploadable = upload.Data("some data", convergence="some convergence string")
 328
 329         d = c.create_dirnode()
 330         def _created(rw_dn):
 331             d2 = rw_dn.set_uri(u"child", filecap, filecap)
 332             d2.addCallback(lambda res: rw_dn)
 333             return d2
 334         d.addCallback(_created)
 335
 336         def _ready(rw_dn):
 337             ro_uri = rw_dn.get_readonly_uri()
 338             ro_dn = c.create_node_from_uri(ro_uri)
 339             self.failUnless(ro_dn.is_readonly())
 340             self.failUnless(ro_dn.is_mutable())
 341
 342             self.shouldFail(dirnode.NotMutableError, "set_uri ro", None,
 343                             ro_dn.set_uri, u"newchild", filecap, filecap)
 344             self.shouldFail(dirnode.NotMutableError, "set_uri ro", None,
 345                             ro_dn.set_node, u"newchild", filenode)
 346             self.shouldFail(dirnode.NotMutableError, "set_nodes ro", None,
 347                             ro_dn.set_nodes, { u"newchild": (filenode, None) })
 348             self.shouldFail(dirnode.NotMutableError, "set_uri ro", None,
 349                             ro_dn.add_file, u"newchild", uploadable)
 350             self.shouldFail(dirnode.NotMutableError, "set_uri ro", None,
 351                             ro_dn.delete, u"child")
 352             self.shouldFail(dirnode.NotMutableError, "set_uri ro", None,
 353                             ro_dn.create_subdirectory, u"newchild")
 354             self.shouldFail(dirnode.NotMutableError, "set_metadata_for ro", None,
 355                             ro_dn.set_metadata_for, u"child", {})
 356             self.shouldFail(dirnode.NotMutableError, "set_uri ro", None,
 357                             ro_dn.move_child_to, u"child", rw_dn)
 358             self.shouldFail(dirnode.NotMutableError, "set_uri ro", None,
 359                             rw_dn.move_child_to, u"child", ro_dn)
 360             return ro_dn.list()
 361         d.addCallback(_ready)
 362         def _listed(children):
 363             self.failUnless(u"child" in children)
 364         d.addCallback(_listed)
 365         return d
 366
 367     def failUnlessGreaterThan(self, a, b):
 368         self.failUnless(a > b, "%r should be > %r" % (a, b))
 369
 370     def failUnlessGreaterOrEqualThan(self, a, b):
 371         self.failUnless(a >= b, "%r should be >= %r" % (a, b))
 372
 373     def test_create(self):
 374         self.basedir = "dirnode/Dirnode/test_create"
 375         self.set_up_grid()
 376         c = self.g.clients[0]
 377
 378         self.expected_manifest = []
 379         self.expected_verifycaps = set()
 380         self.expected_storage_indexes = set()
 381
 382         d = c.create_dirnode()
 383         def _then(n):
 384             # /
 385             self.rootnode = n
 386             self.failUnless(n.is_mutable())
 387             u = n.get_uri()
 388             self.failUnless(u)
 389             self.failUnless(u.startswith("URI:DIR2:"), u)
 390             u_ro = n.get_readonly_uri()
 391             self.failUnless(u_ro.startswith("URI:DIR2-RO:"), u_ro)
 392             u_v = n.get_verify_cap().to_string()
 393             self.failUnless(u_v.startswith("URI:DIR2-Verifier:"), u_v)
 394             u_r = n.get_repair_cap().to_string()
 395             self.failUnlessEqual(u_r, u)
 396             self.expected_manifest.append( ((), u) )
 397             self.expected_verifycaps.add(u_v)
 398             si = n.get_storage_index()
 399             self.expected_storage_indexes.add(base32.b2a(si))
 400             expected_si = n._uri._filenode_uri.storage_index
 401             self.failUnlessEqual(si, expected_si)
 402
 403             d = n.list()
 404             d.addCallback(lambda res: self.failUnlessEqual(res, {}))
 405             d.addCallback(lambda res: n.has_child(u"missing"))
 406             d.addCallback(lambda res: self.failIf(res))
 407
 408             fake_file_uri = make_mutable_file_uri()
 409             other_file_uri = make_mutable_file_uri()
 410             m = c.nodemaker.create_from_cap(fake_file_uri)
 411             ffu_v = m.get_verify_cap().to_string()
 412             self.expected_manifest.append( ((u"child",) , m.get_uri()) )
 413             self.expected_verifycaps.add(ffu_v)
 414             self.expected_storage_indexes.add(base32.b2a(m.get_storage_index()))
 415             d.addCallback(lambda res: n.set_uri(u"child",
 416                                                 fake_file_uri, fake_file_uri))
 417             d.addCallback(lambda res:
 418                           self.shouldFail(ExistingChildError, "set_uri-no",
 419                                           "child 'child' already exists",
 420                                           n.set_uri, u"child",
 421                                           other_file_uri, other_file_uri,
 422                                           overwrite=False))
 423             # /
 424             # /child = mutable
 425
 426             d.addCallback(lambda res: n.create_subdirectory(u"subdir"))
 427
 428             # /
 429             # /child = mutable
 430             # /subdir = directory
 431             def _created(subdir):
 432                 self.failUnless(isinstance(subdir, dirnode.DirectoryNode))
 433                 self.subdir = subdir
 434                 new_v = subdir.get_verify_cap().to_string()
 435                 assert isinstance(new_v, str)
 436                 self.expected_manifest.append( ((u"subdir",), subdir.get_uri()) )
 437                 self.expected_verifycaps.add(new_v)
 438                 si = subdir.get_storage_index()
 439                 self.expected_storage_indexes.add(base32.b2a(si))
 440             d.addCallback(_created)
 441
 442             d.addCallback(lambda res:
 443                           self.shouldFail(ExistingChildError, "mkdir-no",
 444                                           "child 'subdir' already exists",
 445                                           n.create_subdirectory, u"subdir",
 446                                           overwrite=False))
 447
 448             d.addCallback(lambda res: n.list())
 449             d.addCallback(lambda children:
 450                           self.failUnlessEqual(sorted(children.keys()),
 451                                                sorted([u"child", u"subdir"])))
 452
 453             d.addCallback(lambda res: n.start_deep_stats().when_done())
 454             def _check_deepstats(stats):
 455                 self.failUnless(isinstance(stats, dict))
 456                 expected = {"count-immutable-files": 0,
 457                             "count-mutable-files": 1,
 458                             "count-literal-files": 0,
 459                             "count-files": 1,
 460                             "count-directories": 2,
 461                             "size-immutable-files": 0,
 462                             "size-literal-files": 0,
 463                             #"size-directories": 616, # varies
 464                             #"largest-directory": 616,
 465                             "largest-directory-children": 2,
 466                             "largest-immutable-file": 0,
 467                             }
 468                 for k,v in expected.iteritems():
 469                     self.failUnlessEqual(stats[k], v,
 470                                          "stats[%s] was %s, not %s" %
 471                                          (k, stats[k], v))
 472                 self.failUnless(stats["size-directories"] > 500,
 473                                 stats["size-directories"])
 474                 self.failUnless(stats["largest-directory"] > 500,
 475                                 stats["largest-directory"])
 476                 self.failUnlessEqual(stats["size-files-histogram"], [])
 477             d.addCallback(_check_deepstats)
 478
 479             d.addCallback(lambda res: n.build_manifest().when_done())
 480             def _check_manifest(res):
 481                 manifest = res["manifest"]
 482                 self.failUnlessEqual(sorted(manifest),
 483                                      sorted(self.expected_manifest))
 484                 stats = res["stats"]
 485                 _check_deepstats(stats)
 486                 self.failUnlessEqual(self.expected_verifycaps,
 487                                      res["verifycaps"])
 488                 self.failUnlessEqual(self.expected_storage_indexes,
 489                                      res["storage-index"])
 490             d.addCallback(_check_manifest)
 491
 492             def _add_subsubdir(res):
 493                 return self.subdir.create_subdirectory(u"subsubdir")
 494             d.addCallback(_add_subsubdir)
 495             # /
 496             # /child = mutable
 497             # /subdir = directory
 498             # /subdir/subsubdir = directory
 499             d.addCallback(lambda res: n.get_child_at_path(u"subdir/subsubdir"))
 500             d.addCallback(lambda subsubdir:
 501                           self.failUnless(isinstance(subsubdir,
 502                                                      dirnode.DirectoryNode)))
 503             d.addCallback(lambda res: n.get_child_at_path(u""))
 504             d.addCallback(lambda res: self.failUnlessEqual(res.get_uri(),
 505                                                            n.get_uri()))
 506
 507             d.addCallback(lambda res: n.get_metadata_for(u"child"))
 508             d.addCallback(lambda metadata:
 509                           self.failUnlessEqual(set(metadata.keys()),
 510                                                set(["tahoe", "ctime", "mtime"])))
 511
 512             d.addCallback(lambda res:
 513                           self.shouldFail(NoSuchChildError, "gcamap-no",
 514                                           "nope",
 515                                           n.get_child_and_metadata_at_path,
 516                                           u"subdir/nope"))
 517             d.addCallback(lambda res:
 518                           n.get_child_and_metadata_at_path(u""))
 519             def _check_child_and_metadata1(res):
 520                 child, metadata = res
 521                 self.failUnless(isinstance(child, dirnode.DirectoryNode))
 522                 # edge-metadata needs at least one path segment
 523                 self.failUnlessEqual(sorted(metadata.keys()), [])
 524             d.addCallback(_check_child_and_metadata1)
 525             d.addCallback(lambda res:
 526                           n.get_child_and_metadata_at_path(u"child"))
 527
 528             def _check_child_and_metadata2(res):
 529                 child, metadata = res
 530                 self.failUnlessEqual(child.get_uri(),
 531                                      fake_file_uri)
 532                 self.failUnlessEqual(set(metadata.keys()),
 533                                      set(["tahoe", "ctime", "mtime"]))
 534             d.addCallback(_check_child_and_metadata2)
 535
 536             d.addCallback(lambda res:
 537                           n.get_child_and_metadata_at_path(u"subdir/subsubdir"))
 538             def _check_child_and_metadata3(res):
 539                 child, metadata = res
 540                 self.failUnless(isinstance(child, dirnode.DirectoryNode))
 541                 self.failUnlessEqual(set(metadata.keys()),
 542                                      set(["tahoe", "ctime", "mtime"]))
 543             d.addCallback(_check_child_and_metadata3)
 544
 545             # set_uri + metadata
 546             # it should be possible to add a child without any metadata
 547             d.addCallback(lambda res: n.set_uri(u"c2",
 548                                                 fake_file_uri, fake_file_uri,
 549                                                 {}))
 550             d.addCallback(lambda res: n.get_metadata_for(u"c2"))
 551             d.addCallback(lambda metadata: self.failUnlessEqual(metadata.keys(), ['tahoe']))
 552
 553             # You can't override the link timestamps.
 554             d.addCallback(lambda res: n.set_uri(u"c2",
 555                                                 fake_file_uri, fake_file_uri,
 556                                                 { 'tahoe': {'linkcrtime': "bogus"}}))
 557             d.addCallback(lambda res: n.get_metadata_for(u"c2"))
 558             def _has_good_linkcrtime(metadata):
 559                 self.failUnless(metadata.has_key('tahoe'))
 560                 self.failUnless(metadata['tahoe'].has_key('linkcrtime'))
 561                 self.failIfEqual(metadata['tahoe']['linkcrtime'], 'bogus')
 562             d.addCallback(_has_good_linkcrtime)
 563
 564             # if we don't set any defaults, the child should get timestamps
 565             d.addCallback(lambda res: n.set_uri(u"c3",
 566                                                 fake_file_uri, fake_file_uri))
 567             d.addCallback(lambda res: n.get_metadata_for(u"c3"))
 568             d.addCallback(lambda metadata:
 569                           self.failUnlessEqual(set(metadata.keys()),
 570                                                set(["tahoe", "ctime", "mtime"])))
 571
 572             # or we can add specific metadata at set_uri() time, which
 573             # overrides the timestamps
 574             d.addCallback(lambda res: n.set_uri(u"c4",
 575                                                 fake_file_uri, fake_file_uri,
 576                                                 {"key": "value"}))
 577             d.addCallback(lambda res: n.get_metadata_for(u"c4"))
 578             d.addCallback(lambda metadata:
 579                               self.failUnless((set(metadata.keys()) == set(["key", "tahoe"])) and
 580                                               (metadata['key'] == "value"), metadata))
 581
 582             d.addCallback(lambda res: n.delete(u"c2"))
 583             d.addCallback(lambda res: n.delete(u"c3"))
 584             d.addCallback(lambda res: n.delete(u"c4"))
 585
 586             # set_node + metadata
 587             # it should be possible to add a child without any metadata
 588             d.addCallback(lambda res: n.set_node(u"d2", n, {}))
 589             d.addCallback(lambda res: c.create_dirnode())
 590             d.addCallback(lambda n2:
 591                           self.shouldFail(ExistingChildError, "set_node-no",
 592                                           "child 'd2' already exists",
 593                                           n.set_node, u"d2", n2,
 594                                           overwrite=False))
 595             d.addCallback(lambda res: n.get_metadata_for(u"d2"))
 596             d.addCallback(lambda metadata: self.failUnlessEqual(metadata.keys(), ['tahoe']))
 597
 598             # if we don't set any defaults, the child should get timestamps
 599             d.addCallback(lambda res: n.set_node(u"d3", n))
 600             d.addCallback(lambda res: n.get_metadata_for(u"d3"))
 601             d.addCallback(lambda metadata:
 602                           self.failUnlessEqual(set(metadata.keys()),
 603                                                set(["tahoe", "ctime", "mtime"])))
 604
 605             # or we can add specific metadata at set_node() time, which
 606             # overrides the timestamps
 607             d.addCallback(lambda res: n.set_node(u"d4", n,
 608                                                 {"key": "value"}))
 609             d.addCallback(lambda res: n.get_metadata_for(u"d4"))
 610             d.addCallback(lambda metadata:
 611                           self.failUnless((set(metadata.keys()) == set(["key", "tahoe"])) and
 612                                           (metadata['key'] == "value"), metadata))
 613
 614             d.addCallback(lambda res: n.delete(u"d2"))
 615             d.addCallback(lambda res: n.delete(u"d3"))
 616             d.addCallback(lambda res: n.delete(u"d4"))
 617
 618             # metadata through set_children()
 619             d.addCallback(lambda res:
 620                           n.set_children({
 621                               u"e1": (fake_file_uri, fake_file_uri),
 622                               u"e2": (fake_file_uri, fake_file_uri, {}),
 623                               u"e3": (fake_file_uri, fake_file_uri,
 624                                       {"key": "value"}),
 625                               }))
 626             d.addCallback(lambda n2: self.failUnlessIdentical(n2, n))
 627             d.addCallback(lambda res:
 628                           self.shouldFail(ExistingChildError, "set_children-no",
 629                                           "child 'e1' already exists",
 630                                           n.set_children,
 631                                           { u"e1": (other_file_uri,
 632                                                     other_file_uri),
 633                                             u"new": (other_file_uri,
 634                                                      other_file_uri),
 635                                             },
 636                                           overwrite=False))
 637             # and 'new' should not have been created
 638             d.addCallback(lambda res: n.list())
 639             d.addCallback(lambda children: self.failIf(u"new" in children))
 640             d.addCallback(lambda res: n.get_metadata_for(u"e1"))
 641             d.addCallback(lambda metadata:
 642                           self.failUnlessEqual(set(metadata.keys()),
 643                                                set(["tahoe", "ctime", "mtime"])))
 644             d.addCallback(lambda res: n.get_metadata_for(u"e2"))
 645             d.addCallback(lambda metadata:
 646                           self.failUnlessEqual(set(metadata.keys()), set(['tahoe'])))
 647             d.addCallback(lambda res: n.get_metadata_for(u"e3"))
 648             d.addCallback(lambda metadata:
 649                               self.failUnless((set(metadata.keys()) == set(["key", "tahoe"]))
 650                                               and (metadata['key'] == "value"), metadata))
 651
 652             d.addCallback(lambda res: n.delete(u"e1"))
 653             d.addCallback(lambda res: n.delete(u"e2"))
 654             d.addCallback(lambda res: n.delete(u"e3"))
 655
 656             # metadata through set_nodes()
 657             d.addCallback(lambda res:
 658                           n.set_nodes({ u"f1": (n, None),
 659                                         u"f2": (n, {}),
 660                                         u"f3": (n, {"key": "value"}),
 661                                         }))
 662             d.addCallback(lambda n2: self.failUnlessIdentical(n2, n))
 663             d.addCallback(lambda res:
 664                           self.shouldFail(ExistingChildError, "set_nodes-no",
 665                                           "child 'f1' already exists",
 666                                           n.set_nodes, { u"f1": (n, None),
 667                                                          u"new": (n, None), },
 668                                           overwrite=False))
 669             # and 'new' should not have been created
 670             d.addCallback(lambda res: n.list())
 671             d.addCallback(lambda children: self.failIf(u"new" in children))
 672             d.addCallback(lambda res: n.get_metadata_for(u"f1"))
 673             d.addCallback(lambda metadata:
 674                           self.failUnlessEqual(set(metadata.keys()),
 675                                                set(["tahoe", "ctime", "mtime"])))
 676             d.addCallback(lambda res: n.get_metadata_for(u"f2"))
 677             d.addCallback(
 678                 lambda metadata: self.failUnlessEqual(set(metadata.keys()), set(['tahoe'])))
 679             d.addCallback(lambda res: n.get_metadata_for(u"f3"))
 680             d.addCallback(lambda metadata:
 681                               self.failUnless((set(metadata.keys()) == set(["key", "tahoe"])) and
 682                                               (metadata['key'] == "value"), metadata))
 683
 684             d.addCallback(lambda res: n.delete(u"f1"))
 685             d.addCallback(lambda res: n.delete(u"f2"))
 686             d.addCallback(lambda res: n.delete(u"f3"))
 687
 688
 689             d.addCallback(lambda res:
 690                           n.set_metadata_for(u"child",
 691                                              {"tags": ["web2.0-compatible"]}))
 692             d.addCallback(lambda n1: n1.get_metadata_for(u"child"))
 693             d.addCallback(lambda metadata:
 694                           self.failUnlessEqual(metadata,
 695                                                {"tags": ["web2.0-compatible"]}))
 696
 697             def _start(res):
 698                 self._start_timestamp = time.time()
 699             d.addCallback(_start)
 700             # simplejson-1.7.1 (as shipped on Ubuntu 'gutsy') rounds all
 701             # floats to hundredeths (it uses str(num) instead of repr(num)).
 702             # simplejson-1.7.3 does not have this bug. To prevent this bug
 703             # from causing the test to fail, stall for more than a few
 704             # hundrededths of a second.
 705             d.addCallback(self.stall, 0.1)
 706             d.addCallback(lambda res: n.add_file(u"timestamps",
 707                                                  upload.Data("stamp me", convergence="some convergence string")))
 708             d.addCallback(self.stall, 0.1)
 709             def _stop(res):
 710                 self._stop_timestamp = time.time()
 711             d.addCallback(_stop)
 712
 713             d.addCallback(lambda res: n.get_metadata_for(u"timestamps"))
 714             def _check_timestamp1(metadata):
 715                 self.failUnless("ctime" in metadata)
 716                 self.failUnless("mtime" in metadata)
 717                 self.failUnlessGreaterOrEqualThan(metadata["ctime"],
 718                                                   self._start_timestamp)
 719                 self.failUnlessGreaterOrEqualThan(self._stop_timestamp,
 720                                                   metadata["ctime"])
 721                 self.failUnlessGreaterOrEqualThan(metadata["mtime"],
 722                                                   self._start_timestamp)
 723                 self.failUnlessGreaterOrEqualThan(self._stop_timestamp,
 724                                                   metadata["mtime"])
 725                 # Our current timestamp rules say that replacing an existing
 726                 # child should preserve the 'ctime' but update the mtime
 727                 self._old_ctime = metadata["ctime"]
 728                 self._old_mtime = metadata["mtime"]
 729             d.addCallback(_check_timestamp1)
 730             d.addCallback(self.stall, 2.0) # accomodate low-res timestamps
 731             d.addCallback(lambda res: n.set_node(u"timestamps", n))
 732             d.addCallback(lambda res: n.get_metadata_for(u"timestamps"))
 733             def _check_timestamp2(metadata):
 734                 self.failUnlessEqual(metadata["ctime"], self._old_ctime,
 735                                      "%s != %s" % (metadata["ctime"],
 736                                                    self._old_ctime))
 737                 self.failUnlessGreaterThan(metadata["mtime"], self._old_mtime)
 738                 return n.delete(u"timestamps")
 739             d.addCallback(_check_timestamp2)
 740
 741             # also make sure we can add/update timestamps on a
 742             # previously-existing child that didn't have any, since there are
 743             # a lot of 0.7.0-generated edges around out there
 744             d.addCallback(lambda res: n.set_node(u"no_timestamps", n, {}))
 745             d.addCallback(lambda res: n.set_node(u"no_timestamps", n))
 746             d.addCallback(lambda res: n.get_metadata_for(u"no_timestamps"))
 747             d.addCallback(lambda metadata:
 748                           self.failUnlessEqual(set(metadata.keys()),
 749                                                set(["tahoe", "ctime", "mtime"])))
 750             d.addCallback(lambda res: n.delete(u"no_timestamps"))
 751
 752             d.addCallback(lambda res: n.delete(u"subdir"))
 753             d.addCallback(lambda old_child:
 754                           self.failUnlessEqual(old_child.get_uri(),
 755                                                self.subdir.get_uri()))
 756
 757             d.addCallback(lambda res: n.list())
 758             d.addCallback(lambda children:
 759                           self.failUnlessEqual(sorted(children.keys()),
 760                                                sorted([u"child"])))
 761
 762             uploadable1 = upload.Data("some data", convergence="converge")
 763             d.addCallback(lambda res: n.add_file(u"newfile", uploadable1))
 764             d.addCallback(lambda newnode:
 765                           self.failUnless(IImmutableFileNode.providedBy(newnode)))
 766             uploadable2 = upload.Data("some data", convergence="stuff")
 767             d.addCallback(lambda res:
 768                           self.shouldFail(ExistingChildError, "add_file-no",
 769                                           "child 'newfile' already exists",
 770                                           n.add_file, u"newfile",
 771                                           uploadable2,
 772                                           overwrite=False))
 773             d.addCallback(lambda res: n.list())
 774             d.addCallback(lambda children:
 775                           self.failUnlessEqual(sorted(children.keys()),
 776                                                sorted([u"child", u"newfile"])))
 777             d.addCallback(lambda res: n.get_metadata_for(u"newfile"))
 778             d.addCallback(lambda metadata:
 779                           self.failUnlessEqual(set(metadata.keys()),
 780                                                set(["tahoe", "ctime", "mtime"])))
 781
 782             uploadable3 = upload.Data("some data", convergence="converge")
 783             d.addCallback(lambda res: n.add_file(u"newfile-metadata",
 784                                                  uploadable3,
 785                                                  {"key": "value"}))
 786             d.addCallback(lambda newnode:
 787                           self.failUnless(IImmutableFileNode.providedBy(newnode)))
 788             d.addCallback(lambda res: n.get_metadata_for(u"newfile-metadata"))
 789             d.addCallback(lambda metadata:
 790                               self.failUnless((set(metadata.keys()) == set(["key", "tahoe"])) and
 791                                               (metadata['key'] == "value"), metadata))
 792             d.addCallback(lambda res: n.delete(u"newfile-metadata"))
 793
 794             d.addCallback(lambda res: n.create_subdirectory(u"subdir2"))
 795             def _created2(subdir2):
 796                 self.subdir2 = subdir2
 797                 # put something in the way, to make sure it gets overwritten
 798                 return subdir2.add_file(u"child", upload.Data("overwrite me",
 799                                                               "converge"))
 800             d.addCallback(_created2)
 801
 802             d.addCallback(lambda res:
 803                           n.move_child_to(u"child", self.subdir2))
 804             d.addCallback(lambda res: n.list())
 805             d.addCallback(lambda children:
 806                           self.failUnlessEqual(sorted(children.keys()),
 807                                                sorted([u"newfile", u"subdir2"])))
 808             d.addCallback(lambda res: self.subdir2.list())
 809             d.addCallback(lambda children:
 810                           self.failUnlessEqual(sorted(children.keys()),
 811                                                sorted([u"child"])))
 812             d.addCallback(lambda res: self.subdir2.get(u"child"))
 813             d.addCallback(lambda child:
 814                           self.failUnlessEqual(child.get_uri(),
 815                                                fake_file_uri))
 816
 817             # move it back, using new_child_name=
 818             d.addCallback(lambda res:
 819                           self.subdir2.move_child_to(u"child", n, u"newchild"))
 820             d.addCallback(lambda res: n.list())
 821             d.addCallback(lambda children:
 822                           self.failUnlessEqual(sorted(children.keys()),
 823                                                sorted([u"newchild", u"newfile",
 824                                                        u"subdir2"])))
 825             d.addCallback(lambda res: self.subdir2.list())
 826             d.addCallback(lambda children:
 827                           self.failUnlessEqual(sorted(children.keys()), []))
 828
 829             # now make sure that we honor overwrite=False
 830             d.addCallback(lambda res:
 831                           self.subdir2.set_uri(u"newchild",
 832                                                other_file_uri, other_file_uri))
 833
 834             d.addCallback(lambda res:
 835                           self.shouldFail(ExistingChildError, "move_child_to-no",
 836                                           "child 'newchild' already exists",
 837                                           n.move_child_to, u"newchild",
 838                                           self.subdir2,
 839                                           overwrite=False))
 840             d.addCallback(lambda res: self.subdir2.get(u"newchild"))
 841             d.addCallback(lambda child:
 842                           self.failUnlessEqual(child.get_uri(),
 843                                                other_file_uri))
 844
 845             return d
 846
 847         d.addCallback(_then)
 848
 849         d.addErrback(self.explain_error)
 850         return d
 851
 852     def test_create_subdirectory(self):
 853         self.basedir = "dirnode/Dirnode/test_create_subdirectory"
 854         self.set_up_grid()
 855         c = self.g.clients[0]
 856         nm = c.nodemaker
 857
 858         d = c.create_dirnode()
 859         def _then(n):
 860             # /
 861             self.rootnode = n
 862             fake_file_uri = make_mutable_file_uri()
 863             other_file_uri = make_mutable_file_uri()
 864             md = {"metakey": "metavalue"}
 865             kids = {u"kid1": (nm.create_from_cap(fake_file_uri), {}),
 866                     u"kid2": (nm.create_from_cap(other_file_uri), md),
 867                     }
 868             d = n.create_subdirectory(u"subdir", kids)
 869             def _check(sub):
 870                 d = n.get_child_at_path(u"subdir")
 871                 d.addCallback(lambda sub2: self.failUnlessEqual(sub2.get_uri(),
 872                                                                 sub.get_uri()))
 873                 d.addCallback(lambda ign: sub.list())
 874                 return d
 875             d.addCallback(_check)
 876             def _check_kids(kids2):
 877                 self.failUnlessEqual(sorted(kids.keys()), sorted(kids2.keys()))
 878                 self.failUnlessEqual(kids2[u"kid2"][1]["metakey"], "metavalue")
 879             d.addCallback(_check_kids)
 880             return d
 881         d.addCallback(_then)
 882         return d
 883
 884 class MinimalFakeMutableFile:
 885     def get_writekey(self):
 886         return "writekey"
 887
 888 class Packing(unittest.TestCase):
 889     # This is a base32-encoded representation of the directory tree
 890     # root/file1
 891     # root/file2
 892     # root/file3
 893     # as represented after being fed to _pack_contents.
 894     # We have it here so we can decode it, feed it to
 895     # _unpack_contents, and verify that _unpack_contents
 896     # works correctly.
 897
 898     known_tree = "GM4TOORVHJTGS3DFGEWDSNJ2KVJESOSDJBFTU33MPB2GS3LZNVYG6N3GGI3WU5TIORTXC3DOMJ2G4NB2MVWXUZDONBVTE5LNGRZWK2LYN55GY23XGNYXQMTOMZUWU5TENN4DG23ZG5UTO2L2NQ2DO6LFMRWDMZJWGRQTUMZ2GEYDUMJQFQYTIMZ22XZKZORX5XS7CAQCSK3URR6QOHISHRCMGER5LRFSZRNAS5ZSALCS6TWFQAE754IVOIKJVK73WZPP3VUUEDTX3WHTBBZ5YX3CEKHCPG3ZWQLYA4QM6LDRCF7TJQYWLIZHKGN5ROA3AUZPXESBNLQQ6JTC2DBJU2D47IZJTLR3PKZ4RVF57XLPWY7FX7SZV3T6IJ3ORFW37FXUPGOE3ROPFNUX5DCGMAQJ3PGGULBRGM3TU6ZCMN2GS3LFEI5CAMJSGQ3DMNRTHA4TOLRUGI3TKNRWGEWCAITUMFUG6ZJCHIQHWITMNFXGW3LPORUW2ZJCHIQDCMRUGY3DMMZYHE3S4NBSG42TMNRRFQQCE3DJNZVWG4TUNFWWKIR2EAYTENBWGY3DGOBZG4XDIMRXGU3DMML5FQQCE3LUNFWWKIR2EAYTENBWGY3DGOBZG4XDIMRXGU3DMML5FQWDGOJRHI2TUZTJNRSTELBZGQ5FKUSJHJBUQSZ2MFYGKZ3SOBSWQ43IO52WO23CNAZWU3DUGVSWSNTIOE5DK33POVTW4ZLNMNWDK6DHPA2GS2THNF2W25DEN5VGY2LQNFRGG5DKNNRHO5TZPFTWI6LNMRYGQ2LCGJTHM4J2GM5DCMB2GQWDCNBSHKVVQBGRYMACKJ27CVQ6O6B4QPR72RFVTGOZUI76XUSWAX73JRV5PYRHMIFYZIA25MXDPGUGML6M2NMRSG4YD4W4K37ZDYSXHMJ3IUVT4F64YTQQVBJFFFOUC7J7LAB2VFCL5UKKGMR2D3F4EPOYC7UYWQZNR5KXHBSNXLCNBX2SNF22DCXJIHSMEKWEWOG5XCJEVVZ7UW5IB6I64XXQSJ34B5CAYZGZIIMR6LBRGMZTU6ZCMN2GS3LFEI5CAMJSGQ3DMNRTHA4TOLRUGMYDEMJYFQQCE5DBNBXWKIR2EB5SE3DJNZVW233UNFWWKIR2EAYTENBWGY3DGOBZG4XDIMZQGIYTQLBAEJWGS3TLMNZHI2LNMURDUIBRGI2DMNRWGM4DSNZOGQZTAMRRHB6SYIBCNV2GS3LFEI5CAMJSGQ3DMNRTHA4TOLRUGMYDEMJYPUWCYMZZGU5DKOTGNFWGKMZMHE2DUVKSJE5EGSCLHJRW25DDPBYTO2DXPB3GM6DBNYZTI6LJMV3DM2LWNB4TU4LWMNSWW3LKORXWK5DEMN3TI23NNE3WEM3SORRGY5THPA3TKNBUMNZG453BOF2GSZLXMVWWI3DJOFZW623RHIZTUMJQHI2SYMJUGI5BOSHWDPG3WKPAVXCF3XMKA7QVIWPRMWJHDTQHD27AHDCPJWDQENQ5H5ZZILTXQNIXXCIW4LKQABU2GCFRG5FHQN7CHD7HF4EKNRZFIV2ZYQIBM7IQU7F4RGB3XCX3FREPBKQ7UCICHVWPCYFGA6OLH3J45LXQ6GWWICJ3PGWJNLZ7PCRNLAPNYUGU6BENS7OXMBEOOFRIZV3PF2FFWZ5WHDPKXERYP7GNHKRMGEZTOOT3EJRXI2LNMURDUIBRGI2DMNRWGM4DSNZOGQZTGNRSGY4SYIBCORQWQ33FEI5CA6ZCNRUW423NN52GS3LFEI5CAMJSGQ3DMNRTHA4TOLRUGMZTMMRWHEWCAITMNFXGWY3SORUW2ZJCHIQDCMRUGY3DMMZYHE3S4NBTGM3DENRZPUWCAITNORUW2ZJCHIQDCMRUGY3DMMZYHE3S4NBTGM3DENRZPUWCY==="
 899
 900     def test_unpack_and_pack_behavior(self):
 901         known_tree = b32decode(self.known_tree)
 902         nodemaker = NodeMaker(None, None, None,
 903                               None, None, None,
 904                               {"k": 3, "n": 10}, None)
 905         writecap = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
 906         filenode = nodemaker.create_from_cap(writecap)
 907         node = dirnode.DirectoryNode(filenode, nodemaker, None)
 908         children = node._unpack_contents(known_tree)
 909         self._check_children(children)
 910
 911         packed_children = node._pack_contents(children)
 912         children = node._unpack_contents(packed_children)
 913         self._check_children(children)
 914
 915     def _check_children(self, children):
 916         # Are all the expected child nodes there?
 917         self.failUnless(children.has_key(u'file1'))
 918         self.failUnless(children.has_key(u'file2'))
 919         self.failUnless(children.has_key(u'file3'))
 920
 921         # Are the metadata for child 3 right?
 922         file3_rocap = "URI:CHK:cmtcxq7hwxvfxan34yiev6ivhy:qvcekmjtoetdcw4kmi7b3rtblvgx7544crnwaqtiewemdliqsokq:3:10:5"
 923         file3_rwcap = "URI:CHK:cmtcxq7hwxvfxan34yiev6ivhy:qvcekmjtoetdcw4kmi7b3rtblvgx7544crnwaqtiewemdliqsokq:3:10:5"
 924         file3_metadata = {'ctime': 1246663897.4336269, 'tahoe': {'linkmotime': 1246663897.4336269, 'linkcrtime': 1246663897.4336269}, 'mtime': 1246663897.4336269}
 925         self.failUnlessEqual(file3_metadata, children[u'file3'][1])
 926         self.failUnlessEqual(file3_rocap,
 927                              children[u'file3'][0].get_readonly_uri())
 928         self.failUnlessEqual(file3_rwcap,
 929                              children[u'file3'][0].get_uri())
 930
 931         # Are the metadata for child 2 right?
 932         file2_rocap = "URI:CHK:apegrpehshwugkbh3jlt5ei6hq:5oougnemcl5xgx4ijgiumtdojlipibctjkbwvyygdymdphib2fvq:3:10:4"
 933         file2_rwcap = "URI:CHK:apegrpehshwugkbh3jlt5ei6hq:5oougnemcl5xgx4ijgiumtdojlipibctjkbwvyygdymdphib2fvq:3:10:4"
 934         file2_metadata = {'ctime': 1246663897.430218, 'tahoe': {'linkmotime': 1246663897.430218, 'linkcrtime': 1246663897.430218}, 'mtime': 1246663897.430218}
 935         self.failUnlessEqual(file2_metadata, children[u'file2'][1])
 936         self.failUnlessEqual(file2_rocap,
 937                              children[u'file2'][0].get_readonly_uri())
 938         self.failUnlessEqual(file2_rwcap,
 939                              children[u'file2'][0].get_uri())
 940
 941         # Are the metadata for child 1 right?
 942         file1_rocap = "URI:CHK:olxtimympo7f27jvhtgqlnbtn4:emzdnhk2um4seixozlkw3qx2nfijvdkx3ky7i7izl47yedl6e64a:3:10:10"
 943         file1_rwcap = "URI:CHK:olxtimympo7f27jvhtgqlnbtn4:emzdnhk2um4seixozlkw3qx2nfijvdkx3ky7i7izl47yedl6e64a:3:10:10"
 944         file1_metadata = {'ctime': 1246663897.4275661, 'tahoe': {'linkmotime': 1246663897.4275661, 'linkcrtime': 1246663897.4275661}, 'mtime': 1246663897.4275661}
 945         self.failUnlessEqual(file1_metadata, children[u'file1'][1])
 946         self.failUnlessEqual(file1_rocap,
 947                              children[u'file1'][0].get_readonly_uri())
 948         self.failUnlessEqual(file1_rwcap,
 949                              children[u'file1'][0].get_uri())
 950
 951     def _make_kids(self, nm, which):
 952         caps = {"imm": "URI:CHK:n7r3m6wmomelk4sep3kw5cvduq:os7ijw5c3maek7pg65e5254k2fzjflavtpejjyhshpsxuqzhcwwq:3:20:14861",
 953                 "lit": "URI:LIT:n5xgk", # LIT for "one"
 954                 "write": "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq",
 955                 "read": "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q",
 956                 "dirwrite": "URI:DIR2:n6x24zd3seu725yluj75q5boaa:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq",
 957                 "dirread":  "URI:DIR2-RO:b7sr5qsifnicca7cbk3rhrhbvq:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq",
 958                 }
 959         kids = {}
 960         for name in which:
 961             kids[unicode(name)] = (nm.create_from_cap(caps[name]), {})
 962         return kids
 963
 964     def test_deep_immutable(self):
 965         nm = NodeMaker(None, None, None, None, None, None, {"k": 3, "n": 10},
 966                        None)
 967         fn = MinimalFakeMutableFile()
 968
 969         kids = self._make_kids(nm, ["imm", "lit", "write", "read",
 970                                     "dirwrite", "dirread"])
 971         packed = dirnode.pack_children(fn, kids, deep_immutable=False)
 972         self.failUnlessIn("lit", packed)
 973
 974         kids = self._make_kids(nm, ["imm", "lit"])
 975         packed = dirnode.pack_children(fn, kids, deep_immutable=True)
 976         self.failUnlessIn("lit", packed)
 977
 978         kids = self._make_kids(nm, ["imm", "lit", "write"])
 979         e = self.failUnlessRaises(dirnode.MustBeDeepImmutable,
 980                                   dirnode.pack_children,
 981                                   fn, kids, deep_immutable=True)
 982
 983         # read-only is not enough: all children must be immutable
 984         kids = self._make_kids(nm, ["imm", "lit", "read"])
 985         e = self.failUnlessRaises(dirnode.MustBeDeepImmutable,
 986                                   dirnode.pack_children,
 987                                   fn, kids, deep_immutable=True)
 988
 989         kids = self._make_kids(nm, ["imm", "lit", "dirwrite"])
 990         e = self.failUnlessRaises(dirnode.MustBeDeepImmutable,
 991                                   dirnode.pack_children,
 992                                   fn, kids, deep_immutable=True)
 993
 994         kids = self._make_kids(nm, ["imm", "lit", "dirread"])
 995         e = self.failUnlessRaises(dirnode.MustBeDeepImmutable,
 996                                   dirnode.pack_children,
 997                                   fn, kids, deep_immutable=True)
 998
 999 class FakeMutableFile:
1000     implements(IMutableFileNode)
1001     counter = 0
1002     def __init__(self, initial_contents=""):
1003         self.data = self._get_initial_contents(initial_contents)
1004         counter = FakeMutableFile.counter
1005         FakeMutableFile.counter += 1
1006         writekey = hashutil.ssk_writekey_hash(str(counter))
1007         fingerprint = hashutil.ssk_pubkey_fingerprint_hash(str(counter))
1008         self.uri = uri.WriteableSSKFileURI(writekey, fingerprint)
1009
1010     def _get_initial_contents(self, contents):
1011         if isinstance(contents, str):
1012             return contents
1013         if contents is None:
1014             return ""
1015         assert callable(contents), "%s should be callable, not %s" % \
1016                (contents, type(contents))
1017         return contents(self)
1018
1019     def get_cap(self):
1020         return self.uri
1021     def get_uri(self):
1022         return self.uri.to_string()
1023     def download_best_version(self):
1024         return defer.succeed(self.data)
1025     def get_writekey(self):
1026         return "writekey"
1027     def is_readonly(self):
1028         return False
1029     def is_mutable(self):
1030         return True
1031     def modify(self, modifier):
1032         self.data = modifier(self.data, None, True)
1033         return defer.succeed(None)
1034
1035 class FakeNodeMaker(NodeMaker):
1036     def create_mutable_file(self, contents="", keysize=None):
1037         return defer.succeed(FakeMutableFile(contents))
1038
1039 class FakeClient2(Client):
1040     def __init__(self):
1041         self.nodemaker = FakeNodeMaker(None, None, None,
1042                                        None, None, None,
1043                                        {"k":3,"n":10}, None)
1044     def create_node_from_uri(self, rwcap, rocap):
1045         return self.nodemaker.create_from_cap(rwcap, rocap)
1046
1047 class Dirnode2(unittest.TestCase, testutil.ShouldFailMixin):
1048     def setUp(self):
1049         self.client = FakeClient2()
1050         self.nodemaker = self.client.nodemaker
1051
1052     def test_from_future(self):
1053         # create a dirnode that contains unknown URI types, and make sure we
1054         # tolerate them properly. Since dirnodes aren't allowed to add
1055         # unknown node types, we have to be tricky.
1056         d = self.nodemaker.create_new_mutable_directory()
1057         future_writecap = "x-tahoe-crazy://I_am_from_the_future."
1058         future_readcap = "x-tahoe-crazy-readonly://I_am_from_the_future."
1059         future_node = UnknownNode(future_writecap, future_readcap)
1060         def _then(n):
1061             self._node = n
1062             return n.set_node(u"future", future_node)
1063         d.addCallback(_then)
1064
1065         # we should be prohibited from adding an unknown URI to a directory,
1066         # since we don't know how to diminish the cap to a readcap (for the
1067         # dirnode's rocap slot), and we don't want to accidentally grant
1068         # write access to a holder of the dirnode's readcap.
1069         d.addCallback(lambda ign:
1070              self.shouldFail(CannotPackUnknownNodeError,
1071                              "copy unknown",
1072                              "cannot pack unknown node as child add",
1073                              self._node.set_uri, u"add",
1074                              future_writecap, future_readcap))
1075         d.addCallback(lambda ign: self._node.list())
1076         def _check(children):
1077             self.failUnlessEqual(len(children), 1)
1078             (fn, metadata) = children[u"future"]
1079             self.failUnless(isinstance(fn, UnknownNode), fn)
1080             self.failUnlessEqual(fn.get_uri(), future_writecap)
1081             self.failUnlessEqual(fn.get_readonly_uri(), future_readcap)
1082             # but we *should* be allowed to copy this node, because the
1083             # UnknownNode contains all the information that was in the
1084             # original directory (readcap and writecap), so we're preserving
1085             # everything.
1086             return self._node.set_node(u"copy", fn)
1087         d.addCallback(_check)
1088         d.addCallback(lambda ign: self._node.list())
1089         def _check2(children):
1090             self.failUnlessEqual(len(children), 2)
1091             (fn, metadata) = children[u"copy"]
1092             self.failUnless(isinstance(fn, UnknownNode), fn)
1093             self.failUnlessEqual(fn.get_uri(), future_writecap)
1094             self.failUnlessEqual(fn.get_readonly_uri(), future_readcap)
1095         return d
1096
1097 class DeepStats(unittest.TestCase):
1098     timeout = 240 # It takes longer than 120 seconds on Francois's arm box.
1099     def test_stats(self):
1100         ds = dirnode.DeepStats(None)
1101         ds.add("count-files")
1102         ds.add("size-immutable-files", 123)
1103         ds.histogram("size-files-histogram", 123)
1104         ds.max("largest-directory", 444)
1105
1106         s = ds.get_results()
1107         self.failUnlessEqual(s["count-files"], 1)
1108         self.failUnlessEqual(s["size-immutable-files"], 123)
1109         self.failUnlessEqual(s["largest-directory"], 444)
1110         self.failUnlessEqual(s["count-literal-files"], 0)
1111
1112         ds.add("count-files")
1113         ds.add("size-immutable-files", 321)
1114         ds.histogram("size-files-histogram", 321)
1115         ds.max("largest-directory", 2)
1116
1117         s = ds.get_results()
1118         self.failUnlessEqual(s["count-files"], 2)
1119         self.failUnlessEqual(s["size-immutable-files"], 444)
1120         self.failUnlessEqual(s["largest-directory"], 444)
1121         self.failUnlessEqual(s["count-literal-files"], 0)
1122         self.failUnlessEqual(s["size-files-histogram"],
1123                              [ (101, 316, 1), (317, 1000, 1) ])
1124
1125         ds = dirnode.DeepStats(None)
1126         for i in range(1, 1100):
1127             ds.histogram("size-files-histogram", i)
1128         ds.histogram("size-files-histogram", 4*1000*1000*1000*1000) # 4TB
1129         s = ds.get_results()
1130         self.failUnlessEqual(s["size-files-histogram"],
1131                              [ (1, 3, 3),
1132                                (4, 10, 7),
1133                                (11, 31, 21),
1134                                (32, 100, 69),
1135                                (101, 316, 216),
1136                                (317, 1000, 684),
1137                                (1001, 3162, 99),
1138                                (3162277660169L, 10000000000000L, 1),
1139                                ])
1140
1141 class UCWEingMutableFileNode(MutableFileNode):
1142     please_ucwe_after_next_upload = False
1143
1144     def _upload(self, new_contents, servermap):
1145         d = MutableFileNode._upload(self, new_contents, servermap)
1146         def _ucwe(res):
1147             if self.please_ucwe_after_next_upload:
1148                 self.please_ucwe_after_next_upload = False
1149                 raise UncoordinatedWriteError()
1150             return res
1151         d.addCallback(_ucwe)
1152         return d
1153
1154 class UCWEingNodeMaker(NodeMaker):
1155     def _create_mutable(self, cap):
1156         n = UCWEingMutableFileNode(self.storage_broker, self.secret_holder,
1157                                    self.default_encoding_parameters,
1158                                    self.history)
1159         return n.init_from_cap(cap)
1160
1161
1162 class Deleter(GridTestMixin, unittest.TestCase):
1163     timeout = 3600 # It takes longer than 433 seconds on Zandr's ARM box.
1164     def test_retry(self):
1165         # ticket #550, a dirnode.delete which experiences an
1166         # UncoordinatedWriteError will fail with an incorrect "you're
1167         # deleting something which isn't there" NoSuchChildError exception.
1168
1169         # to trigger this, we start by creating a directory with a single
1170         # file in it. Then we create a special dirnode that uses a modified
1171         # MutableFileNode which will raise UncoordinatedWriteError once on
1172         # demand. We then call dirnode.delete, which ought to retry and
1173         # succeed.
1174
1175         self.basedir = self.mktemp()
1176         self.set_up_grid()
1177         c0 = self.g.clients[0]
1178         d = c0.create_dirnode()
1179         small = upload.Data("Small enough for a LIT", None)
1180         def _created_dir(dn):
1181             self.root = dn
1182             self.root_uri = dn.get_uri()
1183             return dn.add_file(u"file", small)
1184         d.addCallback(_created_dir)
1185         def _do_delete(ignored):
1186             nm = UCWEingNodeMaker(c0.storage_broker, c0._secret_holder,
1187                                   c0.get_history(), c0.getServiceNamed("uploader"),
1188                                   c0.downloader,
1189                                   c0.download_cache_dirman,
1190                                   c0.get_encoding_parameters(),
1191                                   c0._key_generator)
1192             n = nm.create_from_cap(self.root_uri)
1193             assert n._node.please_ucwe_after_next_upload == False
1194             n._node.please_ucwe_after_next_upload = True
1195             # This should succeed, not raise an exception
1196             return n.delete(u"file")
1197         d.addCallback(_do_delete)
1198
1199         return d
1200
1201 class Adder(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin):
1202
1203     def test_overwrite(self):
1204         # note: This functionality could be tested without actually creating
1205         # several RSA keys. It would be faster without the GridTestMixin: use
1206         # dn.set_node(nodemaker.create_from_cap(make_chk_file_uri())) instead
1207         # of dn.add_file, and use a special NodeMaker that creates fake
1208         # mutable files.
1209         self.basedir = "dirnode/Adder/test_overwrite"
1210         self.set_up_grid()
1211         c = self.g.clients[0]
1212         fileuri = make_chk_file_uri(1234)
1213         filenode = c.nodemaker.create_from_cap(fileuri)
1214         d = c.create_dirnode()
1215
1216         def _create_directory_tree(root_node):
1217             # Build
1218             # root/file1
1219             # root/file2
1220             # root/dir1
1221             d = root_node.add_file(u'file1', upload.Data("Important Things",
1222                 None))
1223             d.addCallback(lambda res:
1224                 root_node.add_file(u'file2', upload.Data("Sekrit Codes", None)))
1225             d.addCallback(lambda res:
1226                 root_node.create_subdirectory(u"dir1"))
1227             d.addCallback(lambda res: root_node)
1228             return d
1229
1230         d.addCallback(_create_directory_tree)
1231
1232         def _test_adder(root_node):
1233             d = root_node.set_node(u'file1', filenode)
1234             # We've overwritten file1. Let's try it with a directory
1235             d.addCallback(lambda res:
1236                 root_node.create_subdirectory(u'dir2'))
1237             d.addCallback(lambda res:
1238                 root_node.set_node(u'dir2', filenode))
1239             # We try overwriting a file with a child while also specifying
1240             # overwrite=False. We should receive an ExistingChildError
1241             # when we do this.
1242             d.addCallback(lambda res:
1243                 self.shouldFail(ExistingChildError, "set_node",
1244                                 "child 'file1' already exists",
1245                                root_node.set_node, u"file1",
1246                                filenode, overwrite=False))
1247             # If we try with a directory, we should see the same thing
1248             d.addCallback(lambda res:
1249                 self.shouldFail(ExistingChildError, "set_node",
1250                                 "child 'dir1' already exists",
1251                                 root_node.set_node, u'dir1', filenode,
1252                                 overwrite=False))
1253             d.addCallback(lambda res:
1254                  root_node.set_node(u'file1', filenode,
1255                                     overwrite="only-files"))
1256             d.addCallback(lambda res:
1257                  self.shouldFail(ExistingChildError, "set_node",
1258                                 "child 'dir1' already exists",
1259                                 root_node.set_node, u'dir1', filenode,
1260                                 overwrite="only-files"))
1261             return d
1262
1263         d.addCallback(_test_adder)
1264         return d