src/allmydata/test/test_dirnode.py

   1
   2 import time
   3 from twisted.trial import unittest
   4 from twisted.internet import defer
   5 from allmydata import uri, dirnode
   6 from allmydata.client import Client
   7 from allmydata.immutable import upload
   8 from allmydata.interfaces import IFileNode, \
   9      ExistingChildError, NoSuchChildError, \
  10      IDeepCheckResults, IDeepCheckAndRepairResults, CannotPackUnknownNodeError
  11 from allmydata.mutable.filenode import MutableFileNode
  12 from allmydata.mutable.common import UncoordinatedWriteError
  13 from allmydata.util import hashutil, base32
  14 from allmydata.monitor import Monitor
  15 from allmydata.test.common import make_chk_file_uri, make_mutable_file_uri, \
  16      ErrorMixin
  17 from allmydata.test.no_network import GridTestMixin
  18 from allmydata.unknown import UnknownNode
  19 from allmydata.nodemaker import NodeMaker
  20 from base64 import b32decode
  21 import common_util as testutil
  22
  23 class Dirnode(GridTestMixin, unittest.TestCase,
  24               testutil.ShouldFailMixin, testutil.StallMixin, ErrorMixin):
  25     timeout = 240 # It takes longer than 120 seconds on Francois's arm box.
  26
  27     def test_basic(self):
  28         self.basedir = "dirnode/Dirnode/test_basic"
  29         self.set_up_grid()
  30         c = self.g.clients[0]
  31         d = c.create_dirnode()
  32         def _done(res):
  33             self.failUnless(isinstance(res, dirnode.DirectoryNode))
  34             rep = str(res)
  35             self.failUnless("RW-MUT" in rep)
  36         d.addCallback(_done)
  37         return d
  38
  39     def test_initial_children(self):
  40         self.basedir = "dirnode/Dirnode/test_initial_children"
  41         self.set_up_grid()
  42         c = self.g.clients[0]
  43         nm = c.nodemaker
  44         setup_py_uri = "URI:CHK:n7r3m6wmomelk4sep3kw5cvduq:os7ijw5c3maek7pg65e5254k2fzjflavtpejjyhshpsxuqzhcwwq:3:20:14861"
  45         one_uri = "URI:LIT:n5xgk" # LIT for "one"
  46         kids = {u"one": (nm.create_from_cap(one_uri), {}),
  47                 u"two": (nm.create_from_cap(setup_py_uri),
  48                          {"metakey": "metavalue"}),
  49                 }
  50         d = c.create_dirnode(kids)
  51         def _created(dn):
  52             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
  53             rep = str(dn)
  54             self.failUnless("RW-MUT" in rep)
  55             return dn.list()
  56         d.addCallback(_created)
  57         def _check_kids(children):
  58             self.failUnlessEqual(sorted(children.keys()), [u"one", u"two"])
  59             one_node, one_metadata = children[u"one"]
  60             two_node, two_metadata = children[u"two"]
  61             self.failUnlessEqual(one_node.get_size(), 3)
  62             self.failUnlessEqual(two_node.get_size(), 14861)
  63             self.failUnless(isinstance(one_metadata, dict), one_metadata)
  64             self.failUnlessEqual(two_metadata["metakey"], "metavalue")
  65         d.addCallback(_check_kids)
  66         d.addCallback(lambda ign: nm.create_new_mutable_directory(kids))
  67         d.addCallback(lambda dn: dn.list())
  68         d.addCallback(_check_kids)
  69         future_writecap = "x-tahoe-crazy://I_am_from_the_future."
  70         future_readcap = "x-tahoe-crazy-readonly://I_am_from_the_future."
  71         future_node = UnknownNode(future_writecap, future_readcap)
  72         bad_kids1 = {u"one": (future_node, {})}
  73         d.addCallback(lambda ign:
  74                       self.shouldFail(AssertionError, "bad_kids1",
  75                                       "does not accept UnknownNode",
  76                                       nm.create_new_mutable_directory,
  77                                       bad_kids1))
  78         bad_kids2 = {u"one": (nm.create_from_cap(one_uri), None)}
  79         d.addCallback(lambda ign:
  80                       self.shouldFail(AssertionError, "bad_kids2",
  81                                       "requires metadata to be a dict",
  82                                       nm.create_new_mutable_directory,
  83                                       bad_kids2))
  84         return d
  85
  86     def test_immutable(self):
  87         self.basedir = "dirnode/Dirnode/test_immutable"
  88         self.set_up_grid()
  89         c = self.g.clients[0]
  90         nm = c.nodemaker
  91         setup_py_uri = "URI:CHK:n7r3m6wmomelk4sep3kw5cvduq:os7ijw5c3maek7pg65e5254k2fzjflavtpejjyhshpsxuqzhcwwq:3:20:14861"
  92         one_uri = "URI:LIT:n5xgk" # LIT for "one"
  93         mut_readcap = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
  94         mut_writecap = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
  95         kids = {u"one": (nm.create_from_cap(one_uri), {}),
  96                 u"two": (nm.create_from_cap(setup_py_uri),
  97                          {"metakey": "metavalue"}),
  98                 }
  99         d = c.create_immutable_dirnode(kids)
 100         def _created(dn):
 101             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
 102             self.failIf(dn.is_mutable())
 103             self.failUnless(dn.is_readonly())
 104             rep = str(dn)
 105             self.failUnless("RO-IMM" in rep)
 106             cap = dn.get_cap()
 107             self.failUnlessIn("CHK", cap.to_string())
 108             self.cap = cap
 109             return dn.list()
 110         d.addCallback(_created)
 111         def _check_kids(children):
 112             self.failUnlessEqual(sorted(children.keys()), [u"one", u"two"])
 113             one_node, one_metadata = children[u"one"]
 114             two_node, two_metadata = children[u"two"]
 115             self.failUnlessEqual(one_node.get_size(), 3)
 116             self.failUnlessEqual(two_node.get_size(), 14861)
 117             self.failUnless(isinstance(one_metadata, dict), one_metadata)
 118             self.failUnlessEqual(two_metadata["metakey"], "metavalue")
 119         d.addCallback(_check_kids)
 120         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
 121         d.addCallback(lambda dn: dn.list())
 122         d.addCallback(_check_kids)
 123         future_writecap = "x-tahoe-crazy://I_am_from_the_future."
 124         future_readcap = "x-tahoe-crazy-readonly://I_am_from_the_future."
 125         future_node = UnknownNode(future_writecap, future_readcap)
 126         bad_kids1 = {u"one": (future_node, {})}
 127         d.addCallback(lambda ign:
 128                       self.shouldFail(AssertionError, "bad_kids1",
 129                                       "does not accept UnknownNode",
 130                                       c.create_immutable_dirnode,
 131                                       bad_kids1))
 132         bad_kids2 = {u"one": (nm.create_from_cap(one_uri), None)}
 133         d.addCallback(lambda ign:
 134                       self.shouldFail(AssertionError, "bad_kids2",
 135                                       "requires metadata to be a dict",
 136                                       c.create_immutable_dirnode,
 137                                       bad_kids2))
 138         bad_kids3 = {u"one": (nm.create_from_cap(mut_writecap), {})}
 139         d.addCallback(lambda ign:
 140                       self.shouldFail(AssertionError, "bad_kids3",
 141                                       "create_immutable_directory requires immutable children",
 142                                       c.create_immutable_dirnode,
 143                                       bad_kids3))
 144         bad_kids4 = {u"one": (nm.create_from_cap(mut_readcap), {})}
 145         d.addCallback(lambda ign:
 146                       self.shouldFail(AssertionError, "bad_kids4",
 147                                       "create_immutable_directory requires immutable children",
 148                                       c.create_immutable_dirnode,
 149                                       bad_kids4))
 150         d.addCallback(lambda ign: c.create_immutable_dirnode({}))
 151         def _created_empty(dn):
 152             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
 153             self.failIf(dn.is_mutable())
 154             self.failUnless(dn.is_readonly())
 155             rep = str(dn)
 156             self.failUnless("RO-IMM" in rep)
 157             cap = dn.get_cap()
 158             self.failUnlessIn("LIT", cap.to_string())
 159             self.failUnlessEqual(cap.to_string(), "URI:DIR2-LIT:")
 160             self.cap = cap
 161             return dn.list()
 162         d.addCallback(_created_empty)
 163         d.addCallback(lambda kids: self.failUnlessEqual(kids, {}))
 164         smallkids = {u"o": (nm.create_from_cap(one_uri), {})}
 165         d.addCallback(lambda ign: c.create_immutable_dirnode(smallkids))
 166         def _created_small(dn):
 167             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
 168             self.failIf(dn.is_mutable())
 169             self.failUnless(dn.is_readonly())
 170             rep = str(dn)
 171             self.failUnless("RO-IMM" in rep)
 172             cap = dn.get_cap()
 173             self.failUnlessIn("LIT", cap.to_string())
 174             self.failUnlessEqual(cap.to_string(),
 175                                  "URI:DIR2-LIT:gi4tumj2n4wdcmz2kvjesosmjfkdu3rvpbtwwlbqhiwdeot3puwcy")
 176             self.cap = cap
 177             return dn.list()
 178         d.addCallback(_created_small)
 179         d.addCallback(lambda kids: self.failUnlessEqual(kids.keys(), [u"o"]))
 180         return d
 181
 182     def test_check(self):
 183         self.basedir = "dirnode/Dirnode/test_check"
 184         self.set_up_grid()
 185         c = self.g.clients[0]
 186         d = c.create_dirnode()
 187         d.addCallback(lambda dn: dn.check(Monitor()))
 188         def _done(res):
 189             self.failUnless(res.is_healthy())
 190         d.addCallback(_done)
 191         return d
 192
 193     def _test_deepcheck_create(self):
 194         # create a small tree with a loop, and some non-directories
 195         #  root/
 196         #  root/subdir/
 197         #  root/subdir/file1
 198         #  root/subdir/link -> root
 199         #  root/rodir
 200         c = self.g.clients[0]
 201         d = c.create_dirnode()
 202         def _created_root(rootnode):
 203             self._rootnode = rootnode
 204             return rootnode.create_subdirectory(u"subdir")
 205         d.addCallback(_created_root)
 206         def _created_subdir(subdir):
 207             self._subdir = subdir
 208             d = subdir.add_file(u"file1", upload.Data("data"*100, None))
 209             d.addCallback(lambda res: subdir.set_node(u"link", self._rootnode))
 210             d.addCallback(lambda res: c.create_dirnode())
 211             d.addCallback(lambda dn:
 212                           self._rootnode.set_uri(u"rodir",
 213                                                  dn.get_uri(),
 214                                                  dn.get_readonly_uri()))
 215             return d
 216         d.addCallback(_created_subdir)
 217         def _done(res):
 218             return self._rootnode
 219         d.addCallback(_done)
 220         return d
 221
 222     def test_deepcheck(self):
 223         self.basedir = "dirnode/Dirnode/test_deepcheck"
 224         self.set_up_grid()
 225         d = self._test_deepcheck_create()
 226         d.addCallback(lambda rootnode: rootnode.start_deep_check().when_done())
 227         def _check_results(r):
 228             self.failUnless(IDeepCheckResults.providedBy(r))
 229             c = r.get_counters()
 230             self.failUnlessEqual(c,
 231                                  {"count-objects-checked": 4,
 232                                   "count-objects-healthy": 4,
 233                                   "count-objects-unhealthy": 0,
 234                                   "count-objects-unrecoverable": 0,
 235                                   "count-corrupt-shares": 0,
 236                                   })
 237             self.failIf(r.get_corrupt_shares())
 238             self.failUnlessEqual(len(r.get_all_results()), 4)
 239         d.addCallback(_check_results)
 240         return d
 241
 242     def test_deepcheck_and_repair(self):
 243         self.basedir = "dirnode/Dirnode/test_deepcheck_and_repair"
 244         self.set_up_grid()
 245         d = self._test_deepcheck_create()
 246         d.addCallback(lambda rootnode:
 247                       rootnode.start_deep_check_and_repair().when_done())
 248         def _check_results(r):
 249             self.failUnless(IDeepCheckAndRepairResults.providedBy(r))
 250             c = r.get_counters()
 251             self.failUnlessEqual(c,
 252                                  {"count-objects-checked": 4,
 253                                   "count-objects-healthy-pre-repair": 4,
 254                                   "count-objects-unhealthy-pre-repair": 0,
 255                                   "count-objects-unrecoverable-pre-repair": 0,
 256                                   "count-corrupt-shares-pre-repair": 0,
 257                                   "count-objects-healthy-post-repair": 4,
 258                                   "count-objects-unhealthy-post-repair": 0,
 259                                   "count-objects-unrecoverable-post-repair": 0,
 260                                   "count-corrupt-shares-post-repair": 0,
 261                                   "count-repairs-attempted": 0,
 262                                   "count-repairs-successful": 0,
 263                                   "count-repairs-unsuccessful": 0,
 264                                   })
 265             self.failIf(r.get_corrupt_shares())
 266             self.failIf(r.get_remaining_corrupt_shares())
 267             self.failUnlessEqual(len(r.get_all_results()), 4)
 268         d.addCallback(_check_results)
 269         return d
 270
 271     def _mark_file_bad(self, rootnode):
 272         si = rootnode.get_storage_index()
 273         self.delete_shares_numbered(rootnode.get_uri(), [0])
 274         return rootnode
 275
 276     def test_deepcheck_problems(self):
 277         self.basedir = "dirnode/Dirnode/test_deepcheck_problems"
 278         self.set_up_grid()
 279         d = self._test_deepcheck_create()
 280         d.addCallback(lambda rootnode: self._mark_file_bad(rootnode))
 281         d.addCallback(lambda rootnode: rootnode.start_deep_check().when_done())
 282         def _check_results(r):
 283             c = r.get_counters()
 284             self.failUnlessEqual(c,
 285                                  {"count-objects-checked": 4,
 286                                   "count-objects-healthy": 3,
 287                                   "count-objects-unhealthy": 1,
 288                                   "count-objects-unrecoverable": 0,
 289                                   "count-corrupt-shares": 0,
 290                                   })
 291             #self.failUnlessEqual(len(r.get_problems()), 1) # TODO
 292         d.addCallback(_check_results)
 293         return d
 294
 295     def test_readonly(self):
 296         self.basedir = "dirnode/Dirnode/test_readonly"
 297         self.set_up_grid()
 298         c = self.g.clients[0]
 299         nm = c.nodemaker
 300         filecap = make_chk_file_uri(1234)
 301         filenode = nm.create_from_cap(filecap)
 302         uploadable = upload.Data("some data", convergence="some convergence string")
 303
 304         d = c.create_dirnode()
 305         def _created(rw_dn):
 306             d2 = rw_dn.set_uri(u"child", filecap, filecap)
 307             d2.addCallback(lambda res: rw_dn)
 308             return d2
 309         d.addCallback(_created)
 310
 311         def _ready(rw_dn):
 312             ro_uri = rw_dn.get_readonly_uri()
 313             ro_dn = c.create_node_from_uri(ro_uri)
 314             self.failUnless(ro_dn.is_readonly())
 315             self.failUnless(ro_dn.is_mutable())
 316
 317             self.shouldFail(dirnode.NotMutableError, "set_uri ro", None,
 318                             ro_dn.set_uri, u"newchild", filecap, filecap)
 319             self.shouldFail(dirnode.NotMutableError, "set_uri ro", None,
 320                             ro_dn.set_node, u"newchild", filenode)
 321             self.shouldFail(dirnode.NotMutableError, "set_nodes ro", None,
 322                             ro_dn.set_nodes, { u"newchild": (filenode, None) })
 323             self.shouldFail(dirnode.NotMutableError, "set_uri ro", None,
 324                             ro_dn.add_file, u"newchild", uploadable)
 325             self.shouldFail(dirnode.NotMutableError, "set_uri ro", None,
 326                             ro_dn.delete, u"child")
 327             self.shouldFail(dirnode.NotMutableError, "set_uri ro", None,
 328                             ro_dn.create_subdirectory, u"newchild")
 329             self.shouldFail(dirnode.NotMutableError, "set_metadata_for ro", None,
 330                             ro_dn.set_metadata_for, u"child", {})
 331             self.shouldFail(dirnode.NotMutableError, "set_uri ro", None,
 332                             ro_dn.move_child_to, u"child", rw_dn)
 333             self.shouldFail(dirnode.NotMutableError, "set_uri ro", None,
 334                             rw_dn.move_child_to, u"child", ro_dn)
 335             return ro_dn.list()
 336         d.addCallback(_ready)
 337         def _listed(children):
 338             self.failUnless(u"child" in children)
 339         d.addCallback(_listed)
 340         return d
 341
 342     def failUnlessGreaterThan(self, a, b):
 343         self.failUnless(a > b, "%r should be > %r" % (a, b))
 344
 345     def failUnlessGreaterOrEqualThan(self, a, b):
 346         self.failUnless(a >= b, "%r should be >= %r" % (a, b))
 347
 348     def test_create(self):
 349         self.basedir = "dirnode/Dirnode/test_create"
 350         self.set_up_grid()
 351         c = self.g.clients[0]
 352
 353         self.expected_manifest = []
 354         self.expected_verifycaps = set()
 355         self.expected_storage_indexes = set()
 356
 357         d = c.create_dirnode()
 358         def _then(n):
 359             # /
 360             self.rootnode = n
 361             self.failUnless(n.is_mutable())
 362             u = n.get_uri()
 363             self.failUnless(u)
 364             self.failUnless(u.startswith("URI:DIR2:"), u)
 365             u_ro = n.get_readonly_uri()
 366             self.failUnless(u_ro.startswith("URI:DIR2-RO:"), u_ro)
 367             u_v = n.get_verify_cap().to_string()
 368             self.failUnless(u_v.startswith("URI:DIR2-Verifier:"), u_v)
 369             u_r = n.get_repair_cap().to_string()
 370             self.failUnlessEqual(u_r, u)
 371             self.expected_manifest.append( ((), u) )
 372             self.expected_verifycaps.add(u_v)
 373             si = n.get_storage_index()
 374             self.expected_storage_indexes.add(base32.b2a(si))
 375             expected_si = n._uri._filenode_uri.storage_index
 376             self.failUnlessEqual(si, expected_si)
 377
 378             d = n.list()
 379             d.addCallback(lambda res: self.failUnlessEqual(res, {}))
 380             d.addCallback(lambda res: n.has_child(u"missing"))
 381             d.addCallback(lambda res: self.failIf(res))
 382
 383             fake_file_uri = make_mutable_file_uri()
 384             other_file_uri = make_mutable_file_uri()
 385             m = c.nodemaker.create_from_cap(fake_file_uri)
 386             ffu_v = m.get_verify_cap().to_string()
 387             self.expected_manifest.append( ((u"child",) , m.get_uri()) )
 388             self.expected_verifycaps.add(ffu_v)
 389             self.expected_storage_indexes.add(base32.b2a(m.get_storage_index()))
 390             d.addCallback(lambda res: n.set_uri(u"child",
 391                                                 fake_file_uri, fake_file_uri))
 392             d.addCallback(lambda res:
 393                           self.shouldFail(ExistingChildError, "set_uri-no",
 394                                           "child 'child' already exists",
 395                                           n.set_uri, u"child",
 396                                           other_file_uri, other_file_uri,
 397                                           overwrite=False))
 398             # /
 399             # /child = mutable
 400
 401             d.addCallback(lambda res: n.create_subdirectory(u"subdir"))
 402
 403             # /
 404             # /child = mutable
 405             # /subdir = directory
 406             def _created(subdir):
 407                 self.failUnless(isinstance(subdir, dirnode.DirectoryNode))
 408                 self.subdir = subdir
 409                 new_v = subdir.get_verify_cap().to_string()
 410                 assert isinstance(new_v, str)
 411                 self.expected_manifest.append( ((u"subdir",), subdir.get_uri()) )
 412                 self.expected_verifycaps.add(new_v)
 413                 si = subdir.get_storage_index()
 414                 self.expected_storage_indexes.add(base32.b2a(si))
 415             d.addCallback(_created)
 416
 417             d.addCallback(lambda res:
 418                           self.shouldFail(ExistingChildError, "mkdir-no",
 419                                           "child 'subdir' already exists",
 420                                           n.create_subdirectory, u"subdir",
 421                                           overwrite=False))
 422
 423             d.addCallback(lambda res: n.list())
 424             d.addCallback(lambda children:
 425                           self.failUnlessEqual(sorted(children.keys()),
 426                                                sorted([u"child", u"subdir"])))
 427
 428             d.addCallback(lambda res: n.start_deep_stats().when_done())
 429             def _check_deepstats(stats):
 430                 self.failUnless(isinstance(stats, dict))
 431                 expected = {"count-immutable-files": 0,
 432                             "count-mutable-files": 1,
 433                             "count-literal-files": 0,
 434                             "count-files": 1,
 435                             "count-directories": 2,
 436                             "size-immutable-files": 0,
 437                             "size-literal-files": 0,
 438                             #"size-directories": 616, # varies
 439                             #"largest-directory": 616,
 440                             "largest-directory-children": 2,
 441                             "largest-immutable-file": 0,
 442                             }
 443                 for k,v in expected.iteritems():
 444                     self.failUnlessEqual(stats[k], v,
 445                                          "stats[%s] was %s, not %s" %
 446                                          (k, stats[k], v))
 447                 self.failUnless(stats["size-directories"] > 500,
 448                                 stats["size-directories"])
 449                 self.failUnless(stats["largest-directory"] > 500,
 450                                 stats["largest-directory"])
 451                 self.failUnlessEqual(stats["size-files-histogram"], [])
 452             d.addCallback(_check_deepstats)
 453
 454             d.addCallback(lambda res: n.build_manifest().when_done())
 455             def _check_manifest(res):
 456                 manifest = res["manifest"]
 457                 self.failUnlessEqual(sorted(manifest),
 458                                      sorted(self.expected_manifest))
 459                 stats = res["stats"]
 460                 _check_deepstats(stats)
 461                 self.failUnlessEqual(self.expected_verifycaps,
 462                                      res["verifycaps"])
 463                 self.failUnlessEqual(self.expected_storage_indexes,
 464                                      res["storage-index"])
 465             d.addCallback(_check_manifest)
 466
 467             def _add_subsubdir(res):
 468                 return self.subdir.create_subdirectory(u"subsubdir")
 469             d.addCallback(_add_subsubdir)
 470             # /
 471             # /child = mutable
 472             # /subdir = directory
 473             # /subdir/subsubdir = directory
 474             d.addCallback(lambda res: n.get_child_at_path(u"subdir/subsubdir"))
 475             d.addCallback(lambda subsubdir:
 476                           self.failUnless(isinstance(subsubdir,
 477                                                      dirnode.DirectoryNode)))
 478             d.addCallback(lambda res: n.get_child_at_path(u""))
 479             d.addCallback(lambda res: self.failUnlessEqual(res.get_uri(),
 480                                                            n.get_uri()))
 481
 482             d.addCallback(lambda res: n.get_metadata_for(u"child"))
 483             d.addCallback(lambda metadata:
 484                           self.failUnlessEqual(set(metadata.keys()),
 485                                                set(["tahoe", "ctime", "mtime"])))
 486
 487             d.addCallback(lambda res:
 488                           self.shouldFail(NoSuchChildError, "gcamap-no",
 489                                           "nope",
 490                                           n.get_child_and_metadata_at_path,
 491                                           u"subdir/nope"))
 492             d.addCallback(lambda res:
 493                           n.get_child_and_metadata_at_path(u""))
 494             def _check_child_and_metadata1(res):
 495                 child, metadata = res
 496                 self.failUnless(isinstance(child, dirnode.DirectoryNode))
 497                 # edge-metadata needs at least one path segment
 498                 self.failUnlessEqual(sorted(metadata.keys()), [])
 499             d.addCallback(_check_child_and_metadata1)
 500             d.addCallback(lambda res:
 501                           n.get_child_and_metadata_at_path(u"child"))
 502
 503             def _check_child_and_metadata2(res):
 504                 child, metadata = res
 505                 self.failUnlessEqual(child.get_uri(),
 506                                      fake_file_uri)
 507                 self.failUnlessEqual(set(metadata.keys()),
 508                                      set(["tahoe", "ctime", "mtime"]))
 509             d.addCallback(_check_child_and_metadata2)
 510
 511             d.addCallback(lambda res:
 512                           n.get_child_and_metadata_at_path(u"subdir/subsubdir"))
 513             def _check_child_and_metadata3(res):
 514                 child, metadata = res
 515                 self.failUnless(isinstance(child, dirnode.DirectoryNode))
 516                 self.failUnlessEqual(set(metadata.keys()),
 517                                      set(["tahoe", "ctime", "mtime"]))
 518             d.addCallback(_check_child_and_metadata3)
 519
 520             # set_uri + metadata
 521             # it should be possible to add a child without any metadata
 522             d.addCallback(lambda res: n.set_uri(u"c2",
 523                                                 fake_file_uri, fake_file_uri,
 524                                                 {}))
 525             d.addCallback(lambda res: n.get_metadata_for(u"c2"))
 526             d.addCallback(lambda metadata: self.failUnlessEqual(metadata.keys(), ['tahoe']))
 527
 528             # You can't override the link timestamps.
 529             d.addCallback(lambda res: n.set_uri(u"c2",
 530                                                 fake_file_uri, fake_file_uri,
 531                                                 { 'tahoe': {'linkcrtime': "bogus"}}))
 532             d.addCallback(lambda res: n.get_metadata_for(u"c2"))
 533             def _has_good_linkcrtime(metadata):
 534                 self.failUnless(metadata.has_key('tahoe'))
 535                 self.failUnless(metadata['tahoe'].has_key('linkcrtime'))
 536                 self.failIfEqual(metadata['tahoe']['linkcrtime'], 'bogus')
 537             d.addCallback(_has_good_linkcrtime)
 538
 539             # if we don't set any defaults, the child should get timestamps
 540             d.addCallback(lambda res: n.set_uri(u"c3",
 541                                                 fake_file_uri, fake_file_uri))
 542             d.addCallback(lambda res: n.get_metadata_for(u"c3"))
 543             d.addCallback(lambda metadata:
 544                           self.failUnlessEqual(set(metadata.keys()),
 545                                                set(["tahoe", "ctime", "mtime"])))
 546
 547             # or we can add specific metadata at set_uri() time, which
 548             # overrides the timestamps
 549             d.addCallback(lambda res: n.set_uri(u"c4",
 550                                                 fake_file_uri, fake_file_uri,
 551                                                 {"key": "value"}))
 552             d.addCallback(lambda res: n.get_metadata_for(u"c4"))
 553             d.addCallback(lambda metadata:
 554                               self.failUnless((set(metadata.keys()) == set(["key", "tahoe"])) and
 555                                               (metadata['key'] == "value"), metadata))
 556
 557             d.addCallback(lambda res: n.delete(u"c2"))
 558             d.addCallback(lambda res: n.delete(u"c3"))
 559             d.addCallback(lambda res: n.delete(u"c4"))
 560
 561             # set_node + metadata
 562             # it should be possible to add a child without any metadata
 563             d.addCallback(lambda res: n.set_node(u"d2", n, {}))
 564             d.addCallback(lambda res: c.create_dirnode())
 565             d.addCallback(lambda n2:
 566                           self.shouldFail(ExistingChildError, "set_node-no",
 567                                           "child 'd2' already exists",
 568                                           n.set_node, u"d2", n2,
 569                                           overwrite=False))
 570             d.addCallback(lambda res: n.get_metadata_for(u"d2"))
 571             d.addCallback(lambda metadata: self.failUnlessEqual(metadata.keys(), ['tahoe']))
 572
 573             # if we don't set any defaults, the child should get timestamps
 574             d.addCallback(lambda res: n.set_node(u"d3", n))
 575             d.addCallback(lambda res: n.get_metadata_for(u"d3"))
 576             d.addCallback(lambda metadata:
 577                           self.failUnlessEqual(set(metadata.keys()),
 578                                                set(["tahoe", "ctime", "mtime"])))
 579
 580             # or we can add specific metadata at set_node() time, which
 581             # overrides the timestamps
 582             d.addCallback(lambda res: n.set_node(u"d4", n,
 583                                                 {"key": "value"}))
 584             d.addCallback(lambda res: n.get_metadata_for(u"d4"))
 585             d.addCallback(lambda metadata:
 586                           self.failUnless((set(metadata.keys()) == set(["key", "tahoe"])) and
 587                                           (metadata['key'] == "value"), metadata))
 588
 589             d.addCallback(lambda res: n.delete(u"d2"))
 590             d.addCallback(lambda res: n.delete(u"d3"))
 591             d.addCallback(lambda res: n.delete(u"d4"))
 592
 593             # metadata through set_children()
 594             d.addCallback(lambda res:
 595                           n.set_children({
 596                               u"e1": (fake_file_uri, fake_file_uri),
 597                               u"e2": (fake_file_uri, fake_file_uri, {}),
 598                               u"e3": (fake_file_uri, fake_file_uri,
 599                                       {"key": "value"}),
 600                               }))
 601             d.addCallback(lambda n2: self.failUnlessIdentical(n2, n))
 602             d.addCallback(lambda res:
 603                           self.shouldFail(ExistingChildError, "set_children-no",
 604                                           "child 'e1' already exists",
 605                                           n.set_children,
 606                                           { u"e1": (other_file_uri,
 607                                                     other_file_uri),
 608                                             u"new": (other_file_uri,
 609                                                      other_file_uri),
 610                                             },
 611                                           overwrite=False))
 612             # and 'new' should not have been created
 613             d.addCallback(lambda res: n.list())
 614             d.addCallback(lambda children: self.failIf(u"new" in children))
 615             d.addCallback(lambda res: n.get_metadata_for(u"e1"))
 616             d.addCallback(lambda metadata:
 617                           self.failUnlessEqual(set(metadata.keys()),
 618                                                set(["tahoe", "ctime", "mtime"])))
 619             d.addCallback(lambda res: n.get_metadata_for(u"e2"))
 620             d.addCallback(lambda metadata:
 621                           self.failUnlessEqual(set(metadata.keys()), set(['tahoe'])))
 622             d.addCallback(lambda res: n.get_metadata_for(u"e3"))
 623             d.addCallback(lambda metadata:
 624                               self.failUnless((set(metadata.keys()) == set(["key", "tahoe"]))
 625                                               and (metadata['key'] == "value"), metadata))
 626
 627             d.addCallback(lambda res: n.delete(u"e1"))
 628             d.addCallback(lambda res: n.delete(u"e2"))
 629             d.addCallback(lambda res: n.delete(u"e3"))
 630
 631             # metadata through set_nodes()
 632             d.addCallback(lambda res:
 633                           n.set_nodes({ u"f1": (n, None),
 634                                         u"f2": (n, {}),
 635                                         u"f3": (n, {"key": "value"}),
 636                                         }))
 637             d.addCallback(lambda n2: self.failUnlessIdentical(n2, n))
 638             d.addCallback(lambda res:
 639                           self.shouldFail(ExistingChildError, "set_nodes-no",
 640                                           "child 'f1' already exists",
 641                                           n.set_nodes, { u"f1": (n, None),
 642                                                          u"new": (n, None), },
 643                                           overwrite=False))
 644             # and 'new' should not have been created
 645             d.addCallback(lambda res: n.list())
 646             d.addCallback(lambda children: self.failIf(u"new" in children))
 647             d.addCallback(lambda res: n.get_metadata_for(u"f1"))
 648             d.addCallback(lambda metadata:
 649                           self.failUnlessEqual(set(metadata.keys()),
 650                                                set(["tahoe", "ctime", "mtime"])))
 651             d.addCallback(lambda res: n.get_metadata_for(u"f2"))
 652             d.addCallback(
 653                 lambda metadata: self.failUnlessEqual(set(metadata.keys()), set(['tahoe'])))
 654             d.addCallback(lambda res: n.get_metadata_for(u"f3"))
 655             d.addCallback(lambda metadata:
 656                               self.failUnless((set(metadata.keys()) == set(["key", "tahoe"])) and
 657                                               (metadata['key'] == "value"), metadata))
 658
 659             d.addCallback(lambda res: n.delete(u"f1"))
 660             d.addCallback(lambda res: n.delete(u"f2"))
 661             d.addCallback(lambda res: n.delete(u"f3"))
 662
 663
 664             d.addCallback(lambda res:
 665                           n.set_metadata_for(u"child",
 666                                              {"tags": ["web2.0-compatible"]}))
 667             d.addCallback(lambda n1: n1.get_metadata_for(u"child"))
 668             d.addCallback(lambda metadata:
 669                           self.failUnlessEqual(metadata,
 670                                                {"tags": ["web2.0-compatible"]}))
 671
 672             def _start(res):
 673                 self._start_timestamp = time.time()
 674             d.addCallback(_start)
 675             # simplejson-1.7.1 (as shipped on Ubuntu 'gutsy') rounds all
 676             # floats to hundredeths (it uses str(num) instead of repr(num)).
 677             # simplejson-1.7.3 does not have this bug. To prevent this bug
 678             # from causing the test to fail, stall for more than a few
 679             # hundrededths of a second.
 680             d.addCallback(self.stall, 0.1)
 681             d.addCallback(lambda res: n.add_file(u"timestamps",
 682                                                  upload.Data("stamp me", convergence="some convergence string")))
 683             d.addCallback(self.stall, 0.1)
 684             def _stop(res):
 685                 self._stop_timestamp = time.time()
 686             d.addCallback(_stop)
 687
 688             d.addCallback(lambda res: n.get_metadata_for(u"timestamps"))
 689             def _check_timestamp1(metadata):
 690                 self.failUnless("ctime" in metadata)
 691                 self.failUnless("mtime" in metadata)
 692                 self.failUnlessGreaterOrEqualThan(metadata["ctime"],
 693                                                   self._start_timestamp)
 694                 self.failUnlessGreaterOrEqualThan(self._stop_timestamp,
 695                                                   metadata["ctime"])
 696                 self.failUnlessGreaterOrEqualThan(metadata["mtime"],
 697                                                   self._start_timestamp)
 698                 self.failUnlessGreaterOrEqualThan(self._stop_timestamp,
 699                                                   metadata["mtime"])
 700                 # Our current timestamp rules say that replacing an existing
 701                 # child should preserve the 'ctime' but update the mtime
 702                 self._old_ctime = metadata["ctime"]
 703                 self._old_mtime = metadata["mtime"]
 704             d.addCallback(_check_timestamp1)
 705             d.addCallback(self.stall, 2.0) # accomodate low-res timestamps
 706             d.addCallback(lambda res: n.set_node(u"timestamps", n))
 707             d.addCallback(lambda res: n.get_metadata_for(u"timestamps"))
 708             def _check_timestamp2(metadata):
 709                 self.failUnlessEqual(metadata["ctime"], self._old_ctime,
 710                                      "%s != %s" % (metadata["ctime"],
 711                                                    self._old_ctime))
 712                 self.failUnlessGreaterThan(metadata["mtime"], self._old_mtime)
 713                 return n.delete(u"timestamps")
 714             d.addCallback(_check_timestamp2)
 715
 716             # also make sure we can add/update timestamps on a
 717             # previously-existing child that didn't have any, since there are
 718             # a lot of 0.7.0-generated edges around out there
 719             d.addCallback(lambda res: n.set_node(u"no_timestamps", n, {}))
 720             d.addCallback(lambda res: n.set_node(u"no_timestamps", n))
 721             d.addCallback(lambda res: n.get_metadata_for(u"no_timestamps"))
 722             d.addCallback(lambda metadata:
 723                           self.failUnlessEqual(set(metadata.keys()),
 724                                                set(["tahoe", "ctime", "mtime"])))
 725             d.addCallback(lambda res: n.delete(u"no_timestamps"))
 726
 727             d.addCallback(lambda res: n.delete(u"subdir"))
 728             d.addCallback(lambda old_child:
 729                           self.failUnlessEqual(old_child.get_uri(),
 730                                                self.subdir.get_uri()))
 731
 732             d.addCallback(lambda res: n.list())
 733             d.addCallback(lambda children:
 734                           self.failUnlessEqual(sorted(children.keys()),
 735                                                sorted([u"child"])))
 736
 737             uploadable1 = upload.Data("some data", convergence="converge")
 738             d.addCallback(lambda res: n.add_file(u"newfile", uploadable1))
 739             d.addCallback(lambda newnode:
 740                           self.failUnless(IFileNode.providedBy(newnode)))
 741             uploadable2 = upload.Data("some data", convergence="stuff")
 742             d.addCallback(lambda res:
 743                           self.shouldFail(ExistingChildError, "add_file-no",
 744                                           "child 'newfile' already exists",
 745                                           n.add_file, u"newfile",
 746                                           uploadable2,
 747                                           overwrite=False))
 748             d.addCallback(lambda res: n.list())
 749             d.addCallback(lambda children:
 750                           self.failUnlessEqual(sorted(children.keys()),
 751                                                sorted([u"child", u"newfile"])))
 752             d.addCallback(lambda res: n.get_metadata_for(u"newfile"))
 753             d.addCallback(lambda metadata:
 754                           self.failUnlessEqual(set(metadata.keys()),
 755                                                set(["tahoe", "ctime", "mtime"])))
 756
 757             uploadable3 = upload.Data("some data", convergence="converge")
 758             d.addCallback(lambda res: n.add_file(u"newfile-metadata",
 759                                                  uploadable3,
 760                                                  {"key": "value"}))
 761             d.addCallback(lambda newnode:
 762                           self.failUnless(IFileNode.providedBy(newnode)))
 763             d.addCallback(lambda res: n.get_metadata_for(u"newfile-metadata"))
 764             d.addCallback(lambda metadata:
 765                               self.failUnless((set(metadata.keys()) == set(["key", "tahoe"])) and
 766                                               (metadata['key'] == "value"), metadata))
 767             d.addCallback(lambda res: n.delete(u"newfile-metadata"))
 768
 769             d.addCallback(lambda res: n.create_subdirectory(u"subdir2"))
 770             def _created2(subdir2):
 771                 self.subdir2 = subdir2
 772                 # put something in the way, to make sure it gets overwritten
 773                 return subdir2.add_file(u"child", upload.Data("overwrite me",
 774                                                               "converge"))
 775             d.addCallback(_created2)
 776
 777             d.addCallback(lambda res:
 778                           n.move_child_to(u"child", self.subdir2))
 779             d.addCallback(lambda res: n.list())
 780             d.addCallback(lambda children:
 781                           self.failUnlessEqual(sorted(children.keys()),
 782                                                sorted([u"newfile", u"subdir2"])))
 783             d.addCallback(lambda res: self.subdir2.list())
 784             d.addCallback(lambda children:
 785                           self.failUnlessEqual(sorted(children.keys()),
 786                                                sorted([u"child"])))
 787             d.addCallback(lambda res: self.subdir2.get(u"child"))
 788             d.addCallback(lambda child:
 789                           self.failUnlessEqual(child.get_uri(),
 790                                                fake_file_uri))
 791
 792             # move it back, using new_child_name=
 793             d.addCallback(lambda res:
 794                           self.subdir2.move_child_to(u"child", n, u"newchild"))
 795             d.addCallback(lambda res: n.list())
 796             d.addCallback(lambda children:
 797                           self.failUnlessEqual(sorted(children.keys()),
 798                                                sorted([u"newchild", u"newfile",
 799                                                        u"subdir2"])))
 800             d.addCallback(lambda res: self.subdir2.list())
 801             d.addCallback(lambda children:
 802                           self.failUnlessEqual(sorted(children.keys()), []))
 803
 804             # now make sure that we honor overwrite=False
 805             d.addCallback(lambda res:
 806                           self.subdir2.set_uri(u"newchild",
 807                                                other_file_uri, other_file_uri))
 808
 809             d.addCallback(lambda res:
 810                           self.shouldFail(ExistingChildError, "move_child_to-no",
 811                                           "child 'newchild' already exists",
 812                                           n.move_child_to, u"newchild",
 813                                           self.subdir2,
 814                                           overwrite=False))
 815             d.addCallback(lambda res: self.subdir2.get(u"newchild"))
 816             d.addCallback(lambda child:
 817                           self.failUnlessEqual(child.get_uri(),
 818                                                other_file_uri))
 819
 820             return d
 821
 822         d.addCallback(_then)
 823
 824         d.addErrback(self.explain_error)
 825         return d
 826
 827     def test_create_subdirectory(self):
 828         self.basedir = "dirnode/Dirnode/test_create_subdirectory"
 829         self.set_up_grid()
 830         c = self.g.clients[0]
 831         nm = c.nodemaker
 832
 833         d = c.create_dirnode()
 834         def _then(n):
 835             # /
 836             self.rootnode = n
 837             fake_file_uri = make_mutable_file_uri()
 838             other_file_uri = make_mutable_file_uri()
 839             md = {"metakey": "metavalue"}
 840             kids = {u"kid1": (nm.create_from_cap(fake_file_uri), {}),
 841                     u"kid2": (nm.create_from_cap(other_file_uri), md),
 842                     }
 843             d = n.create_subdirectory(u"subdir", kids)
 844             def _check(sub):
 845                 d = n.get_child_at_path(u"subdir")
 846                 d.addCallback(lambda sub2: self.failUnlessEqual(sub2.get_uri(),
 847                                                                 sub.get_uri()))
 848                 d.addCallback(lambda ign: sub.list())
 849                 return d
 850             d.addCallback(_check)
 851             def _check_kids(kids2):
 852                 self.failUnlessEqual(sorted(kids.keys()), sorted(kids2.keys()))
 853                 self.failUnlessEqual(kids2[u"kid2"][1]["metakey"], "metavalue")
 854             d.addCallback(_check_kids)
 855             return d
 856         d.addCallback(_then)
 857         return d
 858
 859 class MinimalFakeMutableFile:
 860     def get_writekey(self):
 861         return "writekey"
 862
 863 class Packing(unittest.TestCase):
 864     # This is a base32-encoded representation of the directory tree
 865     # root/file1
 866     # root/file2
 867     # root/file3
 868     # as represented after being fed to _pack_contents.
 869     # We have it here so we can decode it, feed it to
 870     # _unpack_contents, and verify that _unpack_contents
 871     # works correctly.
 872
 873     known_tree = "GM4TOORVHJTGS3DFGEWDSNJ2KVJESOSDJBFTU33MPB2GS3LZNVYG6N3GGI3WU5TIORTXC3DOMJ2G4NB2MVWXUZDONBVTE5LNGRZWK2LYN55GY23XGNYXQMTOMZUWU5TENN4DG23ZG5UTO2L2NQ2DO6LFMRWDMZJWGRQTUMZ2GEYDUMJQFQYTIMZ22XZKZORX5XS7CAQCSK3URR6QOHISHRCMGER5LRFSZRNAS5ZSALCS6TWFQAE754IVOIKJVK73WZPP3VUUEDTX3WHTBBZ5YX3CEKHCPG3ZWQLYA4QM6LDRCF7TJQYWLIZHKGN5ROA3AUZPXESBNLQQ6JTC2DBJU2D47IZJTLR3PKZ4RVF57XLPWY7FX7SZV3T6IJ3ORFW37FXUPGOE3ROPFNUX5DCGMAQJ3PGGULBRGM3TU6ZCMN2GS3LFEI5CAMJSGQ3DMNRTHA4TOLRUGI3TKNRWGEWCAITUMFUG6ZJCHIQHWITMNFXGW3LPORUW2ZJCHIQDCMRUGY3DMMZYHE3S4NBSG42TMNRRFQQCE3DJNZVWG4TUNFWWKIR2EAYTENBWGY3DGOBZG4XDIMRXGU3DMML5FQQCE3LUNFWWKIR2EAYTENBWGY3DGOBZG4XDIMRXGU3DMML5FQWDGOJRHI2TUZTJNRSTELBZGQ5FKUSJHJBUQSZ2MFYGKZ3SOBSWQ43IO52WO23CNAZWU3DUGVSWSNTIOE5DK33POVTW4ZLNMNWDK6DHPA2GS2THNF2W25DEN5VGY2LQNFRGG5DKNNRHO5TZPFTWI6LNMRYGQ2LCGJTHM4J2GM5DCMB2GQWDCNBSHKVVQBGRYMACKJ27CVQ6O6B4QPR72RFVTGOZUI76XUSWAX73JRV5PYRHMIFYZIA25MXDPGUGML6M2NMRSG4YD4W4K37ZDYSXHMJ3IUVT4F64YTQQVBJFFFOUC7J7LAB2VFCL5UKKGMR2D3F4EPOYC7UYWQZNR5KXHBSNXLCNBX2SNF22DCXJIHSMEKWEWOG5XCJEVVZ7UW5IB6I64XXQSJ34B5CAYZGZIIMR6LBRGMZTU6ZCMN2GS3LFEI5CAMJSGQ3DMNRTHA4TOLRUGMYDEMJYFQQCE5DBNBXWKIR2EB5SE3DJNZVW233UNFWWKIR2EAYTENBWGY3DGOBZG4XDIMZQGIYTQLBAEJWGS3TLMNZHI2LNMURDUIBRGI2DMNRWGM4DSNZOGQZTAMRRHB6SYIBCNV2GS3LFEI5CAMJSGQ3DMNRTHA4TOLRUGMYDEMJYPUWCYMZZGU5DKOTGNFWGKMZMHE2DUVKSJE5EGSCLHJRW25DDPBYTO2DXPB3GM6DBNYZTI6LJMV3DM2LWNB4TU4LWMNSWW3LKORXWK5DEMN3TI23NNE3WEM3SORRGY5THPA3TKNBUMNZG453BOF2GSZLXMVWWI3DJOFZW623RHIZTUMJQHI2SYMJUGI5BOSHWDPG3WKPAVXCF3XMKA7QVIWPRMWJHDTQHD27AHDCPJWDQENQ5H5ZZILTXQNIXXCIW4LKQABU2GCFRG5FHQN7CHD7HF4EKNRZFIV2ZYQIBM7IQU7F4RGB3XCX3FREPBKQ7UCICHVWPCYFGA6OLH3J45LXQ6GWWICJ3PGWJNLZ7PCRNLAPNYUGU6BENS7OXMBEOOFRIZV3PF2FFWZ5WHDPKXERYP7GNHKRMGEZTOOT3EJRXI2LNMURDUIBRGI2DMNRWGM4DSNZOGQZTGNRSGY4SYIBCORQWQ33FEI5CA6ZCNRUW423NN52GS3LFEI5CAMJSGQ3DMNRTHA4TOLRUGMZTMMRWHEWCAITMNFXGWY3SORUW2ZJCHIQDCMRUGY3DMMZYHE3S4NBTGM3DENRZPUWCAITNORUW2ZJCHIQDCMRUGY3DMMZYHE3S4NBTGM3DENRZPUWCY==="
 874
 875     def test_unpack_and_pack_behavior(self):
 876         known_tree = b32decode(self.known_tree)
 877         nodemaker = NodeMaker(None, None, None,
 878                               None, None, None,
 879                               {"k": 3, "n": 10}, None)
 880         writecap = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
 881         filenode = nodemaker.create_from_cap(writecap)
 882         node = dirnode.DirectoryNode(filenode, nodemaker, None)
 883         children = node._unpack_contents(known_tree)
 884         self._check_children(children)
 885
 886         packed_children = node._pack_contents(children)
 887         children = node._unpack_contents(packed_children)
 888         self._check_children(children)
 889
 890     def _check_children(self, children):
 891         # Are all the expected child nodes there?
 892         self.failUnless(children.has_key(u'file1'))
 893         self.failUnless(children.has_key(u'file2'))
 894         self.failUnless(children.has_key(u'file3'))
 895
 896         # Are the metadata for child 3 right?
 897         file3_rocap = "URI:CHK:cmtcxq7hwxvfxan34yiev6ivhy:qvcekmjtoetdcw4kmi7b3rtblvgx7544crnwaqtiewemdliqsokq:3:10:5"
 898         file3_rwcap = "URI:CHK:cmtcxq7hwxvfxan34yiev6ivhy:qvcekmjtoetdcw4kmi7b3rtblvgx7544crnwaqtiewemdliqsokq:3:10:5"
 899         file3_metadata = {'ctime': 1246663897.4336269, 'tahoe': {'linkmotime': 1246663897.4336269, 'linkcrtime': 1246663897.4336269}, 'mtime': 1246663897.4336269}
 900         self.failUnlessEqual(file3_metadata, children[u'file3'][1])
 901         self.failUnlessEqual(file3_rocap,
 902                              children[u'file3'][0].get_readonly_uri())
 903         self.failUnlessEqual(file3_rwcap,
 904                              children[u'file3'][0].get_uri())
 905
 906         # Are the metadata for child 2 right?
 907         file2_rocap = "URI:CHK:apegrpehshwugkbh3jlt5ei6hq:5oougnemcl5xgx4ijgiumtdojlipibctjkbwvyygdymdphib2fvq:3:10:4"
 908         file2_rwcap = "URI:CHK:apegrpehshwugkbh3jlt5ei6hq:5oougnemcl5xgx4ijgiumtdojlipibctjkbwvyygdymdphib2fvq:3:10:4"
 909         file2_metadata = {'ctime': 1246663897.430218, 'tahoe': {'linkmotime': 1246663897.430218, 'linkcrtime': 1246663897.430218}, 'mtime': 1246663897.430218}
 910         self.failUnlessEqual(file2_metadata, children[u'file2'][1])
 911         self.failUnlessEqual(file2_rocap,
 912                              children[u'file2'][0].get_readonly_uri())
 913         self.failUnlessEqual(file2_rwcap,
 914                              children[u'file2'][0].get_uri())
 915
 916         # Are the metadata for child 1 right?
 917         file1_rocap = "URI:CHK:olxtimympo7f27jvhtgqlnbtn4:emzdnhk2um4seixozlkw3qx2nfijvdkx3ky7i7izl47yedl6e64a:3:10:10"
 918         file1_rwcap = "URI:CHK:olxtimympo7f27jvhtgqlnbtn4:emzdnhk2um4seixozlkw3qx2nfijvdkx3ky7i7izl47yedl6e64a:3:10:10"
 919         file1_metadata = {'ctime': 1246663897.4275661, 'tahoe': {'linkmotime': 1246663897.4275661, 'linkcrtime': 1246663897.4275661}, 'mtime': 1246663897.4275661}
 920         self.failUnlessEqual(file1_metadata, children[u'file1'][1])
 921         self.failUnlessEqual(file1_rocap,
 922                              children[u'file1'][0].get_readonly_uri())
 923         self.failUnlessEqual(file1_rwcap,
 924                              children[u'file1'][0].get_uri())
 925
 926     def _make_kids(self, nm, which):
 927         caps = {"imm": "URI:CHK:n7r3m6wmomelk4sep3kw5cvduq:os7ijw5c3maek7pg65e5254k2fzjflavtpejjyhshpsxuqzhcwwq:3:20:14861",
 928                 "lit": "URI:LIT:n5xgk", # LIT for "one"
 929                 "write": "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq",
 930                 "read": "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q",
 931                 "dirwrite": "URI:DIR2:n6x24zd3seu725yluj75q5boaa:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq",
 932                 "dirread":  "URI:DIR2-RO:b7sr5qsifnicca7cbk3rhrhbvq:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq",
 933                 }
 934         kids = {}
 935         for name in which:
 936             kids[unicode(name)] = (nm.create_from_cap(caps[name]), {})
 937         return kids
 938
 939     def test_deep_immutable(self):
 940         nm = NodeMaker(None, None, None, None, None, None, {"k": 3, "n": 10},
 941                        None)
 942         fn = MinimalFakeMutableFile()
 943
 944         kids = self._make_kids(nm, ["imm", "lit", "write", "read",
 945                                     "dirwrite", "dirread"])
 946         packed = dirnode.pack_children(fn, kids, deep_immutable=False)
 947         self.failUnlessIn("lit", packed)
 948
 949         kids = self._make_kids(nm, ["imm", "lit"])
 950         packed = dirnode.pack_children(fn, kids, deep_immutable=True)
 951         self.failUnlessIn("lit", packed)
 952
 953         kids = self._make_kids(nm, ["imm", "lit", "write"])
 954         e = self.failUnlessRaises(dirnode.MustBeDeepImmutable,
 955                                   dirnode.pack_children,
 956                                   fn, kids, deep_immutable=True)
 957
 958         # read-only is not enough: all children must be immutable
 959         kids = self._make_kids(nm, ["imm", "lit", "read"])
 960         e = self.failUnlessRaises(dirnode.MustBeDeepImmutable,
 961                                   dirnode.pack_children,
 962                                   fn, kids, deep_immutable=True)
 963
 964         kids = self._make_kids(nm, ["imm", "lit", "dirwrite"])
 965         e = self.failUnlessRaises(dirnode.MustBeDeepImmutable,
 966                                   dirnode.pack_children,
 967                                   fn, kids, deep_immutable=True)
 968
 969         kids = self._make_kids(nm, ["imm", "lit", "dirread"])
 970         e = self.failUnlessRaises(dirnode.MustBeDeepImmutable,
 971                                   dirnode.pack_children,
 972                                   fn, kids, deep_immutable=True)
 973
 974 class FakeMutableFile:
 975     counter = 0
 976     def __init__(self, initial_contents=""):
 977         self.data = self._get_initial_contents(initial_contents)
 978         counter = FakeMutableFile.counter
 979         FakeMutableFile.counter += 1
 980         writekey = hashutil.ssk_writekey_hash(str(counter))
 981         fingerprint = hashutil.ssk_pubkey_fingerprint_hash(str(counter))
 982         self.uri = uri.WriteableSSKFileURI(writekey, fingerprint)
 983
 984     def _get_initial_contents(self, contents):
 985         if isinstance(contents, str):
 986             return contents
 987         if contents is None:
 988             return ""
 989         assert callable(contents), "%s should be callable, not %s" % \
 990                (contents, type(contents))
 991         return contents(self)
 992
 993     def get_cap(self):
 994         return self.uri
 995     def get_uri(self):
 996         return self.uri.to_string()
 997     def download_best_version(self):
 998         return defer.succeed(self.data)
 999     def get_writekey(self):
1000         return "writekey"
1001     def is_readonly(self):
1002         return False
1003     def is_mutable(self):
1004         return True
1005     def modify(self, modifier):
1006         self.data = modifier(self.data, None, True)
1007         return defer.succeed(None)
1008
1009 class FakeNodeMaker(NodeMaker):
1010     def create_mutable_file(self, contents="", keysize=None):
1011         return defer.succeed(FakeMutableFile(contents))
1012
1013 class FakeClient2(Client):
1014     def __init__(self):
1015         self.nodemaker = FakeNodeMaker(None, None, None,
1016                                        None, None, None,
1017                                        {"k":3,"n":10}, None)
1018     def create_node_from_uri(self, rwcap, rocap):
1019         return self.nodemaker.create_from_cap(rwcap, rocap)
1020
1021 class Dirnode2(unittest.TestCase, testutil.ShouldFailMixin):
1022     def setUp(self):
1023         self.client = FakeClient2()
1024         self.nodemaker = self.client.nodemaker
1025
1026     def test_from_future(self):
1027         # create a dirnode that contains unknown URI types, and make sure we
1028         # tolerate them properly. Since dirnodes aren't allowed to add
1029         # unknown node types, we have to be tricky.
1030         d = self.nodemaker.create_new_mutable_directory()
1031         future_writecap = "x-tahoe-crazy://I_am_from_the_future."
1032         future_readcap = "x-tahoe-crazy-readonly://I_am_from_the_future."
1033         future_node = UnknownNode(future_writecap, future_readcap)
1034         def _then(n):
1035             self._node = n
1036             return n.set_node(u"future", future_node)
1037         d.addCallback(_then)
1038
1039         # we should be prohibited from adding an unknown URI to a directory,
1040         # since we don't know how to diminish the cap to a readcap (for the
1041         # dirnode's rocap slot), and we don't want to accidentally grant
1042         # write access to a holder of the dirnode's readcap.
1043         d.addCallback(lambda ign:
1044              self.shouldFail(CannotPackUnknownNodeError,
1045                              "copy unknown",
1046                              "cannot pack unknown node as child add",
1047                              self._node.set_uri, u"add",
1048                              future_writecap, future_readcap))
1049         d.addCallback(lambda ign: self._node.list())
1050         def _check(children):
1051             self.failUnlessEqual(len(children), 1)
1052             (fn, metadata) = children[u"future"]
1053             self.failUnless(isinstance(fn, UnknownNode), fn)
1054             self.failUnlessEqual(fn.get_uri(), future_writecap)
1055             self.failUnlessEqual(fn.get_readonly_uri(), future_readcap)
1056             # but we *should* be allowed to copy this node, because the
1057             # UnknownNode contains all the information that was in the
1058             # original directory (readcap and writecap), so we're preserving
1059             # everything.
1060             return self._node.set_node(u"copy", fn)
1061         d.addCallback(_check)
1062         d.addCallback(lambda ign: self._node.list())
1063         def _check2(children):
1064             self.failUnlessEqual(len(children), 2)
1065             (fn, metadata) = children[u"copy"]
1066             self.failUnless(isinstance(fn, UnknownNode), fn)
1067             self.failUnlessEqual(fn.get_uri(), future_writecap)
1068             self.failUnlessEqual(fn.get_readonly_uri(), future_readcap)
1069         return d
1070
1071 class DeepStats(unittest.TestCase):
1072     timeout = 240 # It takes longer than 120 seconds on Francois's arm box.
1073     def test_stats(self):
1074         ds = dirnode.DeepStats(None)
1075         ds.add("count-files")
1076         ds.add("size-immutable-files", 123)
1077         ds.histogram("size-files-histogram", 123)
1078         ds.max("largest-directory", 444)
1079
1080         s = ds.get_results()
1081         self.failUnlessEqual(s["count-files"], 1)
1082         self.failUnlessEqual(s["size-immutable-files"], 123)
1083         self.failUnlessEqual(s["largest-directory"], 444)
1084         self.failUnlessEqual(s["count-literal-files"], 0)
1085
1086         ds.add("count-files")
1087         ds.add("size-immutable-files", 321)
1088         ds.histogram("size-files-histogram", 321)
1089         ds.max("largest-directory", 2)
1090
1091         s = ds.get_results()
1092         self.failUnlessEqual(s["count-files"], 2)
1093         self.failUnlessEqual(s["size-immutable-files"], 444)
1094         self.failUnlessEqual(s["largest-directory"], 444)
1095         self.failUnlessEqual(s["count-literal-files"], 0)
1096         self.failUnlessEqual(s["size-files-histogram"],
1097                              [ (101, 316, 1), (317, 1000, 1) ])
1098
1099         ds = dirnode.DeepStats(None)
1100         for i in range(1, 1100):
1101             ds.histogram("size-files-histogram", i)
1102         ds.histogram("size-files-histogram", 4*1000*1000*1000*1000) # 4TB
1103         s = ds.get_results()
1104         self.failUnlessEqual(s["size-files-histogram"],
1105                              [ (1, 3, 3),
1106                                (4, 10, 7),
1107                                (11, 31, 21),
1108                                (32, 100, 69),
1109                                (101, 316, 216),
1110                                (317, 1000, 684),
1111                                (1001, 3162, 99),
1112                                (3162277660169L, 10000000000000L, 1),
1113                                ])
1114
1115 class UCWEingMutableFileNode(MutableFileNode):
1116     please_ucwe_after_next_upload = False
1117
1118     def _upload(self, new_contents, servermap):
1119         d = MutableFileNode._upload(self, new_contents, servermap)
1120         def _ucwe(res):
1121             if self.please_ucwe_after_next_upload:
1122                 self.please_ucwe_after_next_upload = False
1123                 raise UncoordinatedWriteError()
1124             return res
1125         d.addCallback(_ucwe)
1126         return d
1127
1128 class UCWEingNodeMaker(NodeMaker):
1129     def _create_mutable(self, cap):
1130         n = UCWEingMutableFileNode(self.storage_broker, self.secret_holder,
1131                                    self.default_encoding_parameters,
1132                                    self.history)
1133         return n.init_from_cap(cap)
1134
1135
1136 class Deleter(GridTestMixin, unittest.TestCase):
1137     timeout = 3600 # It takes longer than 433 seconds on Zandr's ARM box.
1138     def test_retry(self):
1139         # ticket #550, a dirnode.delete which experiences an
1140         # UncoordinatedWriteError will fail with an incorrect "you're
1141         # deleting something which isn't there" NoSuchChildError exception.
1142
1143         # to trigger this, we start by creating a directory with a single
1144         # file in it. Then we create a special dirnode that uses a modified
1145         # MutableFileNode which will raise UncoordinatedWriteError once on
1146         # demand. We then call dirnode.delete, which ought to retry and
1147         # succeed.
1148
1149         self.basedir = self.mktemp()
1150         self.set_up_grid()
1151         c0 = self.g.clients[0]
1152         d = c0.create_dirnode()
1153         small = upload.Data("Small enough for a LIT", None)
1154         def _created_dir(dn):
1155             self.root = dn
1156             self.root_uri = dn.get_uri()
1157             return dn.add_file(u"file", small)
1158         d.addCallback(_created_dir)
1159         def _do_delete(ignored):
1160             nm = UCWEingNodeMaker(c0.storage_broker, c0._secret_holder,
1161                                   c0.get_history(), c0.getServiceNamed("uploader"),
1162                                   c0.downloader,
1163                                   c0.download_cache_dirman,
1164                                   c0.get_encoding_parameters(),
1165                                   c0._key_generator)
1166             n = nm.create_from_cap(self.root_uri)
1167             assert n._node.please_ucwe_after_next_upload == False
1168             n._node.please_ucwe_after_next_upload = True
1169             # This should succeed, not raise an exception
1170             return n.delete(u"file")
1171         d.addCallback(_do_delete)
1172
1173         return d
1174
1175 class Adder(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin):
1176
1177     def test_overwrite(self):
1178         # note: This functionality could be tested without actually creating
1179         # several RSA keys. It would be faster without the GridTestMixin: use
1180         # dn.set_node(nodemaker.create_from_cap(make_chk_file_uri())) instead
1181         # of dn.add_file, and use a special NodeMaker that creates fake
1182         # mutable files.
1183         self.basedir = "dirnode/Adder/test_overwrite"
1184         self.set_up_grid()
1185         c = self.g.clients[0]
1186         fileuri = make_chk_file_uri(1234)
1187         filenode = c.nodemaker.create_from_cap(fileuri)
1188         d = c.create_dirnode()
1189
1190         def _create_directory_tree(root_node):
1191             # Build
1192             # root/file1
1193             # root/file2
1194             # root/dir1
1195             d = root_node.add_file(u'file1', upload.Data("Important Things",
1196                 None))
1197             d.addCallback(lambda res:
1198                 root_node.add_file(u'file2', upload.Data("Sekrit Codes", None)))
1199             d.addCallback(lambda res:
1200                 root_node.create_subdirectory(u"dir1"))
1201             d.addCallback(lambda res: root_node)
1202             return d
1203
1204         d.addCallback(_create_directory_tree)
1205
1206         def _test_adder(root_node):
1207             d = root_node.set_node(u'file1', filenode)
1208             # We've overwritten file1. Let's try it with a directory
1209             d.addCallback(lambda res:
1210                 root_node.create_subdirectory(u'dir2'))
1211             d.addCallback(lambda res:
1212                 root_node.set_node(u'dir2', filenode))
1213             # We try overwriting a file with a child while also specifying
1214             # overwrite=False. We should receive an ExistingChildError
1215             # when we do this.
1216             d.addCallback(lambda res:
1217                 self.shouldFail(ExistingChildError, "set_node",
1218                                 "child 'file1' already exists",
1219                                root_node.set_node, u"file1",
1220                                filenode, overwrite=False))
1221             # If we try with a directory, we should see the same thing
1222             d.addCallback(lambda res:
1223                 self.shouldFail(ExistingChildError, "set_node",
1224                                 "child 'dir1' already exists",
1225                                 root_node.set_node, u'dir1', filenode,
1226                                 overwrite=False))
1227             d.addCallback(lambda res:
1228                  root_node.set_node(u'file1', filenode,
1229                                     overwrite="only-files"))
1230             d.addCallback(lambda res:
1231                  self.shouldFail(ExistingChildError, "set_node",
1232                                 "child 'dir1' already exists",
1233                                 root_node.set_node, u'dir1', filenode,
1234                                 overwrite="only-files"))
1235             return d
1236
1237         d.addCallback(_test_adder)
1238         return d