src/allmydata/test/test_dirnode.py

   1
   2 import time
   3 from zope.interface import implements
   4 from twisted.trial import unittest
   5 from twisted.internet import defer
   6 from twisted.internet.interfaces import IConsumer
   7 from allmydata import uri, dirnode
   8 from allmydata.client import Client
   9 from allmydata.immutable import upload
  10 from allmydata.interfaces import IImmutableFileNode, IMutableFileNode, \
  11      ExistingChildError, NoSuchChildError, MustNotBeUnknownRWError, \
  12      MustBeDeepImmutableError, MustBeReadonlyError, \
  13      IDeepCheckResults, IDeepCheckAndRepairResults
  14 from allmydata.mutable.filenode import MutableFileNode
  15 from allmydata.mutable.common import UncoordinatedWriteError
  16 from allmydata.util import hashutil, base32
  17 from allmydata.util.netstring import split_netstring
  18 from allmydata.monitor import Monitor
  19 from allmydata.test.common import make_chk_file_uri, make_mutable_file_uri, \
  20      ErrorMixin
  21 from allmydata.test.no_network import GridTestMixin
  22 from allmydata.unknown import UnknownNode, strip_prefix_for_ro
  23 from allmydata.nodemaker import NodeMaker
  24 from base64 import b32decode
  25 import allmydata.test.common_util as testutil
  26
  27 class MemAccum:
  28     implements(IConsumer)
  29     def registerProducer(self, producer, streaming):
  30         self.producer = producer
  31         self.producer.resumeProducing()
  32         pass
  33     def unregisterProducer(self):
  34         pass
  35     def write(self, data):
  36         assert not hasattr(self, 'data')
  37         self.data = data
  38         self.producer.resumeProducing()
  39
  40 setup_py_uri = "URI:CHK:n7r3m6wmomelk4sep3kw5cvduq:os7ijw5c3maek7pg65e5254k2fzjflavtpejjyhshpsxuqzhcwwq:3:20:14861"
  41 one_uri = "URI:LIT:n5xgk" # LIT for "one"
  42 mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
  43 empty_litdir_uri = "URI:DIR2-LIT:"
  44 tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
  45 mut_read_uri = "URI:SSK-RO:jf6wkflosyvntwxqcdo7a54jvm:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
  46 future_write_uri = "x-tahoe-crazy://I_am_from_the_future."
  47 future_read_uri = "x-tahoe-crazy-readonly://I_am_from_the_future."
  48
  49 class Dirnode(GridTestMixin, unittest.TestCase,
  50               testutil.ShouldFailMixin, testutil.StallMixin, ErrorMixin):
  51     timeout = 240 # It takes longer than 120 seconds on Francois's arm box.
  52
  53     def test_basic(self):
  54         self.basedir = "dirnode/Dirnode/test_basic"
  55         self.set_up_grid()
  56         c = self.g.clients[0]
  57         d = c.create_dirnode()
  58         def _done(res):
  59             self.failUnless(isinstance(res, dirnode.DirectoryNode))
  60             self.failUnless(res.is_mutable())
  61             self.failIf(res.is_readonly())
  62             self.failIf(res.is_unknown())
  63             self.failIf(res.is_allowed_in_immutable_directory())
  64             res.raise_error()
  65             rep = str(res)
  66             self.failUnless("RW-MUT" in rep)
  67         d.addCallback(_done)
  68         return d
  69
  70     def test_initial_children(self):
  71         self.basedir = "dirnode/Dirnode/test_initial_children"
  72         self.set_up_grid()
  73         c = self.g.clients[0]
  74         nm = c.nodemaker
  75
  76         kids = {u"one": (nm.create_from_cap(one_uri), {}),
  77                 u"two": (nm.create_from_cap(setup_py_uri),
  78                          {"metakey": "metavalue"}),
  79                 u"mut": (nm.create_from_cap(mut_write_uri, mut_read_uri), {}),
  80                 u"fut": (nm.create_from_cap(future_write_uri, future_read_uri), {}),
  81                 u"fro": (nm.create_from_cap(None, future_read_uri), {}),
  82                 u"empty_litdir": (nm.create_from_cap(empty_litdir_uri), {}),
  83                 u"tiny_litdir": (nm.create_from_cap(tiny_litdir_uri), {}),
  84                 }
  85         d = c.create_dirnode(kids)
  86
  87         def _created(dn):
  88             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
  89             self.failUnless(dn.is_mutable())
  90             self.failIf(dn.is_readonly())
  91             self.failIf(dn.is_unknown())
  92             self.failIf(dn.is_allowed_in_immutable_directory())
  93             dn.raise_error()
  94             rep = str(dn)
  95             self.failUnless("RW-MUT" in rep)
  96             return dn.list()
  97         d.addCallback(_created)
  98
  99         def _check_kids(children):
 100             self.failUnlessEqual(set(children.keys()),
 101                                  set([u"one", u"two", u"mut", u"fut", u"fro", u"empty_litdir", u"tiny_litdir"]))
 102             one_node, one_metadata = children[u"one"]
 103             two_node, two_metadata = children[u"two"]
 104             mut_node, mut_metadata = children[u"mut"]
 105             fut_node, fut_metadata = children[u"fut"]
 106             fro_node, fro_metadata = children[u"fro"]
 107             emptylit_node, emptylit_metadata = children[u"empty_litdir"]
 108             tinylit_node, tinylit_metadata = children[u"tiny_litdir"]
 109
 110             self.failUnlessEqual(one_node.get_size(), 3)
 111             self.failUnlessEqual(one_node.get_uri(), one_uri)
 112             self.failUnlessEqual(one_node.get_readonly_uri(), one_uri)
 113             self.failUnless(isinstance(one_metadata, dict), one_metadata)
 114
 115             self.failUnlessEqual(two_node.get_size(), 14861)
 116             self.failUnlessEqual(two_node.get_uri(), setup_py_uri)
 117             self.failUnlessEqual(two_node.get_readonly_uri(), setup_py_uri)
 118             self.failUnlessEqual(two_metadata["metakey"], "metavalue")
 119
 120             self.failUnlessEqual(mut_node.get_uri(), mut_write_uri)
 121             self.failUnlessEqual(mut_node.get_readonly_uri(), mut_read_uri)
 122             self.failUnless(isinstance(mut_metadata, dict), mut_metadata)
 123
 124             self.failUnless(fut_node.is_unknown())
 125             self.failUnlessEqual(fut_node.get_uri(), future_write_uri)
 126             self.failUnlessEqual(fut_node.get_readonly_uri(), "ro." + future_read_uri)
 127             self.failUnless(isinstance(fut_metadata, dict), fut_metadata)
 128
 129             self.failUnless(fro_node.is_unknown())
 130             self.failUnlessEqual(fro_node.get_uri(), "ro." + future_read_uri)
 131             self.failUnlessEqual(fut_node.get_readonly_uri(), "ro." + future_read_uri)
 132             self.failUnless(isinstance(fro_metadata, dict), fro_metadata)
 133
 134             self.failIf(emptylit_node.is_unknown())
 135             self.failUnlessEqual(emptylit_node.get_storage_index(), None)
 136             self.failIf(tinylit_node.is_unknown())
 137             self.failUnlessEqual(tinylit_node.get_storage_index(), None)
 138
 139             d2 = defer.succeed(None)
 140             d2.addCallback(lambda ignored: emptylit_node.list())
 141             d2.addCallback(lambda children: self.failUnlessEqual(children, {}))
 142             d2.addCallback(lambda ignored: tinylit_node.list())
 143             d2.addCallback(lambda children: self.failUnlessEqual(set(children.keys()),
 144                                                                  set([u"short"])))
 145             d2.addCallback(lambda ignored: tinylit_node.list())
 146             d2.addCallback(lambda children: children[u"short"][0].read(MemAccum()))
 147             d2.addCallback(lambda accum: self.failUnlessEqual(accum.data, "The end."))
 148             return d2
 149
 150         d.addCallback(_check_kids)
 151
 152         d.addCallback(lambda ign: nm.create_new_mutable_directory(kids))
 153         d.addCallback(lambda dn: dn.list())
 154         d.addCallback(_check_kids)
 155
 156         bad_future_node = UnknownNode(future_write_uri, None)
 157         bad_kids1 = {u"one": (bad_future_node, {})}
 158         # This should fail because we don't know how to diminish the future_write_uri
 159         # cap (given in a write slot and not prefixed with "ro." or "imm.") to a readcap.
 160         d.addCallback(lambda ign:
 161                       self.shouldFail(MustNotBeUnknownRWError, "bad_kids1",
 162                                       "cannot attach unknown",
 163                                       nm.create_new_mutable_directory,
 164                                       bad_kids1))
 165         bad_kids2 = {u"one": (nm.create_from_cap(one_uri), None)}
 166         d.addCallback(lambda ign:
 167                       self.shouldFail(AssertionError, "bad_kids2",
 168                                       "requires metadata to be a dict",
 169                                       nm.create_new_mutable_directory,
 170                                       bad_kids2))
 171         return d
 172
 173     def test_immutable(self):
 174         self.basedir = "dirnode/Dirnode/test_immutable"
 175         self.set_up_grid()
 176         c = self.g.clients[0]
 177         nm = c.nodemaker
 178
 179         kids = {u"one": (nm.create_from_cap(one_uri), {}),
 180                 u"two": (nm.create_from_cap(setup_py_uri),
 181                          {"metakey": "metavalue"}),
 182                 u"fut": (nm.create_from_cap(None, future_read_uri), {}),
 183                 u"empty_litdir": (nm.create_from_cap(empty_litdir_uri), {}),
 184                 u"tiny_litdir": (nm.create_from_cap(tiny_litdir_uri), {}),
 185                 }
 186         d = c.create_immutable_dirnode(kids)
 187
 188         def _created(dn):
 189             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
 190             self.failIf(dn.is_mutable())
 191             self.failUnless(dn.is_readonly())
 192             self.failIf(dn.is_unknown())
 193             self.failUnless(dn.is_allowed_in_immutable_directory())
 194             dn.raise_error()
 195             rep = str(dn)
 196             self.failUnless("RO-IMM" in rep)
 197             cap = dn.get_cap()
 198             self.failUnlessIn("CHK", cap.to_string())
 199             self.cap = cap
 200             return dn.list()
 201         d.addCallback(_created)
 202
 203         def _check_kids(children):
 204             self.failUnlessEqual(set(children.keys()),
 205                                  set([u"one", u"two", u"fut", u"empty_litdir", u"tiny_litdir"]))
 206             one_node, one_metadata = children[u"one"]
 207             two_node, two_metadata = children[u"two"]
 208             fut_node, fut_metadata = children[u"fut"]
 209             emptylit_node, emptylit_metadata = children[u"empty_litdir"]
 210             tinylit_node, tinylit_metadata = children[u"tiny_litdir"]
 211
 212             self.failUnlessEqual(one_node.get_size(), 3)
 213             self.failUnlessEqual(one_node.get_uri(), one_uri)
 214             self.failUnlessEqual(one_node.get_readonly_uri(), one_uri)
 215             self.failUnless(isinstance(one_metadata, dict), one_metadata)
 216
 217             self.failUnlessEqual(two_node.get_size(), 14861)
 218             self.failUnlessEqual(two_node.get_uri(), setup_py_uri)
 219             self.failUnlessEqual(two_node.get_readonly_uri(), setup_py_uri)
 220             self.failUnlessEqual(two_metadata["metakey"], "metavalue")
 221
 222             self.failUnless(fut_node.is_unknown())
 223             self.failUnlessEqual(fut_node.get_uri(), "imm." + future_read_uri)
 224             self.failUnlessEqual(fut_node.get_readonly_uri(), "imm." + future_read_uri)
 225             self.failUnless(isinstance(fut_metadata, dict), fut_metadata)
 226
 227             self.failIf(emptylit_node.is_unknown())
 228             self.failUnlessEqual(emptylit_node.get_storage_index(), None)
 229             self.failIf(tinylit_node.is_unknown())
 230             self.failUnlessEqual(tinylit_node.get_storage_index(), None)
 231
 232             d2 = defer.succeed(None)
 233             d2.addCallback(lambda ignored: emptylit_node.list())
 234             d2.addCallback(lambda children: self.failUnlessEqual(children, {}))
 235             d2.addCallback(lambda ignored: tinylit_node.list())
 236             d2.addCallback(lambda children: self.failUnlessEqual(set(children.keys()),
 237                                                                  set([u"short"])))
 238             d2.addCallback(lambda ignored: tinylit_node.list())
 239             d2.addCallback(lambda children: children[u"short"][0].read(MemAccum()))
 240             d2.addCallback(lambda accum: self.failUnlessEqual(accum.data, "The end."))
 241             return d2
 242
 243         d.addCallback(_check_kids)
 244
 245         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
 246         d.addCallback(lambda dn: dn.list())
 247         d.addCallback(_check_kids)
 248
 249         bad_future_node1 = UnknownNode(future_write_uri, None)
 250         bad_kids1 = {u"one": (bad_future_node1, {})}
 251         d.addCallback(lambda ign:
 252                       self.shouldFail(MustNotBeUnknownRWError, "bad_kids1",
 253                                       "cannot attach unknown",
 254                                       c.create_immutable_dirnode,
 255                                       bad_kids1))
 256         bad_future_node2 = UnknownNode(future_write_uri, future_read_uri)
 257         bad_kids2 = {u"one": (bad_future_node2, {})}
 258         d.addCallback(lambda ign:
 259                       self.shouldFail(MustBeDeepImmutableError, "bad_kids2",
 260                                       "is not immutable",
 261                                       c.create_immutable_dirnode,
 262                                       bad_kids2))
 263         bad_kids3 = {u"one": (nm.create_from_cap(one_uri), None)}
 264         d.addCallback(lambda ign:
 265                       self.shouldFail(AssertionError, "bad_kids3",
 266                                       "requires metadata to be a dict",
 267                                       c.create_immutable_dirnode,
 268                                       bad_kids3))
 269         bad_kids4 = {u"one": (nm.create_from_cap(mut_write_uri), {})}
 270         d.addCallback(lambda ign:
 271                       self.shouldFail(MustBeDeepImmutableError, "bad_kids4",
 272                                       "is not immutable",
 273                                       c.create_immutable_dirnode,
 274                                       bad_kids4))
 275         bad_kids5 = {u"one": (nm.create_from_cap(mut_read_uri), {})}
 276         d.addCallback(lambda ign:
 277                       self.shouldFail(MustBeDeepImmutableError, "bad_kids5",
 278                                       "is not immutable",
 279                                       c.create_immutable_dirnode,
 280                                       bad_kids5))
 281         d.addCallback(lambda ign: c.create_immutable_dirnode({}))
 282         def _created_empty(dn):
 283             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
 284             self.failIf(dn.is_mutable())
 285             self.failUnless(dn.is_readonly())
 286             self.failIf(dn.is_unknown())
 287             self.failUnless(dn.is_allowed_in_immutable_directory())
 288             dn.raise_error()
 289             rep = str(dn)
 290             self.failUnless("RO-IMM" in rep)
 291             cap = dn.get_cap()
 292             self.failUnlessIn("LIT", cap.to_string())
 293             self.failUnlessEqual(cap.to_string(), "URI:DIR2-LIT:")
 294             self.cap = cap
 295             return dn.list()
 296         d.addCallback(_created_empty)
 297         d.addCallback(lambda kids: self.failUnlessEqual(kids, {}))
 298         smallkids = {u"o": (nm.create_from_cap(one_uri), {})}
 299         d.addCallback(lambda ign: c.create_immutable_dirnode(smallkids))
 300         def _created_small(dn):
 301             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
 302             self.failIf(dn.is_mutable())
 303             self.failUnless(dn.is_readonly())
 304             self.failIf(dn.is_unknown())
 305             self.failUnless(dn.is_allowed_in_immutable_directory())
 306             dn.raise_error()
 307             rep = str(dn)
 308             self.failUnless("RO-IMM" in rep)
 309             cap = dn.get_cap()
 310             self.failUnlessIn("LIT", cap.to_string())
 311             self.failUnlessEqual(cap.to_string(),
 312                                  "URI:DIR2-LIT:gi4tumj2n4wdcmz2kvjesosmjfkdu3rvpbtwwlbqhiwdeot3puwcy")
 313             self.cap = cap
 314             return dn.list()
 315         d.addCallback(_created_small)
 316         d.addCallback(lambda kids: self.failUnlessEqual(kids.keys(), [u"o"]))
 317
 318         # now test n.create_subdirectory(mutable=False)
 319         d.addCallback(lambda ign: c.create_dirnode())
 320         def _made_parent(n):
 321             d = n.create_subdirectory(u"subdir", kids, mutable=False)
 322             d.addCallback(lambda sd: sd.list())
 323             d.addCallback(_check_kids)
 324             d.addCallback(lambda ign: n.list())
 325             d.addCallback(lambda children:
 326                           self.failUnlessEqual(children.keys(), [u"subdir"]))
 327             d.addCallback(lambda ign: n.get(u"subdir"))
 328             d.addCallback(lambda sd: sd.list())
 329             d.addCallback(_check_kids)
 330             d.addCallback(lambda ign: n.get(u"subdir"))
 331             d.addCallback(lambda sd: self.failIf(sd.is_mutable()))
 332             bad_kids = {u"one": (nm.create_from_cap(mut_write_uri), {})}
 333             d.addCallback(lambda ign:
 334                           self.shouldFail(MustBeDeepImmutableError, "YZ",
 335                                           "is not immutable",
 336                                           n.create_subdirectory,
 337                                           u"sub2", bad_kids, mutable=False))
 338             return d
 339         d.addCallback(_made_parent)
 340         return d
 341
 342     def test_spaces_are_stripped_on_the_way_out(self):
 343         self.basedir = "dirnode/Dirnode/test_spaces_are_stripped_on_the_way_out"
 344         self.set_up_grid()
 345         c = self.g.clients[0]
 346         nm = c.nodemaker
 347
 348         # This test checks that any trailing spaces in URIs are retained in the
 349         # encoded directory, but stripped when we get them out of the directory.
 350         # See ticket #925 for why we want that.
 351
 352         stripped_write_uri = "lafs://from_the_future\t"
 353         stripped_read_uri = "lafs://readonly_from_the_future\t"
 354         spacedout_write_uri = stripped_write_uri + "  "
 355         spacedout_read_uri = stripped_read_uri + "  "
 356
 357         child = nm.create_from_cap(spacedout_write_uri, spacedout_read_uri)
 358         self.failUnlessEqual(child.get_write_uri(), spacedout_write_uri)
 359         self.failUnlessEqual(child.get_readonly_uri(), "ro." + spacedout_read_uri)
 360
 361         kids = {u"child": (child, {})}
 362         d = c.create_dirnode(kids)
 363
 364         def _created(dn):
 365             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
 366             self.failUnless(dn.is_mutable())
 367             self.failIf(dn.is_readonly())
 368             dn.raise_error()
 369             self.cap = dn.get_cap()
 370             self.rootnode = dn
 371             return dn._node.download_best_version()
 372         d.addCallback(_created)
 373
 374         def _check_data(data):
 375             # Decode the netstring representation of the directory to check that the
 376             # spaces are retained when the URIs are stored.
 377             position = 0
 378             numkids = 0
 379             while position < len(data):
 380                 entries, position = split_netstring(data, 1, position)
 381                 entry = entries[0]
 382                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
 383                 name = name_utf8.decode("utf-8")
 384                 rw_uri = self.rootnode._decrypt_rwcapdata(rwcapdata)
 385                 self.failUnless(name in kids)
 386                 (expected_child, ign) = kids[name]
 387                 self.failUnlessEqual(rw_uri, expected_child.get_write_uri())
 388                 self.failUnlessEqual("ro." + ro_uri, expected_child.get_readonly_uri())
 389                 numkids += 1
 390
 391             self.failUnlessEqual(numkids, 1)
 392             return self.rootnode.list()
 393         d.addCallback(_check_data)
 394
 395         # Now when we use the real directory listing code, the trailing spaces
 396         # should have been stripped (and "ro." should have been prepended to the
 397         # ro_uri, since it's unknown).
 398         def _check_kids(children):
 399             self.failUnlessEqual(set(children.keys()), set([u"child"]))
 400             child_node, child_metadata = children[u"child"]
 401
 402             self.failUnlessEqual(child_node.get_write_uri(), stripped_write_uri)
 403             self.failUnlessEqual(child_node.get_readonly_uri(), "ro." + stripped_read_uri)
 404         d.addCallback(_check_kids)
 405
 406         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
 407         d.addCallback(lambda n: n.list())
 408         d.addCallback(_check_kids)  # again with dirnode recreated from cap
 409         return d
 410
 411     def test_check(self):
 412         self.basedir = "dirnode/Dirnode/test_check"
 413         self.set_up_grid()
 414         c = self.g.clients[0]
 415         d = c.create_dirnode()
 416         d.addCallback(lambda dn: dn.check(Monitor()))
 417         def _done(res):
 418             self.failUnless(res.is_healthy())
 419         d.addCallback(_done)
 420         return d
 421
 422     def _test_deepcheck_create(self):
 423         # create a small tree with a loop, and some non-directories
 424         #  root/
 425         #  root/subdir/
 426         #  root/subdir/file1
 427         #  root/subdir/link -> root
 428         #  root/rodir
 429         c = self.g.clients[0]
 430         d = c.create_dirnode()
 431         def _created_root(rootnode):
 432             self._rootnode = rootnode
 433             return rootnode.create_subdirectory(u"subdir")
 434         d.addCallback(_created_root)
 435         def _created_subdir(subdir):
 436             self._subdir = subdir
 437             d = subdir.add_file(u"file1", upload.Data("data"*100, None))
 438             d.addCallback(lambda res: subdir.set_node(u"link", self._rootnode))
 439             d.addCallback(lambda res: c.create_dirnode())
 440             d.addCallback(lambda dn:
 441                           self._rootnode.set_uri(u"rodir",
 442                                                  dn.get_uri(),
 443                                                  dn.get_readonly_uri()))
 444             return d
 445         d.addCallback(_created_subdir)
 446         def _done(res):
 447             return self._rootnode
 448         d.addCallback(_done)
 449         return d
 450
 451     def test_deepcheck(self):
 452         self.basedir = "dirnode/Dirnode/test_deepcheck"
 453         self.set_up_grid()
 454         d = self._test_deepcheck_create()
 455         d.addCallback(lambda rootnode: rootnode.start_deep_check().when_done())
 456         def _check_results(r):
 457             self.failUnless(IDeepCheckResults.providedBy(r))
 458             c = r.get_counters()
 459             self.failUnlessEqual(c,
 460                                  {"count-objects-checked": 4,
 461                                   "count-objects-healthy": 4,
 462                                   "count-objects-unhealthy": 0,
 463                                   "count-objects-unrecoverable": 0,
 464                                   "count-corrupt-shares": 0,
 465                                   })
 466             self.failIf(r.get_corrupt_shares())
 467             self.failUnlessEqual(len(r.get_all_results()), 4)
 468         d.addCallback(_check_results)
 469         return d
 470
 471     def test_deepcheck_and_repair(self):
 472         self.basedir = "dirnode/Dirnode/test_deepcheck_and_repair"
 473         self.set_up_grid()
 474         d = self._test_deepcheck_create()
 475         d.addCallback(lambda rootnode:
 476                       rootnode.start_deep_check_and_repair().when_done())
 477         def _check_results(r):
 478             self.failUnless(IDeepCheckAndRepairResults.providedBy(r))
 479             c = r.get_counters()
 480             self.failUnlessEqual(c,
 481                                  {"count-objects-checked": 4,
 482                                   "count-objects-healthy-pre-repair": 4,
 483                                   "count-objects-unhealthy-pre-repair": 0,
 484                                   "count-objects-unrecoverable-pre-repair": 0,
 485                                   "count-corrupt-shares-pre-repair": 0,
 486                                   "count-objects-healthy-post-repair": 4,
 487                                   "count-objects-unhealthy-post-repair": 0,
 488                                   "count-objects-unrecoverable-post-repair": 0,
 489                                   "count-corrupt-shares-post-repair": 0,
 490                                   "count-repairs-attempted": 0,
 491                                   "count-repairs-successful": 0,
 492                                   "count-repairs-unsuccessful": 0,
 493                                   })
 494             self.failIf(r.get_corrupt_shares())
 495             self.failIf(r.get_remaining_corrupt_shares())
 496             self.failUnlessEqual(len(r.get_all_results()), 4)
 497         d.addCallback(_check_results)
 498         return d
 499
 500     def _mark_file_bad(self, rootnode):
 501         self.delete_shares_numbered(rootnode.get_uri(), [0])
 502         return rootnode
 503
 504     def test_deepcheck_problems(self):
 505         self.basedir = "dirnode/Dirnode/test_deepcheck_problems"
 506         self.set_up_grid()
 507         d = self._test_deepcheck_create()
 508         d.addCallback(lambda rootnode: self._mark_file_bad(rootnode))
 509         d.addCallback(lambda rootnode: rootnode.start_deep_check().when_done())
 510         def _check_results(r):
 511             c = r.get_counters()
 512             self.failUnlessEqual(c,
 513                                  {"count-objects-checked": 4,
 514                                   "count-objects-healthy": 3,
 515                                   "count-objects-unhealthy": 1,
 516                                   "count-objects-unrecoverable": 0,
 517                                   "count-corrupt-shares": 0,
 518                                   })
 519             #self.failUnlessEqual(len(r.get_problems()), 1) # TODO
 520         d.addCallback(_check_results)
 521         return d
 522
 523     def test_readonly(self):
 524         self.basedir = "dirnode/Dirnode/test_readonly"
 525         self.set_up_grid()
 526         c = self.g.clients[0]
 527         nm = c.nodemaker
 528         filecap = make_chk_file_uri(1234)
 529         filenode = nm.create_from_cap(filecap)
 530         uploadable = upload.Data("some data", convergence="some convergence string")
 531
 532         d = c.create_dirnode()
 533         def _created(rw_dn):
 534             d2 = rw_dn.set_uri(u"child", filecap, filecap)
 535             d2.addCallback(lambda res: rw_dn)
 536             return d2
 537         d.addCallback(_created)
 538
 539         def _ready(rw_dn):
 540             ro_uri = rw_dn.get_readonly_uri()
 541             ro_dn = c.create_node_from_uri(ro_uri)
 542             self.failUnless(ro_dn.is_readonly())
 543             self.failUnless(ro_dn.is_mutable())
 544             self.failIf(ro_dn.is_unknown())
 545             self.failIf(ro_dn.is_allowed_in_immutable_directory())
 546             ro_dn.raise_error()
 547
 548             self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
 549                             ro_dn.set_uri, u"newchild", filecap, filecap)
 550             self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
 551                             ro_dn.set_node, u"newchild", filenode)
 552             self.shouldFail(dirnode.NotWriteableError, "set_nodes ro", None,
 553                             ro_dn.set_nodes, { u"newchild": (filenode, None) })
 554             self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
 555                             ro_dn.add_file, u"newchild", uploadable)
 556             self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
 557                             ro_dn.delete, u"child")
 558             self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
 559                             ro_dn.create_subdirectory, u"newchild")
 560             self.shouldFail(dirnode.NotWriteableError, "set_metadata_for ro", None,
 561                             ro_dn.set_metadata_for, u"child", {})
 562             self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
 563                             ro_dn.move_child_to, u"child", rw_dn)
 564             self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
 565                             rw_dn.move_child_to, u"child", ro_dn)
 566             return ro_dn.list()
 567         d.addCallback(_ready)
 568         def _listed(children):
 569             self.failUnless(u"child" in children)
 570         d.addCallback(_listed)
 571         return d
 572
 573     def failUnlessGreaterThan(self, a, b):
 574         self.failUnless(a > b, "%r should be > %r" % (a, b))
 575
 576     def failUnlessGreaterOrEqualThan(self, a, b):
 577         self.failUnless(a >= b, "%r should be >= %r" % (a, b))
 578
 579     def test_create(self):
 580         self.basedir = "dirnode/Dirnode/test_create"
 581         self.set_up_grid()
 582         c = self.g.clients[0]
 583
 584         self.expected_manifest = []
 585         self.expected_verifycaps = set()
 586         self.expected_storage_indexes = set()
 587
 588         d = c.create_dirnode()
 589         def _then(n):
 590             # /
 591             self.rootnode = n
 592             self.failUnless(n.is_mutable())
 593             u = n.get_uri()
 594             self.failUnless(u)
 595             self.failUnless(u.startswith("URI:DIR2:"), u)
 596             u_ro = n.get_readonly_uri()
 597             self.failUnless(u_ro.startswith("URI:DIR2-RO:"), u_ro)
 598             u_v = n.get_verify_cap().to_string()
 599             self.failUnless(u_v.startswith("URI:DIR2-Verifier:"), u_v)
 600             u_r = n.get_repair_cap().to_string()
 601             self.failUnlessEqual(u_r, u)
 602             self.expected_manifest.append( ((), u) )
 603             self.expected_verifycaps.add(u_v)
 604             si = n.get_storage_index()
 605             self.expected_storage_indexes.add(base32.b2a(si))
 606             expected_si = n._uri.get_storage_index()
 607             self.failUnlessEqual(si, expected_si)
 608
 609             d = n.list()
 610             d.addCallback(lambda res: self.failUnlessEqual(res, {}))
 611             d.addCallback(lambda res: n.has_child(u"missing"))
 612             d.addCallback(lambda res: self.failIf(res))
 613
 614             fake_file_uri = make_mutable_file_uri()
 615             other_file_uri = make_mutable_file_uri()
 616             m = c.nodemaker.create_from_cap(fake_file_uri)
 617             ffu_v = m.get_verify_cap().to_string()
 618             self.expected_manifest.append( ((u"child",) , m.get_uri()) )
 619             self.expected_verifycaps.add(ffu_v)
 620             self.expected_storage_indexes.add(base32.b2a(m.get_storage_index()))
 621             d.addCallback(lambda res: n.set_uri(u"child",
 622                                                 fake_file_uri, fake_file_uri))
 623             d.addCallback(lambda res:
 624                           self.shouldFail(ExistingChildError, "set_uri-no",
 625                                           "child 'child' already exists",
 626                                           n.set_uri, u"child",
 627                                           other_file_uri, other_file_uri,
 628                                           overwrite=False))
 629             # /
 630             # /child = mutable
 631
 632             d.addCallback(lambda res: n.create_subdirectory(u"subdir"))
 633
 634             # /
 635             # /child = mutable
 636             # /subdir = directory
 637             def _created(subdir):
 638                 self.failUnless(isinstance(subdir, dirnode.DirectoryNode))
 639                 self.subdir = subdir
 640                 new_v = subdir.get_verify_cap().to_string()
 641                 assert isinstance(new_v, str)
 642                 self.expected_manifest.append( ((u"subdir",), subdir.get_uri()) )
 643                 self.expected_verifycaps.add(new_v)
 644                 si = subdir.get_storage_index()
 645                 self.expected_storage_indexes.add(base32.b2a(si))
 646             d.addCallback(_created)
 647
 648             d.addCallback(lambda res:
 649                           self.shouldFail(ExistingChildError, "mkdir-no",
 650                                           "child 'subdir' already exists",
 651                                           n.create_subdirectory, u"subdir",
 652                                           overwrite=False))
 653
 654             d.addCallback(lambda res: n.list())
 655             d.addCallback(lambda children:
 656                           self.failUnlessEqual(set(children.keys()),
 657                                                set([u"child", u"subdir"])))
 658
 659             d.addCallback(lambda res: n.start_deep_stats().when_done())
 660             def _check_deepstats(stats):
 661                 self.failUnless(isinstance(stats, dict))
 662                 expected = {"count-immutable-files": 0,
 663                             "count-mutable-files": 1,
 664                             "count-literal-files": 0,
 665                             "count-files": 1,
 666                             "count-directories": 2,
 667                             "size-immutable-files": 0,
 668                             "size-literal-files": 0,
 669                             #"size-directories": 616, # varies
 670                             #"largest-directory": 616,
 671                             "largest-directory-children": 2,
 672                             "largest-immutable-file": 0,
 673                             }
 674                 for k,v in expected.iteritems():
 675                     self.failUnlessEqual(stats[k], v,
 676                                          "stats[%s] was %s, not %s" %
 677                                          (k, stats[k], v))
 678                 self.failUnless(stats["size-directories"] > 500,
 679                                 stats["size-directories"])
 680                 self.failUnless(stats["largest-directory"] > 500,
 681                                 stats["largest-directory"])
 682                 self.failUnlessEqual(stats["size-files-histogram"], [])
 683             d.addCallback(_check_deepstats)
 684
 685             d.addCallback(lambda res: n.build_manifest().when_done())
 686             def _check_manifest(res):
 687                 manifest = res["manifest"]
 688                 self.failUnlessEqual(sorted(manifest),
 689                                      sorted(self.expected_manifest))
 690                 stats = res["stats"]
 691                 _check_deepstats(stats)
 692                 self.failUnlessEqual(self.expected_verifycaps,
 693                                      res["verifycaps"])
 694                 self.failUnlessEqual(self.expected_storage_indexes,
 695                                      res["storage-index"])
 696             d.addCallback(_check_manifest)
 697
 698             def _add_subsubdir(res):
 699                 return self.subdir.create_subdirectory(u"subsubdir")
 700             d.addCallback(_add_subsubdir)
 701             # /
 702             # /child = mutable
 703             # /subdir = directory
 704             # /subdir/subsubdir = directory
 705             d.addCallback(lambda res: n.get_child_at_path(u"subdir/subsubdir"))
 706             d.addCallback(lambda subsubdir:
 707                           self.failUnless(isinstance(subsubdir,
 708                                                      dirnode.DirectoryNode)))
 709             d.addCallback(lambda res: n.get_child_at_path(u""))
 710             d.addCallback(lambda res: self.failUnlessEqual(res.get_uri(),
 711                                                            n.get_uri()))
 712
 713             d.addCallback(lambda res: n.get_metadata_for(u"child"))
 714             d.addCallback(lambda metadata:
 715                           self.failUnlessEqual(set(metadata.keys()),
 716                                                set(["tahoe", "ctime", "mtime"])))
 717
 718             d.addCallback(lambda res:
 719                           self.shouldFail(NoSuchChildError, "gcamap-no",
 720                                           "nope",
 721                                           n.get_child_and_metadata_at_path,
 722                                           u"subdir/nope"))
 723             d.addCallback(lambda res:
 724                           n.get_child_and_metadata_at_path(u""))
 725             def _check_child_and_metadata1(res):
 726                 child, metadata = res
 727                 self.failUnless(isinstance(child, dirnode.DirectoryNode))
 728                 # edge-metadata needs at least one path segment
 729                 self.failUnlessEqual(set(metadata.keys()), set([]))
 730             d.addCallback(_check_child_and_metadata1)
 731             d.addCallback(lambda res:
 732                           n.get_child_and_metadata_at_path(u"child"))
 733
 734             def _check_child_and_metadata2(res):
 735                 child, metadata = res
 736                 self.failUnlessEqual(child.get_uri(),
 737                                      fake_file_uri)
 738                 self.failUnlessEqual(set(metadata.keys()),
 739                                      set(["tahoe", "ctime", "mtime"]))
 740             d.addCallback(_check_child_and_metadata2)
 741
 742             d.addCallback(lambda res:
 743                           n.get_child_and_metadata_at_path(u"subdir/subsubdir"))
 744             def _check_child_and_metadata3(res):
 745                 child, metadata = res
 746                 self.failUnless(isinstance(child, dirnode.DirectoryNode))
 747                 self.failUnlessEqual(set(metadata.keys()),
 748                                      set(["tahoe", "ctime", "mtime"]))
 749             d.addCallback(_check_child_and_metadata3)
 750
 751             # set_uri + metadata
 752             # it should be possible to add a child without any metadata
 753             d.addCallback(lambda res: n.set_uri(u"c2",
 754                                                 fake_file_uri, fake_file_uri,
 755                                                 {}))
 756             d.addCallback(lambda res: n.get_metadata_for(u"c2"))
 757             d.addCallback(lambda metadata:
 758                           self.failUnlessEqual(set(metadata.keys()), set(["tahoe", "ctime", "mtime"])))
 759
 760             # You can't override the link timestamps.
 761             d.addCallback(lambda res: n.set_uri(u"c2",
 762                                                 fake_file_uri, fake_file_uri,
 763                                                 { 'tahoe': {'linkcrtime': "bogus"}}))
 764             d.addCallback(lambda res: n.get_metadata_for(u"c2"))
 765             def _has_good_linkcrtime(metadata):
 766                 self.failUnless(metadata.has_key('tahoe'))
 767                 self.failUnless(metadata['tahoe'].has_key('linkcrtime'))
 768                 self.failIfEqual(metadata['tahoe']['linkcrtime'], 'bogus')
 769             d.addCallback(_has_good_linkcrtime)
 770
 771             # if we don't set any defaults, the child should get timestamps
 772             d.addCallback(lambda res: n.set_uri(u"c3",
 773                                                 fake_file_uri, fake_file_uri))
 774             d.addCallback(lambda res: n.get_metadata_for(u"c3"))
 775             d.addCallback(lambda metadata:
 776                           self.failUnlessEqual(set(metadata.keys()), set(["tahoe", "ctime", "mtime"])))
 777
 778             # or we can add specific metadata at set_uri() time, which
 779             # overrides the timestamps
 780             d.addCallback(lambda res: n.set_uri(u"c4",
 781                                                 fake_file_uri, fake_file_uri,
 782                                                 {"key": "value"}))
 783             d.addCallback(lambda res: n.get_metadata_for(u"c4"))
 784             d.addCallback(lambda metadata:
 785                               self.failUnless((set(metadata.keys()) == set(["key", "tahoe", "ctime", "mtime"])) and
 786                                               (metadata['key'] == "value"), metadata))
 787
 788             d.addCallback(lambda res: n.delete(u"c2"))
 789             d.addCallback(lambda res: n.delete(u"c3"))
 790             d.addCallback(lambda res: n.delete(u"c4"))
 791
 792             # set_node + metadata
 793             # it should be possible to add a child without any metadata
 794             d.addCallback(lambda res: n.set_node(u"d2", n, {}))
 795             d.addCallback(lambda res: c.create_dirnode())
 796             d.addCallback(lambda n2:
 797                           self.shouldFail(ExistingChildError, "set_node-no",
 798                                           "child 'd2' already exists",
 799                                           n.set_node, u"d2", n2,
 800                                           overwrite=False))
 801             d.addCallback(lambda res: n.get_metadata_for(u"d2"))
 802             d.addCallback(lambda metadata:
 803                           self.failUnlessEqual(set(metadata.keys()), set(["tahoe", "ctime", "mtime"])))
 804
 805             # if we don't set any defaults, the child should get timestamps
 806             d.addCallback(lambda res: n.set_node(u"d3", n))
 807             d.addCallback(lambda res: n.get_metadata_for(u"d3"))
 808             d.addCallback(lambda metadata:
 809                           self.failUnlessEqual(set(metadata.keys()), set(["tahoe", "ctime", "mtime"])))
 810
 811             # or we can add specific metadata at set_node() time, which
 812             # overrides the timestamps
 813             d.addCallback(lambda res: n.set_node(u"d4", n,
 814                                                 {"key": "value"}))
 815             d.addCallback(lambda res: n.get_metadata_for(u"d4"))
 816             d.addCallback(lambda metadata:
 817                           self.failUnless((set(metadata.keys()) == set(["key", "tahoe", "ctime", "mtime"])) and
 818                                           (metadata["key"] == "value"), metadata))
 819
 820             d.addCallback(lambda res: n.delete(u"d2"))
 821             d.addCallback(lambda res: n.delete(u"d3"))
 822             d.addCallback(lambda res: n.delete(u"d4"))
 823
 824             # metadata through set_children()
 825             d.addCallback(lambda res:
 826                           n.set_children({
 827                               u"e1": (fake_file_uri, fake_file_uri),
 828                               u"e2": (fake_file_uri, fake_file_uri, {}),
 829                               u"e3": (fake_file_uri, fake_file_uri,
 830                                       {"key": "value"}),
 831                               }))
 832             d.addCallback(lambda n2: self.failUnlessIdentical(n2, n))
 833             d.addCallback(lambda res:
 834                           self.shouldFail(ExistingChildError, "set_children-no",
 835                                           "child 'e1' already exists",
 836                                           n.set_children,
 837                                           { u"e1": (other_file_uri,
 838                                                     other_file_uri),
 839                                             u"new": (other_file_uri,
 840                                                      other_file_uri),
 841                                             },
 842                                           overwrite=False))
 843             # and 'new' should not have been created
 844             d.addCallback(lambda res: n.list())
 845             d.addCallback(lambda children: self.failIf(u"new" in children))
 846             d.addCallback(lambda res: n.get_metadata_for(u"e1"))
 847             d.addCallback(lambda metadata:
 848                           self.failUnlessEqual(set(metadata.keys()), set(["tahoe", "ctime", "mtime"])))
 849             d.addCallback(lambda res: n.get_metadata_for(u"e2"))
 850             d.addCallback(lambda metadata:
 851                           self.failUnlessEqual(set(metadata.keys()), set(["tahoe", "ctime", "mtime"])))
 852             d.addCallback(lambda res: n.get_metadata_for(u"e3"))
 853             d.addCallback(lambda metadata:
 854                           self.failUnless((set(metadata.keys()) == set(["key", "tahoe", "ctime", "mtime"])) and
 855                                           (metadata["key"] == "value"), metadata))
 856
 857             d.addCallback(lambda res: n.delete(u"e1"))
 858             d.addCallback(lambda res: n.delete(u"e2"))
 859             d.addCallback(lambda res: n.delete(u"e3"))
 860
 861             # metadata through set_nodes()
 862             d.addCallback(lambda res:
 863                           n.set_nodes({ u"f1": (n, None),
 864                                         u"f2": (n, {}),
 865                                         u"f3": (n, {"key": "value"}),
 866                                         }))
 867             d.addCallback(lambda n2: self.failUnlessIdentical(n2, n))
 868             d.addCallback(lambda res:
 869                           self.shouldFail(ExistingChildError, "set_nodes-no",
 870                                           "child 'f1' already exists",
 871                                           n.set_nodes, { u"f1": (n, None),
 872                                                          u"new": (n, None), },
 873                                           overwrite=False))
 874             # and 'new' should not have been created
 875             d.addCallback(lambda res: n.list())
 876             d.addCallback(lambda children: self.failIf(u"new" in children))
 877             d.addCallback(lambda res: n.get_metadata_for(u"f1"))
 878             d.addCallback(lambda metadata:
 879                           self.failUnlessEqual(set(metadata.keys()), set(["tahoe", "ctime", "mtime"])))
 880             d.addCallback(lambda res: n.get_metadata_for(u"f2"))
 881             d.addCallback(lambda metadata:
 882                           self.failUnlessEqual(set(metadata.keys()), set(["tahoe", "ctime", "mtime"])))
 883             d.addCallback(lambda res: n.get_metadata_for(u"f3"))
 884             d.addCallback(lambda metadata:
 885                           self.failUnless((set(metadata.keys()) == set(["key", "tahoe", "ctime", "mtime"])) and
 886                                           (metadata["key"] == "value"), metadata))
 887
 888             d.addCallback(lambda res: n.delete(u"f1"))
 889             d.addCallback(lambda res: n.delete(u"f2"))
 890             d.addCallback(lambda res: n.delete(u"f3"))
 891
 892
 893             d.addCallback(lambda res:
 894                           n.set_metadata_for(u"child",
 895                                              {"tags": ["web2.0-compatible"], "tahoe": {"bad": "mojo"}}))
 896             d.addCallback(lambda n1: n1.get_metadata_for(u"child"))
 897             d.addCallback(lambda metadata:
 898                           self.failUnless((set(metadata.keys()) == set(["tags", "tahoe", "ctime", "mtime"])) and
 899                                           metadata["tags"] == ["web2.0-compatible"] and
 900                                           "bad" not in metadata["tahoe"], metadata))
 901
 902             def _start(res):
 903                 self._start_timestamp = time.time()
 904             d.addCallback(_start)
 905             # simplejson-1.7.1 (as shipped on Ubuntu 'gutsy') rounds all
 906             # floats to hundredeths (it uses str(num) instead of repr(num)).
 907             # simplejson-1.7.3 does not have this bug. To prevent this bug
 908             # from causing the test to fail, stall for more than a few
 909             # hundrededths of a second.
 910             d.addCallback(self.stall, 0.1)
 911             d.addCallback(lambda res: n.add_file(u"timestamps",
 912                                                  upload.Data("stamp me", convergence="some convergence string")))
 913             d.addCallback(self.stall, 0.1)
 914             def _stop(res):
 915                 self._stop_timestamp = time.time()
 916             d.addCallback(_stop)
 917
 918             d.addCallback(lambda res: n.get_metadata_for(u"timestamps"))
 919             def _check_timestamp1(metadata):
 920                 self.failUnless("ctime" in metadata)
 921                 self.failUnless("mtime" in metadata)
 922                 self.failUnlessGreaterOrEqualThan(metadata["ctime"],
 923                                                   self._start_timestamp)
 924                 self.failUnlessGreaterOrEqualThan(self._stop_timestamp,
 925                                                   metadata["ctime"])
 926                 self.failUnlessGreaterOrEqualThan(metadata["mtime"],
 927                                                   self._start_timestamp)
 928                 self.failUnlessGreaterOrEqualThan(self._stop_timestamp,
 929                                                   metadata["mtime"])
 930                 # Our current timestamp rules say that replacing an existing
 931                 # child should preserve the 'ctime' but update the mtime
 932                 self._old_ctime = metadata["ctime"]
 933                 self._old_mtime = metadata["mtime"]
 934             d.addCallback(_check_timestamp1)
 935             d.addCallback(self.stall, 2.0) # accomodate low-res timestamps
 936             d.addCallback(lambda res: n.set_node(u"timestamps", n))
 937             d.addCallback(lambda res: n.get_metadata_for(u"timestamps"))
 938             def _check_timestamp2(metadata):
 939                 self.failUnlessEqual(metadata["ctime"], self._old_ctime,
 940                                      "%s != %s" % (metadata["ctime"],
 941                                                    self._old_ctime))
 942                 self.failUnlessGreaterThan(metadata["mtime"], self._old_mtime)
 943                 return n.delete(u"timestamps")
 944             d.addCallback(_check_timestamp2)
 945
 946             # also make sure we can add/update timestamps on a
 947             # previously-existing child that didn't have any, since there are
 948             # a lot of 0.7.0-generated edges around out there
 949             d.addCallback(lambda res: n.set_node(u"no_timestamps", n, {}))
 950             d.addCallback(lambda res: n.set_node(u"no_timestamps", n))
 951             d.addCallback(lambda res: n.get_metadata_for(u"no_timestamps"))
 952             d.addCallback(lambda metadata:
 953                           self.failUnlessEqual(set(metadata.keys()),
 954                                                set(["tahoe", "ctime", "mtime"])))
 955             d.addCallback(lambda res: n.delete(u"no_timestamps"))
 956
 957             d.addCallback(lambda res: n.delete(u"subdir"))
 958             d.addCallback(lambda old_child:
 959                           self.failUnlessEqual(old_child.get_uri(),
 960                                                self.subdir.get_uri()))
 961
 962             d.addCallback(lambda res: n.list())
 963             d.addCallback(lambda children:
 964                           self.failUnlessEqual(set(children.keys()),
 965                                                set([u"child"])))
 966
 967             uploadable1 = upload.Data("some data", convergence="converge")
 968             d.addCallback(lambda res: n.add_file(u"newfile", uploadable1))
 969             d.addCallback(lambda newnode:
 970                           self.failUnless(IImmutableFileNode.providedBy(newnode)))
 971             uploadable2 = upload.Data("some data", convergence="stuff")
 972             d.addCallback(lambda res:
 973                           self.shouldFail(ExistingChildError, "add_file-no",
 974                                           "child 'newfile' already exists",
 975                                           n.add_file, u"newfile",
 976                                           uploadable2,
 977                                           overwrite=False))
 978             d.addCallback(lambda res: n.list())
 979             d.addCallback(lambda children:
 980                           self.failUnlessEqual(set(children.keys()),
 981                                                set([u"child", u"newfile"])))
 982             d.addCallback(lambda res: n.get_metadata_for(u"newfile"))
 983             d.addCallback(lambda metadata:
 984                           self.failUnlessEqual(set(metadata.keys()),
 985                                                set(["tahoe", "ctime", "mtime"])))
 986
 987             uploadable3 = upload.Data("some data", convergence="converge")
 988             d.addCallback(lambda res: n.add_file(u"newfile-metadata",
 989                                                  uploadable3,
 990                                                  {"key": "value"}))
 991             d.addCallback(lambda newnode:
 992                           self.failUnless(IImmutableFileNode.providedBy(newnode)))
 993             d.addCallback(lambda res: n.get_metadata_for(u"newfile-metadata"))
 994             d.addCallback(lambda metadata:
 995                               self.failUnless((set(metadata.keys()) == set(["key", "tahoe", "ctime", "mtime"])) and
 996                                               (metadata['key'] == "value"), metadata))
 997             d.addCallback(lambda res: n.delete(u"newfile-metadata"))
 998
 999             d.addCallback(lambda res: n.create_subdirectory(u"subdir2"))
1000             def _created2(subdir2):
1001                 self.subdir2 = subdir2
1002                 # put something in the way, to make sure it gets overwritten
1003                 return subdir2.add_file(u"child", upload.Data("overwrite me",
1004                                                               "converge"))
1005             d.addCallback(_created2)
1006
1007             d.addCallback(lambda res:
1008                           n.move_child_to(u"child", self.subdir2))
1009             d.addCallback(lambda res: n.list())
1010             d.addCallback(lambda children:
1011                           self.failUnlessEqual(set(children.keys()),
1012                                                set([u"newfile", u"subdir2"])))
1013             d.addCallback(lambda res: self.subdir2.list())
1014             d.addCallback(lambda children:
1015                           self.failUnlessEqual(set(children.keys()),
1016                                                set([u"child"])))
1017             d.addCallback(lambda res: self.subdir2.get(u"child"))
1018             d.addCallback(lambda child:
1019                           self.failUnlessEqual(child.get_uri(),
1020                                                fake_file_uri))
1021
1022             # move it back, using new_child_name=
1023             d.addCallback(lambda res:
1024                           self.subdir2.move_child_to(u"child", n, u"newchild"))
1025             d.addCallback(lambda res: n.list())
1026             d.addCallback(lambda children:
1027                           self.failUnlessEqual(set(children.keys()),
1028                                                set([u"newchild", u"newfile",
1029                                                     u"subdir2"])))
1030             d.addCallback(lambda res: self.subdir2.list())
1031             d.addCallback(lambda children:
1032                           self.failUnlessEqual(set(children.keys()), set([])))
1033
1034             # now make sure that we honor overwrite=False
1035             d.addCallback(lambda res:
1036                           self.subdir2.set_uri(u"newchild",
1037                                                other_file_uri, other_file_uri))
1038
1039             d.addCallback(lambda res:
1040                           self.shouldFail(ExistingChildError, "move_child_to-no",
1041                                           "child 'newchild' already exists",
1042                                           n.move_child_to, u"newchild",
1043                                           self.subdir2,
1044                                           overwrite=False))
1045             d.addCallback(lambda res: self.subdir2.get(u"newchild"))
1046             d.addCallback(lambda child:
1047                           self.failUnlessEqual(child.get_uri(),
1048                                                other_file_uri))
1049
1050             return d
1051
1052         d.addCallback(_then)
1053
1054         d.addErrback(self.explain_error)
1055         return d
1056
1057     def test_update_metadata(self):
1058         (t1, t2, t3) = (626644800, 634745640, 892226160)
1059
1060         md1 = dirnode.update_metadata({}, {"ctime": t1}, t2)
1061         self.failUnlessEqual(md1, {"ctime": t1, "mtime": t2,
1062                                    "tahoe":{"linkcrtime": t1, "linkmotime": t2}})
1063
1064         md2 = dirnode.update_metadata(md1, {"key": "value", "tahoe": {"bad": "mojo"}}, t3)
1065         self.failUnlessEqual(md2, {"key": "value", "ctime": t1, "mtime": t3,
1066                                    "tahoe":{"linkcrtime": t1, "linkmotime": t3}})
1067
1068         md3 = dirnode.update_metadata({}, None, t3)
1069         self.failUnlessEqual(md3, {"ctime": t3, "mtime": t3,
1070                                    "tahoe":{"linkcrtime": t3, "linkmotime": t3}})
1071
1072     def test_create_subdirectory(self):
1073         self.basedir = "dirnode/Dirnode/test_create_subdirectory"
1074         self.set_up_grid()
1075         c = self.g.clients[0]
1076         nm = c.nodemaker
1077
1078         d = c.create_dirnode()
1079         def _then(n):
1080             # /
1081             self.rootnode = n
1082             fake_file_uri = make_mutable_file_uri()
1083             other_file_uri = make_mutable_file_uri()
1084             md = {"metakey": "metavalue"}
1085             kids = {u"kid1": (nm.create_from_cap(fake_file_uri), {}),
1086                     u"kid2": (nm.create_from_cap(other_file_uri), md),
1087                     }
1088             d = n.create_subdirectory(u"subdir", kids)
1089             def _check(sub):
1090                 d = n.get_child_at_path(u"subdir")
1091                 d.addCallback(lambda sub2: self.failUnlessEqual(sub2.get_uri(),
1092                                                                 sub.get_uri()))
1093                 d.addCallback(lambda ign: sub.list())
1094                 return d
1095             d.addCallback(_check)
1096             def _check_kids(kids2):
1097                 self.failUnlessEqual(set(kids.keys()), set(kids2.keys()))
1098                 self.failUnlessEqual(kids2[u"kid2"][1]["metakey"], "metavalue")
1099             d.addCallback(_check_kids)
1100             return d
1101         d.addCallback(_then)
1102         return d
1103
1104 class MinimalFakeMutableFile:
1105     def get_writekey(self):
1106         return "writekey"
1107
1108 class Packing(unittest.TestCase):
1109     # This is a base32-encoded representation of the directory tree
1110     # root/file1
1111     # root/file2
1112     # root/file3
1113     # as represented after being fed to _pack_contents.
1114     # We have it here so we can decode it, feed it to
1115     # _unpack_contents, and verify that _unpack_contents
1116     # works correctly.
1117
1118     known_tree = "GM4TOORVHJTGS3DFGEWDSNJ2KVJESOSDJBFTU33MPB2GS3LZNVYG6N3GGI3WU5TIORTXC3DOMJ2G4NB2MVWXUZDONBVTE5LNGRZWK2LYN55GY23XGNYXQMTOMZUWU5TENN4DG23ZG5UTO2L2NQ2DO6LFMRWDMZJWGRQTUMZ2GEYDUMJQFQYTIMZ22XZKZORX5XS7CAQCSK3URR6QOHISHRCMGER5LRFSZRNAS5ZSALCS6TWFQAE754IVOIKJVK73WZPP3VUUEDTX3WHTBBZ5YX3CEKHCPG3ZWQLYA4QM6LDRCF7TJQYWLIZHKGN5ROA3AUZPXESBNLQQ6JTC2DBJU2D47IZJTLR3PKZ4RVF57XLPWY7FX7SZV3T6IJ3ORFW37FXUPGOE3ROPFNUX5DCGMAQJ3PGGULBRGM3TU6ZCMN2GS3LFEI5CAMJSGQ3DMNRTHA4TOLRUGI3TKNRWGEWCAITUMFUG6ZJCHIQHWITMNFXGW3LPORUW2ZJCHIQDCMRUGY3DMMZYHE3S4NBSG42TMNRRFQQCE3DJNZVWG4TUNFWWKIR2EAYTENBWGY3DGOBZG4XDIMRXGU3DMML5FQQCE3LUNFWWKIR2EAYTENBWGY3DGOBZG4XDIMRXGU3DMML5FQWDGOJRHI2TUZTJNRSTELBZGQ5FKUSJHJBUQSZ2MFYGKZ3SOBSWQ43IO52WO23CNAZWU3DUGVSWSNTIOE5DK33POVTW4ZLNMNWDK6DHPA2GS2THNF2W25DEN5VGY2LQNFRGG5DKNNRHO5TZPFTWI6LNMRYGQ2LCGJTHM4J2GM5DCMB2GQWDCNBSHKVVQBGRYMACKJ27CVQ6O6B4QPR72RFVTGOZUI76XUSWAX73JRV5PYRHMIFYZIA25MXDPGUGML6M2NMRSG4YD4W4K37ZDYSXHMJ3IUVT4F64YTQQVBJFFFOUC7J7LAB2VFCL5UKKGMR2D3F4EPOYC7UYWQZNR5KXHBSNXLCNBX2SNF22DCXJIHSMEKWEWOG5XCJEVVZ7UW5IB6I64XXQSJ34B5CAYZGZIIMR6LBRGMZTU6ZCMN2GS3LFEI5CAMJSGQ3DMNRTHA4TOLRUGMYDEMJYFQQCE5DBNBXWKIR2EB5SE3DJNZVW233UNFWWKIR2EAYTENBWGY3DGOBZG4XDIMZQGIYTQLBAEJWGS3TLMNZHI2LNMURDUIBRGI2DMNRWGM4DSNZOGQZTAMRRHB6SYIBCNV2GS3LFEI5CAMJSGQ3DMNRTHA4TOLRUGMYDEMJYPUWCYMZZGU5DKOTGNFWGKMZMHE2DUVKSJE5EGSCLHJRW25DDPBYTO2DXPB3GM6DBNYZTI6LJMV3DM2LWNB4TU4LWMNSWW3LKORXWK5DEMN3TI23NNE3WEM3SORRGY5THPA3TKNBUMNZG453BOF2GSZLXMVWWI3DJOFZW623RHIZTUMJQHI2SYMJUGI5BOSHWDPG3WKPAVXCF3XMKA7QVIWPRMWJHDTQHD27AHDCPJWDQENQ5H5ZZILTXQNIXXCIW4LKQABU2GCFRG5FHQN7CHD7HF4EKNRZFIV2ZYQIBM7IQU7F4RGB3XCX3FREPBKQ7UCICHVWPCYFGA6OLH3J45LXQ6GWWICJ3PGWJNLZ7PCRNLAPNYUGU6BENS7OXMBEOOFRIZV3PF2FFWZ5WHDPKXERYP7GNHKRMGEZTOOT3EJRXI2LNMURDUIBRGI2DMNRWGM4DSNZOGQZTGNRSGY4SYIBCORQWQ33FEI5CA6ZCNRUW423NN52GS3LFEI5CAMJSGQ3DMNRTHA4TOLRUGMZTMMRWHEWCAITMNFXGWY3SORUW2ZJCHIQDCMRUGY3DMMZYHE3S4NBTGM3DENRZPUWCAITNORUW2ZJCHIQDCMRUGY3DMMZYHE3S4NBTGM3DENRZPUWCY==="
1119
1120     def test_unpack_and_pack_behavior(self):
1121         known_tree = b32decode(self.known_tree)
1122         nodemaker = NodeMaker(None, None, None,
1123                               None, None, None,
1124                               {"k": 3, "n": 10}, None)
1125         write_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
1126         filenode = nodemaker.create_from_cap(write_uri)
1127         node = dirnode.DirectoryNode(filenode, nodemaker, None)
1128         children = node._unpack_contents(known_tree)
1129         self._check_children(children)
1130
1131         packed_children = node._pack_contents(children)
1132         children = node._unpack_contents(packed_children)
1133         self._check_children(children)
1134
1135     def _check_children(self, children):
1136         # Are all the expected child nodes there?
1137         self.failUnless(children.has_key(u'file1'))
1138         self.failUnless(children.has_key(u'file2'))
1139         self.failUnless(children.has_key(u'file3'))
1140
1141         # Are the metadata for child 3 right?
1142         file3_rocap = "URI:CHK:cmtcxq7hwxvfxan34yiev6ivhy:qvcekmjtoetdcw4kmi7b3rtblvgx7544crnwaqtiewemdliqsokq:3:10:5"
1143         file3_rwcap = "URI:CHK:cmtcxq7hwxvfxan34yiev6ivhy:qvcekmjtoetdcw4kmi7b3rtblvgx7544crnwaqtiewemdliqsokq:3:10:5"
1144         file3_metadata = {'ctime': 1246663897.4336269, 'tahoe': {'linkmotime': 1246663897.4336269, 'linkcrtime': 1246663897.4336269}, 'mtime': 1246663897.4336269}
1145         self.failUnlessEqual(file3_metadata, children[u'file3'][1])
1146         self.failUnlessEqual(file3_rocap,
1147                              children[u'file3'][0].get_readonly_uri())
1148         self.failUnlessEqual(file3_rwcap,
1149                              children[u'file3'][0].get_uri())
1150
1151         # Are the metadata for child 2 right?
1152         file2_rocap = "URI:CHK:apegrpehshwugkbh3jlt5ei6hq:5oougnemcl5xgx4ijgiumtdojlipibctjkbwvyygdymdphib2fvq:3:10:4"
1153         file2_rwcap = "URI:CHK:apegrpehshwugkbh3jlt5ei6hq:5oougnemcl5xgx4ijgiumtdojlipibctjkbwvyygdymdphib2fvq:3:10:4"
1154         file2_metadata = {'ctime': 1246663897.430218, 'tahoe': {'linkmotime': 1246663897.430218, 'linkcrtime': 1246663897.430218}, 'mtime': 1246663897.430218}
1155         self.failUnlessEqual(file2_metadata, children[u'file2'][1])
1156         self.failUnlessEqual(file2_rocap,
1157                              children[u'file2'][0].get_readonly_uri())
1158         self.failUnlessEqual(file2_rwcap,
1159                              children[u'file2'][0].get_uri())
1160
1161         # Are the metadata for child 1 right?
1162         file1_rocap = "URI:CHK:olxtimympo7f27jvhtgqlnbtn4:emzdnhk2um4seixozlkw3qx2nfijvdkx3ky7i7izl47yedl6e64a:3:10:10"
1163         file1_rwcap = "URI:CHK:olxtimympo7f27jvhtgqlnbtn4:emzdnhk2um4seixozlkw3qx2nfijvdkx3ky7i7izl47yedl6e64a:3:10:10"
1164         file1_metadata = {'ctime': 1246663897.4275661, 'tahoe': {'linkmotime': 1246663897.4275661, 'linkcrtime': 1246663897.4275661}, 'mtime': 1246663897.4275661}
1165         self.failUnlessEqual(file1_metadata, children[u'file1'][1])
1166         self.failUnlessEqual(file1_rocap,
1167                              children[u'file1'][0].get_readonly_uri())
1168         self.failUnlessEqual(file1_rwcap,
1169                              children[u'file1'][0].get_uri())
1170
1171     def _make_kids(self, nm, which):
1172         caps = {"imm": "URI:CHK:n7r3m6wmomelk4sep3kw5cvduq:os7ijw5c3maek7pg65e5254k2fzjflavtpejjyhshpsxuqzhcwwq:3:20:14861",
1173                 "lit": "URI:LIT:n5xgk", # LIT for "one"
1174                 "write": "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq",
1175                 "read": "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q",
1176                 "dirwrite": "URI:DIR2:n6x24zd3seu725yluj75q5boaa:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq",
1177                 "dirread":  "URI:DIR2-RO:b7sr5qsifnicca7cbk3rhrhbvq:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq",
1178                 }
1179         kids = {}
1180         for name in which:
1181             kids[unicode(name)] = (nm.create_from_cap(caps[name]), {})
1182         return kids
1183
1184     def test_deep_immutable(self):
1185         nm = NodeMaker(None, None, None, None, None, None, {"k": 3, "n": 10},
1186                        None)
1187         fn = MinimalFakeMutableFile()
1188
1189         kids = self._make_kids(nm, ["imm", "lit", "write", "read",
1190                                     "dirwrite", "dirread"])
1191         packed = dirnode.pack_children(fn, kids, deep_immutable=False)
1192         self.failUnlessIn("lit", packed)
1193
1194         kids = self._make_kids(nm, ["imm", "lit"])
1195         packed = dirnode.pack_children(fn, kids, deep_immutable=True)
1196         self.failUnlessIn("lit", packed)
1197
1198         kids = self._make_kids(nm, ["imm", "lit", "write"])
1199         self.failUnlessRaises(dirnode.MustBeDeepImmutableError,
1200                               dirnode.pack_children,
1201                               fn, kids, deep_immutable=True)
1202
1203         # read-only is not enough: all children must be immutable
1204         kids = self._make_kids(nm, ["imm", "lit", "read"])
1205         self.failUnlessRaises(dirnode.MustBeDeepImmutableError,
1206                               dirnode.pack_children,
1207                               fn, kids, deep_immutable=True)
1208
1209         kids = self._make_kids(nm, ["imm", "lit", "dirwrite"])
1210         self.failUnlessRaises(dirnode.MustBeDeepImmutableError,
1211                               dirnode.pack_children,
1212                               fn, kids, deep_immutable=True)
1213
1214         kids = self._make_kids(nm, ["imm", "lit", "dirread"])
1215         self.failUnlessRaises(dirnode.MustBeDeepImmutableError,
1216                               dirnode.pack_children,
1217                               fn, kids, deep_immutable=True)
1218
1219 class FakeMutableFile:
1220     implements(IMutableFileNode)
1221     counter = 0
1222     def __init__(self, initial_contents=""):
1223         self.data = self._get_initial_contents(initial_contents)
1224         counter = FakeMutableFile.counter
1225         FakeMutableFile.counter += 1
1226         writekey = hashutil.ssk_writekey_hash(str(counter))
1227         fingerprint = hashutil.ssk_pubkey_fingerprint_hash(str(counter))
1228         self.uri = uri.WriteableSSKFileURI(writekey, fingerprint)
1229
1230     def _get_initial_contents(self, contents):
1231         if isinstance(contents, str):
1232             return contents
1233         if contents is None:
1234             return ""
1235         assert callable(contents), "%s should be callable, not %s" % \
1236                (contents, type(contents))
1237         return contents(self)
1238
1239     def get_cap(self):
1240         return self.uri
1241
1242     def get_uri(self):
1243         return self.uri.to_string()
1244
1245     def get_write_uri(self):
1246         return self.uri.to_string()
1247
1248     def download_best_version(self):
1249         return defer.succeed(self.data)
1250
1251     def get_writekey(self):
1252         return "writekey"
1253
1254     def is_readonly(self):
1255         return False
1256
1257     def is_mutable(self):
1258         return True
1259
1260     def is_unknown(self):
1261         return False
1262
1263     def is_allowed_in_immutable_directory(self):
1264         return False
1265
1266     def raise_error(self):
1267         pass
1268
1269     def modify(self, modifier):
1270         self.data = modifier(self.data, None, True)
1271         return defer.succeed(None)
1272
1273 class FakeNodeMaker(NodeMaker):
1274     def create_mutable_file(self, contents="", keysize=None):
1275         return defer.succeed(FakeMutableFile(contents))
1276
1277 class FakeClient2(Client):
1278     def __init__(self):
1279         self.nodemaker = FakeNodeMaker(None, None, None,
1280                                        None, None, None,
1281                                        {"k":3,"n":10}, None)
1282     def create_node_from_uri(self, rwcap, rocap):
1283         return self.nodemaker.create_from_cap(rwcap, rocap)
1284
1285 class Dirnode2(unittest.TestCase, testutil.ShouldFailMixin):
1286     def setUp(self):
1287         client = FakeClient2()
1288         self.nodemaker = client.nodemaker
1289
1290     def test_from_future(self):
1291         # Create a mutable directory that contains unknown URI types, and make sure
1292         # we tolerate them properly.
1293         d = self.nodemaker.create_new_mutable_directory()
1294         future_write_uri = "x-tahoe-crazy://I_am_from_the_future."
1295         future_read_uri = "x-tahoe-crazy-readonly://I_am_from_the_future."
1296         future_imm_uri = "x-tahoe-crazy-immutable://I_am_from_the_future."
1297         future_node = UnknownNode(future_write_uri, future_read_uri)
1298         def _then(n):
1299             self._node = n
1300             return n.set_node(u"future", future_node)
1301         d.addCallback(_then)
1302
1303         # We should be prohibited from adding an unknown URI to a directory
1304         # just in the rw_uri slot, since we don't know how to diminish the cap
1305         # to a readcap (for the ro_uri slot).
1306         d.addCallback(lambda ign:
1307              self.shouldFail(MustNotBeUnknownRWError,
1308                              "copy unknown",
1309                              "cannot attach unknown rw cap as child",
1310                              self._node.set_uri, u"add",
1311                              future_write_uri, None))
1312
1313         # However, we should be able to add both rw_uri and ro_uri as a pair of
1314         # unknown URIs.
1315         d.addCallback(lambda ign: self._node.set_uri(u"add-pair",
1316                                                      future_write_uri, future_read_uri))
1317
1318         # and to add an URI prefixed with "ro." or "imm." when it is given in a
1319         # write slot (or URL parameter).
1320         d.addCallback(lambda ign: self._node.set_uri(u"add-ro",
1321                                                      "ro." + future_read_uri, None))
1322         d.addCallback(lambda ign: self._node.set_uri(u"add-imm",
1323                                                      "imm." + future_imm_uri, None))
1324
1325         d.addCallback(lambda ign: self._node.list())
1326         def _check(children):
1327             self.failUnlessEqual(len(children), 4)
1328             (fn, metadata) = children[u"future"]
1329             self.failUnless(isinstance(fn, UnknownNode), fn)
1330             self.failUnlessEqual(fn.get_uri(), future_write_uri)
1331             self.failUnlessEqual(fn.get_write_uri(), future_write_uri)
1332             self.failUnlessEqual(fn.get_readonly_uri(), "ro." + future_read_uri)
1333
1334             (fn2, metadata2) = children[u"add-pair"]
1335             self.failUnless(isinstance(fn2, UnknownNode), fn2)
1336             self.failUnlessEqual(fn2.get_uri(), future_write_uri)
1337             self.failUnlessEqual(fn2.get_write_uri(), future_write_uri)
1338             self.failUnlessEqual(fn2.get_readonly_uri(), "ro." + future_read_uri)
1339
1340             (fn3, metadata3) = children[u"add-ro"]
1341             self.failUnless(isinstance(fn3, UnknownNode), fn3)
1342             self.failUnlessEqual(fn3.get_uri(), "ro." + future_read_uri)
1343             self.failUnlessEqual(fn3.get_write_uri(), None)
1344             self.failUnlessEqual(fn3.get_readonly_uri(), "ro." + future_read_uri)
1345
1346             (fn4, metadata4) = children[u"add-imm"]
1347             self.failUnless(isinstance(fn4, UnknownNode), fn4)
1348             self.failUnlessEqual(fn4.get_uri(), "imm." + future_imm_uri)
1349             self.failUnlessEqual(fn4.get_write_uri(), None)
1350             self.failUnlessEqual(fn4.get_readonly_uri(), "imm." + future_imm_uri)
1351
1352             # We should also be allowed to copy the "future" UnknownNode, because
1353             # it contains all the information that was in the original directory
1354             # (readcap and writecap), so we're preserving everything.
1355             return self._node.set_node(u"copy", fn)
1356         d.addCallback(_check)
1357
1358         d.addCallback(lambda ign: self._node.list())
1359         def _check2(children):
1360             self.failUnlessEqual(len(children), 5)
1361             (fn, metadata) = children[u"copy"]
1362             self.failUnless(isinstance(fn, UnknownNode), fn)
1363             self.failUnlessEqual(fn.get_uri(), future_write_uri)
1364             self.failUnlessEqual(fn.get_write_uri(), future_write_uri)
1365             self.failUnlessEqual(fn.get_readonly_uri(), "ro." + future_read_uri)
1366         d.addCallback(_check2)
1367         return d
1368
1369     def test_unknown_strip_prefix_for_ro(self):
1370         self.failUnlessEqual(strip_prefix_for_ro("foo",     False), "foo")
1371         self.failUnlessEqual(strip_prefix_for_ro("ro.foo",  False), "foo")
1372         self.failUnlessEqual(strip_prefix_for_ro("imm.foo", False), "imm.foo")
1373         self.failUnlessEqual(strip_prefix_for_ro("foo",     True),  "foo")
1374         self.failUnlessEqual(strip_prefix_for_ro("ro.foo",  True),  "foo")
1375         self.failUnlessEqual(strip_prefix_for_ro("imm.foo", True),  "foo")
1376
1377     def test_unknownnode(self):
1378         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
1379         mut_read_uri = "URI:SSK-RO:jf6wkflosyvntwxqcdo7a54jvm:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
1380         lit_uri = "URI:LIT:n5xgk"
1381
1382         # This does not attempt to be exhaustive.
1383         no_no        = [# Opaque node, but not an error.
1384                         ( 0, UnknownNode(None, None)),
1385                         ( 1, UnknownNode(None, None, deep_immutable=True)),
1386                        ]
1387         unknown_rw   = [# These are errors because we're only given a rw_uri, and we can't
1388                         # diminish it.
1389                         ( 2, UnknownNode("foo", None)),
1390                         ( 3, UnknownNode("foo", None, deep_immutable=True)),
1391                         ( 4, UnknownNode("ro.foo", None, deep_immutable=True)),
1392                         ( 5, UnknownNode("ro." + mut_read_uri, None, deep_immutable=True)),
1393                         ( 6, UnknownNode("URI:SSK-RO:foo", None, deep_immutable=True)),
1394                         ( 7, UnknownNode("URI:SSK:foo", None)),
1395                        ]
1396         must_be_ro   = [# These are errors because a readonly constraint is not met.
1397                         ( 8, UnknownNode("ro." + mut_write_uri, None)),
1398                         ( 9, UnknownNode(None, "ro." + mut_write_uri)),
1399                        ]
1400         must_be_imm  = [# These are errors because an immutable constraint is not met.
1401                         (10, UnknownNode(None, "ro.URI:SSK-RO:foo", deep_immutable=True)),
1402                         (11, UnknownNode(None, "imm.URI:SSK:foo")),
1403                         (12, UnknownNode(None, "imm.URI:SSK-RO:foo")),
1404                         (13, UnknownNode("bar", "ro.foo", deep_immutable=True)),
1405                         (14, UnknownNode("bar", "imm.foo", deep_immutable=True)),
1406                         (15, UnknownNode("bar", "imm." + lit_uri, deep_immutable=True)),
1407                         (16, UnknownNode("imm." + mut_write_uri, None)),
1408                         (17, UnknownNode("imm." + mut_read_uri, None)),
1409                         (18, UnknownNode("bar", "imm.foo")),
1410                        ]
1411         bad_uri      = [# These are errors because the URI is bad once we've stripped the prefix.
1412                         (19, UnknownNode("ro.URI:SSK-RO:foo", None)),
1413                         (20, UnknownNode("imm.URI:CHK:foo", None, deep_immutable=True)),
1414                         (21, UnknownNode(None, "URI:CHK:foo")),
1415                         (22, UnknownNode(None, "URI:CHK:foo", deep_immutable=True)),
1416                        ]
1417         ro_prefixed  = [# These are valid, and the readcap should end up with a ro. prefix.
1418                         (23, UnknownNode(None, "foo")),
1419                         (24, UnknownNode(None, "ro.foo")),
1420                         (25, UnknownNode(None, "ro." + lit_uri)),
1421                         (26, UnknownNode("bar", "foo")),
1422                         (27, UnknownNode("bar", "ro.foo")),
1423                         (28, UnknownNode("bar", "ro." + lit_uri)),
1424                         (29, UnknownNode("ro.foo", None)),
1425                         (30, UnknownNode("ro." + lit_uri, None)),
1426                        ]
1427         imm_prefixed = [# These are valid, and the readcap should end up with an imm. prefix.
1428                         (31, UnknownNode(None, "foo", deep_immutable=True)),
1429                         (32, UnknownNode(None, "ro.foo", deep_immutable=True)),
1430                         (33, UnknownNode(None, "imm.foo")),
1431                         (34, UnknownNode(None, "imm.foo", deep_immutable=True)),
1432                         (35, UnknownNode("imm." + lit_uri, None)),
1433                         (36, UnknownNode("imm." + lit_uri, None, deep_immutable=True)),
1434                         (37, UnknownNode(None, "imm." + lit_uri)),
1435                         (38, UnknownNode(None, "imm." + lit_uri, deep_immutable=True)),
1436                        ]
1437         error = unknown_rw + must_be_ro + must_be_imm + bad_uri
1438         ok = ro_prefixed + imm_prefixed
1439
1440         for (i, n) in no_no + error + ok:
1441             self.failUnless(n.is_unknown(), i)
1442
1443         for (i, n) in no_no + error:
1444             self.failUnless(n.get_uri() is None, i)
1445             self.failUnless(n.get_write_uri() is None, i)
1446             self.failUnless(n.get_readonly_uri() is None, i)
1447
1448         for (i, n) in no_no + ok:
1449             n.raise_error()
1450
1451         for (i, n) in unknown_rw:
1452             self.failUnlessRaises(MustNotBeUnknownRWError, lambda: n.raise_error())
1453
1454         for (i, n) in must_be_ro:
1455             self.failUnlessRaises(MustBeReadonlyError, lambda: n.raise_error())
1456
1457         for (i, n) in must_be_imm:
1458             self.failUnlessRaises(MustBeDeepImmutableError, lambda: n.raise_error())
1459
1460         for (i, n) in bad_uri:
1461             self.failUnlessRaises(uri.BadURIError, lambda: n.raise_error())
1462
1463         for (i, n) in ok:
1464             self.failIf(n.get_readonly_uri() is None, i)
1465
1466         for (i, n) in ro_prefixed:
1467             self.failUnless(n.get_readonly_uri().startswith("ro."), i)
1468
1469         for (i, n) in imm_prefixed:
1470             self.failUnless(n.get_readonly_uri().startswith("imm."), i)
1471
1472
1473 class DeepStats(unittest.TestCase):
1474     timeout = 240 # It takes longer than 120 seconds on Francois's arm box.
1475     def test_stats(self):
1476         ds = dirnode.DeepStats(None)
1477         ds.add("count-files")
1478         ds.add("size-immutable-files", 123)
1479         ds.histogram("size-files-histogram", 123)
1480         ds.max("largest-directory", 444)
1481
1482         s = ds.get_results()
1483         self.failUnlessEqual(s["count-files"], 1)
1484         self.failUnlessEqual(s["size-immutable-files"], 123)
1485         self.failUnlessEqual(s["largest-directory"], 444)
1486         self.failUnlessEqual(s["count-literal-files"], 0)
1487
1488         ds.add("count-files")
1489         ds.add("size-immutable-files", 321)
1490         ds.histogram("size-files-histogram", 321)
1491         ds.max("largest-directory", 2)
1492
1493         s = ds.get_results()
1494         self.failUnlessEqual(s["count-files"], 2)
1495         self.failUnlessEqual(s["size-immutable-files"], 444)
1496         self.failUnlessEqual(s["largest-directory"], 444)
1497         self.failUnlessEqual(s["count-literal-files"], 0)
1498         self.failUnlessEqual(s["size-files-histogram"],
1499                              [ (101, 316, 1), (317, 1000, 1) ])
1500
1501         ds = dirnode.DeepStats(None)
1502         for i in range(1, 1100):
1503             ds.histogram("size-files-histogram", i)
1504         ds.histogram("size-files-histogram", 4*1000*1000*1000*1000) # 4TB
1505         s = ds.get_results()
1506         self.failUnlessEqual(s["size-files-histogram"],
1507                              [ (1, 3, 3),
1508                                (4, 10, 7),
1509                                (11, 31, 21),
1510                                (32, 100, 69),
1511                                (101, 316, 216),
1512                                (317, 1000, 684),
1513                                (1001, 3162, 99),
1514                                (3162277660169L, 10000000000000L, 1),
1515                                ])
1516
1517 class UCWEingMutableFileNode(MutableFileNode):
1518     please_ucwe_after_next_upload = False
1519
1520     def _upload(self, new_contents, servermap):
1521         d = MutableFileNode._upload(self, new_contents, servermap)
1522         def _ucwe(res):
1523             if self.please_ucwe_after_next_upload:
1524                 self.please_ucwe_after_next_upload = False
1525                 raise UncoordinatedWriteError()
1526             return res
1527         d.addCallback(_ucwe)
1528         return d
1529
1530 class UCWEingNodeMaker(NodeMaker):
1531     def _create_mutable(self, cap):
1532         n = UCWEingMutableFileNode(self.storage_broker, self.secret_holder,
1533                                    self.default_encoding_parameters,
1534                                    self.history)
1535         return n.init_from_cap(cap)
1536
1537
1538 class Deleter(GridTestMixin, unittest.TestCase):
1539     timeout = 3600 # It takes longer than 433 seconds on Zandr's ARM box.
1540     def test_retry(self):
1541         # ticket #550, a dirnode.delete which experiences an
1542         # UncoordinatedWriteError will fail with an incorrect "you're
1543         # deleting something which isn't there" NoSuchChildError exception.
1544
1545         # to trigger this, we start by creating a directory with a single
1546         # file in it. Then we create a special dirnode that uses a modified
1547         # MutableFileNode which will raise UncoordinatedWriteError once on
1548         # demand. We then call dirnode.delete, which ought to retry and
1549         # succeed.
1550
1551         self.basedir = self.mktemp()
1552         self.set_up_grid()
1553         c0 = self.g.clients[0]
1554         d = c0.create_dirnode()
1555         small = upload.Data("Small enough for a LIT", None)
1556         def _created_dir(dn):
1557             self.root = dn
1558             self.root_uri = dn.get_uri()
1559             return dn.add_file(u"file", small)
1560         d.addCallback(_created_dir)
1561         def _do_delete(ignored):
1562             nm = UCWEingNodeMaker(c0.storage_broker, c0._secret_holder,
1563                                   c0.get_history(), c0.getServiceNamed("uploader"),
1564                                   c0.downloader,
1565                                   c0.download_cache_dirman,
1566                                   c0.get_encoding_parameters(),
1567                                   c0._key_generator)
1568             n = nm.create_from_cap(self.root_uri)
1569             assert n._node.please_ucwe_after_next_upload == False
1570             n._node.please_ucwe_after_next_upload = True
1571             # This should succeed, not raise an exception
1572             return n.delete(u"file")
1573         d.addCallback(_do_delete)
1574
1575         return d
1576
1577 class Adder(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin):
1578
1579     def test_overwrite(self):
1580         # note: This functionality could be tested without actually creating
1581         # several RSA keys. It would be faster without the GridTestMixin: use
1582         # dn.set_node(nodemaker.create_from_cap(make_chk_file_uri())) instead
1583         # of dn.add_file, and use a special NodeMaker that creates fake
1584         # mutable files.
1585         self.basedir = "dirnode/Adder/test_overwrite"
1586         self.set_up_grid()
1587         c = self.g.clients[0]
1588         fileuri = make_chk_file_uri(1234)
1589         filenode = c.nodemaker.create_from_cap(fileuri)
1590         d = c.create_dirnode()
1591
1592         def _create_directory_tree(root_node):
1593             # Build
1594             # root/file1
1595             # root/file2
1596             # root/dir1
1597             d = root_node.add_file(u'file1', upload.Data("Important Things",
1598                 None))
1599             d.addCallback(lambda res:
1600                 root_node.add_file(u'file2', upload.Data("Sekrit Codes", None)))
1601             d.addCallback(lambda res:
1602                 root_node.create_subdirectory(u"dir1"))
1603             d.addCallback(lambda res: root_node)
1604             return d
1605
1606         d.addCallback(_create_directory_tree)
1607
1608         def _test_adder(root_node):
1609             d = root_node.set_node(u'file1', filenode)
1610             # We've overwritten file1. Let's try it with a directory
1611             d.addCallback(lambda res:
1612                 root_node.create_subdirectory(u'dir2'))
1613             d.addCallback(lambda res:
1614                 root_node.set_node(u'dir2', filenode))
1615             # We try overwriting a file with a child while also specifying
1616             # overwrite=False. We should receive an ExistingChildError
1617             # when we do this.
1618             d.addCallback(lambda res:
1619                 self.shouldFail(ExistingChildError, "set_node",
1620                                 "child 'file1' already exists",
1621                                 root_node.set_node, u"file1",
1622                                 filenode, overwrite=False))
1623             # If we try with a directory, we should see the same thing
1624             d.addCallback(lambda res:
1625                 self.shouldFail(ExistingChildError, "set_node",
1626                                 "child 'dir1' already exists",
1627                                 root_node.set_node, u'dir1', filenode,
1628                                 overwrite=False))
1629             d.addCallback(lambda res:
1630                 root_node.set_node(u'file1', filenode,
1631                                    overwrite="only-files"))
1632             d.addCallback(lambda res:
1633                 self.shouldFail(ExistingChildError, "set_node",
1634                                 "child 'dir1' already exists",
1635                                 root_node.set_node, u'dir1', filenode,
1636                                 overwrite="only-files"))
1637             return d
1638
1639         d.addCallback(_test_adder)
1640         return d