src/allmydata/test/test_dirnode.py

   1
   2 import time
   3 import unicodedata
   4 from zope.interface import implements
   5 from twisted.trial import unittest
   6 from twisted.internet import defer
   7 from twisted.internet.interfaces import IConsumer
   8 from allmydata import uri, dirnode
   9 from allmydata.client import Client
  10 from allmydata.immutable import upload
  11 from allmydata.interfaces import IImmutableFileNode, IMutableFileNode, \
  12      ExistingChildError, NoSuchChildError, MustNotBeUnknownRWError, \
  13      MustBeDeepImmutableError, MustBeReadonlyError, \
  14      IDeepCheckResults, IDeepCheckAndRepairResults
  15 from allmydata.mutable.filenode import MutableFileNode
  16 from allmydata.mutable.common import UncoordinatedWriteError
  17 from allmydata.util import hashutil, base32
  18 from allmydata.util.netstring import split_netstring
  19 from allmydata.monitor import Monitor
  20 from allmydata.test.common import make_chk_file_uri, make_mutable_file_uri, \
  21      ErrorMixin
  22 from allmydata.test.no_network import GridTestMixin
  23 from allmydata.unknown import UnknownNode, strip_prefix_for_ro
  24 from allmydata.nodemaker import NodeMaker
  25 from base64 import b32decode
  26 import allmydata.test.common_util as testutil
  27
  28 class MemAccum:
  29     implements(IConsumer)
  30     def registerProducer(self, producer, streaming):
  31         self.producer = producer
  32         self.producer.resumeProducing()
  33         pass
  34     def unregisterProducer(self):
  35         pass
  36     def write(self, data):
  37         assert not hasattr(self, 'data')
  38         self.data = data
  39         self.producer.resumeProducing()
  40
  41 setup_py_uri = "URI:CHK:n7r3m6wmomelk4sep3kw5cvduq:os7ijw5c3maek7pg65e5254k2fzjflavtpejjyhshpsxuqzhcwwq:3:20:14861"
  42 one_uri = "URI:LIT:n5xgk" # LIT for "one"
  43 mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
  44 empty_litdir_uri = "URI:DIR2-LIT:"
  45 tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
  46 mut_read_uri = "URI:SSK-RO:jf6wkflosyvntwxqcdo7a54jvm:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
  47 future_write_uri = "x-tahoe-crazy://I_am_from_the_future."
  48 future_read_uri = "x-tahoe-crazy-readonly://I_am_from_the_future."
  49
  50 # 'o' 'n' 'e-macron'
  51 one_nfc = u"on\u0113"
  52 one_nfd = u"one\u0304"
  53
  54 class Dirnode(GridTestMixin, unittest.TestCase,
  55               testutil.ShouldFailMixin, testutil.StallMixin, ErrorMixin):
  56     timeout = 240 # It takes longer than 120 seconds on Francois's arm box.
  57
  58     def test_basic(self):
  59         self.basedir = "dirnode/Dirnode/test_basic"
  60         self.set_up_grid()
  61         c = self.g.clients[0]
  62         d = c.create_dirnode()
  63         def _done(res):
  64             self.failUnless(isinstance(res, dirnode.DirectoryNode))
  65             self.failUnless(res.is_mutable())
  66             self.failIf(res.is_readonly())
  67             self.failIf(res.is_unknown())
  68             self.failIf(res.is_allowed_in_immutable_directory())
  69             res.raise_error()
  70             rep = str(res)
  71             self.failUnless("RW-MUT" in rep)
  72         d.addCallback(_done)
  73         return d
  74
  75     def test_initial_children(self):
  76         self.basedir = "dirnode/Dirnode/test_initial_children"
  77         self.set_up_grid()
  78         c = self.g.clients[0]
  79         nm = c.nodemaker
  80
  81         kids = {one_nfd: (nm.create_from_cap(one_uri), {}),
  82                 u"two": (nm.create_from_cap(setup_py_uri),
  83                          {"metakey": "metavalue"}),
  84                 u"mut": (nm.create_from_cap(mut_write_uri, mut_read_uri), {}),
  85                 u"fut": (nm.create_from_cap(future_write_uri, future_read_uri), {}),
  86                 u"fro": (nm.create_from_cap(None, future_read_uri), {}),
  87                 u"empty_litdir": (nm.create_from_cap(empty_litdir_uri), {}),
  88                 u"tiny_litdir": (nm.create_from_cap(tiny_litdir_uri), {}),
  89                 }
  90         d = c.create_dirnode(kids)
  91
  92         def _created(dn):
  93             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
  94             self.failUnless(dn.is_mutable())
  95             self.failIf(dn.is_readonly())
  96             self.failIf(dn.is_unknown())
  97             self.failIf(dn.is_allowed_in_immutable_directory())
  98             dn.raise_error()
  99             rep = str(dn)
 100             self.failUnless("RW-MUT" in rep)
 101             return dn.list()
 102         d.addCallback(_created)
 103
 104         def _check_kids(children):
 105             self.failUnlessEqual(set(children.keys()),
 106                                  set([one_nfc, u"two", u"mut", u"fut", u"fro", u"empty_litdir", u"tiny_litdir"]))
 107             one_node, one_metadata = children[one_nfc]
 108             two_node, two_metadata = children[u"two"]
 109             mut_node, mut_metadata = children[u"mut"]
 110             fut_node, fut_metadata = children[u"fut"]
 111             fro_node, fro_metadata = children[u"fro"]
 112             emptylit_node, emptylit_metadata = children[u"empty_litdir"]
 113             tinylit_node, tinylit_metadata = children[u"tiny_litdir"]
 114
 115             self.failUnlessEqual(one_node.get_size(), 3)
 116             self.failUnlessEqual(one_node.get_uri(), one_uri)
 117             self.failUnlessEqual(one_node.get_readonly_uri(), one_uri)
 118             self.failUnless(isinstance(one_metadata, dict), one_metadata)
 119
 120             self.failUnlessEqual(two_node.get_size(), 14861)
 121             self.failUnlessEqual(two_node.get_uri(), setup_py_uri)
 122             self.failUnlessEqual(two_node.get_readonly_uri(), setup_py_uri)
 123             self.failUnlessEqual(two_metadata["metakey"], "metavalue")
 124
 125             self.failUnlessEqual(mut_node.get_uri(), mut_write_uri)
 126             self.failUnlessEqual(mut_node.get_readonly_uri(), mut_read_uri)
 127             self.failUnless(isinstance(mut_metadata, dict), mut_metadata)
 128
 129             self.failUnless(fut_node.is_unknown())
 130             self.failUnlessEqual(fut_node.get_uri(), future_write_uri)
 131             self.failUnlessEqual(fut_node.get_readonly_uri(), "ro." + future_read_uri)
 132             self.failUnless(isinstance(fut_metadata, dict), fut_metadata)
 133
 134             self.failUnless(fro_node.is_unknown())
 135             self.failUnlessEqual(fro_node.get_uri(), "ro." + future_read_uri)
 136             self.failUnlessEqual(fut_node.get_readonly_uri(), "ro." + future_read_uri)
 137             self.failUnless(isinstance(fro_metadata, dict), fro_metadata)
 138
 139             self.failIf(emptylit_node.is_unknown())
 140             self.failUnlessEqual(emptylit_node.get_storage_index(), None)
 141             self.failIf(tinylit_node.is_unknown())
 142             self.failUnlessEqual(tinylit_node.get_storage_index(), None)
 143
 144             d2 = defer.succeed(None)
 145             d2.addCallback(lambda ignored: emptylit_node.list())
 146             d2.addCallback(lambda children: self.failUnlessEqual(children, {}))
 147             d2.addCallback(lambda ignored: tinylit_node.list())
 148             d2.addCallback(lambda children: self.failUnlessEqual(set(children.keys()),
 149                                                                  set([u"short"])))
 150             d2.addCallback(lambda ignored: tinylit_node.list())
 151             d2.addCallback(lambda children: children[u"short"][0].read(MemAccum()))
 152             d2.addCallback(lambda accum: self.failUnlessEqual(accum.data, "The end."))
 153             return d2
 154
 155         d.addCallback(_check_kids)
 156
 157         d.addCallback(lambda ign: nm.create_new_mutable_directory(kids))
 158         d.addCallback(lambda dn: dn.list())
 159         d.addCallback(_check_kids)
 160
 161         bad_future_node = UnknownNode(future_write_uri, None)
 162         bad_kids1 = {one_nfd: (bad_future_node, {})}
 163         # This should fail because we don't know how to diminish the future_write_uri
 164         # cap (given in a write slot and not prefixed with "ro." or "imm.") to a readcap.
 165         d.addCallback(lambda ign:
 166                       self.shouldFail(MustNotBeUnknownRWError, "bad_kids1",
 167                                       "cannot attach unknown",
 168                                       nm.create_new_mutable_directory,
 169                                       bad_kids1))
 170         bad_kids2 = {one_nfd: (nm.create_from_cap(one_uri), None)}
 171         d.addCallback(lambda ign:
 172                       self.shouldFail(AssertionError, "bad_kids2",
 173                                       "requires metadata to be a dict",
 174                                       nm.create_new_mutable_directory,
 175                                       bad_kids2))
 176         return d
 177
 178     def test_immutable(self):
 179         self.basedir = "dirnode/Dirnode/test_immutable"
 180         self.set_up_grid()
 181         c = self.g.clients[0]
 182         nm = c.nodemaker
 183
 184         kids = {one_nfd: (nm.create_from_cap(one_uri), {}),
 185                 u"two": (nm.create_from_cap(setup_py_uri),
 186                          {"metakey": "metavalue"}),
 187                 u"fut": (nm.create_from_cap(None, future_read_uri), {}),
 188                 u"empty_litdir": (nm.create_from_cap(empty_litdir_uri), {}),
 189                 u"tiny_litdir": (nm.create_from_cap(tiny_litdir_uri), {}),
 190                 }
 191         d = c.create_immutable_dirnode(kids)
 192
 193         def _created(dn):
 194             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
 195             self.failIf(dn.is_mutable())
 196             self.failUnless(dn.is_readonly())
 197             self.failIf(dn.is_unknown())
 198             self.failUnless(dn.is_allowed_in_immutable_directory())
 199             dn.raise_error()
 200             rep = str(dn)
 201             self.failUnless("RO-IMM" in rep)
 202             cap = dn.get_cap()
 203             self.failUnlessIn("CHK", cap.to_string())
 204             self.cap = cap
 205             return dn.list()
 206         d.addCallback(_created)
 207
 208         def _check_kids(children):
 209             self.failUnlessEqual(set(children.keys()),
 210                                  set([one_nfc, u"two", u"fut", u"empty_litdir", u"tiny_litdir"]))
 211             one_node, one_metadata = children[one_nfc]
 212             two_node, two_metadata = children[u"two"]
 213             fut_node, fut_metadata = children[u"fut"]
 214             emptylit_node, emptylit_metadata = children[u"empty_litdir"]
 215             tinylit_node, tinylit_metadata = children[u"tiny_litdir"]
 216
 217             self.failUnlessEqual(one_node.get_size(), 3)
 218             self.failUnlessEqual(one_node.get_uri(), one_uri)
 219             self.failUnlessEqual(one_node.get_readonly_uri(), one_uri)
 220             self.failUnless(isinstance(one_metadata, dict), one_metadata)
 221
 222             self.failUnlessEqual(two_node.get_size(), 14861)
 223             self.failUnlessEqual(two_node.get_uri(), setup_py_uri)
 224             self.failUnlessEqual(two_node.get_readonly_uri(), setup_py_uri)
 225             self.failUnlessEqual(two_metadata["metakey"], "metavalue")
 226
 227             self.failUnless(fut_node.is_unknown())
 228             self.failUnlessEqual(fut_node.get_uri(), "imm." + future_read_uri)
 229             self.failUnlessEqual(fut_node.get_readonly_uri(), "imm." + future_read_uri)
 230             self.failUnless(isinstance(fut_metadata, dict), fut_metadata)
 231
 232             self.failIf(emptylit_node.is_unknown())
 233             self.failUnlessEqual(emptylit_node.get_storage_index(), None)
 234             self.failIf(tinylit_node.is_unknown())
 235             self.failUnlessEqual(tinylit_node.get_storage_index(), None)
 236
 237             d2 = defer.succeed(None)
 238             d2.addCallback(lambda ignored: emptylit_node.list())
 239             d2.addCallback(lambda children: self.failUnlessEqual(children, {}))
 240             d2.addCallback(lambda ignored: tinylit_node.list())
 241             d2.addCallback(lambda children: self.failUnlessEqual(set(children.keys()),
 242                                                                  set([u"short"])))
 243             d2.addCallback(lambda ignored: tinylit_node.list())
 244             d2.addCallback(lambda children: children[u"short"][0].read(MemAccum()))
 245             d2.addCallback(lambda accum: self.failUnlessEqual(accum.data, "The end."))
 246             return d2
 247
 248         d.addCallback(_check_kids)
 249
 250         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
 251         d.addCallback(lambda dn: dn.list())
 252         d.addCallback(_check_kids)
 253
 254         bad_future_node1 = UnknownNode(future_write_uri, None)
 255         bad_kids1 = {one_nfd: (bad_future_node1, {})}
 256         d.addCallback(lambda ign:
 257                       self.shouldFail(MustNotBeUnknownRWError, "bad_kids1",
 258                                       "cannot attach unknown",
 259                                       c.create_immutable_dirnode,
 260                                       bad_kids1))
 261         bad_future_node2 = UnknownNode(future_write_uri, future_read_uri)
 262         bad_kids2 = {one_nfd: (bad_future_node2, {})}
 263         d.addCallback(lambda ign:
 264                       self.shouldFail(MustBeDeepImmutableError, "bad_kids2",
 265                                       "is not allowed in an immutable directory",
 266                                       c.create_immutable_dirnode,
 267                                       bad_kids2))
 268         bad_kids3 = {one_nfd: (nm.create_from_cap(one_uri), None)}
 269         d.addCallback(lambda ign:
 270                       self.shouldFail(AssertionError, "bad_kids3",
 271                                       "requires metadata to be a dict",
 272                                       c.create_immutable_dirnode,
 273                                       bad_kids3))
 274         bad_kids4 = {one_nfd: (nm.create_from_cap(mut_write_uri), {})}
 275         d.addCallback(lambda ign:
 276                       self.shouldFail(MustBeDeepImmutableError, "bad_kids4",
 277                                       "is not allowed in an immutable directory",
 278                                       c.create_immutable_dirnode,
 279                                       bad_kids4))
 280         bad_kids5 = {one_nfd: (nm.create_from_cap(mut_read_uri), {})}
 281         d.addCallback(lambda ign:
 282                       self.shouldFail(MustBeDeepImmutableError, "bad_kids5",
 283                                       "is not allowed in an immutable directory",
 284                                       c.create_immutable_dirnode,
 285                                       bad_kids5))
 286         d.addCallback(lambda ign: c.create_immutable_dirnode({}))
 287         def _created_empty(dn):
 288             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
 289             self.failIf(dn.is_mutable())
 290             self.failUnless(dn.is_readonly())
 291             self.failIf(dn.is_unknown())
 292             self.failUnless(dn.is_allowed_in_immutable_directory())
 293             dn.raise_error()
 294             rep = str(dn)
 295             self.failUnless("RO-IMM" in rep)
 296             cap = dn.get_cap()
 297             self.failUnlessIn("LIT", cap.to_string())
 298             self.failUnlessEqual(cap.to_string(), "URI:DIR2-LIT:")
 299             self.cap = cap
 300             return dn.list()
 301         d.addCallback(_created_empty)
 302         d.addCallback(lambda kids: self.failUnlessEqual(kids, {}))
 303         smallkids = {u"o": (nm.create_from_cap(one_uri), {})}
 304         d.addCallback(lambda ign: c.create_immutable_dirnode(smallkids))
 305         def _created_small(dn):
 306             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
 307             self.failIf(dn.is_mutable())
 308             self.failUnless(dn.is_readonly())
 309             self.failIf(dn.is_unknown())
 310             self.failUnless(dn.is_allowed_in_immutable_directory())
 311             dn.raise_error()
 312             rep = str(dn)
 313             self.failUnless("RO-IMM" in rep)
 314             cap = dn.get_cap()
 315             self.failUnlessIn("LIT", cap.to_string())
 316             self.failUnlessEqual(cap.to_string(),
 317                                  "URI:DIR2-LIT:gi4tumj2n4wdcmz2kvjesosmjfkdu3rvpbtwwlbqhiwdeot3puwcy")
 318             self.cap = cap
 319             return dn.list()
 320         d.addCallback(_created_small)
 321         d.addCallback(lambda kids: self.failUnlessEqual(kids.keys(), [u"o"]))
 322
 323         # now test n.create_subdirectory(mutable=False)
 324         d.addCallback(lambda ign: c.create_dirnode())
 325         def _made_parent(n):
 326             d = n.create_subdirectory(u"subdir", kids, mutable=False)
 327             d.addCallback(lambda sd: sd.list())
 328             d.addCallback(_check_kids)
 329             d.addCallback(lambda ign: n.list())
 330             d.addCallback(lambda children:
 331                           self.failUnlessEqual(children.keys(), [u"subdir"]))
 332             d.addCallback(lambda ign: n.get(u"subdir"))
 333             d.addCallback(lambda sd: sd.list())
 334             d.addCallback(_check_kids)
 335             d.addCallback(lambda ign: n.get(u"subdir"))
 336             d.addCallback(lambda sd: self.failIf(sd.is_mutable()))
 337             bad_kids = {one_nfd: (nm.create_from_cap(mut_write_uri), {})}
 338             d.addCallback(lambda ign:
 339                           self.shouldFail(MustBeDeepImmutableError, "YZ",
 340                                           "is not allowed in an immutable directory",
 341                                           n.create_subdirectory,
 342                                           u"sub2", bad_kids, mutable=False))
 343             return d
 344         d.addCallback(_made_parent)
 345         return d
 346
 347     def test_directory_representation(self):
 348         self.basedir = "dirnode/Dirnode/test_directory_representation"
 349         self.set_up_grid()
 350         c = self.g.clients[0]
 351         nm = c.nodemaker
 352
 353         # This test checks that any trailing spaces in URIs are retained in the
 354         # encoded directory, but stripped when we get them out of the directory.
 355         # See ticket #925 for why we want that.
 356         # It also tests that we store child names as UTF-8 NFC, and normalize
 357         # them again when retrieving them.
 358
 359         stripped_write_uri = "lafs://from_the_future\t"
 360         stripped_read_uri = "lafs://readonly_from_the_future\t"
 361         spacedout_write_uri = stripped_write_uri + "  "
 362         spacedout_read_uri = stripped_read_uri + "  "
 363
 364         child = nm.create_from_cap(spacedout_write_uri, spacedout_read_uri)
 365         self.failUnlessEqual(child.get_write_uri(), spacedout_write_uri)
 366         self.failUnlessEqual(child.get_readonly_uri(), "ro." + spacedout_read_uri)
 367
 368         child_dottedi = u"ch\u0131\u0307ld"
 369
 370         kids_in   = {child_dottedi: (child, {}), one_nfd: (child, {})}
 371         kids_out  = {child_dottedi: (child, {}), one_nfc: (child, {})}
 372         kids_norm = {u"child":      (child, {}), one_nfc: (child, {})}
 373         d = c.create_dirnode(kids_in)
 374
 375         def _created(dn):
 376             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
 377             self.failUnless(dn.is_mutable())
 378             self.failIf(dn.is_readonly())
 379             dn.raise_error()
 380             self.cap = dn.get_cap()
 381             self.rootnode = dn
 382             return dn._node.download_best_version()
 383         d.addCallback(_created)
 384
 385         def _check_data(data):
 386             # Decode the netstring representation of the directory to check that the
 387             # spaces are retained when the URIs are stored, and that the names are stored
 388             # as NFC.
 389             position = 0
 390             numkids = 0
 391             while position < len(data):
 392                 entries, position = split_netstring(data, 1, position)
 393                 entry = entries[0]
 394                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
 395                 name = name_utf8.decode("utf-8")
 396                 rw_uri = self.rootnode._decrypt_rwcapdata(rwcapdata)
 397                 self.failUnlessIn(name, kids_out)
 398                 (expected_child, ign) = kids_out[name]
 399                 self.failUnlessEqual(rw_uri, expected_child.get_write_uri())
 400                 self.failUnlessEqual("ro." + ro_uri, expected_child.get_readonly_uri())
 401                 numkids += 1
 402
 403             self.failUnlessEqual(numkids, len(kids_out))
 404             return self.rootnode
 405         d.addCallback(_check_data)
 406
 407         # Mock up a hypothetical future version of Unicode that adds a canonical equivalence
 408         # between dotless-i + dot-above, and 'i'. That would actually be prohibited by the
 409         # stability rules, but similar additions involving currently-unassigned characters
 410         # would not be.
 411         old_normalize = unicodedata.normalize
 412         def future_normalize(form, s):
 413             assert form == 'NFC', form
 414             return old_normalize(form, s).replace(u"\u0131\u0307", u"i")
 415
 416         def _list(node):
 417             unicodedata.normalize = future_normalize
 418             d2 = node.list()
 419             def _undo_mock(res):
 420                 unicodedata.normalize = old_normalize
 421                 return res
 422             d2.addBoth(_undo_mock)
 423             return d2
 424         d.addCallback(_list)
 425
 426         def _check_kids(children):
 427             # Now when we use the real directory listing code, the trailing spaces
 428             # should have been stripped (and "ro." should have been prepended to the
 429             # ro_uri, since it's unknown). Also the dotless-i + dot-above should have been
 430             # normalized to 'i'.
 431
 432             self.failUnlessEqual(set(children.keys()), set(kids_norm.keys()))
 433             child_node, child_metadata = children[u"child"]
 434
 435             self.failUnlessEqual(child_node.get_write_uri(), stripped_write_uri)
 436             self.failUnlessEqual(child_node.get_readonly_uri(), "ro." + stripped_read_uri)
 437         d.addCallback(_check_kids)
 438
 439         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
 440         d.addCallback(_list)
 441         d.addCallback(_check_kids)  # again with dirnode recreated from cap
 442         return d
 443
 444     def test_check(self):
 445         self.basedir = "dirnode/Dirnode/test_check"
 446         self.set_up_grid()
 447         c = self.g.clients[0]
 448         d = c.create_dirnode()
 449         d.addCallback(lambda dn: dn.check(Monitor()))
 450         def _done(res):
 451             self.failUnless(res.is_healthy())
 452         d.addCallback(_done)
 453         return d
 454
 455     def _test_deepcheck_create(self):
 456         # create a small tree with a loop, and some non-directories
 457         #  root/
 458         #  root/subdir/
 459         #  root/subdir/file1
 460         #  root/subdir/link -> root
 461         #  root/rodir
 462         c = self.g.clients[0]
 463         d = c.create_dirnode()
 464         def _created_root(rootnode):
 465             self._rootnode = rootnode
 466             return rootnode.create_subdirectory(u"subdir")
 467         d.addCallback(_created_root)
 468         def _created_subdir(subdir):
 469             self._subdir = subdir
 470             d = subdir.add_file(u"file1", upload.Data("data"*100, None))
 471             d.addCallback(lambda res: subdir.set_node(u"link", self._rootnode))
 472             d.addCallback(lambda res: c.create_dirnode())
 473             d.addCallback(lambda dn:
 474                           self._rootnode.set_uri(u"rodir",
 475                                                  dn.get_uri(),
 476                                                  dn.get_readonly_uri()))
 477             return d
 478         d.addCallback(_created_subdir)
 479         def _done(res):
 480             return self._rootnode
 481         d.addCallback(_done)
 482         return d
 483
 484     def test_deepcheck(self):
 485         self.basedir = "dirnode/Dirnode/test_deepcheck"
 486         self.set_up_grid()
 487         d = self._test_deepcheck_create()
 488         d.addCallback(lambda rootnode: rootnode.start_deep_check().when_done())
 489         def _check_results(r):
 490             self.failUnless(IDeepCheckResults.providedBy(r))
 491             c = r.get_counters()
 492             self.failUnlessEqual(c,
 493                                  {"count-objects-checked": 4,
 494                                   "count-objects-healthy": 4,
 495                                   "count-objects-unhealthy": 0,
 496                                   "count-objects-unrecoverable": 0,
 497                                   "count-corrupt-shares": 0,
 498                                   })
 499             self.failIf(r.get_corrupt_shares())
 500             self.failUnlessEqual(len(r.get_all_results()), 4)
 501         d.addCallback(_check_results)
 502         return d
 503
 504     def test_deepcheck_and_repair(self):
 505         self.basedir = "dirnode/Dirnode/test_deepcheck_and_repair"
 506         self.set_up_grid()
 507         d = self._test_deepcheck_create()
 508         d.addCallback(lambda rootnode:
 509                       rootnode.start_deep_check_and_repair().when_done())
 510         def _check_results(r):
 511             self.failUnless(IDeepCheckAndRepairResults.providedBy(r))
 512             c = r.get_counters()
 513             self.failUnlessEqual(c,
 514                                  {"count-objects-checked": 4,
 515                                   "count-objects-healthy-pre-repair": 4,
 516                                   "count-objects-unhealthy-pre-repair": 0,
 517                                   "count-objects-unrecoverable-pre-repair": 0,
 518                                   "count-corrupt-shares-pre-repair": 0,
 519                                   "count-objects-healthy-post-repair": 4,
 520                                   "count-objects-unhealthy-post-repair": 0,
 521                                   "count-objects-unrecoverable-post-repair": 0,
 522                                   "count-corrupt-shares-post-repair": 0,
 523                                   "count-repairs-attempted": 0,
 524                                   "count-repairs-successful": 0,
 525                                   "count-repairs-unsuccessful": 0,
 526                                   })
 527             self.failIf(r.get_corrupt_shares())
 528             self.failIf(r.get_remaining_corrupt_shares())
 529             self.failUnlessEqual(len(r.get_all_results()), 4)
 530         d.addCallback(_check_results)
 531         return d
 532
 533     def _mark_file_bad(self, rootnode):
 534         self.delete_shares_numbered(rootnode.get_uri(), [0])
 535         return rootnode
 536
 537     def test_deepcheck_problems(self):
 538         self.basedir = "dirnode/Dirnode/test_deepcheck_problems"
 539         self.set_up_grid()
 540         d = self._test_deepcheck_create()
 541         d.addCallback(lambda rootnode: self._mark_file_bad(rootnode))
 542         d.addCallback(lambda rootnode: rootnode.start_deep_check().when_done())
 543         def _check_results(r):
 544             c = r.get_counters()
 545             self.failUnlessEqual(c,
 546                                  {"count-objects-checked": 4,
 547                                   "count-objects-healthy": 3,
 548                                   "count-objects-unhealthy": 1,
 549                                   "count-objects-unrecoverable": 0,
 550                                   "count-corrupt-shares": 0,
 551                                   })
 552             #self.failUnlessEqual(len(r.get_problems()), 1) # TODO
 553         d.addCallback(_check_results)
 554         return d
 555
 556     def test_readonly(self):
 557         self.basedir = "dirnode/Dirnode/test_readonly"
 558         self.set_up_grid()
 559         c = self.g.clients[0]
 560         nm = c.nodemaker
 561         filecap = make_chk_file_uri(1234)
 562         filenode = nm.create_from_cap(filecap)
 563         uploadable = upload.Data("some data", convergence="some convergence string")
 564
 565         d = c.create_dirnode()
 566         def _created(rw_dn):
 567             d2 = rw_dn.set_uri(u"child", filecap, filecap)
 568             d2.addCallback(lambda res: rw_dn)
 569             return d2
 570         d.addCallback(_created)
 571
 572         def _ready(rw_dn):
 573             ro_uri = rw_dn.get_readonly_uri()
 574             ro_dn = c.create_node_from_uri(ro_uri)
 575             self.failUnless(ro_dn.is_readonly())
 576             self.failUnless(ro_dn.is_mutable())
 577             self.failIf(ro_dn.is_unknown())
 578             self.failIf(ro_dn.is_allowed_in_immutable_directory())
 579             ro_dn.raise_error()
 580
 581             self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
 582                             ro_dn.set_uri, u"newchild", filecap, filecap)
 583             self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
 584                             ro_dn.set_node, u"newchild", filenode)
 585             self.shouldFail(dirnode.NotWriteableError, "set_nodes ro", None,
 586                             ro_dn.set_nodes, { u"newchild": (filenode, None) })
 587             self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
 588                             ro_dn.add_file, u"newchild", uploadable)
 589             self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
 590                             ro_dn.delete, u"child")
 591             self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
 592                             ro_dn.create_subdirectory, u"newchild")
 593             self.shouldFail(dirnode.NotWriteableError, "set_metadata_for ro", None,
 594                             ro_dn.set_metadata_for, u"child", {})
 595             self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
 596                             ro_dn.move_child_to, u"child", rw_dn)
 597             self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
 598                             rw_dn.move_child_to, u"child", ro_dn)
 599             return ro_dn.list()
 600         d.addCallback(_ready)
 601         def _listed(children):
 602             self.failUnless(u"child" in children)
 603         d.addCallback(_listed)
 604         return d
 605
 606     def failUnlessGreaterThan(self, a, b):
 607         self.failUnless(a > b, "%r should be > %r" % (a, b))
 608
 609     def failUnlessGreaterOrEqualThan(self, a, b):
 610         self.failUnless(a >= b, "%r should be >= %r" % (a, b))
 611
 612     def test_create(self):
 613         self.basedir = "dirnode/Dirnode/test_create"
 614         self.set_up_grid()
 615         c = self.g.clients[0]
 616
 617         self.expected_manifest = []
 618         self.expected_verifycaps = set()
 619         self.expected_storage_indexes = set()
 620
 621         d = c.create_dirnode()
 622         def _then(n):
 623             # /
 624             self.rootnode = n
 625             self.failUnless(n.is_mutable())
 626             u = n.get_uri()
 627             self.failUnless(u)
 628             self.failUnless(u.startswith("URI:DIR2:"), u)
 629             u_ro = n.get_readonly_uri()
 630             self.failUnless(u_ro.startswith("URI:DIR2-RO:"), u_ro)
 631             u_v = n.get_verify_cap().to_string()
 632             self.failUnless(u_v.startswith("URI:DIR2-Verifier:"), u_v)
 633             u_r = n.get_repair_cap().to_string()
 634             self.failUnlessEqual(u_r, u)
 635             self.expected_manifest.append( ((), u) )
 636             self.expected_verifycaps.add(u_v)
 637             si = n.get_storage_index()
 638             self.expected_storage_indexes.add(base32.b2a(si))
 639             expected_si = n._uri.get_storage_index()
 640             self.failUnlessEqual(si, expected_si)
 641
 642             d = n.list()
 643             d.addCallback(lambda res: self.failUnlessEqual(res, {}))
 644             d.addCallback(lambda res: n.has_child(u"missing"))
 645             d.addCallback(lambda res: self.failIf(res))
 646
 647             fake_file_uri = make_mutable_file_uri()
 648             other_file_uri = make_mutable_file_uri()
 649             m = c.nodemaker.create_from_cap(fake_file_uri)
 650             ffu_v = m.get_verify_cap().to_string()
 651             self.expected_manifest.append( ((u"child",) , m.get_uri()) )
 652             self.expected_verifycaps.add(ffu_v)
 653             self.expected_storage_indexes.add(base32.b2a(m.get_storage_index()))
 654             d.addCallback(lambda res: n.set_uri(u"child",
 655                                                 fake_file_uri, fake_file_uri))
 656             d.addCallback(lambda res:
 657                           self.shouldFail(ExistingChildError, "set_uri-no",
 658                                           "child 'child' already exists",
 659                                           n.set_uri, u"child",
 660                                           other_file_uri, other_file_uri,
 661                                           overwrite=False))
 662             # /
 663             # /child = mutable
 664
 665             d.addCallback(lambda res: n.create_subdirectory(u"subdir"))
 666
 667             # /
 668             # /child = mutable
 669             # /subdir = directory
 670             def _created(subdir):
 671                 self.failUnless(isinstance(subdir, dirnode.DirectoryNode))
 672                 self.subdir = subdir
 673                 new_v = subdir.get_verify_cap().to_string()
 674                 assert isinstance(new_v, str)
 675                 self.expected_manifest.append( ((u"subdir",), subdir.get_uri()) )
 676                 self.expected_verifycaps.add(new_v)
 677                 si = subdir.get_storage_index()
 678                 self.expected_storage_indexes.add(base32.b2a(si))
 679             d.addCallback(_created)
 680
 681             d.addCallback(lambda res:
 682                           self.shouldFail(ExistingChildError, "mkdir-no",
 683                                           "child 'subdir' already exists",
 684                                           n.create_subdirectory, u"subdir",
 685                                           overwrite=False))
 686
 687             d.addCallback(lambda res: n.list())
 688             d.addCallback(lambda children:
 689                           self.failUnlessEqual(set(children.keys()),
 690                                                set([u"child", u"subdir"])))
 691
 692             d.addCallback(lambda res: n.start_deep_stats().when_done())
 693             def _check_deepstats(stats):
 694                 self.failUnless(isinstance(stats, dict))
 695                 expected = {"count-immutable-files": 0,
 696                             "count-mutable-files": 1,
 697                             "count-literal-files": 0,
 698                             "count-files": 1,
 699                             "count-directories": 2,
 700                             "size-immutable-files": 0,
 701                             "size-literal-files": 0,
 702                             #"size-directories": 616, # varies
 703                             #"largest-directory": 616,
 704                             "largest-directory-children": 2,
 705                             "largest-immutable-file": 0,
 706                             }
 707                 for k,v in expected.iteritems():
 708                     self.failUnlessEqual(stats[k], v,
 709                                          "stats[%s] was %s, not %s" %
 710                                          (k, stats[k], v))
 711                 self.failUnless(stats["size-directories"] > 500,
 712                                 stats["size-directories"])
 713                 self.failUnless(stats["largest-directory"] > 500,
 714                                 stats["largest-directory"])
 715                 self.failUnlessEqual(stats["size-files-histogram"], [])
 716             d.addCallback(_check_deepstats)
 717
 718             d.addCallback(lambda res: n.build_manifest().when_done())
 719             def _check_manifest(res):
 720                 manifest = res["manifest"]
 721                 self.failUnlessEqual(sorted(manifest),
 722                                      sorted(self.expected_manifest))
 723                 stats = res["stats"]
 724                 _check_deepstats(stats)
 725                 self.failUnlessEqual(self.expected_verifycaps,
 726                                      res["verifycaps"])
 727                 self.failUnlessEqual(self.expected_storage_indexes,
 728                                      res["storage-index"])
 729             d.addCallback(_check_manifest)
 730
 731             def _add_subsubdir(res):
 732                 return self.subdir.create_subdirectory(u"subsubdir")
 733             d.addCallback(_add_subsubdir)
 734             # /
 735             # /child = mutable
 736             # /subdir = directory
 737             # /subdir/subsubdir = directory
 738             d.addCallback(lambda res: n.get_child_at_path(u"subdir/subsubdir"))
 739             d.addCallback(lambda subsubdir:
 740                           self.failUnless(isinstance(subsubdir,
 741                                                      dirnode.DirectoryNode)))
 742             d.addCallback(lambda res: n.get_child_at_path(u""))
 743             d.addCallback(lambda res: self.failUnlessEqual(res.get_uri(),
 744                                                            n.get_uri()))
 745
 746             d.addCallback(lambda res: n.get_metadata_for(u"child"))
 747             d.addCallback(lambda metadata:
 748                           self.failUnlessEqual(set(metadata.keys()),
 749                                                set(["tahoe", "ctime", "mtime"])))
 750
 751             d.addCallback(lambda res:
 752                           self.shouldFail(NoSuchChildError, "gcamap-no",
 753                                           "nope",
 754                                           n.get_child_and_metadata_at_path,
 755                                           u"subdir/nope"))
 756             d.addCallback(lambda res:
 757                           n.get_child_and_metadata_at_path(u""))
 758             def _check_child_and_metadata1(res):
 759                 child, metadata = res
 760                 self.failUnless(isinstance(child, dirnode.DirectoryNode))
 761                 # edge-metadata needs at least one path segment
 762                 self.failUnlessEqual(set(metadata.keys()), set([]))
 763             d.addCallback(_check_child_and_metadata1)
 764             d.addCallback(lambda res:
 765                           n.get_child_and_metadata_at_path(u"child"))
 766
 767             def _check_child_and_metadata2(res):
 768                 child, metadata = res
 769                 self.failUnlessEqual(child.get_uri(),
 770                                      fake_file_uri)
 771                 self.failUnlessEqual(set(metadata.keys()),
 772                                      set(["tahoe", "ctime", "mtime"]))
 773             d.addCallback(_check_child_and_metadata2)
 774
 775             d.addCallback(lambda res:
 776                           n.get_child_and_metadata_at_path(u"subdir/subsubdir"))
 777             def _check_child_and_metadata3(res):
 778                 child, metadata = res
 779                 self.failUnless(isinstance(child, dirnode.DirectoryNode))
 780                 self.failUnlessEqual(set(metadata.keys()),
 781                                      set(["tahoe", "ctime", "mtime"]))
 782             d.addCallback(_check_child_and_metadata3)
 783
 784             # set_uri + metadata
 785             # it should be possible to add a child without any metadata
 786             d.addCallback(lambda res: n.set_uri(u"c2",
 787                                                 fake_file_uri, fake_file_uri,
 788                                                 {}))
 789             d.addCallback(lambda res: n.get_metadata_for(u"c2"))
 790             d.addCallback(lambda metadata:
 791                           self.failUnlessEqual(set(metadata.keys()), set(["tahoe", "ctime", "mtime"])))
 792
 793             # You can't override the link timestamps.
 794             d.addCallback(lambda res: n.set_uri(u"c2",
 795                                                 fake_file_uri, fake_file_uri,
 796                                                 { 'tahoe': {'linkcrtime': "bogus"}}))
 797             d.addCallback(lambda res: n.get_metadata_for(u"c2"))
 798             def _has_good_linkcrtime(metadata):
 799                 self.failUnless(metadata.has_key('tahoe'))
 800                 self.failUnless(metadata['tahoe'].has_key('linkcrtime'))
 801                 self.failIfEqual(metadata['tahoe']['linkcrtime'], 'bogus')
 802             d.addCallback(_has_good_linkcrtime)
 803
 804             # if we don't set any defaults, the child should get timestamps
 805             d.addCallback(lambda res: n.set_uri(u"c3",
 806                                                 fake_file_uri, fake_file_uri))
 807             d.addCallback(lambda res: n.get_metadata_for(u"c3"))
 808             d.addCallback(lambda metadata:
 809                           self.failUnlessEqual(set(metadata.keys()), set(["tahoe", "ctime", "mtime"])))
 810
 811             # we can also add specific metadata at set_uri() time
 812             d.addCallback(lambda res: n.set_uri(u"c4",
 813                                                 fake_file_uri, fake_file_uri,
 814                                                 {"key": "value"}))
 815             d.addCallback(lambda res: n.get_metadata_for(u"c4"))
 816             d.addCallback(lambda metadata:
 817                               self.failUnless((set(metadata.keys()) == set(["key", "tahoe", "ctime", "mtime"])) and
 818                                               (metadata['key'] == "value"), metadata))
 819
 820             d.addCallback(lambda res: n.delete(u"c2"))
 821             d.addCallback(lambda res: n.delete(u"c3"))
 822             d.addCallback(lambda res: n.delete(u"c4"))
 823
 824             # set_node + metadata
 825             # it should be possible to add a child without any metadata except for timestamps
 826             d.addCallback(lambda res: n.set_node(u"d2", n, {}))
 827             d.addCallback(lambda res: c.create_dirnode())
 828             d.addCallback(lambda n2:
 829                           self.shouldFail(ExistingChildError, "set_node-no",
 830                                           "child 'd2' already exists",
 831                                           n.set_node, u"d2", n2,
 832                                           overwrite=False))
 833             d.addCallback(lambda res: n.get_metadata_for(u"d2"))
 834             d.addCallback(lambda metadata:
 835                           self.failUnlessEqual(set(metadata.keys()), set(["tahoe", "ctime", "mtime"])))
 836
 837             # if we don't set any defaults, the child should get timestamps
 838             d.addCallback(lambda res: n.set_node(u"d3", n))
 839             d.addCallback(lambda res: n.get_metadata_for(u"d3"))
 840             d.addCallback(lambda metadata:
 841                           self.failUnlessEqual(set(metadata.keys()), set(["tahoe", "ctime", "mtime"])))
 842
 843             # we can also add specific metadata at set_node() time
 844             d.addCallback(lambda res: n.set_node(u"d4", n,
 845                                                 {"key": "value"}))
 846             d.addCallback(lambda res: n.get_metadata_for(u"d4"))
 847             d.addCallback(lambda metadata:
 848                           self.failUnless((set(metadata.keys()) == set(["key", "tahoe", "ctime", "mtime"])) and
 849                                           (metadata["key"] == "value"), metadata))
 850
 851             d.addCallback(lambda res: n.delete(u"d2"))
 852             d.addCallback(lambda res: n.delete(u"d3"))
 853             d.addCallback(lambda res: n.delete(u"d4"))
 854
 855             # metadata through set_children()
 856             d.addCallback(lambda res:
 857                           n.set_children({
 858                               u"e1": (fake_file_uri, fake_file_uri),
 859                               u"e2": (fake_file_uri, fake_file_uri, {}),
 860                               u"e3": (fake_file_uri, fake_file_uri,
 861                                       {"key": "value"}),
 862                               }))
 863             d.addCallback(lambda n2: self.failUnlessIdentical(n2, n))
 864             d.addCallback(lambda res:
 865                           self.shouldFail(ExistingChildError, "set_children-no",
 866                                           "child 'e1' already exists",
 867                                           n.set_children,
 868                                           { u"e1": (other_file_uri,
 869                                                     other_file_uri),
 870                                             u"new": (other_file_uri,
 871                                                      other_file_uri),
 872                                             },
 873                                           overwrite=False))
 874             # and 'new' should not have been created
 875             d.addCallback(lambda res: n.list())
 876             d.addCallback(lambda children: self.failIf(u"new" in children))
 877             d.addCallback(lambda res: n.get_metadata_for(u"e1"))
 878             d.addCallback(lambda metadata:
 879                           self.failUnlessEqual(set(metadata.keys()), set(["tahoe", "ctime", "mtime"])))
 880             d.addCallback(lambda res: n.get_metadata_for(u"e2"))
 881             d.addCallback(lambda metadata:
 882                           self.failUnlessEqual(set(metadata.keys()), set(["tahoe", "ctime", "mtime"])))
 883             d.addCallback(lambda res: n.get_metadata_for(u"e3"))
 884             d.addCallback(lambda metadata:
 885                           self.failUnless((set(metadata.keys()) == set(["key", "tahoe", "ctime", "mtime"])) and
 886                                           (metadata["key"] == "value"), metadata))
 887
 888             d.addCallback(lambda res: n.delete(u"e1"))
 889             d.addCallback(lambda res: n.delete(u"e2"))
 890             d.addCallback(lambda res: n.delete(u"e3"))
 891
 892             # metadata through set_nodes()
 893             d.addCallback(lambda res:
 894                           n.set_nodes({ u"f1": (n, None),
 895                                         u"f2": (n, {}),
 896                                         u"f3": (n, {"key": "value"}),
 897                                         }))
 898             d.addCallback(lambda n2: self.failUnlessIdentical(n2, n))
 899             d.addCallback(lambda res:
 900                           self.shouldFail(ExistingChildError, "set_nodes-no",
 901                                           "child 'f1' already exists",
 902                                           n.set_nodes, { u"f1": (n, None),
 903                                                          u"new": (n, None), },
 904                                           overwrite=False))
 905             # and 'new' should not have been created
 906             d.addCallback(lambda res: n.list())
 907             d.addCallback(lambda children: self.failIf(u"new" in children))
 908             d.addCallback(lambda res: n.get_metadata_for(u"f1"))
 909             d.addCallback(lambda metadata:
 910                           self.failUnlessEqual(set(metadata.keys()), set(["tahoe", "ctime", "mtime"])))
 911             d.addCallback(lambda res: n.get_metadata_for(u"f2"))
 912             d.addCallback(lambda metadata:
 913                           self.failUnlessEqual(set(metadata.keys()), set(["tahoe", "ctime", "mtime"])))
 914             d.addCallback(lambda res: n.get_metadata_for(u"f3"))
 915             d.addCallback(lambda metadata:
 916                           self.failUnless((set(metadata.keys()) == set(["key", "tahoe", "ctime", "mtime"])) and
 917                                           (metadata["key"] == "value"), metadata))
 918
 919             d.addCallback(lambda res: n.delete(u"f1"))
 920             d.addCallback(lambda res: n.delete(u"f2"))
 921             d.addCallback(lambda res: n.delete(u"f3"))
 922
 923
 924             d.addCallback(lambda res:
 925                           n.set_metadata_for(u"child",
 926                                              {"tags": ["web2.0-compatible"], "tahoe": {"bad": "mojo"}}))
 927             d.addCallback(lambda n1: n1.get_metadata_for(u"child"))
 928             d.addCallback(lambda metadata:
 929                           self.failUnless((set(metadata.keys()) == set(["tags", "tahoe", "ctime", "mtime"])) and
 930                                           metadata["tags"] == ["web2.0-compatible"] and
 931                                           "bad" not in metadata["tahoe"], metadata))
 932
 933             d.addCallback(lambda res:
 934                           self.shouldFail(NoSuchChildError, "set_metadata_for-nosuch", "",
 935                                           n.set_metadata_for, u"nosuch", {}))
 936
 937
 938             def _start(res):
 939                 self._start_timestamp = time.time()
 940             d.addCallback(_start)
 941             # simplejson-1.7.1 (as shipped on Ubuntu 'gutsy') rounds all
 942             # floats to hundredeths (it uses str(num) instead of repr(num)).
 943             # simplejson-1.7.3 does not have this bug. To prevent this bug
 944             # from causing the test to fail, stall for more than a few
 945             # hundrededths of a second.
 946             d.addCallback(self.stall, 0.1)
 947             d.addCallback(lambda res: n.add_file(u"timestamps",
 948                                                  upload.Data("stamp me", convergence="some convergence string")))
 949             d.addCallback(self.stall, 0.1)
 950             def _stop(res):
 951                 self._stop_timestamp = time.time()
 952             d.addCallback(_stop)
 953
 954             d.addCallback(lambda res: n.get_metadata_for(u"timestamps"))
 955             def _check_timestamp1(metadata):
 956                 self.failUnless("ctime" in metadata)
 957                 self.failUnless("mtime" in metadata)
 958                 self.failUnlessGreaterOrEqualThan(metadata["ctime"],
 959                                                   self._start_timestamp)
 960                 self.failUnlessGreaterOrEqualThan(self._stop_timestamp,
 961                                                   metadata["ctime"])
 962                 self.failUnlessGreaterOrEqualThan(metadata["mtime"],
 963                                                   self._start_timestamp)
 964                 self.failUnlessGreaterOrEqualThan(self._stop_timestamp,
 965                                                   metadata["mtime"])
 966                 # Our current timestamp rules say that replacing an existing
 967                 # child should preserve the 'ctime' but update the mtime
 968                 self._old_ctime = metadata["ctime"]
 969                 self._old_mtime = metadata["mtime"]
 970             d.addCallback(_check_timestamp1)
 971             d.addCallback(self.stall, 2.0) # accomodate low-res timestamps
 972             d.addCallback(lambda res: n.set_node(u"timestamps", n))
 973             d.addCallback(lambda res: n.get_metadata_for(u"timestamps"))
 974             def _check_timestamp2(metadata):
 975                 self.failUnlessEqual(metadata["ctime"], self._old_ctime,
 976                                      "%s != %s" % (metadata["ctime"],
 977                                                    self._old_ctime))
 978                 self.failUnlessGreaterThan(metadata["mtime"], self._old_mtime)
 979                 return n.delete(u"timestamps")
 980             d.addCallback(_check_timestamp2)
 981
 982             # also make sure we can add/update timestamps on a
 983             # previously-existing child that didn't have any, since there are
 984             # a lot of 0.7.0-generated edges around out there
 985             d.addCallback(lambda res: n.set_node(u"no_timestamps", n, {}))
 986             d.addCallback(lambda res: n.set_node(u"no_timestamps", n))
 987             d.addCallback(lambda res: n.get_metadata_for(u"no_timestamps"))
 988             d.addCallback(lambda metadata:
 989                           self.failUnlessEqual(set(metadata.keys()),
 990                                                set(["tahoe", "ctime", "mtime"])))
 991             d.addCallback(lambda res: n.delete(u"no_timestamps"))
 992
 993             d.addCallback(lambda res: n.delete(u"subdir"))
 994             d.addCallback(lambda old_child:
 995                           self.failUnlessEqual(old_child.get_uri(),
 996                                                self.subdir.get_uri()))
 997
 998             d.addCallback(lambda res: n.list())
 999             d.addCallback(lambda children:
1000                           self.failUnlessEqual(set(children.keys()),
1001                                                set([u"child"])))
1002
1003             uploadable1 = upload.Data("some data", convergence="converge")
1004             d.addCallback(lambda res: n.add_file(u"newfile", uploadable1))
1005             d.addCallback(lambda newnode:
1006                           self.failUnless(IImmutableFileNode.providedBy(newnode)))
1007             uploadable2 = upload.Data("some data", convergence="stuff")
1008             d.addCallback(lambda res:
1009                           self.shouldFail(ExistingChildError, "add_file-no",
1010                                           "child 'newfile' already exists",
1011                                           n.add_file, u"newfile",
1012                                           uploadable2,
1013                                           overwrite=False))
1014             d.addCallback(lambda res: n.list())
1015             d.addCallback(lambda children:
1016                           self.failUnlessEqual(set(children.keys()),
1017                                                set([u"child", u"newfile"])))
1018             d.addCallback(lambda res: n.get_metadata_for(u"newfile"))
1019             d.addCallback(lambda metadata:
1020                           self.failUnlessEqual(set(metadata.keys()),
1021                                                set(["tahoe", "ctime", "mtime"])))
1022
1023             uploadable3 = upload.Data("some data", convergence="converge")
1024             d.addCallback(lambda res: n.add_file(u"newfile-metadata",
1025                                                  uploadable3,
1026                                                  {"key": "value"}))
1027             d.addCallback(lambda newnode:
1028                           self.failUnless(IImmutableFileNode.providedBy(newnode)))
1029             d.addCallback(lambda res: n.get_metadata_for(u"newfile-metadata"))
1030             d.addCallback(lambda metadata:
1031                               self.failUnless((set(metadata.keys()) == set(["key", "tahoe", "ctime", "mtime"])) and
1032                                               (metadata['key'] == "value"), metadata))
1033             d.addCallback(lambda res: n.delete(u"newfile-metadata"))
1034
1035             d.addCallback(lambda res: n.create_subdirectory(u"subdir2"))
1036             def _created2(subdir2):
1037                 self.subdir2 = subdir2
1038                 # put something in the way, to make sure it gets overwritten
1039                 return subdir2.add_file(u"child", upload.Data("overwrite me",
1040                                                               "converge"))
1041             d.addCallback(_created2)
1042
1043             d.addCallback(lambda res:
1044                           n.move_child_to(u"child", self.subdir2))
1045             d.addCallback(lambda res: n.list())
1046             d.addCallback(lambda children:
1047                           self.failUnlessEqual(set(children.keys()),
1048                                                set([u"newfile", u"subdir2"])))
1049             d.addCallback(lambda res: self.subdir2.list())
1050             d.addCallback(lambda children:
1051                           self.failUnlessEqual(set(children.keys()),
1052                                                set([u"child"])))
1053             d.addCallback(lambda res: self.subdir2.get(u"child"))
1054             d.addCallback(lambda child:
1055                           self.failUnlessEqual(child.get_uri(),
1056                                                fake_file_uri))
1057
1058             # move it back, using new_child_name=
1059             d.addCallback(lambda res:
1060                           self.subdir2.move_child_to(u"child", n, u"newchild"))
1061             d.addCallback(lambda res: n.list())
1062             d.addCallback(lambda children:
1063                           self.failUnlessEqual(set(children.keys()),
1064                                                set([u"newchild", u"newfile",
1065                                                     u"subdir2"])))
1066             d.addCallback(lambda res: self.subdir2.list())
1067             d.addCallback(lambda children:
1068                           self.failUnlessEqual(set(children.keys()), set([])))
1069
1070             # now make sure that we honor overwrite=False
1071             d.addCallback(lambda res:
1072                           self.subdir2.set_uri(u"newchild",
1073                                                other_file_uri, other_file_uri))
1074
1075             d.addCallback(lambda res:
1076                           self.shouldFail(ExistingChildError, "move_child_to-no",
1077                                           "child 'newchild' already exists",
1078                                           n.move_child_to, u"newchild",
1079                                           self.subdir2,
1080                                           overwrite=False))
1081             d.addCallback(lambda res: self.subdir2.get(u"newchild"))
1082             d.addCallback(lambda child:
1083                           self.failUnlessEqual(child.get_uri(),
1084                                                other_file_uri))
1085
1086
1087             # Setting the no-write field should diminish a mutable cap to read-only
1088             # (for both files and directories).
1089
1090             d.addCallback(lambda ign: n.set_uri(u"mutable", other_file_uri, other_file_uri))
1091             d.addCallback(lambda ign: n.get(u"mutable"))
1092             d.addCallback(lambda mutable: self.failIf(mutable.is_readonly(), mutable))
1093             d.addCallback(lambda ign: n.set_metadata_for(u"mutable", {"no-write": True}))
1094             d.addCallback(lambda ign: n.get(u"mutable"))
1095             d.addCallback(lambda mutable: self.failUnless(mutable.is_readonly(), mutable))
1096             d.addCallback(lambda ign: n.set_metadata_for(u"mutable", {"no-write": True}))
1097             d.addCallback(lambda ign: n.get(u"mutable"))
1098             d.addCallback(lambda mutable: self.failUnless(mutable.is_readonly(), mutable))
1099
1100             d.addCallback(lambda ign: n.get(u"subdir2"))
1101             d.addCallback(lambda subdir2: self.failIf(subdir2.is_readonly()))
1102             d.addCallback(lambda ign: n.set_metadata_for(u"subdir2", {"no-write": True}))
1103             d.addCallback(lambda ign: n.get(u"subdir2"))
1104             d.addCallback(lambda subdir2: self.failUnless(subdir2.is_readonly(), subdir2))
1105
1106             d.addCallback(lambda ign: n.set_uri(u"mutable_ro", other_file_uri, other_file_uri,
1107                                                 metadata={"no-write": True}))
1108             d.addCallback(lambda ign: n.get(u"mutable_ro"))
1109             d.addCallback(lambda mutable_ro: self.failUnless(mutable_ro.is_readonly(), mutable_ro))
1110
1111             d.addCallback(lambda ign: n.create_subdirectory(u"subdir_ro", metadata={"no-write": True}))
1112             d.addCallback(lambda ign: n.get(u"subdir_ro"))
1113             d.addCallback(lambda subdir_ro: self.failUnless(subdir_ro.is_readonly(), subdir_ro))
1114
1115             return d
1116
1117         d.addCallback(_then)
1118
1119         d.addErrback(self.explain_error)
1120         return d
1121
1122     def test_update_metadata(self):
1123         (t1, t2, t3) = (626644800, 634745640, 892226160)
1124
1125         md1 = dirnode.update_metadata({}, {"ctime": t1}, t2)
1126         self.failUnlessEqual(md1, {"ctime": t1, "mtime": t2,
1127                                    "tahoe":{"linkcrtime": t1, "linkmotime": t2}})
1128
1129         md2 = dirnode.update_metadata(md1, {"key": "value", "tahoe": {"bad": "mojo"}}, t3)
1130         self.failUnlessEqual(md2, {"key": "value", "ctime": t1, "mtime": t3,
1131                                    "tahoe":{"linkcrtime": t1, "linkmotime": t3}})
1132
1133         md3 = dirnode.update_metadata({}, None, t3)
1134         self.failUnlessEqual(md3, {"ctime": t3, "mtime": t3,
1135                                    "tahoe":{"linkcrtime": t3, "linkmotime": t3}})
1136
1137     def test_create_subdirectory(self):
1138         self.basedir = "dirnode/Dirnode/test_create_subdirectory"
1139         self.set_up_grid()
1140         c = self.g.clients[0]
1141         nm = c.nodemaker
1142
1143         d = c.create_dirnode()
1144         def _then(n):
1145             # /
1146             self.rootnode = n
1147             fake_file_uri = make_mutable_file_uri()
1148             other_file_uri = make_mutable_file_uri()
1149             md = {"metakey": "metavalue"}
1150             kids = {u"kid1": (nm.create_from_cap(fake_file_uri), {}),
1151                     u"kid2": (nm.create_from_cap(other_file_uri), md),
1152                     }
1153             d = n.create_subdirectory(u"subdir", kids)
1154             def _check(sub):
1155                 d = n.get_child_at_path(u"subdir")
1156                 d.addCallback(lambda sub2: self.failUnlessEqual(sub2.get_uri(),
1157                                                                 sub.get_uri()))
1158                 d.addCallback(lambda ign: sub.list())
1159                 return d
1160             d.addCallback(_check)
1161             def _check_kids(kids2):
1162                 self.failUnlessEqual(set(kids.keys()), set(kids2.keys()))
1163                 self.failUnlessEqual(kids2[u"kid2"][1]["metakey"], "metavalue")
1164             d.addCallback(_check_kids)
1165             return d
1166         d.addCallback(_then)
1167         return d
1168
1169 class MinimalFakeMutableFile:
1170     def get_writekey(self):
1171         return "writekey"
1172
1173 class Packing(unittest.TestCase):
1174     # This is a base32-encoded representation of the directory tree
1175     # root/file1
1176     # root/file2
1177     # root/file3
1178     # as represented after being fed to _pack_contents.
1179     # We have it here so we can decode it, feed it to
1180     # _unpack_contents, and verify that _unpack_contents
1181     # works correctly.
1182
1183     known_tree = "GM4TOORVHJTGS3DFGEWDSNJ2KVJESOSDJBFTU33MPB2GS3LZNVYG6N3GGI3WU5TIORTXC3DOMJ2G4NB2MVWXUZDONBVTE5LNGRZWK2LYN55GY23XGNYXQMTOMZUWU5TENN4DG23ZG5UTO2L2NQ2DO6LFMRWDMZJWGRQTUMZ2GEYDUMJQFQYTIMZ22XZKZORX5XS7CAQCSK3URR6QOHISHRCMGER5LRFSZRNAS5ZSALCS6TWFQAE754IVOIKJVK73WZPP3VUUEDTX3WHTBBZ5YX3CEKHCPG3ZWQLYA4QM6LDRCF7TJQYWLIZHKGN5ROA3AUZPXESBNLQQ6JTC2DBJU2D47IZJTLR3PKZ4RVF57XLPWY7FX7SZV3T6IJ3ORFW37FXUPGOE3ROPFNUX5DCGMAQJ3PGGULBRGM3TU6ZCMN2GS3LFEI5CAMJSGQ3DMNRTHA4TOLRUGI3TKNRWGEWCAITUMFUG6ZJCHIQHWITMNFXGW3LPORUW2ZJCHIQDCMRUGY3DMMZYHE3S4NBSG42TMNRRFQQCE3DJNZVWG4TUNFWWKIR2EAYTENBWGY3DGOBZG4XDIMRXGU3DMML5FQQCE3LUNFWWKIR2EAYTENBWGY3DGOBZG4XDIMRXGU3DMML5FQWDGOJRHI2TUZTJNRSTELBZGQ5FKUSJHJBUQSZ2MFYGKZ3SOBSWQ43IO52WO23CNAZWU3DUGVSWSNTIOE5DK33POVTW4ZLNMNWDK6DHPA2GS2THNF2W25DEN5VGY2LQNFRGG5DKNNRHO5TZPFTWI6LNMRYGQ2LCGJTHM4J2GM5DCMB2GQWDCNBSHKVVQBGRYMACKJ27CVQ6O6B4QPR72RFVTGOZUI76XUSWAX73JRV5PYRHMIFYZIA25MXDPGUGML6M2NMRSG4YD4W4K37ZDYSXHMJ3IUVT4F64YTQQVBJFFFOUC7J7LAB2VFCL5UKKGMR2D3F4EPOYC7UYWQZNR5KXHBSNXLCNBX2SNF22DCXJIHSMEKWEWOG5XCJEVVZ7UW5IB6I64XXQSJ34B5CAYZGZIIMR6LBRGMZTU6ZCMN2GS3LFEI5CAMJSGQ3DMNRTHA4TOLRUGMYDEMJYFQQCE5DBNBXWKIR2EB5SE3DJNZVW233UNFWWKIR2EAYTENBWGY3DGOBZG4XDIMZQGIYTQLBAEJWGS3TLMNZHI2LNMURDUIBRGI2DMNRWGM4DSNZOGQZTAMRRHB6SYIBCNV2GS3LFEI5CAMJSGQ3DMNRTHA4TOLRUGMYDEMJYPUWCYMZZGU5DKOTGNFWGKMZMHE2DUVKSJE5EGSCLHJRW25DDPBYTO2DXPB3GM6DBNYZTI6LJMV3DM2LWNB4TU4LWMNSWW3LKORXWK5DEMN3TI23NNE3WEM3SORRGY5THPA3TKNBUMNZG453BOF2GSZLXMVWWI3DJOFZW623RHIZTUMJQHI2SYMJUGI5BOSHWDPG3WKPAVXCF3XMKA7QVIWPRMWJHDTQHD27AHDCPJWDQENQ5H5ZZILTXQNIXXCIW4LKQABU2GCFRG5FHQN7CHD7HF4EKNRZFIV2ZYQIBM7IQU7F4RGB3XCX3FREPBKQ7UCICHVWPCYFGA6OLH3J45LXQ6GWWICJ3PGWJNLZ7PCRNLAPNYUGU6BENS7OXMBEOOFRIZV3PF2FFWZ5WHDPKXERYP7GNHKRMGEZTOOT3EJRXI2LNMURDUIBRGI2DMNRWGM4DSNZOGQZTGNRSGY4SYIBCORQWQ33FEI5CA6ZCNRUW423NN52GS3LFEI5CAMJSGQ3DMNRTHA4TOLRUGMZTMMRWHEWCAITMNFXGWY3SORUW2ZJCHIQDCMRUGY3DMMZYHE3S4NBTGM3DENRZPUWCAITNORUW2ZJCHIQDCMRUGY3DMMZYHE3S4NBTGM3DENRZPUWCY==="
1184
1185     def test_unpack_and_pack_behavior(self):
1186         known_tree = b32decode(self.known_tree)
1187         nodemaker = NodeMaker(None, None, None,
1188                               None, None, None,
1189                               {"k": 3, "n": 10}, None)
1190         write_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
1191         filenode = nodemaker.create_from_cap(write_uri)
1192         node = dirnode.DirectoryNode(filenode, nodemaker, None)
1193         children = node._unpack_contents(known_tree)
1194         self._check_children(children)
1195
1196         packed_children = node._pack_contents(children)
1197         children = node._unpack_contents(packed_children)
1198         self._check_children(children)
1199
1200     def _check_children(self, children):
1201         # Are all the expected child nodes there?
1202         self.failUnless(children.has_key(u'file1'))
1203         self.failUnless(children.has_key(u'file2'))
1204         self.failUnless(children.has_key(u'file3'))
1205
1206         # Are the metadata for child 3 right?
1207         file3_rocap = "URI:CHK:cmtcxq7hwxvfxan34yiev6ivhy:qvcekmjtoetdcw4kmi7b3rtblvgx7544crnwaqtiewemdliqsokq:3:10:5"
1208         file3_rwcap = "URI:CHK:cmtcxq7hwxvfxan34yiev6ivhy:qvcekmjtoetdcw4kmi7b3rtblvgx7544crnwaqtiewemdliqsokq:3:10:5"
1209         file3_metadata = {'ctime': 1246663897.4336269, 'tahoe': {'linkmotime': 1246663897.4336269, 'linkcrtime': 1246663897.4336269}, 'mtime': 1246663897.4336269}
1210         self.failUnlessEqual(file3_metadata, children[u'file3'][1])
1211         self.failUnlessEqual(file3_rocap,
1212                              children[u'file3'][0].get_readonly_uri())
1213         self.failUnlessEqual(file3_rwcap,
1214                              children[u'file3'][0].get_uri())
1215
1216         # Are the metadata for child 2 right?
1217         file2_rocap = "URI:CHK:apegrpehshwugkbh3jlt5ei6hq:5oougnemcl5xgx4ijgiumtdojlipibctjkbwvyygdymdphib2fvq:3:10:4"
1218         file2_rwcap = "URI:CHK:apegrpehshwugkbh3jlt5ei6hq:5oougnemcl5xgx4ijgiumtdojlipibctjkbwvyygdymdphib2fvq:3:10:4"
1219         file2_metadata = {'ctime': 1246663897.430218, 'tahoe': {'linkmotime': 1246663897.430218, 'linkcrtime': 1246663897.430218}, 'mtime': 1246663897.430218}
1220         self.failUnlessEqual(file2_metadata, children[u'file2'][1])
1221         self.failUnlessEqual(file2_rocap,
1222                              children[u'file2'][0].get_readonly_uri())
1223         self.failUnlessEqual(file2_rwcap,
1224                              children[u'file2'][0].get_uri())
1225
1226         # Are the metadata for child 1 right?
1227         file1_rocap = "URI:CHK:olxtimympo7f27jvhtgqlnbtn4:emzdnhk2um4seixozlkw3qx2nfijvdkx3ky7i7izl47yedl6e64a:3:10:10"
1228         file1_rwcap = "URI:CHK:olxtimympo7f27jvhtgqlnbtn4:emzdnhk2um4seixozlkw3qx2nfijvdkx3ky7i7izl47yedl6e64a:3:10:10"
1229         file1_metadata = {'ctime': 1246663897.4275661, 'tahoe': {'linkmotime': 1246663897.4275661, 'linkcrtime': 1246663897.4275661}, 'mtime': 1246663897.4275661}
1230         self.failUnlessEqual(file1_metadata, children[u'file1'][1])
1231         self.failUnlessEqual(file1_rocap,
1232                              children[u'file1'][0].get_readonly_uri())
1233         self.failUnlessEqual(file1_rwcap,
1234                              children[u'file1'][0].get_uri())
1235
1236     def _make_kids(self, nm, which):
1237         caps = {"imm": "URI:CHK:n7r3m6wmomelk4sep3kw5cvduq:os7ijw5c3maek7pg65e5254k2fzjflavtpejjyhshpsxuqzhcwwq:3:20:14861",
1238                 "lit": "URI:LIT:n5xgk", # LIT for "one"
1239                 "write": "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq",
1240                 "read": "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q",
1241                 "dirwrite": "URI:DIR2:n6x24zd3seu725yluj75q5boaa:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq",
1242                 "dirread":  "URI:DIR2-RO:b7sr5qsifnicca7cbk3rhrhbvq:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq",
1243                 }
1244         kids = {}
1245         for name in which:
1246             kids[unicode(name)] = (nm.create_from_cap(caps[name]), {})
1247         return kids
1248
1249     def test_deep_immutable(self):
1250         nm = NodeMaker(None, None, None, None, None, None, {"k": 3, "n": 10},
1251                        None)
1252         fn = MinimalFakeMutableFile()
1253
1254         kids = self._make_kids(nm, ["imm", "lit", "write", "read",
1255                                     "dirwrite", "dirread"])
1256         packed = dirnode.pack_children(fn, kids, deep_immutable=False)
1257         self.failUnlessIn("lit", packed)
1258
1259         kids = self._make_kids(nm, ["imm", "lit"])
1260         packed = dirnode.pack_children(fn, kids, deep_immutable=True)
1261         self.failUnlessIn("lit", packed)
1262
1263         kids = self._make_kids(nm, ["imm", "lit", "write"])
1264         self.failUnlessRaises(dirnode.MustBeDeepImmutableError,
1265                               dirnode.pack_children,
1266                               fn, kids, deep_immutable=True)
1267
1268         # read-only is not enough: all children must be immutable
1269         kids = self._make_kids(nm, ["imm", "lit", "read"])
1270         self.failUnlessRaises(dirnode.MustBeDeepImmutableError,
1271                               dirnode.pack_children,
1272                               fn, kids, deep_immutable=True)
1273
1274         kids = self._make_kids(nm, ["imm", "lit", "dirwrite"])
1275         self.failUnlessRaises(dirnode.MustBeDeepImmutableError,
1276                               dirnode.pack_children,
1277                               fn, kids, deep_immutable=True)
1278
1279         kids = self._make_kids(nm, ["imm", "lit", "dirread"])
1280         self.failUnlessRaises(dirnode.MustBeDeepImmutableError,
1281                               dirnode.pack_children,
1282                               fn, kids, deep_immutable=True)
1283
1284 class FakeMutableFile:
1285     implements(IMutableFileNode)
1286     counter = 0
1287     def __init__(self, initial_contents=""):
1288         self.data = self._get_initial_contents(initial_contents)
1289         counter = FakeMutableFile.counter
1290         FakeMutableFile.counter += 1
1291         writekey = hashutil.ssk_writekey_hash(str(counter))
1292         fingerprint = hashutil.ssk_pubkey_fingerprint_hash(str(counter))
1293         self.uri = uri.WriteableSSKFileURI(writekey, fingerprint)
1294
1295     def _get_initial_contents(self, contents):
1296         if isinstance(contents, str):
1297             return contents
1298         if contents is None:
1299             return ""
1300         assert callable(contents), "%s should be callable, not %s" % \
1301                (contents, type(contents))
1302         return contents(self)
1303
1304     def get_cap(self):
1305         return self.uri
1306
1307     def get_uri(self):
1308         return self.uri.to_string()
1309
1310     def get_write_uri(self):
1311         return self.uri.to_string()
1312
1313     def download_best_version(self):
1314         return defer.succeed(self.data)
1315
1316     def get_writekey(self):
1317         return "writekey"
1318
1319     def is_readonly(self):
1320         return False
1321
1322     def is_mutable(self):
1323         return True
1324
1325     def is_unknown(self):
1326         return False
1327
1328     def is_allowed_in_immutable_directory(self):
1329         return False
1330
1331     def raise_error(self):
1332         pass
1333
1334     def modify(self, modifier):
1335         self.data = modifier(self.data, None, True)
1336         return defer.succeed(None)
1337
1338 class FakeNodeMaker(NodeMaker):
1339     def create_mutable_file(self, contents="", keysize=None):
1340         return defer.succeed(FakeMutableFile(contents))
1341
1342 class FakeClient2(Client):
1343     def __init__(self):
1344         self.nodemaker = FakeNodeMaker(None, None, None,
1345                                        None, None, None,
1346                                        {"k":3,"n":10}, None)
1347     def create_node_from_uri(self, rwcap, rocap):
1348         return self.nodemaker.create_from_cap(rwcap, rocap)
1349
1350 class Dirnode2(unittest.TestCase, testutil.ShouldFailMixin):
1351     def setUp(self):
1352         client = FakeClient2()
1353         self.nodemaker = client.nodemaker
1354
1355     def test_from_future(self):
1356         # Create a mutable directory that contains unknown URI types, and make sure
1357         # we tolerate them properly.
1358         d = self.nodemaker.create_new_mutable_directory()
1359         future_write_uri = "x-tahoe-crazy://I_am_from_the_future."
1360         future_read_uri = "x-tahoe-crazy-readonly://I_am_from_the_future."
1361         future_imm_uri = "x-tahoe-crazy-immutable://I_am_from_the_future."
1362         future_node = UnknownNode(future_write_uri, future_read_uri)
1363         def _then(n):
1364             self._node = n
1365             return n.set_node(u"future", future_node)
1366         d.addCallback(_then)
1367
1368         # We should be prohibited from adding an unknown URI to a directory
1369         # just in the rw_uri slot, since we don't know how to diminish the cap
1370         # to a readcap (for the ro_uri slot).
1371         d.addCallback(lambda ign:
1372              self.shouldFail(MustNotBeUnknownRWError,
1373                              "copy unknown",
1374                              "cannot attach unknown rw cap as child",
1375                              self._node.set_uri, u"add",
1376                              future_write_uri, None))
1377
1378         # However, we should be able to add both rw_uri and ro_uri as a pair of
1379         # unknown URIs.
1380         d.addCallback(lambda ign: self._node.set_uri(u"add-pair",
1381                                                      future_write_uri, future_read_uri))
1382
1383         # and to add an URI prefixed with "ro." or "imm." when it is given in a
1384         # write slot (or URL parameter).
1385         d.addCallback(lambda ign: self._node.set_uri(u"add-ro",
1386                                                      "ro." + future_read_uri, None))
1387         d.addCallback(lambda ign: self._node.set_uri(u"add-imm",
1388                                                      "imm." + future_imm_uri, None))
1389
1390         d.addCallback(lambda ign: self._node.list())
1391         def _check(children):
1392             self.failUnlessEqual(len(children), 4)
1393             (fn, metadata) = children[u"future"]
1394             self.failUnless(isinstance(fn, UnknownNode), fn)
1395             self.failUnlessEqual(fn.get_uri(), future_write_uri)
1396             self.failUnlessEqual(fn.get_write_uri(), future_write_uri)
1397             self.failUnlessEqual(fn.get_readonly_uri(), "ro." + future_read_uri)
1398
1399             (fn2, metadata2) = children[u"add-pair"]
1400             self.failUnless(isinstance(fn2, UnknownNode), fn2)
1401             self.failUnlessEqual(fn2.get_uri(), future_write_uri)
1402             self.failUnlessEqual(fn2.get_write_uri(), future_write_uri)
1403             self.failUnlessEqual(fn2.get_readonly_uri(), "ro." + future_read_uri)
1404
1405             (fn3, metadata3) = children[u"add-ro"]
1406             self.failUnless(isinstance(fn3, UnknownNode), fn3)
1407             self.failUnlessEqual(fn3.get_uri(), "ro." + future_read_uri)
1408             self.failUnlessEqual(fn3.get_write_uri(), None)
1409             self.failUnlessEqual(fn3.get_readonly_uri(), "ro." + future_read_uri)
1410
1411             (fn4, metadata4) = children[u"add-imm"]
1412             self.failUnless(isinstance(fn4, UnknownNode), fn4)
1413             self.failUnlessEqual(fn4.get_uri(), "imm." + future_imm_uri)
1414             self.failUnlessEqual(fn4.get_write_uri(), None)
1415             self.failUnlessEqual(fn4.get_readonly_uri(), "imm." + future_imm_uri)
1416
1417             # We should also be allowed to copy the "future" UnknownNode, because
1418             # it contains all the information that was in the original directory
1419             # (readcap and writecap), so we're preserving everything.
1420             return self._node.set_node(u"copy", fn)
1421         d.addCallback(_check)
1422
1423         d.addCallback(lambda ign: self._node.list())
1424         def _check2(children):
1425             self.failUnlessEqual(len(children), 5)
1426             (fn, metadata) = children[u"copy"]
1427             self.failUnless(isinstance(fn, UnknownNode), fn)
1428             self.failUnlessEqual(fn.get_uri(), future_write_uri)
1429             self.failUnlessEqual(fn.get_write_uri(), future_write_uri)
1430             self.failUnlessEqual(fn.get_readonly_uri(), "ro." + future_read_uri)
1431         d.addCallback(_check2)
1432         return d
1433
1434     def test_unknown_strip_prefix_for_ro(self):
1435         self.failUnlessEqual(strip_prefix_for_ro("foo",     False), "foo")
1436         self.failUnlessEqual(strip_prefix_for_ro("ro.foo",  False), "foo")
1437         self.failUnlessEqual(strip_prefix_for_ro("imm.foo", False), "imm.foo")
1438         self.failUnlessEqual(strip_prefix_for_ro("foo",     True),  "foo")
1439         self.failUnlessEqual(strip_prefix_for_ro("ro.foo",  True),  "foo")
1440         self.failUnlessEqual(strip_prefix_for_ro("imm.foo", True),  "foo")
1441
1442     def test_unknownnode(self):
1443         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
1444         mut_read_uri = "URI:SSK-RO:jf6wkflosyvntwxqcdo7a54jvm:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
1445         lit_uri = "URI:LIT:n5xgk"
1446
1447         # This does not attempt to be exhaustive.
1448         no_no        = [# Opaque node, but not an error.
1449                         ( 0, UnknownNode(None, None)),
1450                         ( 1, UnknownNode(None, None, deep_immutable=True)),
1451                        ]
1452         unknown_rw   = [# These are errors because we're only given a rw_uri, and we can't
1453                         # diminish it.
1454                         ( 2, UnknownNode("foo", None)),
1455                         ( 3, UnknownNode("foo", None, deep_immutable=True)),
1456                         ( 4, UnknownNode("ro.foo", None, deep_immutable=True)),
1457                         ( 5, UnknownNode("ro." + mut_read_uri, None, deep_immutable=True)),
1458                         ( 6, UnknownNode("URI:SSK-RO:foo", None, deep_immutable=True)),
1459                         ( 7, UnknownNode("URI:SSK:foo", None)),
1460                        ]
1461         must_be_ro   = [# These are errors because a readonly constraint is not met.
1462                         ( 8, UnknownNode("ro." + mut_write_uri, None)),
1463                         ( 9, UnknownNode(None, "ro." + mut_write_uri)),
1464                        ]
1465         must_be_imm  = [# These are errors because an immutable constraint is not met.
1466                         (10, UnknownNode(None, "ro.URI:SSK-RO:foo", deep_immutable=True)),
1467                         (11, UnknownNode(None, "imm.URI:SSK:foo")),
1468                         (12, UnknownNode(None, "imm.URI:SSK-RO:foo")),
1469                         (13, UnknownNode("bar", "ro.foo", deep_immutable=True)),
1470                         (14, UnknownNode("bar", "imm.foo", deep_immutable=True)),
1471                         (15, UnknownNode("bar", "imm." + lit_uri, deep_immutable=True)),
1472                         (16, UnknownNode("imm." + mut_write_uri, None)),
1473                         (17, UnknownNode("imm." + mut_read_uri, None)),
1474                         (18, UnknownNode("bar", "imm.foo")),
1475                        ]
1476         bad_uri      = [# These are errors because the URI is bad once we've stripped the prefix.
1477                         (19, UnknownNode("ro.URI:SSK-RO:foo", None)),
1478                         (20, UnknownNode("imm.URI:CHK:foo", None, deep_immutable=True)),
1479                         (21, UnknownNode(None, "URI:CHK:foo")),
1480                         (22, UnknownNode(None, "URI:CHK:foo", deep_immutable=True)),
1481                        ]
1482         ro_prefixed  = [# These are valid, and the readcap should end up with a ro. prefix.
1483                         (23, UnknownNode(None, "foo")),
1484                         (24, UnknownNode(None, "ro.foo")),
1485                         (25, UnknownNode(None, "ro." + lit_uri)),
1486                         (26, UnknownNode("bar", "foo")),
1487                         (27, UnknownNode("bar", "ro.foo")),
1488                         (28, UnknownNode("bar", "ro." + lit_uri)),
1489                         (29, UnknownNode("ro.foo", None)),
1490                         (30, UnknownNode("ro." + lit_uri, None)),
1491                        ]
1492         imm_prefixed = [# These are valid, and the readcap should end up with an imm. prefix.
1493                         (31, UnknownNode(None, "foo", deep_immutable=True)),
1494                         (32, UnknownNode(None, "ro.foo", deep_immutable=True)),
1495                         (33, UnknownNode(None, "imm.foo")),
1496                         (34, UnknownNode(None, "imm.foo", deep_immutable=True)),
1497                         (35, UnknownNode("imm." + lit_uri, None)),
1498                         (36, UnknownNode("imm." + lit_uri, None, deep_immutable=True)),
1499                         (37, UnknownNode(None, "imm." + lit_uri)),
1500                         (38, UnknownNode(None, "imm." + lit_uri, deep_immutable=True)),
1501                        ]
1502         error = unknown_rw + must_be_ro + must_be_imm + bad_uri
1503         ok = ro_prefixed + imm_prefixed
1504
1505         for (i, n) in no_no + error + ok:
1506             self.failUnless(n.is_unknown(), i)
1507
1508         for (i, n) in no_no + error:
1509             self.failUnless(n.get_uri() is None, i)
1510             self.failUnless(n.get_write_uri() is None, i)
1511             self.failUnless(n.get_readonly_uri() is None, i)
1512
1513         for (i, n) in no_no + ok:
1514             n.raise_error()
1515
1516         for (i, n) in unknown_rw:
1517             self.failUnlessRaises(MustNotBeUnknownRWError, lambda: n.raise_error())
1518
1519         for (i, n) in must_be_ro:
1520             self.failUnlessRaises(MustBeReadonlyError, lambda: n.raise_error())
1521
1522         for (i, n) in must_be_imm:
1523             self.failUnlessRaises(MustBeDeepImmutableError, lambda: n.raise_error())
1524
1525         for (i, n) in bad_uri:
1526             self.failUnlessRaises(uri.BadURIError, lambda: n.raise_error())
1527
1528         for (i, n) in ok:
1529             self.failIf(n.get_readonly_uri() is None, i)
1530
1531         for (i, n) in ro_prefixed:
1532             self.failUnless(n.get_readonly_uri().startswith("ro."), i)
1533
1534         for (i, n) in imm_prefixed:
1535             self.failUnless(n.get_readonly_uri().startswith("imm."), i)
1536
1537
1538 class DeepStats(unittest.TestCase):
1539     timeout = 240 # It takes longer than 120 seconds on Francois's arm box.
1540     def test_stats(self):
1541         ds = dirnode.DeepStats(None)
1542         ds.add("count-files")
1543         ds.add("size-immutable-files", 123)
1544         ds.histogram("size-files-histogram", 123)
1545         ds.max("largest-directory", 444)
1546
1547         s = ds.get_results()
1548         self.failUnlessEqual(s["count-files"], 1)
1549         self.failUnlessEqual(s["size-immutable-files"], 123)
1550         self.failUnlessEqual(s["largest-directory"], 444)
1551         self.failUnlessEqual(s["count-literal-files"], 0)
1552
1553         ds.add("count-files")
1554         ds.add("size-immutable-files", 321)
1555         ds.histogram("size-files-histogram", 321)
1556         ds.max("largest-directory", 2)
1557
1558         s = ds.get_results()
1559         self.failUnlessEqual(s["count-files"], 2)
1560         self.failUnlessEqual(s["size-immutable-files"], 444)
1561         self.failUnlessEqual(s["largest-directory"], 444)
1562         self.failUnlessEqual(s["count-literal-files"], 0)
1563         self.failUnlessEqual(s["size-files-histogram"],
1564                              [ (101, 316, 1), (317, 1000, 1) ])
1565
1566         ds = dirnode.DeepStats(None)
1567         for i in range(1, 1100):
1568             ds.histogram("size-files-histogram", i)
1569         ds.histogram("size-files-histogram", 4*1000*1000*1000*1000) # 4TB
1570         s = ds.get_results()
1571         self.failUnlessEqual(s["size-files-histogram"],
1572                              [ (1, 3, 3),
1573                                (4, 10, 7),
1574                                (11, 31, 21),
1575                                (32, 100, 69),
1576                                (101, 316, 216),
1577                                (317, 1000, 684),
1578                                (1001, 3162, 99),
1579                                (3162277660169L, 10000000000000L, 1),
1580                                ])
1581
1582 class UCWEingMutableFileNode(MutableFileNode):
1583     please_ucwe_after_next_upload = False
1584
1585     def _upload(self, new_contents, servermap):
1586         d = MutableFileNode._upload(self, new_contents, servermap)
1587         def _ucwe(res):
1588             if self.please_ucwe_after_next_upload:
1589                 self.please_ucwe_after_next_upload = False
1590                 raise UncoordinatedWriteError()
1591             return res
1592         d.addCallback(_ucwe)
1593         return d
1594
1595 class UCWEingNodeMaker(NodeMaker):
1596     def _create_mutable(self, cap):
1597         n = UCWEingMutableFileNode(self.storage_broker, self.secret_holder,
1598                                    self.default_encoding_parameters,
1599                                    self.history)
1600         return n.init_from_cap(cap)
1601
1602
1603 class Deleter(GridTestMixin, unittest.TestCase):
1604     timeout = 3600 # It takes longer than 433 seconds on Zandr's ARM box.
1605     def test_retry(self):
1606         # ticket #550, a dirnode.delete which experiences an
1607         # UncoordinatedWriteError will fail with an incorrect "you're
1608         # deleting something which isn't there" NoSuchChildError exception.
1609
1610         # to trigger this, we start by creating a directory with a single
1611         # file in it. Then we create a special dirnode that uses a modified
1612         # MutableFileNode which will raise UncoordinatedWriteError once on
1613         # demand. We then call dirnode.delete, which ought to retry and
1614         # succeed.
1615
1616         self.basedir = self.mktemp()
1617         self.set_up_grid()
1618         c0 = self.g.clients[0]
1619         d = c0.create_dirnode()
1620         small = upload.Data("Small enough for a LIT", None)
1621         def _created_dir(dn):
1622             self.root = dn
1623             self.root_uri = dn.get_uri()
1624             return dn.add_file(u"file", small)
1625         d.addCallback(_created_dir)
1626         def _do_delete(ignored):
1627             nm = UCWEingNodeMaker(c0.storage_broker, c0._secret_holder,
1628                                   c0.get_history(), c0.getServiceNamed("uploader"),
1629                                   c0.downloader,
1630                                   c0.download_cache_dirman,
1631                                   c0.get_encoding_parameters(),
1632                                   c0._key_generator)
1633             n = nm.create_from_cap(self.root_uri)
1634             assert n._node.please_ucwe_after_next_upload == False
1635             n._node.please_ucwe_after_next_upload = True
1636             # This should succeed, not raise an exception
1637             return n.delete(u"file")
1638         d.addCallback(_do_delete)
1639
1640         return d
1641
1642 class Adder(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin):
1643
1644     def test_overwrite(self):
1645         # note: This functionality could be tested without actually creating
1646         # several RSA keys. It would be faster without the GridTestMixin: use
1647         # dn.set_node(nodemaker.create_from_cap(make_chk_file_uri())) instead
1648         # of dn.add_file, and use a special NodeMaker that creates fake
1649         # mutable files.
1650         self.basedir = "dirnode/Adder/test_overwrite"
1651         self.set_up_grid()
1652         c = self.g.clients[0]
1653         fileuri = make_chk_file_uri(1234)
1654         filenode = c.nodemaker.create_from_cap(fileuri)
1655         d = c.create_dirnode()
1656
1657         def _create_directory_tree(root_node):
1658             # Build
1659             # root/file1
1660             # root/file2
1661             # root/dir1
1662             d = root_node.add_file(u'file1', upload.Data("Important Things",
1663                 None))
1664             d.addCallback(lambda res:
1665                 root_node.add_file(u'file2', upload.Data("Sekrit Codes", None)))
1666             d.addCallback(lambda res:
1667                 root_node.create_subdirectory(u"dir1"))
1668             d.addCallback(lambda res: root_node)
1669             return d
1670
1671         d.addCallback(_create_directory_tree)
1672
1673         def _test_adder(root_node):
1674             d = root_node.set_node(u'file1', filenode)
1675             # We've overwritten file1. Let's try it with a directory
1676             d.addCallback(lambda res:
1677                 root_node.create_subdirectory(u'dir2'))
1678             d.addCallback(lambda res:
1679                 root_node.set_node(u'dir2', filenode))
1680             # We try overwriting a file with a child while also specifying
1681             # overwrite=False. We should receive an ExistingChildError
1682             # when we do this.
1683             d.addCallback(lambda res:
1684                 self.shouldFail(ExistingChildError, "set_node",
1685                                 "child 'file1' already exists",
1686                                 root_node.set_node, u"file1",
1687                                 filenode, overwrite=False))
1688             # If we try with a directory, we should see the same thing
1689             d.addCallback(lambda res:
1690                 self.shouldFail(ExistingChildError, "set_node",
1691                                 "child 'dir1' already exists",
1692                                 root_node.set_node, u'dir1', filenode,
1693                                 overwrite=False))
1694             d.addCallback(lambda res:
1695                 root_node.set_node(u'file1', filenode,
1696                                    overwrite="only-files"))
1697             d.addCallback(lambda res:
1698                 self.shouldFail(ExistingChildError, "set_node",
1699                                 "child 'dir1' already exists",
1700                                 root_node.set_node, u'dir1', filenode,
1701                                 overwrite="only-files"))
1702             return d
1703
1704         d.addCallback(_test_adder)
1705         return d