src/allmydata/test/test_dirnode.py

   1
   2 import time
   3 import unicodedata
   4 from zope.interface import implements
   5 from twisted.trial import unittest
   6 from twisted.internet import defer
   7 from twisted.internet.interfaces import IConsumer
   8 from allmydata import uri, dirnode
   9 from allmydata.client import Client
  10 from allmydata.immutable import upload
  11 from allmydata.interfaces import IImmutableFileNode, IMutableFileNode, \
  12      ExistingChildError, NoSuchChildError, MustNotBeUnknownRWError, \
  13      MustBeDeepImmutableError, MustBeReadonlyError, \
  14      IDeepCheckResults, IDeepCheckAndRepairResults
  15 from allmydata.mutable.filenode import MutableFileNode
  16 from allmydata.mutable.common import UncoordinatedWriteError
  17 from allmydata.util import hashutil, base32
  18 from allmydata.util.netstring import split_netstring
  19 from allmydata.monitor import Monitor
  20 from allmydata.test.common import make_chk_file_uri, make_mutable_file_uri, \
  21      ErrorMixin
  22 from allmydata.test.no_network import GridTestMixin
  23 from allmydata.unknown import UnknownNode, strip_prefix_for_ro
  24 from allmydata.nodemaker import NodeMaker
  25 from base64 import b32decode
  26 import allmydata.test.common_util as testutil
  27
  28 class MemAccum:
  29     implements(IConsumer)
  30     def registerProducer(self, producer, streaming):
  31         self.producer = producer
  32         self.producer.resumeProducing()
  33         pass
  34     def unregisterProducer(self):
  35         pass
  36     def write(self, data):
  37         assert not hasattr(self, 'data')
  38         self.data = data
  39         self.producer.resumeProducing()
  40
  41 setup_py_uri = "URI:CHK:n7r3m6wmomelk4sep3kw5cvduq:os7ijw5c3maek7pg65e5254k2fzjflavtpejjyhshpsxuqzhcwwq:3:20:14861"
  42 one_uri = "URI:LIT:n5xgk" # LIT for "one"
  43 mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
  44 empty_litdir_uri = "URI:DIR2-LIT:"
  45 tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
  46 mut_read_uri = "URI:SSK-RO:jf6wkflosyvntwxqcdo7a54jvm:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
  47 future_write_uri = "x-tahoe-crazy://I_am_from_the_future."
  48 future_read_uri = "x-tahoe-crazy-readonly://I_am_from_the_future."
  49 future_nonascii_write_uri = u"x-tahoe-even-more-crazy://I_am_from_the_future_rw_\u263A".encode('utf-8')
  50 future_nonascii_read_uri = u"x-tahoe-even-more-crazy-readonly://I_am_from_the_future_ro_\u263A".encode('utf-8')
  51
  52 # 'o' 'n' 'e-macron'
  53 one_nfc = u"on\u0113"
  54 one_nfd = u"one\u0304"
  55
  56 class Dirnode(GridTestMixin, unittest.TestCase,
  57               testutil.ReallyEqualMixin, testutil.ShouldFailMixin, testutil.StallMixin, ErrorMixin):
  58     timeout = 240 # It takes longer than 120 seconds on Francois's arm box.
  59
  60     def test_basic(self):
  61         self.basedir = "dirnode/Dirnode/test_basic"
  62         self.set_up_grid()
  63         c = self.g.clients[0]
  64         d = c.create_dirnode()
  65         def _done(res):
  66             self.failUnless(isinstance(res, dirnode.DirectoryNode))
  67             self.failUnless(res.is_mutable())
  68             self.failIf(res.is_readonly())
  69             self.failIf(res.is_unknown())
  70             self.failIf(res.is_allowed_in_immutable_directory())
  71             res.raise_error()
  72             rep = str(res)
  73             self.failUnless("RW-MUT" in rep)
  74         d.addCallback(_done)
  75         return d
  76
  77     def test_initial_children(self):
  78         self.basedir = "dirnode/Dirnode/test_initial_children"
  79         self.set_up_grid()
  80         c = self.g.clients[0]
  81         nm = c.nodemaker
  82
  83         kids = {one_nfd: (nm.create_from_cap(one_uri), {}),
  84                 u"two": (nm.create_from_cap(setup_py_uri),
  85                          {"metakey": "metavalue"}),
  86                 u"mut": (nm.create_from_cap(mut_write_uri, mut_read_uri), {}),
  87                 u"fut": (nm.create_from_cap(future_write_uri, future_read_uri), {}),
  88                 u"fro": (nm.create_from_cap(None, future_read_uri), {}),
  89                 u"fut-unic": (nm.create_from_cap(future_nonascii_write_uri, future_nonascii_read_uri), {}),
  90                 u"fro-unic": (nm.create_from_cap(None, future_nonascii_read_uri), {}),
  91                 u"empty_litdir": (nm.create_from_cap(empty_litdir_uri), {}),
  92                 u"tiny_litdir": (nm.create_from_cap(tiny_litdir_uri), {}),
  93                 }
  94         d = c.create_dirnode(kids)
  95
  96         def _created(dn):
  97             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
  98             self.failUnless(dn.is_mutable())
  99             self.failIf(dn.is_readonly())
 100             self.failIf(dn.is_unknown())
 101             self.failIf(dn.is_allowed_in_immutable_directory())
 102             dn.raise_error()
 103             rep = str(dn)
 104             self.failUnless("RW-MUT" in rep)
 105             return dn.list()
 106         d.addCallback(_created)
 107
 108         def _check_kids(children):
 109             self.failUnlessReallyEqual(set(children.keys()),
 110                                        set([one_nfc, u"two", u"mut", u"fut", u"fro",
 111                                             u"fut-unic", u"fro-unic", u"empty_litdir", u"tiny_litdir"]))
 112             one_node, one_metadata = children[one_nfc]
 113             two_node, two_metadata = children[u"two"]
 114             mut_node, mut_metadata = children[u"mut"]
 115             fut_node, fut_metadata = children[u"fut"]
 116             fro_node, fro_metadata = children[u"fro"]
 117             futna_node, futna_metadata = children[u"fut-unic"]
 118             frona_node, frona_metadata = children[u"fro-unic"]
 119             emptylit_node, emptylit_metadata = children[u"empty_litdir"]
 120             tinylit_node, tinylit_metadata = children[u"tiny_litdir"]
 121
 122             self.failUnlessReallyEqual(one_node.get_size(), 3)
 123             self.failUnlessReallyEqual(one_node.get_uri(), one_uri)
 124             self.failUnlessReallyEqual(one_node.get_readonly_uri(), one_uri)
 125             self.failUnless(isinstance(one_metadata, dict), one_metadata)
 126
 127             self.failUnlessReallyEqual(two_node.get_size(), 14861)
 128             self.failUnlessReallyEqual(two_node.get_uri(), setup_py_uri)
 129             self.failUnlessReallyEqual(two_node.get_readonly_uri(), setup_py_uri)
 130             self.failUnlessEqual(two_metadata["metakey"], "metavalue")
 131
 132             self.failUnlessReallyEqual(mut_node.get_uri(), mut_write_uri)
 133             self.failUnlessReallyEqual(mut_node.get_readonly_uri(), mut_read_uri)
 134             self.failUnless(isinstance(mut_metadata, dict), mut_metadata)
 135
 136             self.failUnless(fut_node.is_unknown())
 137             self.failUnlessReallyEqual(fut_node.get_uri(), future_write_uri)
 138             self.failUnlessReallyEqual(fut_node.get_readonly_uri(), "ro." + future_read_uri)
 139             self.failUnless(isinstance(fut_metadata, dict), fut_metadata)
 140
 141             self.failUnless(futna_node.is_unknown())
 142             self.failUnlessReallyEqual(futna_node.get_uri(), future_nonascii_write_uri)
 143             self.failUnlessReallyEqual(futna_node.get_readonly_uri(), "ro." + future_nonascii_read_uri)
 144             self.failUnless(isinstance(futna_metadata, dict), futna_metadata)
 145
 146             self.failUnless(fro_node.is_unknown())
 147             self.failUnlessReallyEqual(fro_node.get_uri(), "ro." + future_read_uri)
 148             self.failUnlessReallyEqual(fut_node.get_readonly_uri(), "ro." + future_read_uri)
 149             self.failUnless(isinstance(fro_metadata, dict), fro_metadata)
 150
 151             self.failUnless(frona_node.is_unknown())
 152             self.failUnlessReallyEqual(frona_node.get_uri(), "ro." + future_nonascii_read_uri)
 153             self.failUnlessReallyEqual(futna_node.get_readonly_uri(), "ro." + future_nonascii_read_uri)
 154             self.failUnless(isinstance(frona_metadata, dict), frona_metadata)
 155
 156             self.failIf(emptylit_node.is_unknown())
 157             self.failUnlessReallyEqual(emptylit_node.get_storage_index(), None)
 158             self.failIf(tinylit_node.is_unknown())
 159             self.failUnlessReallyEqual(tinylit_node.get_storage_index(), None)
 160
 161             d2 = defer.succeed(None)
 162             d2.addCallback(lambda ignored: emptylit_node.list())
 163             d2.addCallback(lambda children: self.failUnlessEqual(children, {}))
 164             d2.addCallback(lambda ignored: tinylit_node.list())
 165             d2.addCallback(lambda children: self.failUnlessReallyEqual(set(children.keys()),
 166                                                                        set([u"short"])))
 167             d2.addCallback(lambda ignored: tinylit_node.list())
 168             d2.addCallback(lambda children: children[u"short"][0].read(MemAccum()))
 169             d2.addCallback(lambda accum: self.failUnlessReallyEqual(accum.data, "The end."))
 170             return d2
 171
 172         d.addCallback(_check_kids)
 173
 174         d.addCallback(lambda ign: nm.create_new_mutable_directory(kids))
 175         d.addCallback(lambda dn: dn.list())
 176         d.addCallback(_check_kids)
 177
 178         bad_future_node = UnknownNode(future_write_uri, None)
 179         bad_kids1 = {one_nfd: (bad_future_node, {})}
 180         # This should fail because we don't know how to diminish the future_write_uri
 181         # cap (given in a write slot and not prefixed with "ro." or "imm.") to a readcap.
 182         d.addCallback(lambda ign:
 183                       self.shouldFail(MustNotBeUnknownRWError, "bad_kids1",
 184                                       "cannot attach unknown",
 185                                       nm.create_new_mutable_directory,
 186                                       bad_kids1))
 187         bad_kids2 = {one_nfd: (nm.create_from_cap(one_uri), None)}
 188         d.addCallback(lambda ign:
 189                       self.shouldFail(AssertionError, "bad_kids2",
 190                                       "requires metadata to be a dict",
 191                                       nm.create_new_mutable_directory,
 192                                       bad_kids2))
 193         return d
 194
 195     def test_immutable(self):
 196         self.basedir = "dirnode/Dirnode/test_immutable"
 197         self.set_up_grid()
 198         c = self.g.clients[0]
 199         nm = c.nodemaker
 200
 201         kids = {one_nfd: (nm.create_from_cap(one_uri), {}),
 202                 u"two": (nm.create_from_cap(setup_py_uri),
 203                          {"metakey": "metavalue"}),
 204                 u"fut": (nm.create_from_cap(None, future_read_uri), {}),
 205                 u"futna": (nm.create_from_cap(None, future_nonascii_read_uri), {}),
 206                 u"empty_litdir": (nm.create_from_cap(empty_litdir_uri), {}),
 207                 u"tiny_litdir": (nm.create_from_cap(tiny_litdir_uri), {}),
 208                 }
 209         d = c.create_immutable_dirnode(kids)
 210
 211         def _created(dn):
 212             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
 213             self.failIf(dn.is_mutable())
 214             self.failUnless(dn.is_readonly())
 215             self.failIf(dn.is_unknown())
 216             self.failUnless(dn.is_allowed_in_immutable_directory())
 217             dn.raise_error()
 218             rep = str(dn)
 219             self.failUnless("RO-IMM" in rep)
 220             cap = dn.get_cap()
 221             self.failUnlessIn("CHK", cap.to_string())
 222             self.cap = cap
 223             return dn.list()
 224         d.addCallback(_created)
 225
 226         def _check_kids(children):
 227             self.failUnlessReallyEqual(set(children.keys()),
 228                                        set([one_nfc, u"two", u"fut", u"futna", u"empty_litdir", u"tiny_litdir"]))
 229             one_node, one_metadata = children[one_nfc]
 230             two_node, two_metadata = children[u"two"]
 231             fut_node, fut_metadata = children[u"fut"]
 232             futna_node, futna_metadata = children[u"futna"]
 233             emptylit_node, emptylit_metadata = children[u"empty_litdir"]
 234             tinylit_node, tinylit_metadata = children[u"tiny_litdir"]
 235
 236             self.failUnlessReallyEqual(one_node.get_size(), 3)
 237             self.failUnlessReallyEqual(one_node.get_uri(), one_uri)
 238             self.failUnlessReallyEqual(one_node.get_readonly_uri(), one_uri)
 239             self.failUnless(isinstance(one_metadata, dict), one_metadata)
 240
 241             self.failUnlessReallyEqual(two_node.get_size(), 14861)
 242             self.failUnlessReallyEqual(two_node.get_uri(), setup_py_uri)
 243             self.failUnlessReallyEqual(two_node.get_readonly_uri(), setup_py_uri)
 244             self.failUnlessEqual(two_metadata["metakey"], "metavalue")
 245
 246             self.failUnless(fut_node.is_unknown())
 247             self.failUnlessReallyEqual(fut_node.get_uri(), "imm." + future_read_uri)
 248             self.failUnlessReallyEqual(fut_node.get_readonly_uri(), "imm." + future_read_uri)
 249             self.failUnless(isinstance(fut_metadata, dict), fut_metadata)
 250
 251             self.failUnless(futna_node.is_unknown())
 252             self.failUnlessReallyEqual(futna_node.get_uri(), "imm." + future_nonascii_read_uri)
 253             self.failUnlessReallyEqual(futna_node.get_readonly_uri(), "imm." + future_nonascii_read_uri)
 254             self.failUnless(isinstance(futna_metadata, dict), futna_metadata)
 255
 256             self.failIf(emptylit_node.is_unknown())
 257             self.failUnlessReallyEqual(emptylit_node.get_storage_index(), None)
 258             self.failIf(tinylit_node.is_unknown())
 259             self.failUnlessReallyEqual(tinylit_node.get_storage_index(), None)
 260
 261             d2 = defer.succeed(None)
 262             d2.addCallback(lambda ignored: emptylit_node.list())
 263             d2.addCallback(lambda children: self.failUnlessEqual(children, {}))
 264             d2.addCallback(lambda ignored: tinylit_node.list())
 265             d2.addCallback(lambda children: self.failUnlessReallyEqual(set(children.keys()),
 266                                                                        set([u"short"])))
 267             d2.addCallback(lambda ignored: tinylit_node.list())
 268             d2.addCallback(lambda children: children[u"short"][0].read(MemAccum()))
 269             d2.addCallback(lambda accum: self.failUnlessReallyEqual(accum.data, "The end."))
 270             return d2
 271
 272         d.addCallback(_check_kids)
 273
 274         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
 275         d.addCallback(lambda dn: dn.list())
 276         d.addCallback(_check_kids)
 277
 278         bad_future_node1 = UnknownNode(future_write_uri, None)
 279         bad_kids1 = {one_nfd: (bad_future_node1, {})}
 280         d.addCallback(lambda ign:
 281                       self.shouldFail(MustNotBeUnknownRWError, "bad_kids1",
 282                                       "cannot attach unknown",
 283                                       c.create_immutable_dirnode,
 284                                       bad_kids1))
 285         bad_future_node2 = UnknownNode(future_write_uri, future_read_uri)
 286         bad_kids2 = {one_nfd: (bad_future_node2, {})}
 287         d.addCallback(lambda ign:
 288                       self.shouldFail(MustBeDeepImmutableError, "bad_kids2",
 289                                       "is not allowed in an immutable directory",
 290                                       c.create_immutable_dirnode,
 291                                       bad_kids2))
 292         bad_kids3 = {one_nfd: (nm.create_from_cap(one_uri), None)}
 293         d.addCallback(lambda ign:
 294                       self.shouldFail(AssertionError, "bad_kids3",
 295                                       "requires metadata to be a dict",
 296                                       c.create_immutable_dirnode,
 297                                       bad_kids3))
 298         bad_kids4 = {one_nfd: (nm.create_from_cap(mut_write_uri), {})}
 299         d.addCallback(lambda ign:
 300                       self.shouldFail(MustBeDeepImmutableError, "bad_kids4",
 301                                       "is not allowed in an immutable directory",
 302                                       c.create_immutable_dirnode,
 303                                       bad_kids4))
 304         bad_kids5 = {one_nfd: (nm.create_from_cap(mut_read_uri), {})}
 305         d.addCallback(lambda ign:
 306                       self.shouldFail(MustBeDeepImmutableError, "bad_kids5",
 307                                       "is not allowed in an immutable directory",
 308                                       c.create_immutable_dirnode,
 309                                       bad_kids5))
 310         d.addCallback(lambda ign: c.create_immutable_dirnode({}))
 311         def _created_empty(dn):
 312             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
 313             self.failIf(dn.is_mutable())
 314             self.failUnless(dn.is_readonly())
 315             self.failIf(dn.is_unknown())
 316             self.failUnless(dn.is_allowed_in_immutable_directory())
 317             dn.raise_error()
 318             rep = str(dn)
 319             self.failUnless("RO-IMM" in rep)
 320             cap = dn.get_cap()
 321             self.failUnlessIn("LIT", cap.to_string())
 322             self.failUnlessReallyEqual(cap.to_string(), "URI:DIR2-LIT:")
 323             self.cap = cap
 324             return dn.list()
 325         d.addCallback(_created_empty)
 326         d.addCallback(lambda kids: self.failUnlessEqual(kids, {}))
 327         smallkids = {u"o": (nm.create_from_cap(one_uri), {})}
 328         d.addCallback(lambda ign: c.create_immutable_dirnode(smallkids))
 329         def _created_small(dn):
 330             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
 331             self.failIf(dn.is_mutable())
 332             self.failUnless(dn.is_readonly())
 333             self.failIf(dn.is_unknown())
 334             self.failUnless(dn.is_allowed_in_immutable_directory())
 335             dn.raise_error()
 336             rep = str(dn)
 337             self.failUnless("RO-IMM" in rep)
 338             cap = dn.get_cap()
 339             self.failUnlessIn("LIT", cap.to_string())
 340             self.failUnlessReallyEqual(cap.to_string(),
 341                                        "URI:DIR2-LIT:gi4tumj2n4wdcmz2kvjesosmjfkdu3rvpbtwwlbqhiwdeot3puwcy")
 342             self.cap = cap
 343             return dn.list()
 344         d.addCallback(_created_small)
 345         d.addCallback(lambda kids: self.failUnlessReallyEqual(kids.keys(), [u"o"]))
 346
 347         # now test n.create_subdirectory(mutable=False)
 348         d.addCallback(lambda ign: c.create_dirnode())
 349         def _made_parent(n):
 350             d = n.create_subdirectory(u"subdir", kids, mutable=False)
 351             d.addCallback(lambda sd: sd.list())
 352             d.addCallback(_check_kids)
 353             d.addCallback(lambda ign: n.list())
 354             d.addCallback(lambda children:
 355                           self.failUnlessReallyEqual(children.keys(), [u"subdir"]))
 356             d.addCallback(lambda ign: n.get(u"subdir"))
 357             d.addCallback(lambda sd: sd.list())
 358             d.addCallback(_check_kids)
 359             d.addCallback(lambda ign: n.get(u"subdir"))
 360             d.addCallback(lambda sd: self.failIf(sd.is_mutable()))
 361             bad_kids = {one_nfd: (nm.create_from_cap(mut_write_uri), {})}
 362             d.addCallback(lambda ign:
 363                           self.shouldFail(MustBeDeepImmutableError, "YZ",
 364                                           "is not allowed in an immutable directory",
 365                                           n.create_subdirectory,
 366                                           u"sub2", bad_kids, mutable=False))
 367             return d
 368         d.addCallback(_made_parent)
 369         return d
 370
 371     def test_directory_representation(self):
 372         self.basedir = "dirnode/Dirnode/test_directory_representation"
 373         self.set_up_grid()
 374         c = self.g.clients[0]
 375         nm = c.nodemaker
 376
 377         # This test checks that any trailing spaces in URIs are retained in the
 378         # encoded directory, but stripped when we get them out of the directory.
 379         # See ticket #925 for why we want that.
 380         # It also tests that we store child names as UTF-8 NFC, and normalize
 381         # them again when retrieving them.
 382
 383         stripped_write_uri = "lafs://from_the_future\t"
 384         stripped_read_uri = "lafs://readonly_from_the_future\t"
 385         spacedout_write_uri = stripped_write_uri + "  "
 386         spacedout_read_uri = stripped_read_uri + "  "
 387
 388         child = nm.create_from_cap(spacedout_write_uri, spacedout_read_uri)
 389         self.failUnlessReallyEqual(child.get_write_uri(), spacedout_write_uri)
 390         self.failUnlessReallyEqual(child.get_readonly_uri(), "ro." + spacedout_read_uri)
 391
 392         child_dottedi = u"ch\u0131\u0307ld"
 393
 394         kids_in   = {child_dottedi: (child, {}), one_nfd: (child, {})}
 395         kids_out  = {child_dottedi: (child, {}), one_nfc: (child, {})}
 396         kids_norm = {u"child":      (child, {}), one_nfc: (child, {})}
 397         d = c.create_dirnode(kids_in)
 398
 399         def _created(dn):
 400             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
 401             self.failUnless(dn.is_mutable())
 402             self.failIf(dn.is_readonly())
 403             dn.raise_error()
 404             self.cap = dn.get_cap()
 405             self.rootnode = dn
 406             return dn._node.download_best_version()
 407         d.addCallback(_created)
 408
 409         def _check_data(data):
 410             # Decode the netstring representation of the directory to check that the
 411             # spaces are retained when the URIs are stored, and that the names are stored
 412             # as NFC.
 413             position = 0
 414             numkids = 0
 415             while position < len(data):
 416                 entries, position = split_netstring(data, 1, position)
 417                 entry = entries[0]
 418                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
 419                 name = name_utf8.decode("utf-8")
 420                 rw_uri = self.rootnode._decrypt_rwcapdata(rwcapdata)
 421                 self.failUnlessIn(name, kids_out)
 422                 (expected_child, ign) = kids_out[name]
 423                 self.failUnlessReallyEqual(rw_uri, expected_child.get_write_uri())
 424                 self.failUnlessReallyEqual("ro." + ro_uri, expected_child.get_readonly_uri())
 425                 numkids += 1
 426
 427             self.failUnlessReallyEqual(numkids, len(kids_out))
 428             return self.rootnode
 429         d.addCallback(_check_data)
 430
 431         # Mock up a hypothetical future version of Unicode that adds a canonical equivalence
 432         # between dotless-i + dot-above, and 'i'. That would actually be prohibited by the
 433         # stability rules, but similar additions involving currently-unassigned characters
 434         # would not be.
 435         old_normalize = unicodedata.normalize
 436         def future_normalize(form, s):
 437             assert form == 'NFC', form
 438             return old_normalize(form, s).replace(u"\u0131\u0307", u"i")
 439
 440         def _list(node):
 441             unicodedata.normalize = future_normalize
 442             d2 = node.list()
 443             def _undo_mock(res):
 444                 unicodedata.normalize = old_normalize
 445                 return res
 446             d2.addBoth(_undo_mock)
 447             return d2
 448         d.addCallback(_list)
 449
 450         def _check_kids(children):
 451             # Now when we use the real directory listing code, the trailing spaces
 452             # should have been stripped (and "ro." should have been prepended to the
 453             # ro_uri, since it's unknown). Also the dotless-i + dot-above should have been
 454             # normalized to 'i'.
 455
 456             self.failUnlessReallyEqual(set(children.keys()), set(kids_norm.keys()))
 457             child_node, child_metadata = children[u"child"]
 458
 459             self.failUnlessReallyEqual(child_node.get_write_uri(), stripped_write_uri)
 460             self.failUnlessReallyEqual(child_node.get_readonly_uri(), "ro." + stripped_read_uri)
 461         d.addCallback(_check_kids)
 462
 463         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
 464         d.addCallback(_list)
 465         d.addCallback(_check_kids)  # again with dirnode recreated from cap
 466         return d
 467
 468     def test_check(self):
 469         self.basedir = "dirnode/Dirnode/test_check"
 470         self.set_up_grid()
 471         c = self.g.clients[0]
 472         d = c.create_dirnode()
 473         d.addCallback(lambda dn: dn.check(Monitor()))
 474         def _done(res):
 475             self.failUnless(res.is_healthy())
 476         d.addCallback(_done)
 477         return d
 478
 479     def _test_deepcheck_create(self):
 480         # create a small tree with a loop, and some non-directories
 481         #  root/
 482         #  root/subdir/
 483         #  root/subdir/file1
 484         #  root/subdir/link -> root
 485         #  root/rodir
 486         c = self.g.clients[0]
 487         d = c.create_dirnode()
 488         def _created_root(rootnode):
 489             self._rootnode = rootnode
 490             return rootnode.create_subdirectory(u"subdir")
 491         d.addCallback(_created_root)
 492         def _created_subdir(subdir):
 493             self._subdir = subdir
 494             d = subdir.add_file(u"file1", upload.Data("data"*100, None))
 495             d.addCallback(lambda res: subdir.set_node(u"link", self._rootnode))
 496             d.addCallback(lambda res: c.create_dirnode())
 497             d.addCallback(lambda dn:
 498                           self._rootnode.set_uri(u"rodir",
 499                                                  dn.get_uri(),
 500                                                  dn.get_readonly_uri()))
 501             return d
 502         d.addCallback(_created_subdir)
 503         def _done(res):
 504             return self._rootnode
 505         d.addCallback(_done)
 506         return d
 507
 508     def test_deepcheck(self):
 509         self.basedir = "dirnode/Dirnode/test_deepcheck"
 510         self.set_up_grid()
 511         d = self._test_deepcheck_create()
 512         d.addCallback(lambda rootnode: rootnode.start_deep_check().when_done())
 513         def _check_results(r):
 514             self.failUnless(IDeepCheckResults.providedBy(r))
 515             c = r.get_counters()
 516             self.failUnlessReallyEqual(c,
 517                                        {"count-objects-checked": 4,
 518                                         "count-objects-healthy": 4,
 519                                         "count-objects-unhealthy": 0,
 520                                         "count-objects-unrecoverable": 0,
 521                                         "count-corrupt-shares": 0,
 522                                         })
 523             self.failIf(r.get_corrupt_shares())
 524             self.failUnlessReallyEqual(len(r.get_all_results()), 4)
 525         d.addCallback(_check_results)
 526         return d
 527
 528     def test_deepcheck_and_repair(self):
 529         self.basedir = "dirnode/Dirnode/test_deepcheck_and_repair"
 530         self.set_up_grid()
 531         d = self._test_deepcheck_create()
 532         d.addCallback(lambda rootnode:
 533                       rootnode.start_deep_check_and_repair().when_done())
 534         def _check_results(r):
 535             self.failUnless(IDeepCheckAndRepairResults.providedBy(r))
 536             c = r.get_counters()
 537             self.failUnlessReallyEqual(c,
 538                                        {"count-objects-checked": 4,
 539                                         "count-objects-healthy-pre-repair": 4,
 540                                         "count-objects-unhealthy-pre-repair": 0,
 541                                         "count-objects-unrecoverable-pre-repair": 0,
 542                                         "count-corrupt-shares-pre-repair": 0,
 543                                         "count-objects-healthy-post-repair": 4,
 544                                         "count-objects-unhealthy-post-repair": 0,
 545                                         "count-objects-unrecoverable-post-repair": 0,
 546                                         "count-corrupt-shares-post-repair": 0,
 547                                         "count-repairs-attempted": 0,
 548                                         "count-repairs-successful": 0,
 549                                         "count-repairs-unsuccessful": 0,
 550                                         })
 551             self.failIf(r.get_corrupt_shares())
 552             self.failIf(r.get_remaining_corrupt_shares())
 553             self.failUnlessReallyEqual(len(r.get_all_results()), 4)
 554         d.addCallback(_check_results)
 555         return d
 556
 557     def _mark_file_bad(self, rootnode):
 558         self.delete_shares_numbered(rootnode.get_uri(), [0])
 559         return rootnode
 560
 561     def test_deepcheck_problems(self):
 562         self.basedir = "dirnode/Dirnode/test_deepcheck_problems"
 563         self.set_up_grid()
 564         d = self._test_deepcheck_create()
 565         d.addCallback(lambda rootnode: self._mark_file_bad(rootnode))
 566         d.addCallback(lambda rootnode: rootnode.start_deep_check().when_done())
 567         def _check_results(r):
 568             c = r.get_counters()
 569             self.failUnlessReallyEqual(c,
 570                                        {"count-objects-checked": 4,
 571                                         "count-objects-healthy": 3,
 572                                         "count-objects-unhealthy": 1,
 573                                         "count-objects-unrecoverable": 0,
 574                                         "count-corrupt-shares": 0,
 575                                         })
 576             #self.failUnlessReallyEqual(len(r.get_problems()), 1) # TODO
 577         d.addCallback(_check_results)
 578         return d
 579
 580     def test_readonly(self):
 581         self.basedir = "dirnode/Dirnode/test_readonly"
 582         self.set_up_grid()
 583         c = self.g.clients[0]
 584         nm = c.nodemaker
 585         filecap = make_chk_file_uri(1234)
 586         filenode = nm.create_from_cap(filecap)
 587         uploadable = upload.Data("some data", convergence="some convergence string")
 588
 589         d = c.create_dirnode()
 590         def _created(rw_dn):
 591             d2 = rw_dn.set_uri(u"child", filecap, filecap)
 592             d2.addCallback(lambda res: rw_dn)
 593             return d2
 594         d.addCallback(_created)
 595
 596         def _ready(rw_dn):
 597             ro_uri = rw_dn.get_readonly_uri()
 598             ro_dn = c.create_node_from_uri(ro_uri)
 599             self.failUnless(ro_dn.is_readonly())
 600             self.failUnless(ro_dn.is_mutable())
 601             self.failIf(ro_dn.is_unknown())
 602             self.failIf(ro_dn.is_allowed_in_immutable_directory())
 603             ro_dn.raise_error()
 604
 605             self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
 606                             ro_dn.set_uri, u"newchild", filecap, filecap)
 607             self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
 608                             ro_dn.set_node, u"newchild", filenode)
 609             self.shouldFail(dirnode.NotWriteableError, "set_nodes ro", None,
 610                             ro_dn.set_nodes, { u"newchild": (filenode, None) })
 611             self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
 612                             ro_dn.add_file, u"newchild", uploadable)
 613             self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
 614                             ro_dn.delete, u"child")
 615             self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
 616                             ro_dn.create_subdirectory, u"newchild")
 617             self.shouldFail(dirnode.NotWriteableError, "set_metadata_for ro", None,
 618                             ro_dn.set_metadata_for, u"child", {})
 619             self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
 620                             ro_dn.move_child_to, u"child", rw_dn)
 621             self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
 622                             rw_dn.move_child_to, u"child", ro_dn)
 623             return ro_dn.list()
 624         d.addCallback(_ready)
 625         def _listed(children):
 626             self.failUnless(u"child" in children)
 627         d.addCallback(_listed)
 628         return d
 629
 630     def failUnlessGreaterThan(self, a, b):
 631         self.failUnless(a > b, "%r should be > %r" % (a, b))
 632
 633     def failUnlessGreaterOrEqualThan(self, a, b):
 634         self.failUnless(a >= b, "%r should be >= %r" % (a, b))
 635
 636     def test_create(self):
 637         self.basedir = "dirnode/Dirnode/test_create"
 638         self.set_up_grid()
 639         c = self.g.clients[0]
 640
 641         self.expected_manifest = []
 642         self.expected_verifycaps = set()
 643         self.expected_storage_indexes = set()
 644
 645         d = c.create_dirnode()
 646         def _then(n):
 647             # /
 648             self.rootnode = n
 649             self.failUnless(n.is_mutable())
 650             u = n.get_uri()
 651             self.failUnless(u)
 652             self.failUnless(u.startswith("URI:DIR2:"), u)
 653             u_ro = n.get_readonly_uri()
 654             self.failUnless(u_ro.startswith("URI:DIR2-RO:"), u_ro)
 655             u_v = n.get_verify_cap().to_string()
 656             self.failUnless(u_v.startswith("URI:DIR2-Verifier:"), u_v)
 657             u_r = n.get_repair_cap().to_string()
 658             self.failUnlessReallyEqual(u_r, u)
 659             self.expected_manifest.append( ((), u) )
 660             self.expected_verifycaps.add(u_v)
 661             si = n.get_storage_index()
 662             self.expected_storage_indexes.add(base32.b2a(si))
 663             expected_si = n._uri.get_storage_index()
 664             self.failUnlessReallyEqual(si, expected_si)
 665
 666             d = n.list()
 667             d.addCallback(lambda res: self.failUnlessEqual(res, {}))
 668             d.addCallback(lambda res: n.has_child(u"missing"))
 669             d.addCallback(lambda res: self.failIf(res))
 670
 671             fake_file_uri = make_mutable_file_uri()
 672             other_file_uri = make_mutable_file_uri()
 673             m = c.nodemaker.create_from_cap(fake_file_uri)
 674             ffu_v = m.get_verify_cap().to_string()
 675             self.expected_manifest.append( ((u"child",) , m.get_uri()) )
 676             self.expected_verifycaps.add(ffu_v)
 677             self.expected_storage_indexes.add(base32.b2a(m.get_storage_index()))
 678             d.addCallback(lambda res: n.set_uri(u"child",
 679                                                 fake_file_uri, fake_file_uri))
 680             d.addCallback(lambda res:
 681                           self.shouldFail(ExistingChildError, "set_uri-no",
 682                                           "child 'child' already exists",
 683                                           n.set_uri, u"child",
 684                                           other_file_uri, other_file_uri,
 685                                           overwrite=False))
 686             # /
 687             # /child = mutable
 688
 689             d.addCallback(lambda res: n.create_subdirectory(u"subdir"))
 690
 691             # /
 692             # /child = mutable
 693             # /subdir = directory
 694             def _created(subdir):
 695                 self.failUnless(isinstance(subdir, dirnode.DirectoryNode))
 696                 self.subdir = subdir
 697                 new_v = subdir.get_verify_cap().to_string()
 698                 assert isinstance(new_v, str)
 699                 self.expected_manifest.append( ((u"subdir",), subdir.get_uri()) )
 700                 self.expected_verifycaps.add(new_v)
 701                 si = subdir.get_storage_index()
 702                 self.expected_storage_indexes.add(base32.b2a(si))
 703             d.addCallback(_created)
 704
 705             d.addCallback(lambda res:
 706                           self.shouldFail(ExistingChildError, "mkdir-no",
 707                                           "child 'subdir' already exists",
 708                                           n.create_subdirectory, u"subdir",
 709                                           overwrite=False))
 710
 711             d.addCallback(lambda res: n.list())
 712             d.addCallback(lambda children:
 713                           self.failUnlessReallyEqual(set(children.keys()),
 714                                                      set([u"child", u"subdir"])))
 715
 716             d.addCallback(lambda res: n.start_deep_stats().when_done())
 717             def _check_deepstats(stats):
 718                 self.failUnless(isinstance(stats, dict))
 719                 expected = {"count-immutable-files": 0,
 720                             "count-mutable-files": 1,
 721                             "count-literal-files": 0,
 722                             "count-files": 1,
 723                             "count-directories": 2,
 724                             "size-immutable-files": 0,
 725                             "size-literal-files": 0,
 726                             #"size-directories": 616, # varies
 727                             #"largest-directory": 616,
 728                             "largest-directory-children": 2,
 729                             "largest-immutable-file": 0,
 730                             }
 731                 for k,v in expected.iteritems():
 732                     self.failUnlessReallyEqual(stats[k], v,
 733                                                "stats[%s] was %s, not %s" %
 734                                                (k, stats[k], v))
 735                 self.failUnless(stats["size-directories"] > 500,
 736                                 stats["size-directories"])
 737                 self.failUnless(stats["largest-directory"] > 500,
 738                                 stats["largest-directory"])
 739                 self.failUnlessReallyEqual(stats["size-files-histogram"], [])
 740             d.addCallback(_check_deepstats)
 741
 742             d.addCallback(lambda res: n.build_manifest().when_done())
 743             def _check_manifest(res):
 744                 manifest = res["manifest"]
 745                 self.failUnlessReallyEqual(sorted(manifest),
 746                                            sorted(self.expected_manifest))
 747                 stats = res["stats"]
 748                 _check_deepstats(stats)
 749                 self.failUnlessReallyEqual(self.expected_verifycaps,
 750                                            res["verifycaps"])
 751                 self.failUnlessReallyEqual(self.expected_storage_indexes,
 752                                            res["storage-index"])
 753             d.addCallback(_check_manifest)
 754
 755             def _add_subsubdir(res):
 756                 return self.subdir.create_subdirectory(u"subsubdir")
 757             d.addCallback(_add_subsubdir)
 758             # /
 759             # /child = mutable
 760             # /subdir = directory
 761             # /subdir/subsubdir = directory
 762             d.addCallback(lambda res: n.get_child_at_path(u"subdir/subsubdir"))
 763             d.addCallback(lambda subsubdir:
 764                           self.failUnless(isinstance(subsubdir,
 765                                                      dirnode.DirectoryNode)))
 766             d.addCallback(lambda res: n.get_child_at_path(u""))
 767             d.addCallback(lambda res: self.failUnlessReallyEqual(res.get_uri(),
 768                                                                  n.get_uri()))
 769
 770             d.addCallback(lambda res: n.get_metadata_for(u"child"))
 771             d.addCallback(lambda metadata:
 772                           self.failUnlessEqual(set(metadata.keys()),
 773                                                set(["tahoe"])))
 774
 775             d.addCallback(lambda res:
 776                           self.shouldFail(NoSuchChildError, "gcamap-no",
 777                                           "nope",
 778                                           n.get_child_and_metadata_at_path,
 779                                           u"subdir/nope"))
 780             d.addCallback(lambda res:
 781                           n.get_child_and_metadata_at_path(u""))
 782             def _check_child_and_metadata1(res):
 783                 child, metadata = res
 784                 self.failUnless(isinstance(child, dirnode.DirectoryNode))
 785                 # edge-metadata needs at least one path segment
 786                 self.failUnlessEqual(set(metadata.keys()), set([]))
 787             d.addCallback(_check_child_and_metadata1)
 788             d.addCallback(lambda res:
 789                           n.get_child_and_metadata_at_path(u"child"))
 790
 791             def _check_child_and_metadata2(res):
 792                 child, metadata = res
 793                 self.failUnlessReallyEqual(child.get_uri(),
 794                                            fake_file_uri)
 795                 self.failUnlessEqual(set(metadata.keys()), set(["tahoe"]))
 796             d.addCallback(_check_child_and_metadata2)
 797
 798             d.addCallback(lambda res:
 799                           n.get_child_and_metadata_at_path(u"subdir/subsubdir"))
 800             def _check_child_and_metadata3(res):
 801                 child, metadata = res
 802                 self.failUnless(isinstance(child, dirnode.DirectoryNode))
 803                 self.failUnlessEqual(set(metadata.keys()), set(["tahoe"]))
 804             d.addCallback(_check_child_and_metadata3)
 805
 806             # set_uri + metadata
 807             # it should be possible to add a child without any metadata
 808             d.addCallback(lambda res: n.set_uri(u"c2",
 809                                                 fake_file_uri, fake_file_uri,
 810                                                 {}))
 811             d.addCallback(lambda res: n.get_metadata_for(u"c2"))
 812             d.addCallback(lambda metadata:
 813                           self.failUnlessEqual(set(metadata.keys()), set(["tahoe"])))
 814
 815             # You can't override the link timestamps.
 816             d.addCallback(lambda res: n.set_uri(u"c2",
 817                                                 fake_file_uri, fake_file_uri,
 818                                                 { 'tahoe': {'linkcrtime': "bogus"}}))
 819             d.addCallback(lambda res: n.get_metadata_for(u"c2"))
 820             def _has_good_linkcrtime(metadata):
 821                 self.failUnless(metadata.has_key('tahoe'))
 822                 self.failUnless(metadata['tahoe'].has_key('linkcrtime'))
 823                 self.failIfEqual(metadata['tahoe']['linkcrtime'], 'bogus')
 824             d.addCallback(_has_good_linkcrtime)
 825
 826             # if we don't set any defaults, the child should get timestamps
 827             d.addCallback(lambda res: n.set_uri(u"c3",
 828                                                 fake_file_uri, fake_file_uri))
 829             d.addCallback(lambda res: n.get_metadata_for(u"c3"))
 830             d.addCallback(lambda metadata:
 831                           self.failUnlessEqual(set(metadata.keys()), set(["tahoe"])))
 832
 833             # we can also add specific metadata at set_uri() time
 834             d.addCallback(lambda res: n.set_uri(u"c4",
 835                                                 fake_file_uri, fake_file_uri,
 836                                                 {"key": "value"}))
 837             d.addCallback(lambda res: n.get_metadata_for(u"c4"))
 838             d.addCallback(lambda metadata:
 839                               self.failUnless((set(metadata.keys()) == set(["key", "tahoe"])) and
 840                                               (metadata['key'] == "value"), metadata))
 841
 842             d.addCallback(lambda res: n.delete(u"c2"))
 843             d.addCallback(lambda res: n.delete(u"c3"))
 844             d.addCallback(lambda res: n.delete(u"c4"))
 845
 846             # set_node + metadata
 847             # it should be possible to add a child without any metadata except for timestamps
 848             d.addCallback(lambda res: n.set_node(u"d2", n, {}))
 849             d.addCallback(lambda res: c.create_dirnode())
 850             d.addCallback(lambda n2:
 851                           self.shouldFail(ExistingChildError, "set_node-no",
 852                                           "child 'd2' already exists",
 853                                           n.set_node, u"d2", n2,
 854                                           overwrite=False))
 855             d.addCallback(lambda res: n.get_metadata_for(u"d2"))
 856             d.addCallback(lambda metadata:
 857                           self.failUnlessEqual(set(metadata.keys()), set(["tahoe"])))
 858
 859             # if we don't set any defaults, the child should get timestamps
 860             d.addCallback(lambda res: n.set_node(u"d3", n))
 861             d.addCallback(lambda res: n.get_metadata_for(u"d3"))
 862             d.addCallback(lambda metadata:
 863                           self.failUnlessEqual(set(metadata.keys()), set(["tahoe"])))
 864
 865             # we can also add specific metadata at set_node() time
 866             d.addCallback(lambda res: n.set_node(u"d4", n,
 867                                                 {"key": "value"}))
 868             d.addCallback(lambda res: n.get_metadata_for(u"d4"))
 869             d.addCallback(lambda metadata:
 870                           self.failUnless((set(metadata.keys()) == set(["key", "tahoe"])) and
 871                                           (metadata["key"] == "value"), metadata))
 872
 873             d.addCallback(lambda res: n.delete(u"d2"))
 874             d.addCallback(lambda res: n.delete(u"d3"))
 875             d.addCallback(lambda res: n.delete(u"d4"))
 876
 877             # metadata through set_children()
 878             d.addCallback(lambda res:
 879                           n.set_children({
 880                               u"e1": (fake_file_uri, fake_file_uri),
 881                               u"e2": (fake_file_uri, fake_file_uri, {}),
 882                               u"e3": (fake_file_uri, fake_file_uri,
 883                                       {"key": "value"}),
 884                               }))
 885             d.addCallback(lambda n2: self.failUnlessIdentical(n2, n))
 886             d.addCallback(lambda res:
 887                           self.shouldFail(ExistingChildError, "set_children-no",
 888                                           "child 'e1' already exists",
 889                                           n.set_children,
 890                                           { u"e1": (other_file_uri,
 891                                                     other_file_uri),
 892                                             u"new": (other_file_uri,
 893                                                      other_file_uri),
 894                                             },
 895                                           overwrite=False))
 896             # and 'new' should not have been created
 897             d.addCallback(lambda res: n.list())
 898             d.addCallback(lambda children: self.failIf(u"new" in children))
 899             d.addCallback(lambda res: n.get_metadata_for(u"e1"))
 900             d.addCallback(lambda metadata:
 901                           self.failUnlessEqual(set(metadata.keys()), set(["tahoe"])))
 902             d.addCallback(lambda res: n.get_metadata_for(u"e2"))
 903             d.addCallback(lambda metadata:
 904                           self.failUnlessEqual(set(metadata.keys()), set(["tahoe"])))
 905             d.addCallback(lambda res: n.get_metadata_for(u"e3"))
 906             d.addCallback(lambda metadata:
 907                           self.failUnless((set(metadata.keys()) == set(["key", "tahoe"])) and
 908                                           (metadata["key"] == "value"), metadata))
 909
 910             d.addCallback(lambda res: n.delete(u"e1"))
 911             d.addCallback(lambda res: n.delete(u"e2"))
 912             d.addCallback(lambda res: n.delete(u"e3"))
 913
 914             # metadata through set_nodes()
 915             d.addCallback(lambda res:
 916                           n.set_nodes({ u"f1": (n, None),
 917                                         u"f2": (n, {}),
 918                                         u"f3": (n, {"key": "value"}),
 919                                         }))
 920             d.addCallback(lambda n2: self.failUnlessIdentical(n2, n))
 921             d.addCallback(lambda res:
 922                           self.shouldFail(ExistingChildError, "set_nodes-no",
 923                                           "child 'f1' already exists",
 924                                           n.set_nodes, { u"f1": (n, None),
 925                                                          u"new": (n, None), },
 926                                           overwrite=False))
 927             # and 'new' should not have been created
 928             d.addCallback(lambda res: n.list())
 929             d.addCallback(lambda children: self.failIf(u"new" in children))
 930             d.addCallback(lambda res: n.get_metadata_for(u"f1"))
 931             d.addCallback(lambda metadata:
 932                           self.failUnlessEqual(set(metadata.keys()), set(["tahoe"])))
 933             d.addCallback(lambda res: n.get_metadata_for(u"f2"))
 934             d.addCallback(lambda metadata:
 935                           self.failUnlessEqual(set(metadata.keys()), set(["tahoe"])))
 936             d.addCallback(lambda res: n.get_metadata_for(u"f3"))
 937             d.addCallback(lambda metadata:
 938                           self.failUnless((set(metadata.keys()) == set(["key", "tahoe"])) and
 939                                           (metadata["key"] == "value"), metadata))
 940
 941             d.addCallback(lambda res: n.delete(u"f1"))
 942             d.addCallback(lambda res: n.delete(u"f2"))
 943             d.addCallback(lambda res: n.delete(u"f3"))
 944
 945
 946             d.addCallback(lambda res:
 947                           n.set_metadata_for(u"child",
 948                                              {"tags": ["web2.0-compatible"], "tahoe": {"bad": "mojo"}}))
 949             d.addCallback(lambda n1: n1.get_metadata_for(u"child"))
 950             d.addCallback(lambda metadata:
 951                           self.failUnless((set(metadata.keys()) == set(["tags", "tahoe"])) and
 952                                           metadata["tags"] == ["web2.0-compatible"] and
 953                                           "bad" not in metadata["tahoe"], metadata))
 954
 955             d.addCallback(lambda res:
 956                           self.shouldFail(NoSuchChildError, "set_metadata_for-nosuch", "",
 957                                           n.set_metadata_for, u"nosuch", {}))
 958
 959
 960             def _start(res):
 961                 self._start_timestamp = time.time()
 962             d.addCallback(_start)
 963             # simplejson-1.7.1 (as shipped on Ubuntu 'gutsy') rounds all
 964             # floats to hundredeths (it uses str(num) instead of repr(num)).
 965             # simplejson-1.7.3 does not have this bug. To prevent this bug
 966             # from causing the test to fail, stall for more than a few
 967             # hundrededths of a second.
 968             d.addCallback(self.stall, 0.1)
 969             d.addCallback(lambda res: n.add_file(u"timestamps",
 970                                                  upload.Data("stamp me", convergence="some convergence string")))
 971             d.addCallback(self.stall, 0.1)
 972             def _stop(res):
 973                 self._stop_timestamp = time.time()
 974             d.addCallback(_stop)
 975
 976             d.addCallback(lambda res: n.get_metadata_for(u"timestamps"))
 977             def _check_timestamp1(metadata):
 978                 self.failUnlessEqual(set(metadata.keys()), set(["tahoe"]))
 979                 tahoe_md = metadata["tahoe"]
 980                 self.failUnlessEqual(set(tahoe_md.keys()), set(["linkcrtime", "linkmotime"]))
 981
 982                 self.failUnlessGreaterOrEqualThan(tahoe_md["linkcrtime"],
 983                                                   self._start_timestamp)
 984                 self.failUnlessGreaterOrEqualThan(self._stop_timestamp,
 985                                                   tahoe_md["linkcrtime"])
 986                 self.failUnlessGreaterOrEqualThan(tahoe_md["linkmotime"],
 987                                                   self._start_timestamp)
 988                 self.failUnlessGreaterOrEqualThan(self._stop_timestamp,
 989                                                   tahoe_md["linkmotime"])
 990                 # Our current timestamp rules say that replacing an existing
 991                 # child should preserve the 'linkcrtime' but update the
 992                 # 'linkmotime'
 993                 self._old_linkcrtime = tahoe_md["linkcrtime"]
 994                 self._old_linkmotime = tahoe_md["linkmotime"]
 995             d.addCallback(_check_timestamp1)
 996             d.addCallback(self.stall, 2.0) # accomodate low-res timestamps
 997             d.addCallback(lambda res: n.set_node(u"timestamps", n))
 998             d.addCallback(lambda res: n.get_metadata_for(u"timestamps"))
 999             def _check_timestamp2(metadata):
1000                 self.failUnlessIn("tahoe", metadata)
1001                 tahoe_md = metadata["tahoe"]
1002                 self.failUnlessEqual(set(tahoe_md.keys()), set(["linkcrtime", "linkmotime"]))
1003
1004                 self.failUnlessReallyEqual(tahoe_md["linkcrtime"], self._old_linkcrtime)
1005                 self.failUnlessGreaterThan(tahoe_md["linkmotime"], self._old_linkmotime)
1006                 return n.delete(u"timestamps")
1007             d.addCallback(_check_timestamp2)
1008
1009             d.addCallback(lambda res: n.delete(u"subdir"))
1010             d.addCallback(lambda old_child:
1011                           self.failUnlessReallyEqual(old_child.get_uri(),
1012                                                      self.subdir.get_uri()))
1013
1014             d.addCallback(lambda res: n.list())
1015             d.addCallback(lambda children:
1016                           self.failUnlessReallyEqual(set(children.keys()),
1017                                                      set([u"child"])))
1018
1019             uploadable1 = upload.Data("some data", convergence="converge")
1020             d.addCallback(lambda res: n.add_file(u"newfile", uploadable1))
1021             d.addCallback(lambda newnode:
1022                           self.failUnless(IImmutableFileNode.providedBy(newnode)))
1023             uploadable2 = upload.Data("some data", convergence="stuff")
1024             d.addCallback(lambda res:
1025                           self.shouldFail(ExistingChildError, "add_file-no",
1026                                           "child 'newfile' already exists",
1027                                           n.add_file, u"newfile",
1028                                           uploadable2,
1029                                           overwrite=False))
1030             d.addCallback(lambda res: n.list())
1031             d.addCallback(lambda children:
1032                           self.failUnlessReallyEqual(set(children.keys()),
1033                                                      set([u"child", u"newfile"])))
1034             d.addCallback(lambda res: n.get_metadata_for(u"newfile"))
1035             d.addCallback(lambda metadata:
1036                           self.failUnlessEqual(set(metadata.keys()), set(["tahoe"])))
1037
1038             uploadable3 = upload.Data("some data", convergence="converge")
1039             d.addCallback(lambda res: n.add_file(u"newfile-metadata",
1040                                                  uploadable3,
1041                                                  {"key": "value"}))
1042             d.addCallback(lambda newnode:
1043                           self.failUnless(IImmutableFileNode.providedBy(newnode)))
1044             d.addCallback(lambda res: n.get_metadata_for(u"newfile-metadata"))
1045             d.addCallback(lambda metadata:
1046                               self.failUnless((set(metadata.keys()) == set(["key", "tahoe"])) and
1047                                               (metadata['key'] == "value"), metadata))
1048             d.addCallback(lambda res: n.delete(u"newfile-metadata"))
1049
1050             d.addCallback(lambda res: n.create_subdirectory(u"subdir2"))
1051             def _created2(subdir2):
1052                 self.subdir2 = subdir2
1053                 # put something in the way, to make sure it gets overwritten
1054                 return subdir2.add_file(u"child", upload.Data("overwrite me",
1055                                                               "converge"))
1056             d.addCallback(_created2)
1057
1058             d.addCallback(lambda res:
1059                           n.move_child_to(u"child", self.subdir2))
1060             d.addCallback(lambda res: n.list())
1061             d.addCallback(lambda children:
1062                           self.failUnlessReallyEqual(set(children.keys()),
1063                                                      set([u"newfile", u"subdir2"])))
1064             d.addCallback(lambda res: self.subdir2.list())
1065             d.addCallback(lambda children:
1066                           self.failUnlessReallyEqual(set(children.keys()),
1067                                                      set([u"child"])))
1068             d.addCallback(lambda res: self.subdir2.get(u"child"))
1069             d.addCallback(lambda child:
1070                           self.failUnlessReallyEqual(child.get_uri(),
1071                                                      fake_file_uri))
1072
1073             # move it back, using new_child_name=
1074             d.addCallback(lambda res:
1075                           self.subdir2.move_child_to(u"child", n, u"newchild"))
1076             d.addCallback(lambda res: n.list())
1077             d.addCallback(lambda children:
1078                           self.failUnlessReallyEqual(set(children.keys()),
1079                                                      set([u"newchild", u"newfile",
1080                                                           u"subdir2"])))
1081             d.addCallback(lambda res: self.subdir2.list())
1082             d.addCallback(lambda children:
1083                           self.failUnlessReallyEqual(set(children.keys()), set([])))
1084
1085             # now make sure that we honor overwrite=False
1086             d.addCallback(lambda res:
1087                           self.subdir2.set_uri(u"newchild",
1088                                                other_file_uri, other_file_uri))
1089
1090             d.addCallback(lambda res:
1091                           self.shouldFail(ExistingChildError, "move_child_to-no",
1092                                           "child 'newchild' already exists",
1093                                           n.move_child_to, u"newchild",
1094                                           self.subdir2,
1095                                           overwrite=False))
1096             d.addCallback(lambda res: self.subdir2.get(u"newchild"))
1097             d.addCallback(lambda child:
1098                           self.failUnlessReallyEqual(child.get_uri(),
1099                                                      other_file_uri))
1100
1101
1102             # Setting the no-write field should diminish a mutable cap to read-only
1103             # (for both files and directories).
1104
1105             d.addCallback(lambda ign: n.set_uri(u"mutable", other_file_uri, other_file_uri))
1106             d.addCallback(lambda ign: n.get(u"mutable"))
1107             d.addCallback(lambda mutable: self.failIf(mutable.is_readonly(), mutable))
1108             d.addCallback(lambda ign: n.set_metadata_for(u"mutable", {"no-write": True}))
1109             d.addCallback(lambda ign: n.get(u"mutable"))
1110             d.addCallback(lambda mutable: self.failUnless(mutable.is_readonly(), mutable))
1111             d.addCallback(lambda ign: n.set_metadata_for(u"mutable", {"no-write": True}))
1112             d.addCallback(lambda ign: n.get(u"mutable"))
1113             d.addCallback(lambda mutable: self.failUnless(mutable.is_readonly(), mutable))
1114
1115             d.addCallback(lambda ign: n.get(u"subdir2"))
1116             d.addCallback(lambda subdir2: self.failIf(subdir2.is_readonly()))
1117             d.addCallback(lambda ign: n.set_metadata_for(u"subdir2", {"no-write": True}))
1118             d.addCallback(lambda ign: n.get(u"subdir2"))
1119             d.addCallback(lambda subdir2: self.failUnless(subdir2.is_readonly(), subdir2))
1120
1121             d.addCallback(lambda ign: n.set_uri(u"mutable_ro", other_file_uri, other_file_uri,
1122                                                 metadata={"no-write": True}))
1123             d.addCallback(lambda ign: n.get(u"mutable_ro"))
1124             d.addCallback(lambda mutable_ro: self.failUnless(mutable_ro.is_readonly(), mutable_ro))
1125
1126             d.addCallback(lambda ign: n.create_subdirectory(u"subdir_ro", metadata={"no-write": True}))
1127             d.addCallback(lambda ign: n.get(u"subdir_ro"))
1128             d.addCallback(lambda subdir_ro: self.failUnless(subdir_ro.is_readonly(), subdir_ro))
1129
1130             return d
1131
1132         d.addCallback(_then)
1133
1134         d.addErrback(self.explain_error)
1135         return d
1136
1137     def test_update_metadata(self):
1138         (t1, t2, t3) = (626644800.0, 634745640.0, 892226160.0)
1139
1140         md1 = dirnode.update_metadata({"ctime": t1}, {}, t2)
1141         self.failUnlessEqual(md1, {"tahoe":{"linkcrtime": t1, "linkmotime": t2}})
1142
1143         md2 = dirnode.update_metadata(md1, {"key": "value", "tahoe": {"bad": "mojo"}}, t3)
1144         self.failUnlessEqual(md2, {"key": "value",
1145                                    "tahoe":{"linkcrtime": t1, "linkmotime": t3}})
1146
1147         md3 = dirnode.update_metadata({}, None, t3)
1148         self.failUnlessEqual(md3, {"tahoe":{"linkcrtime": t3, "linkmotime": t3}})
1149
1150         md4 = dirnode.update_metadata({}, {"bool": True, "number": 42}, t1)
1151         self.failUnlessEqual(md4, {"bool": True, "number": 42,
1152                                    "tahoe":{"linkcrtime": t1, "linkmotime": t1}})
1153
1154     def test_create_subdirectory(self):
1155         self.basedir = "dirnode/Dirnode/test_create_subdirectory"
1156         self.set_up_grid()
1157         c = self.g.clients[0]
1158         nm = c.nodemaker
1159
1160         d = c.create_dirnode()
1161         def _then(n):
1162             # /
1163             self.rootnode = n
1164             fake_file_uri = make_mutable_file_uri()
1165             other_file_uri = make_mutable_file_uri()
1166             md = {"metakey": "metavalue"}
1167             kids = {u"kid1": (nm.create_from_cap(fake_file_uri), {}),
1168                     u"kid2": (nm.create_from_cap(other_file_uri), md),
1169                     }
1170             d = n.create_subdirectory(u"subdir", kids)
1171             def _check(sub):
1172                 d = n.get_child_at_path(u"subdir")
1173                 d.addCallback(lambda sub2: self.failUnlessReallyEqual(sub2.get_uri(),
1174                                                                       sub.get_uri()))
1175                 d.addCallback(lambda ign: sub.list())
1176                 return d
1177             d.addCallback(_check)
1178             def _check_kids(kids2):
1179                 self.failUnlessEqual(set(kids.keys()), set(kids2.keys()))
1180                 self.failUnlessEqual(kids2[u"kid2"][1]["metakey"], "metavalue")
1181             d.addCallback(_check_kids)
1182             return d
1183         d.addCallback(_then)
1184         return d
1185
1186 class MinimalFakeMutableFile:
1187     def get_writekey(self):
1188         return "writekey"
1189
1190 class Packing(testutil.ReallyEqualMixin, unittest.TestCase):
1191     # This is a base32-encoded representation of the directory tree
1192     # root/file1
1193     # root/file2
1194     # root/file3
1195     # as represented after being fed to _pack_contents.
1196     # We have it here so we can decode it, feed it to
1197     # _unpack_contents, and verify that _unpack_contents
1198     # works correctly.
1199
1200     known_tree = "GM4TOORVHJTGS3DFGEWDSNJ2KVJESOSDJBFTU33MPB2GS3LZNVYG6N3GGI3WU5TIORTXC3DOMJ2G4NB2MVWXUZDONBVTE5LNGRZWK2LYN55GY23XGNYXQMTOMZUWU5TENN4DG23ZG5UTO2L2NQ2DO6LFMRWDMZJWGRQTUMZ2GEYDUMJQFQYTIMZ22XZKZORX5XS7CAQCSK3URR6QOHISHRCMGER5LRFSZRNAS5ZSALCS6TWFQAE754IVOIKJVK73WZPP3VUUEDTX3WHTBBZ5YX3CEKHCPG3ZWQLYA4QM6LDRCF7TJQYWLIZHKGN5ROA3AUZPXESBNLQQ6JTC2DBJU2D47IZJTLR3PKZ4RVF57XLPWY7FX7SZV3T6IJ3ORFW37FXUPGOE3ROPFNUX5DCGMAQJ3PGGULBRGM3TU6ZCMN2GS3LFEI5CAMJSGQ3DMNRTHA4TOLRUGI3TKNRWGEWCAITUMFUG6ZJCHIQHWITMNFXGW3LPORUW2ZJCHIQDCMRUGY3DMMZYHE3S4NBSG42TMNRRFQQCE3DJNZVWG4TUNFWWKIR2EAYTENBWGY3DGOBZG4XDIMRXGU3DMML5FQQCE3LUNFWWKIR2EAYTENBWGY3DGOBZG4XDIMRXGU3DMML5FQWDGOJRHI2TUZTJNRSTELBZGQ5FKUSJHJBUQSZ2MFYGKZ3SOBSWQ43IO52WO23CNAZWU3DUGVSWSNTIOE5DK33POVTW4ZLNMNWDK6DHPA2GS2THNF2W25DEN5VGY2LQNFRGG5DKNNRHO5TZPFTWI6LNMRYGQ2LCGJTHM4J2GM5DCMB2GQWDCNBSHKVVQBGRYMACKJ27CVQ6O6B4QPR72RFVTGOZUI76XUSWAX73JRV5PYRHMIFYZIA25MXDPGUGML6M2NMRSG4YD4W4K37ZDYSXHMJ3IUVT4F64YTQQVBJFFFOUC7J7LAB2VFCL5UKKGMR2D3F4EPOYC7UYWQZNR5KXHBSNXLCNBX2SNF22DCXJIHSMEKWEWOG5XCJEVVZ7UW5IB6I64XXQSJ34B5CAYZGZIIMR6LBRGMZTU6ZCMN2GS3LFEI5CAMJSGQ3DMNRTHA4TOLRUGMYDEMJYFQQCE5DBNBXWKIR2EB5SE3DJNZVW233UNFWWKIR2EAYTENBWGY3DGOBZG4XDIMZQGIYTQLBAEJWGS3TLMNZHI2LNMURDUIBRGI2DMNRWGM4DSNZOGQZTAMRRHB6SYIBCNV2GS3LFEI5CAMJSGQ3DMNRTHA4TOLRUGMYDEMJYPUWCYMZZGU5DKOTGNFWGKMZMHE2DUVKSJE5EGSCLHJRW25DDPBYTO2DXPB3GM6DBNYZTI6LJMV3DM2LWNB4TU4LWMNSWW3LKORXWK5DEMN3TI23NNE3WEM3SORRGY5THPA3TKNBUMNZG453BOF2GSZLXMVWWI3DJOFZW623RHIZTUMJQHI2SYMJUGI5BOSHWDPG3WKPAVXCF3XMKA7QVIWPRMWJHDTQHD27AHDCPJWDQENQ5H5ZZILTXQNIXXCIW4LKQABU2GCFRG5FHQN7CHD7HF4EKNRZFIV2ZYQIBM7IQU7F4RGB3XCX3FREPBKQ7UCICHVWPCYFGA6OLH3J45LXQ6GWWICJ3PGWJNLZ7PCRNLAPNYUGU6BENS7OXMBEOOFRIZV3PF2FFWZ5WHDPKXERYP7GNHKRMGEZTOOT3EJRXI2LNMURDUIBRGI2DMNRWGM4DSNZOGQZTGNRSGY4SYIBCORQWQ33FEI5CA6ZCNRUW423NN52GS3LFEI5CAMJSGQ3DMNRTHA4TOLRUGMZTMMRWHEWCAITMNFXGWY3SORUW2ZJCHIQDCMRUGY3DMMZYHE3S4NBTGM3DENRZPUWCAITNORUW2ZJCHIQDCMRUGY3DMMZYHE3S4NBTGM3DENRZPUWCY==="
1201
1202     def test_unpack_and_pack_behavior(self):
1203         known_tree = b32decode(self.known_tree)
1204         nodemaker = NodeMaker(None, None, None,
1205                               None, None, None,
1206                               {"k": 3, "n": 10}, None)
1207         write_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
1208         filenode = nodemaker.create_from_cap(write_uri)
1209         node = dirnode.DirectoryNode(filenode, nodemaker, None)
1210         children = node._unpack_contents(known_tree)
1211         self._check_children(children)
1212
1213         packed_children = node._pack_contents(children)
1214         children = node._unpack_contents(packed_children)
1215         self._check_children(children)
1216
1217     def _check_children(self, children):
1218         # Are all the expected child nodes there?
1219         self.failUnless(children.has_key(u'file1'))
1220         self.failUnless(children.has_key(u'file2'))
1221         self.failUnless(children.has_key(u'file3'))
1222
1223         # Are the metadata for child 3 right?
1224         file3_rocap = "URI:CHK:cmtcxq7hwxvfxan34yiev6ivhy:qvcekmjtoetdcw4kmi7b3rtblvgx7544crnwaqtiewemdliqsokq:3:10:5"
1225         file3_rwcap = "URI:CHK:cmtcxq7hwxvfxan34yiev6ivhy:qvcekmjtoetdcw4kmi7b3rtblvgx7544crnwaqtiewemdliqsokq:3:10:5"
1226         file3_metadata = {'ctime': 1246663897.4336269, 'tahoe': {'linkmotime': 1246663897.4336269, 'linkcrtime': 1246663897.4336269}, 'mtime': 1246663897.4336269}
1227         self.failUnlessEqual(file3_metadata, children[u'file3'][1])
1228         self.failUnlessReallyEqual(file3_rocap,
1229                                    children[u'file3'][0].get_readonly_uri())
1230         self.failUnlessReallyEqual(file3_rwcap,
1231                                    children[u'file3'][0].get_uri())
1232
1233         # Are the metadata for child 2 right?
1234         file2_rocap = "URI:CHK:apegrpehshwugkbh3jlt5ei6hq:5oougnemcl5xgx4ijgiumtdojlipibctjkbwvyygdymdphib2fvq:3:10:4"
1235         file2_rwcap = "URI:CHK:apegrpehshwugkbh3jlt5ei6hq:5oougnemcl5xgx4ijgiumtdojlipibctjkbwvyygdymdphib2fvq:3:10:4"
1236         file2_metadata = {'ctime': 1246663897.430218, 'tahoe': {'linkmotime': 1246663897.430218, 'linkcrtime': 1246663897.430218}, 'mtime': 1246663897.430218}
1237         self.failUnlessEqual(file2_metadata, children[u'file2'][1])
1238         self.failUnlessReallyEqual(file2_rocap,
1239                                    children[u'file2'][0].get_readonly_uri())
1240         self.failUnlessReallyEqual(file2_rwcap,
1241                                    children[u'file2'][0].get_uri())
1242
1243         # Are the metadata for child 1 right?
1244         file1_rocap = "URI:CHK:olxtimympo7f27jvhtgqlnbtn4:emzdnhk2um4seixozlkw3qx2nfijvdkx3ky7i7izl47yedl6e64a:3:10:10"
1245         file1_rwcap = "URI:CHK:olxtimympo7f27jvhtgqlnbtn4:emzdnhk2um4seixozlkw3qx2nfijvdkx3ky7i7izl47yedl6e64a:3:10:10"
1246         file1_metadata = {'ctime': 1246663897.4275661, 'tahoe': {'linkmotime': 1246663897.4275661, 'linkcrtime': 1246663897.4275661}, 'mtime': 1246663897.4275661}
1247         self.failUnlessEqual(file1_metadata, children[u'file1'][1])
1248         self.failUnlessReallyEqual(file1_rocap,
1249                                    children[u'file1'][0].get_readonly_uri())
1250         self.failUnlessReallyEqual(file1_rwcap,
1251                                    children[u'file1'][0].get_uri())
1252
1253     def _make_kids(self, nm, which):
1254         caps = {"imm": "URI:CHK:n7r3m6wmomelk4sep3kw5cvduq:os7ijw5c3maek7pg65e5254k2fzjflavtpejjyhshpsxuqzhcwwq:3:20:14861",
1255                 "lit": "URI:LIT:n5xgk", # LIT for "one"
1256                 "write": "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq",
1257                 "read": "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q",
1258                 "dirwrite": "URI:DIR2:n6x24zd3seu725yluj75q5boaa:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq",
1259                 "dirread":  "URI:DIR2-RO:b7sr5qsifnicca7cbk3rhrhbvq:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq",
1260                 }
1261         kids = {}
1262         for name in which:
1263             kids[unicode(name)] = (nm.create_from_cap(caps[name]), {})
1264         return kids
1265
1266     def test_deep_immutable(self):
1267         nm = NodeMaker(None, None, None, None, None, None, {"k": 3, "n": 10},
1268                        None)
1269         fn = MinimalFakeMutableFile()
1270
1271         kids = self._make_kids(nm, ["imm", "lit", "write", "read",
1272                                     "dirwrite", "dirread"])
1273         packed = dirnode.pack_children(kids, fn.get_writekey(), deep_immutable=False)
1274         self.failUnlessIn("lit", packed)
1275
1276         kids = self._make_kids(nm, ["imm", "lit"])
1277         packed = dirnode.pack_children(kids, fn.get_writekey(), deep_immutable=True)
1278         self.failUnlessIn("lit", packed)
1279
1280         kids = self._make_kids(nm, ["imm", "lit", "write"])
1281         self.failUnlessRaises(dirnode.MustBeDeepImmutableError,
1282                               dirnode.pack_children,
1283                               kids, fn.get_writekey(), deep_immutable=True)
1284
1285         # read-only is not enough: all children must be immutable
1286         kids = self._make_kids(nm, ["imm", "lit", "read"])
1287         self.failUnlessRaises(dirnode.MustBeDeepImmutableError,
1288                               dirnode.pack_children,
1289                               kids, fn.get_writekey(), deep_immutable=True)
1290
1291         kids = self._make_kids(nm, ["imm", "lit", "dirwrite"])
1292         self.failUnlessRaises(dirnode.MustBeDeepImmutableError,
1293                               dirnode.pack_children,
1294                               kids, fn.get_writekey(), deep_immutable=True)
1295
1296         kids = self._make_kids(nm, ["imm", "lit", "dirread"])
1297         self.failUnlessRaises(dirnode.MustBeDeepImmutableError,
1298                               dirnode.pack_children,
1299                               kids, fn.get_writekey(), deep_immutable=True)
1300
1301 class FakeMutableFile:
1302     implements(IMutableFileNode)
1303     counter = 0
1304     def __init__(self, initial_contents=""):
1305         self.data = self._get_initial_contents(initial_contents)
1306         counter = FakeMutableFile.counter
1307         FakeMutableFile.counter += 1
1308         writekey = hashutil.ssk_writekey_hash(str(counter))
1309         fingerprint = hashutil.ssk_pubkey_fingerprint_hash(str(counter))
1310         self.uri = uri.WriteableSSKFileURI(writekey, fingerprint)
1311
1312     def _get_initial_contents(self, contents):
1313         if isinstance(contents, str):
1314             return contents
1315         if contents is None:
1316             return ""
1317         assert callable(contents), "%s should be callable, not %s" % \
1318                (contents, type(contents))
1319         return contents(self)
1320
1321     def get_cap(self):
1322         return self.uri
1323
1324     def get_uri(self):
1325         return self.uri.to_string()
1326
1327     def get_write_uri(self):
1328         return self.uri.to_string()
1329
1330     def download_best_version(self):
1331         return defer.succeed(self.data)
1332
1333     def get_writekey(self):
1334         return "writekey"
1335
1336     def is_readonly(self):
1337         return False
1338
1339     def is_mutable(self):
1340         return True
1341
1342     def is_unknown(self):
1343         return False
1344
1345     def is_allowed_in_immutable_directory(self):
1346         return False
1347
1348     def raise_error(self):
1349         pass
1350
1351     def modify(self, modifier):
1352         self.data = modifier(self.data, None, True)
1353         return defer.succeed(None)
1354
1355 class FakeNodeMaker(NodeMaker):
1356     def create_mutable_file(self, contents="", keysize=None):
1357         return defer.succeed(FakeMutableFile(contents))
1358
1359 class FakeClient2(Client):
1360     def __init__(self):
1361         self.nodemaker = FakeNodeMaker(None, None, None,
1362                                        None, None, None,
1363                                        {"k":3,"n":10}, None)
1364     def create_node_from_uri(self, rwcap, rocap):
1365         return self.nodemaker.create_from_cap(rwcap, rocap)
1366
1367 class Dirnode2(testutil.ReallyEqualMixin, testutil.ShouldFailMixin, unittest.TestCase):
1368     def setUp(self):
1369         client = FakeClient2()
1370         self.nodemaker = client.nodemaker
1371
1372     def test_from_future(self):
1373         # Create a mutable directory that contains unknown URI types, and make sure
1374         # we tolerate them properly.
1375         d = self.nodemaker.create_new_mutable_directory()
1376         future_write_uri = u"x-tahoe-crazy://I_am_from_the_future_rw_\u263A".encode('utf-8')
1377         future_read_uri = u"x-tahoe-crazy-readonly://I_am_from_the_future_ro_\u263A".encode('utf-8')
1378         future_imm_uri = u"x-tahoe-crazy-immutable://I_am_from_the_future_imm_\u263A".encode('utf-8')
1379         future_node = UnknownNode(future_write_uri, future_read_uri)
1380         def _then(n):
1381             self._node = n
1382             return n.set_node(u"future", future_node)
1383         d.addCallback(_then)
1384
1385         # We should be prohibited from adding an unknown URI to a directory
1386         # just in the rw_uri slot, since we don't know how to diminish the cap
1387         # to a readcap (for the ro_uri slot).
1388         d.addCallback(lambda ign:
1389              self.shouldFail(MustNotBeUnknownRWError,
1390                              "copy unknown",
1391                              "cannot attach unknown rw cap as child",
1392                              self._node.set_uri, u"add",
1393                              future_write_uri, None))
1394
1395         # However, we should be able to add both rw_uri and ro_uri as a pair of
1396         # unknown URIs.
1397         d.addCallback(lambda ign: self._node.set_uri(u"add-pair",
1398                                                      future_write_uri, future_read_uri))
1399
1400         # and to add an URI prefixed with "ro." or "imm." when it is given in a
1401         # write slot (or URL parameter).
1402         d.addCallback(lambda ign: self._node.set_uri(u"add-ro",
1403                                                      "ro." + future_read_uri, None))
1404         d.addCallback(lambda ign: self._node.set_uri(u"add-imm",
1405                                                      "imm." + future_imm_uri, None))
1406
1407         d.addCallback(lambda ign: self._node.list())
1408         def _check(children):
1409             self.failUnlessReallyEqual(len(children), 4)
1410             (fn, metadata) = children[u"future"]
1411             self.failUnless(isinstance(fn, UnknownNode), fn)
1412             self.failUnlessReallyEqual(fn.get_uri(), future_write_uri)
1413             self.failUnlessReallyEqual(fn.get_write_uri(), future_write_uri)
1414             self.failUnlessReallyEqual(fn.get_readonly_uri(), "ro." + future_read_uri)
1415
1416             (fn2, metadata2) = children[u"add-pair"]
1417             self.failUnless(isinstance(fn2, UnknownNode), fn2)
1418             self.failUnlessReallyEqual(fn2.get_uri(), future_write_uri)
1419             self.failUnlessReallyEqual(fn2.get_write_uri(), future_write_uri)
1420             self.failUnlessReallyEqual(fn2.get_readonly_uri(), "ro." + future_read_uri)
1421
1422             (fn3, metadata3) = children[u"add-ro"]
1423             self.failUnless(isinstance(fn3, UnknownNode), fn3)
1424             self.failUnlessReallyEqual(fn3.get_uri(), "ro." + future_read_uri)
1425             self.failUnlessReallyEqual(fn3.get_write_uri(), None)
1426             self.failUnlessReallyEqual(fn3.get_readonly_uri(), "ro." + future_read_uri)
1427
1428             (fn4, metadata4) = children[u"add-imm"]
1429             self.failUnless(isinstance(fn4, UnknownNode), fn4)
1430             self.failUnlessReallyEqual(fn4.get_uri(), "imm." + future_imm_uri)
1431             self.failUnlessReallyEqual(fn4.get_write_uri(), None)
1432             self.failUnlessReallyEqual(fn4.get_readonly_uri(), "imm." + future_imm_uri)
1433
1434             # We should also be allowed to copy the "future" UnknownNode, because
1435             # it contains all the information that was in the original directory
1436             # (readcap and writecap), so we're preserving everything.
1437             return self._node.set_node(u"copy", fn)
1438         d.addCallback(_check)
1439
1440         d.addCallback(lambda ign: self._node.list())
1441         def _check2(children):
1442             self.failUnlessReallyEqual(len(children), 5)
1443             (fn, metadata) = children[u"copy"]
1444             self.failUnless(isinstance(fn, UnknownNode), fn)
1445             self.failUnlessReallyEqual(fn.get_uri(), future_write_uri)
1446             self.failUnlessReallyEqual(fn.get_write_uri(), future_write_uri)
1447             self.failUnlessReallyEqual(fn.get_readonly_uri(), "ro." + future_read_uri)
1448         d.addCallback(_check2)
1449         return d
1450
1451     def test_unknown_strip_prefix_for_ro(self):
1452         self.failUnlessReallyEqual(strip_prefix_for_ro("foo",     False), "foo")
1453         self.failUnlessReallyEqual(strip_prefix_for_ro("ro.foo",  False), "foo")
1454         self.failUnlessReallyEqual(strip_prefix_for_ro("imm.foo", False), "imm.foo")
1455         self.failUnlessReallyEqual(strip_prefix_for_ro("foo",     True),  "foo")
1456         self.failUnlessReallyEqual(strip_prefix_for_ro("ro.foo",  True),  "foo")
1457         self.failUnlessReallyEqual(strip_prefix_for_ro("imm.foo", True),  "foo")
1458
1459     def test_unknownnode(self):
1460         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
1461         mut_read_uri = "URI:SSK-RO:jf6wkflosyvntwxqcdo7a54jvm:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
1462         lit_uri = "URI:LIT:n5xgk"
1463
1464         # This does not attempt to be exhaustive.
1465         no_no        = [# Opaque node, but not an error.
1466                         ( 0, UnknownNode(None, None)),
1467                         ( 1, UnknownNode(None, None, deep_immutable=True)),
1468                        ]
1469         unknown_rw   = [# These are errors because we're only given a rw_uri, and we can't
1470                         # diminish it.
1471                         ( 2, UnknownNode("foo", None)),
1472                         ( 3, UnknownNode("foo", None, deep_immutable=True)),
1473                         ( 4, UnknownNode("ro.foo", None, deep_immutable=True)),
1474                         ( 5, UnknownNode("ro." + mut_read_uri, None, deep_immutable=True)),
1475                         ( 6, UnknownNode("URI:SSK-RO:foo", None, deep_immutable=True)),
1476                         ( 7, UnknownNode("URI:SSK:foo", None)),
1477                        ]
1478         must_be_ro   = [# These are errors because a readonly constraint is not met.
1479                         ( 8, UnknownNode("ro." + mut_write_uri, None)),
1480                         ( 9, UnknownNode(None, "ro." + mut_write_uri)),
1481                        ]
1482         must_be_imm  = [# These are errors because an immutable constraint is not met.
1483                         (10, UnknownNode(None, "ro.URI:SSK-RO:foo", deep_immutable=True)),
1484                         (11, UnknownNode(None, "imm.URI:SSK:foo")),
1485                         (12, UnknownNode(None, "imm.URI:SSK-RO:foo")),
1486                         (13, UnknownNode("bar", "ro.foo", deep_immutable=True)),
1487                         (14, UnknownNode("bar", "imm.foo", deep_immutable=True)),
1488                         (15, UnknownNode("bar", "imm." + lit_uri, deep_immutable=True)),
1489                         (16, UnknownNode("imm." + mut_write_uri, None)),
1490                         (17, UnknownNode("imm." + mut_read_uri, None)),
1491                         (18, UnknownNode("bar", "imm.foo")),
1492                        ]
1493         bad_uri      = [# These are errors because the URI is bad once we've stripped the prefix.
1494                         (19, UnknownNode("ro.URI:SSK-RO:foo", None)),
1495                         (20, UnknownNode("imm.URI:CHK:foo", None, deep_immutable=True)),
1496                         (21, UnknownNode(None, "URI:CHK:foo")),
1497                         (22, UnknownNode(None, "URI:CHK:foo", deep_immutable=True)),
1498                        ]
1499         ro_prefixed  = [# These are valid, and the readcap should end up with a ro. prefix.
1500                         (23, UnknownNode(None, "foo")),
1501                         (24, UnknownNode(None, "ro.foo")),
1502                         (25, UnknownNode(None, "ro." + lit_uri)),
1503                         (26, UnknownNode("bar", "foo")),
1504                         (27, UnknownNode("bar", "ro.foo")),
1505                         (28, UnknownNode("bar", "ro." + lit_uri)),
1506                         (29, UnknownNode("ro.foo", None)),
1507                         (30, UnknownNode("ro." + lit_uri, None)),
1508                        ]
1509         imm_prefixed = [# These are valid, and the readcap should end up with an imm. prefix.
1510                         (31, UnknownNode(None, "foo", deep_immutable=True)),
1511                         (32, UnknownNode(None, "ro.foo", deep_immutable=True)),
1512                         (33, UnknownNode(None, "imm.foo")),
1513                         (34, UnknownNode(None, "imm.foo", deep_immutable=True)),
1514                         (35, UnknownNode("imm." + lit_uri, None)),
1515                         (36, UnknownNode("imm." + lit_uri, None, deep_immutable=True)),
1516                         (37, UnknownNode(None, "imm." + lit_uri)),
1517                         (38, UnknownNode(None, "imm." + lit_uri, deep_immutable=True)),
1518                        ]
1519         error = unknown_rw + must_be_ro + must_be_imm + bad_uri
1520         ok = ro_prefixed + imm_prefixed
1521
1522         for (i, n) in no_no + error + ok:
1523             self.failUnless(n.is_unknown(), i)
1524
1525         for (i, n) in no_no + error:
1526             self.failUnless(n.get_uri() is None, i)
1527             self.failUnless(n.get_write_uri() is None, i)
1528             self.failUnless(n.get_readonly_uri() is None, i)
1529
1530         for (i, n) in no_no + ok:
1531             n.raise_error()
1532
1533         for (i, n) in unknown_rw:
1534             self.failUnlessRaises(MustNotBeUnknownRWError, lambda: n.raise_error())
1535
1536         for (i, n) in must_be_ro:
1537             self.failUnlessRaises(MustBeReadonlyError, lambda: n.raise_error())
1538
1539         for (i, n) in must_be_imm:
1540             self.failUnlessRaises(MustBeDeepImmutableError, lambda: n.raise_error())
1541
1542         for (i, n) in bad_uri:
1543             self.failUnlessRaises(uri.BadURIError, lambda: n.raise_error())
1544
1545         for (i, n) in ok:
1546             self.failIf(n.get_readonly_uri() is None, i)
1547
1548         for (i, n) in ro_prefixed:
1549             self.failUnless(n.get_readonly_uri().startswith("ro."), i)
1550
1551         for (i, n) in imm_prefixed:
1552             self.failUnless(n.get_readonly_uri().startswith("imm."), i)
1553
1554
1555 class DeepStats(testutil.ReallyEqualMixin, unittest.TestCase):
1556     timeout = 240 # It takes longer than 120 seconds on Francois's arm box.
1557     def test_stats(self):
1558         ds = dirnode.DeepStats(None)
1559         ds.add("count-files")
1560         ds.add("size-immutable-files", 123)
1561         ds.histogram("size-files-histogram", 123)
1562         ds.max("largest-directory", 444)
1563
1564         s = ds.get_results()
1565         self.failUnlessReallyEqual(s["count-files"], 1)
1566         self.failUnlessReallyEqual(s["size-immutable-files"], 123)
1567         self.failUnlessReallyEqual(s["largest-directory"], 444)
1568         self.failUnlessReallyEqual(s["count-literal-files"], 0)
1569
1570         ds.add("count-files")
1571         ds.add("size-immutable-files", 321)
1572         ds.histogram("size-files-histogram", 321)
1573         ds.max("largest-directory", 2)
1574
1575         s = ds.get_results()
1576         self.failUnlessReallyEqual(s["count-files"], 2)
1577         self.failUnlessReallyEqual(s["size-immutable-files"], 444)
1578         self.failUnlessReallyEqual(s["largest-directory"], 444)
1579         self.failUnlessReallyEqual(s["count-literal-files"], 0)
1580         self.failUnlessReallyEqual(s["size-files-histogram"],
1581                                    [ (101, 316, 1), (317, 1000, 1) ])
1582
1583         ds = dirnode.DeepStats(None)
1584         for i in range(1, 1100):
1585             ds.histogram("size-files-histogram", i)
1586         ds.histogram("size-files-histogram", 4*1000*1000*1000*1000) # 4TB
1587         s = ds.get_results()
1588         self.failUnlessReallyEqual(s["size-files-histogram"],
1589                                    [ (1, 3, 3),
1590                                      (4, 10, 7),
1591                                      (11, 31, 21),
1592                                      (32, 100, 69),
1593                                      (101, 316, 216),
1594                                      (317, 1000, 684),
1595                                      (1001, 3162, 99),
1596                                      (3162277660169L, 10000000000000L, 1),
1597                                      ])
1598
1599 class UCWEingMutableFileNode(MutableFileNode):
1600     please_ucwe_after_next_upload = False
1601
1602     def _upload(self, new_contents, servermap):
1603         d = MutableFileNode._upload(self, new_contents, servermap)
1604         def _ucwe(res):
1605             if self.please_ucwe_after_next_upload:
1606                 self.please_ucwe_after_next_upload = False
1607                 raise UncoordinatedWriteError()
1608             return res
1609         d.addCallback(_ucwe)
1610         return d
1611
1612 class UCWEingNodeMaker(NodeMaker):
1613     def _create_mutable(self, cap):
1614         n = UCWEingMutableFileNode(self.storage_broker, self.secret_holder,
1615                                    self.default_encoding_parameters,
1616                                    self.history)
1617         return n.init_from_cap(cap)
1618
1619
1620 class Deleter(GridTestMixin, testutil.ReallyEqualMixin, unittest.TestCase):
1621     timeout = 3600 # It takes longer than 433 seconds on Zandr's ARM box.
1622     def test_retry(self):
1623         # ticket #550, a dirnode.delete which experiences an
1624         # UncoordinatedWriteError will fail with an incorrect "you're
1625         # deleting something which isn't there" NoSuchChildError exception.
1626
1627         # to trigger this, we start by creating a directory with a single
1628         # file in it. Then we create a special dirnode that uses a modified
1629         # MutableFileNode which will raise UncoordinatedWriteError once on
1630         # demand. We then call dirnode.delete, which ought to retry and
1631         # succeed.
1632
1633         self.basedir = self.mktemp()
1634         self.set_up_grid()
1635         c0 = self.g.clients[0]
1636         d = c0.create_dirnode()
1637         small = upload.Data("Small enough for a LIT", None)
1638         def _created_dir(dn):
1639             self.root = dn
1640             self.root_uri = dn.get_uri()
1641             return dn.add_file(u"file", small)
1642         d.addCallback(_created_dir)
1643         def _do_delete(ignored):
1644             nm = UCWEingNodeMaker(c0.storage_broker, c0._secret_holder,
1645                                   c0.get_history(), c0.getServiceNamed("uploader"),
1646                                   c0.downloader,
1647                                   c0.download_cache_dirman,
1648                                   c0.get_encoding_parameters(),
1649                                   c0._key_generator)
1650             n = nm.create_from_cap(self.root_uri)
1651             assert n._node.please_ucwe_after_next_upload == False
1652             n._node.please_ucwe_after_next_upload = True
1653             # This should succeed, not raise an exception
1654             return n.delete(u"file")
1655         d.addCallback(_do_delete)
1656
1657         return d
1658
1659 class Adder(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin):
1660
1661     def test_overwrite(self):
1662         # note: This functionality could be tested without actually creating
1663         # several RSA keys. It would be faster without the GridTestMixin: use
1664         # dn.set_node(nodemaker.create_from_cap(make_chk_file_uri())) instead
1665         # of dn.add_file, and use a special NodeMaker that creates fake
1666         # mutable files.
1667         self.basedir = "dirnode/Adder/test_overwrite"
1668         self.set_up_grid()
1669         c = self.g.clients[0]
1670         fileuri = make_chk_file_uri(1234)
1671         filenode = c.nodemaker.create_from_cap(fileuri)
1672         d = c.create_dirnode()
1673
1674         def _create_directory_tree(root_node):
1675             # Build
1676             # root/file1
1677             # root/file2
1678             # root/dir1
1679             d = root_node.add_file(u'file1', upload.Data("Important Things",
1680                 None))
1681             d.addCallback(lambda res:
1682                 root_node.add_file(u'file2', upload.Data("Sekrit Codes", None)))
1683             d.addCallback(lambda res:
1684                 root_node.create_subdirectory(u"dir1"))
1685             d.addCallback(lambda res: root_node)
1686             return d
1687
1688         d.addCallback(_create_directory_tree)
1689
1690         def _test_adder(root_node):
1691             d = root_node.set_node(u'file1', filenode)
1692             # We've overwritten file1. Let's try it with a directory
1693             d.addCallback(lambda res:
1694                 root_node.create_subdirectory(u'dir2'))
1695             d.addCallback(lambda res:
1696                 root_node.set_node(u'dir2', filenode))
1697             # We try overwriting a file with a child while also specifying
1698             # overwrite=False. We should receive an ExistingChildError
1699             # when we do this.
1700             d.addCallback(lambda res:
1701                 self.shouldFail(ExistingChildError, "set_node",
1702                                 "child 'file1' already exists",
1703                                 root_node.set_node, u"file1",
1704                                 filenode, overwrite=False))
1705             # If we try with a directory, we should see the same thing
1706             d.addCallback(lambda res:
1707                 self.shouldFail(ExistingChildError, "set_node",
1708                                 "child 'dir1' already exists",
1709                                 root_node.set_node, u'dir1', filenode,
1710                                 overwrite=False))
1711             d.addCallback(lambda res:
1712                 root_node.set_node(u'file1', filenode,
1713                                    overwrite="only-files"))
1714             d.addCallback(lambda res:
1715                 self.shouldFail(ExistingChildError, "set_node",
1716                                 "child 'dir1' already exists",
1717                                 root_node.set_node, u'dir1', filenode,
1718                                 overwrite="only-files"))
1719             return d
1720
1721         d.addCallback(_test_adder)
1722         return d