src/allmydata/test/test_dirnode.py

   1 import time
   2 import unicodedata
   3 from zope.interface import implements
   4 from twisted.trial import unittest
   5 from twisted.internet import defer
   6 from twisted.internet.interfaces import IConsumer
   7 from allmydata import uri, dirnode
   8 from allmydata.client import Client
   9 from allmydata.immutable import upload
  10 from allmydata.interfaces import IImmutableFileNode, IMutableFileNode, \
  11      ExistingChildError, NoSuchChildError, MustNotBeUnknownRWError, \
  12      MustBeDeepImmutableError, MustBeReadonlyError, \
  13      IDeepCheckResults, IDeepCheckAndRepairResults, \
  14      MDMF_VERSION, SDMF_VERSION
  15 from allmydata.mutable.filenode import MutableFileNode
  16 from allmydata.mutable.common import UncoordinatedWriteError
  17 from allmydata.util import hashutil, base32
  18 from allmydata.util.netstring import split_netstring
  19 from allmydata.monitor import Monitor
  20 from allmydata.test.common import make_chk_file_uri, make_mutable_file_uri, \
  21      ErrorMixin
  22 from allmydata.test.no_network import GridTestMixin
  23 from allmydata.unknown import UnknownNode, strip_prefix_for_ro
  24 from allmydata.nodemaker import NodeMaker
  25 from base64 import b32decode
  26 import allmydata.test.common_util as testutil
  27
  28 class MemAccum:
  29     implements(IConsumer)
  30     def registerProducer(self, producer, streaming):
  31         self.producer = producer
  32         self.producer.resumeProducing()
  33         pass
  34     def unregisterProducer(self):
  35         pass
  36     def write(self, data):
  37         assert not hasattr(self, 'data')
  38         self.data = data
  39         self.producer.resumeProducing()
  40
  41 setup_py_uri = "URI:CHK:n7r3m6wmomelk4sep3kw5cvduq:os7ijw5c3maek7pg65e5254k2fzjflavtpejjyhshpsxuqzhcwwq:3:20:14861"
  42 one_uri = "URI:LIT:n5xgk" # LIT for "one"
  43 mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
  44 mdmf_write_uri = "URI:MDMF:x533rhbm6kiehzl5kj3s44n5ie:4gif5rhneyd763ouo5qjrgnsoa3bg43xycy4robj2rf3tvmhdl3a"
  45 empty_litdir_uri = "URI:DIR2-LIT:"
  46 tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
  47 mut_read_uri = "URI:SSK-RO:jf6wkflosyvntwxqcdo7a54jvm:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
  48 mdmf_read_uri = "URI:MDMF-RO:d4cydxselputycfzkw6qgz4zv4:4gif5rhneyd763ouo5qjrgnsoa3bg43xycy4robj2rf3tvmhdl3a"
  49 future_write_uri = "x-tahoe-crazy://I_am_from_the_future."
  50 future_read_uri = "x-tahoe-crazy-readonly://I_am_from_the_future."
  51 future_nonascii_write_uri = u"x-tahoe-even-more-crazy://I_am_from_the_future_rw_\u263A".encode('utf-8')
  52 future_nonascii_read_uri = u"x-tahoe-even-more-crazy-readonly://I_am_from_the_future_ro_\u263A".encode('utf-8')
  53
  54 # 'o' 'n' 'e-macron'
  55 one_nfc = u"on\u0113"
  56 one_nfd = u"one\u0304"
  57
  58 class Dirnode(GridTestMixin, unittest.TestCase,
  59               testutil.ReallyEqualMixin, testutil.ShouldFailMixin, testutil.StallMixin, ErrorMixin):
  60     timeout = 480 # It occasionally takes longer than 240 seconds on Francois's arm box.
  61
  62     def _do_create_test(self, mdmf=False):
  63         c = self.g.clients[0]
  64
  65         self.expected_manifest = []
  66         self.expected_verifycaps = set()
  67         self.expected_storage_indexes = set()
  68
  69         d = None
  70         if mdmf:
  71             d = c.create_dirnode(version=MDMF_VERSION)
  72         else:
  73             d = c.create_dirnode()
  74         def _then(n):
  75             # /
  76             self.rootnode = n
  77             backing_node = n._node
  78             if mdmf:
  79                 self.failUnlessEqual(backing_node.get_version(),
  80                                      MDMF_VERSION)
  81             else:
  82                 self.failUnlessEqual(backing_node.get_version(),
  83                                      SDMF_VERSION)
  84             self.failUnless(n.is_mutable())
  85             u = n.get_uri()
  86             self.failUnless(u)
  87             cap_formats = []
  88             if mdmf:
  89                 cap_formats = ["URI:DIR2-MDMF:",
  90                                "URI:DIR2-MDMF-RO:",
  91                                "URI:DIR2-MDMF-Verifier:"]
  92             else:
  93                 cap_formats = ["URI:DIR2:",
  94                                "URI:DIR2-RO",
  95                                "URI:DIR2-Verifier:"]
  96             rw, ro, v = cap_formats
  97             self.failUnless(u.startswith(rw), u)
  98             u_ro = n.get_readonly_uri()
  99             self.failUnless(u_ro.startswith(ro), u_ro)
 100             u_v = n.get_verify_cap().to_string()
 101             self.failUnless(u_v.startswith(v), u_v)
 102             u_r = n.get_repair_cap().to_string()
 103             self.failUnlessReallyEqual(u_r, u)
 104             self.expected_manifest.append( ((), u) )
 105             self.expected_verifycaps.add(u_v)
 106             si = n.get_storage_index()
 107             self.expected_storage_indexes.add(base32.b2a(si))
 108             expected_si = n._uri.get_storage_index()
 109             self.failUnlessReallyEqual(si, expected_si)
 110
 111             d = n.list()
 112             d.addCallback(lambda res: self.failUnlessEqual(res, {}))
 113             d.addCallback(lambda res: n.has_child(u"missing"))
 114             d.addCallback(lambda res: self.failIf(res))
 115
 116             fake_file_uri = make_mutable_file_uri()
 117             other_file_uri = make_mutable_file_uri()
 118             m = c.nodemaker.create_from_cap(fake_file_uri)
 119             ffu_v = m.get_verify_cap().to_string()
 120             self.expected_manifest.append( ((u"child",) , m.get_uri()) )
 121             self.expected_verifycaps.add(ffu_v)
 122             self.expected_storage_indexes.add(base32.b2a(m.get_storage_index()))
 123             d.addCallback(lambda res: n.set_uri(u"child",
 124                                                 fake_file_uri, fake_file_uri))
 125             d.addCallback(lambda res:
 126                           self.shouldFail(ExistingChildError, "set_uri-no",
 127                                           "child 'child' already exists",
 128                                           n.set_uri, u"child",
 129                                           other_file_uri, other_file_uri,
 130                                           overwrite=False))
 131             # /
 132             # /child = mutable
 133
 134             d.addCallback(lambda res: n.create_subdirectory(u"subdir"))
 135
 136             # /
 137             # /child = mutable
 138             # /subdir = directory
 139             def _created(subdir):
 140                 self.failUnless(isinstance(subdir, dirnode.DirectoryNode))
 141                 self.subdir = subdir
 142                 new_v = subdir.get_verify_cap().to_string()
 143                 assert isinstance(new_v, str)
 144                 self.expected_manifest.append( ((u"subdir",), subdir.get_uri()) )
 145                 self.expected_verifycaps.add(new_v)
 146                 si = subdir.get_storage_index()
 147                 self.expected_storage_indexes.add(base32.b2a(si))
 148             d.addCallback(_created)
 149
 150             d.addCallback(lambda res:
 151                           self.shouldFail(ExistingChildError, "mkdir-no",
 152                                           "child 'subdir' already exists",
 153                                           n.create_subdirectory, u"subdir",
 154                                           overwrite=False))
 155
 156             d.addCallback(lambda res: n.list())
 157             d.addCallback(lambda children:
 158                           self.failUnlessReallyEqual(set(children.keys()),
 159                                                      set([u"child", u"subdir"])))
 160
 161             d.addCallback(lambda res: n.start_deep_stats().when_done())
 162             def _check_deepstats(stats):
 163                 self.failUnless(isinstance(stats, dict))
 164                 expected = {"count-immutable-files": 0,
 165                             "count-mutable-files": 1,
 166                             "count-literal-files": 0,
 167                             "count-files": 1,
 168                             "count-directories": 2,
 169                             "size-immutable-files": 0,
 170                             "size-literal-files": 0,
 171                             #"size-directories": 616, # varies
 172                             #"largest-directory": 616,
 173                             "largest-directory-children": 2,
 174                             "largest-immutable-file": 0,
 175                             }
 176                 for k,v in expected.iteritems():
 177                     self.failUnlessReallyEqual(stats[k], v,
 178                                                "stats[%s] was %s, not %s" %
 179                                                (k, stats[k], v))
 180                 self.failUnless(stats["size-directories"] > 500,
 181                                 stats["size-directories"])
 182                 self.failUnless(stats["largest-directory"] > 500,
 183                                 stats["largest-directory"])
 184                 self.failUnlessReallyEqual(stats["size-files-histogram"], [])
 185             d.addCallback(_check_deepstats)
 186
 187             d.addCallback(lambda res: n.build_manifest().when_done())
 188             def _check_manifest(res):
 189                 manifest = res["manifest"]
 190                 self.failUnlessReallyEqual(sorted(manifest),
 191                                            sorted(self.expected_manifest))
 192                 stats = res["stats"]
 193                 _check_deepstats(stats)
 194                 self.failUnlessReallyEqual(self.expected_verifycaps,
 195                                            res["verifycaps"])
 196                 self.failUnlessReallyEqual(self.expected_storage_indexes,
 197                                            res["storage-index"])
 198             d.addCallback(_check_manifest)
 199
 200             def _add_subsubdir(res):
 201                 return self.subdir.create_subdirectory(u"subsubdir")
 202             d.addCallback(_add_subsubdir)
 203             # /
 204             # /child = mutable
 205             # /subdir = directory
 206             # /subdir/subsubdir = directory
 207             d.addCallback(lambda res: n.get_child_at_path(u"subdir/subsubdir"))
 208             d.addCallback(lambda subsubdir:
 209                           self.failUnless(isinstance(subsubdir,
 210                                                      dirnode.DirectoryNode)))
 211             d.addCallback(lambda res: n.get_child_at_path(u""))
 212             d.addCallback(lambda res: self.failUnlessReallyEqual(res.get_uri(),
 213                                                                  n.get_uri()))
 214
 215             d.addCallback(lambda res: n.get_metadata_for(u"child"))
 216             d.addCallback(lambda metadata:
 217                           self.failUnlessEqual(set(metadata.keys()),
 218                                                set(["tahoe"])))
 219
 220             d.addCallback(lambda res:
 221                           self.shouldFail(NoSuchChildError, "gcamap-no",
 222                                           "nope",
 223                                           n.get_child_and_metadata_at_path,
 224                                           u"subdir/nope"))
 225             d.addCallback(lambda res:
 226                           n.get_child_and_metadata_at_path(u""))
 227             def _check_child_and_metadata1(res):
 228                 child, metadata = res
 229                 self.failUnless(isinstance(child, dirnode.DirectoryNode))
 230                 # edge-metadata needs at least one path segment
 231                 self.failUnlessEqual(set(metadata.keys()), set([]))
 232             d.addCallback(_check_child_and_metadata1)
 233             d.addCallback(lambda res:
 234                           n.get_child_and_metadata_at_path(u"child"))
 235
 236             def _check_child_and_metadata2(res):
 237                 child, metadata = res
 238                 self.failUnlessReallyEqual(child.get_uri(),
 239                                            fake_file_uri)
 240                 self.failUnlessEqual(set(metadata.keys()), set(["tahoe"]))
 241             d.addCallback(_check_child_and_metadata2)
 242
 243             d.addCallback(lambda res:
 244                           n.get_child_and_metadata_at_path(u"subdir/subsubdir"))
 245             def _check_child_and_metadata3(res):
 246                 child, metadata = res
 247                 self.failUnless(isinstance(child, dirnode.DirectoryNode))
 248                 self.failUnlessEqual(set(metadata.keys()), set(["tahoe"]))
 249             d.addCallback(_check_child_and_metadata3)
 250
 251             # set_uri + metadata
 252             # it should be possible to add a child without any metadata
 253             d.addCallback(lambda res: n.set_uri(u"c2",
 254                                                 fake_file_uri, fake_file_uri,
 255                                                 {}))
 256             d.addCallback(lambda res: n.get_metadata_for(u"c2"))
 257             d.addCallback(lambda metadata:
 258                           self.failUnlessEqual(set(metadata.keys()), set(["tahoe"])))
 259
 260             # You can't override the link timestamps.
 261             d.addCallback(lambda res: n.set_uri(u"c2",
 262                                                 fake_file_uri, fake_file_uri,
 263                                                 { 'tahoe': {'linkcrtime': "bogus"}}))
 264             d.addCallback(lambda res: n.get_metadata_for(u"c2"))
 265             def _has_good_linkcrtime(metadata):
 266                 self.failUnless(metadata.has_key('tahoe'))
 267                 self.failUnless(metadata['tahoe'].has_key('linkcrtime'))
 268                 self.failIfEqual(metadata['tahoe']['linkcrtime'], 'bogus')
 269             d.addCallback(_has_good_linkcrtime)
 270
 271             # if we don't set any defaults, the child should get timestamps
 272             d.addCallback(lambda res: n.set_uri(u"c3",
 273                                                 fake_file_uri, fake_file_uri))
 274             d.addCallback(lambda res: n.get_metadata_for(u"c3"))
 275             d.addCallback(lambda metadata:
 276                           self.failUnlessEqual(set(metadata.keys()), set(["tahoe"])))
 277
 278             # we can also add specific metadata at set_uri() time
 279             d.addCallback(lambda res: n.set_uri(u"c4",
 280                                                 fake_file_uri, fake_file_uri,
 281                                                 {"key": "value"}))
 282             d.addCallback(lambda res: n.get_metadata_for(u"c4"))
 283             d.addCallback(lambda metadata:
 284                               self.failUnless((set(metadata.keys()) == set(["key", "tahoe"])) and
 285                                               (metadata['key'] == "value"), metadata))
 286
 287             d.addCallback(lambda res: n.delete(u"c2"))
 288             d.addCallback(lambda res: n.delete(u"c3"))
 289             d.addCallback(lambda res: n.delete(u"c4"))
 290
 291             # set_node + metadata
 292             # it should be possible to add a child without any metadata except for timestamps
 293             d.addCallback(lambda res: n.set_node(u"d2", n, {}))
 294             d.addCallback(lambda res: c.create_dirnode())
 295             d.addCallback(lambda n2:
 296                           self.shouldFail(ExistingChildError, "set_node-no",
 297                                           "child 'd2' already exists",
 298                                           n.set_node, u"d2", n2,
 299                                           overwrite=False))
 300             d.addCallback(lambda res: n.get_metadata_for(u"d2"))
 301             d.addCallback(lambda metadata:
 302                           self.failUnlessEqual(set(metadata.keys()), set(["tahoe"])))
 303
 304             # if we don't set any defaults, the child should get timestamps
 305             d.addCallback(lambda res: n.set_node(u"d3", n))
 306             d.addCallback(lambda res: n.get_metadata_for(u"d3"))
 307             d.addCallback(lambda metadata:
 308                           self.failUnlessEqual(set(metadata.keys()), set(["tahoe"])))
 309
 310             # we can also add specific metadata at set_node() time
 311             d.addCallback(lambda res: n.set_node(u"d4", n,
 312                                                 {"key": "value"}))
 313             d.addCallback(lambda res: n.get_metadata_for(u"d4"))
 314             d.addCallback(lambda metadata:
 315                           self.failUnless((set(metadata.keys()) == set(["key", "tahoe"])) and
 316                                           (metadata["key"] == "value"), metadata))
 317
 318             d.addCallback(lambda res: n.delete(u"d2"))
 319             d.addCallback(lambda res: n.delete(u"d3"))
 320             d.addCallback(lambda res: n.delete(u"d4"))
 321
 322             # metadata through set_children()
 323             d.addCallback(lambda res:
 324                           n.set_children({
 325                               u"e1": (fake_file_uri, fake_file_uri),
 326                               u"e2": (fake_file_uri, fake_file_uri, {}),
 327                               u"e3": (fake_file_uri, fake_file_uri,
 328                                       {"key": "value"}),
 329                               }))
 330             d.addCallback(lambda n2: self.failUnlessIdentical(n2, n))
 331             d.addCallback(lambda res:
 332                           self.shouldFail(ExistingChildError, "set_children-no",
 333                                           "child 'e1' already exists",
 334                                           n.set_children,
 335                                           { u"e1": (other_file_uri,
 336                                                     other_file_uri),
 337                                             u"new": (other_file_uri,
 338                                                      other_file_uri),
 339                                             },
 340                                           overwrite=False))
 341             # and 'new' should not have been created
 342             d.addCallback(lambda res: n.list())
 343             d.addCallback(lambda children: self.failIf(u"new" in children))
 344             d.addCallback(lambda res: n.get_metadata_for(u"e1"))
 345             d.addCallback(lambda metadata:
 346                           self.failUnlessEqual(set(metadata.keys()), set(["tahoe"])))
 347             d.addCallback(lambda res: n.get_metadata_for(u"e2"))
 348             d.addCallback(lambda metadata:
 349                           self.failUnlessEqual(set(metadata.keys()), set(["tahoe"])))
 350             d.addCallback(lambda res: n.get_metadata_for(u"e3"))
 351             d.addCallback(lambda metadata:
 352                           self.failUnless((set(metadata.keys()) == set(["key", "tahoe"])) and
 353                                           (metadata["key"] == "value"), metadata))
 354
 355             d.addCallback(lambda res: n.delete(u"e1"))
 356             d.addCallback(lambda res: n.delete(u"e2"))
 357             d.addCallback(lambda res: n.delete(u"e3"))
 358
 359             # metadata through set_nodes()
 360             d.addCallback(lambda res:
 361                           n.set_nodes({ u"f1": (n, None),
 362                                         u"f2": (n, {}),
 363                                         u"f3": (n, {"key": "value"}),
 364                                         }))
 365             d.addCallback(lambda n2: self.failUnlessIdentical(n2, n))
 366             d.addCallback(lambda res:
 367                           self.shouldFail(ExistingChildError, "set_nodes-no",
 368                                           "child 'f1' already exists",
 369                                           n.set_nodes, { u"f1": (n, None),
 370                                                          u"new": (n, None), },
 371                                           overwrite=False))
 372             # and 'new' should not have been created
 373             d.addCallback(lambda res: n.list())
 374             d.addCallback(lambda children: self.failIf(u"new" in children))
 375             d.addCallback(lambda res: n.get_metadata_for(u"f1"))
 376             d.addCallback(lambda metadata:
 377                           self.failUnlessEqual(set(metadata.keys()), set(["tahoe"])))
 378             d.addCallback(lambda res: n.get_metadata_for(u"f2"))
 379             d.addCallback(lambda metadata:
 380                           self.failUnlessEqual(set(metadata.keys()), set(["tahoe"])))
 381             d.addCallback(lambda res: n.get_metadata_for(u"f3"))
 382             d.addCallback(lambda metadata:
 383                           self.failUnless((set(metadata.keys()) == set(["key", "tahoe"])) and
 384                                           (metadata["key"] == "value"), metadata))
 385
 386             d.addCallback(lambda res: n.delete(u"f1"))
 387             d.addCallback(lambda res: n.delete(u"f2"))
 388             d.addCallback(lambda res: n.delete(u"f3"))
 389
 390
 391             d.addCallback(lambda res:
 392                           n.set_metadata_for(u"child",
 393                                              {"tags": ["web2.0-compatible"], "tahoe": {"bad": "mojo"}}))
 394             d.addCallback(lambda n1: n1.get_metadata_for(u"child"))
 395             d.addCallback(lambda metadata:
 396                           self.failUnless((set(metadata.keys()) == set(["tags", "tahoe"])) and
 397                                           metadata["tags"] == ["web2.0-compatible"] and
 398                                           "bad" not in metadata["tahoe"], metadata))
 399
 400             d.addCallback(lambda res:
 401                           self.shouldFail(NoSuchChildError, "set_metadata_for-nosuch", "",
 402                                           n.set_metadata_for, u"nosuch", {}))
 403
 404
 405             def _start(res):
 406                 self._start_timestamp = time.time()
 407             d.addCallback(_start)
 408             # simplejson-1.7.1 (as shipped on Ubuntu 'gutsy') rounds all
 409             # floats to hundredeths (it uses str(num) instead of repr(num)).
 410             # simplejson-1.7.3 does not have this bug. To prevent this bug
 411             # from causing the test to fail, stall for more than a few
 412             # hundrededths of a second.
 413             d.addCallback(self.stall, 0.1)
 414             d.addCallback(lambda res: n.add_file(u"timestamps",
 415                                                  upload.Data("stamp me", convergence="some convergence string")))
 416             d.addCallback(self.stall, 0.1)
 417             def _stop(res):
 418                 self._stop_timestamp = time.time()
 419             d.addCallback(_stop)
 420
 421             d.addCallback(lambda res: n.get_metadata_for(u"timestamps"))
 422             def _check_timestamp1(metadata):
 423                 self.failUnlessEqual(set(metadata.keys()), set(["tahoe"]))
 424                 tahoe_md = metadata["tahoe"]
 425                 self.failUnlessEqual(set(tahoe_md.keys()), set(["linkcrtime", "linkmotime"]))
 426
 427                 self.failUnlessGreaterOrEqualThan(tahoe_md["linkcrtime"],
 428                                                   self._start_timestamp)
 429                 self.failUnlessGreaterOrEqualThan(self._stop_timestamp,
 430                                                   tahoe_md["linkcrtime"])
 431                 self.failUnlessGreaterOrEqualThan(tahoe_md["linkmotime"],
 432                                                   self._start_timestamp)
 433                 self.failUnlessGreaterOrEqualThan(self._stop_timestamp,
 434                                                   tahoe_md["linkmotime"])
 435                 # Our current timestamp rules say that replacing an existing
 436                 # child should preserve the 'linkcrtime' but update the
 437                 # 'linkmotime'
 438                 self._old_linkcrtime = tahoe_md["linkcrtime"]
 439                 self._old_linkmotime = tahoe_md["linkmotime"]
 440             d.addCallback(_check_timestamp1)
 441             d.addCallback(self.stall, 2.0) # accomodate low-res timestamps
 442             d.addCallback(lambda res: n.set_node(u"timestamps", n))
 443             d.addCallback(lambda res: n.get_metadata_for(u"timestamps"))
 444             def _check_timestamp2(metadata):
 445                 self.failUnlessIn("tahoe", metadata)
 446                 tahoe_md = metadata["tahoe"]
 447                 self.failUnlessEqual(set(tahoe_md.keys()), set(["linkcrtime", "linkmotime"]))
 448
 449                 self.failUnlessReallyEqual(tahoe_md["linkcrtime"], self._old_linkcrtime)
 450                 self.failUnlessGreaterThan(tahoe_md["linkmotime"], self._old_linkmotime)
 451                 return n.delete(u"timestamps")
 452             d.addCallback(_check_timestamp2)
 453
 454             d.addCallback(lambda res: n.delete(u"subdir"))
 455             d.addCallback(lambda old_child:
 456                           self.failUnlessReallyEqual(old_child.get_uri(),
 457                                                      self.subdir.get_uri()))
 458
 459             d.addCallback(lambda res: n.list())
 460             d.addCallback(lambda children:
 461                           self.failUnlessReallyEqual(set(children.keys()),
 462                                                      set([u"child"])))
 463
 464             uploadable1 = upload.Data("some data", convergence="converge")
 465             d.addCallback(lambda res: n.add_file(u"newfile", uploadable1))
 466             d.addCallback(lambda newnode:
 467                           self.failUnless(IImmutableFileNode.providedBy(newnode)))
 468             uploadable2 = upload.Data("some data", convergence="stuff")
 469             d.addCallback(lambda res:
 470                           self.shouldFail(ExistingChildError, "add_file-no",
 471                                           "child 'newfile' already exists",
 472                                           n.add_file, u"newfile",
 473                                           uploadable2,
 474                                           overwrite=False))
 475             d.addCallback(lambda res: n.list())
 476             d.addCallback(lambda children:
 477                           self.failUnlessReallyEqual(set(children.keys()),
 478                                                      set([u"child", u"newfile"])))
 479             d.addCallback(lambda res: n.get_metadata_for(u"newfile"))
 480             d.addCallback(lambda metadata:
 481                           self.failUnlessEqual(set(metadata.keys()), set(["tahoe"])))
 482
 483             uploadable3 = upload.Data("some data", convergence="converge")
 484             d.addCallback(lambda res: n.add_file(u"newfile-metadata",
 485                                                  uploadable3,
 486                                                  {"key": "value"}))
 487             d.addCallback(lambda newnode:
 488                           self.failUnless(IImmutableFileNode.providedBy(newnode)))
 489             d.addCallback(lambda res: n.get_metadata_for(u"newfile-metadata"))
 490             d.addCallback(lambda metadata:
 491                               self.failUnless((set(metadata.keys()) == set(["key", "tahoe"])) and
 492                                               (metadata['key'] == "value"), metadata))
 493             d.addCallback(lambda res: n.delete(u"newfile-metadata"))
 494
 495             d.addCallback(lambda res: n.create_subdirectory(u"subdir2"))
 496             def _created2(subdir2):
 497                 self.subdir2 = subdir2
 498                 # put something in the way, to make sure it gets overwritten
 499                 return subdir2.add_file(u"child", upload.Data("overwrite me",
 500                                                               "converge"))
 501             d.addCallback(_created2)
 502
 503             d.addCallback(lambda res:
 504                           n.move_child_to(u"child", self.subdir2))
 505             d.addCallback(lambda res: n.list())
 506             d.addCallback(lambda children:
 507                           self.failUnlessReallyEqual(set(children.keys()),
 508                                                      set([u"newfile", u"subdir2"])))
 509             d.addCallback(lambda res: self.subdir2.list())
 510             d.addCallback(lambda children:
 511                           self.failUnlessReallyEqual(set(children.keys()),
 512                                                      set([u"child"])))
 513             d.addCallback(lambda res: self.subdir2.get(u"child"))
 514             d.addCallback(lambda child:
 515                           self.failUnlessReallyEqual(child.get_uri(),
 516                                                      fake_file_uri))
 517
 518             # move it back, using new_child_name=
 519             d.addCallback(lambda res:
 520                           self.subdir2.move_child_to(u"child", n, u"newchild"))
 521             d.addCallback(lambda res: n.list())
 522             d.addCallback(lambda children:
 523                           self.failUnlessReallyEqual(set(children.keys()),
 524                                                      set([u"newchild", u"newfile",
 525                                                           u"subdir2"])))
 526             d.addCallback(lambda res: self.subdir2.list())
 527             d.addCallback(lambda children:
 528                           self.failUnlessReallyEqual(set(children.keys()), set([])))
 529
 530             # now make sure that we honor overwrite=False
 531             d.addCallback(lambda res:
 532                           self.subdir2.set_uri(u"newchild",
 533                                                other_file_uri, other_file_uri))
 534
 535             d.addCallback(lambda res:
 536                           self.shouldFail(ExistingChildError, "move_child_to-no",
 537                                           "child 'newchild' already exists",
 538                                           n.move_child_to, u"newchild",
 539                                           self.subdir2,
 540                                           overwrite=False))
 541             d.addCallback(lambda res: self.subdir2.get(u"newchild"))
 542             d.addCallback(lambda child:
 543                           self.failUnlessReallyEqual(child.get_uri(),
 544                                                      other_file_uri))
 545
 546
 547             # Setting the no-write field should diminish a mutable cap to read-only
 548             # (for both files and directories).
 549
 550             d.addCallback(lambda ign: n.set_uri(u"mutable", other_file_uri, other_file_uri))
 551             d.addCallback(lambda ign: n.get(u"mutable"))
 552             d.addCallback(lambda mutable: self.failIf(mutable.is_readonly(), mutable))
 553             d.addCallback(lambda ign: n.set_metadata_for(u"mutable", {"no-write": True}))
 554             d.addCallback(lambda ign: n.get(u"mutable"))
 555             d.addCallback(lambda mutable: self.failUnless(mutable.is_readonly(), mutable))
 556             d.addCallback(lambda ign: n.set_metadata_for(u"mutable", {"no-write": True}))
 557             d.addCallback(lambda ign: n.get(u"mutable"))
 558             d.addCallback(lambda mutable: self.failUnless(mutable.is_readonly(), mutable))
 559
 560             d.addCallback(lambda ign: n.get(u"subdir2"))
 561             d.addCallback(lambda subdir2: self.failIf(subdir2.is_readonly()))
 562             d.addCallback(lambda ign: n.set_metadata_for(u"subdir2", {"no-write": True}))
 563             d.addCallback(lambda ign: n.get(u"subdir2"))
 564             d.addCallback(lambda subdir2: self.failUnless(subdir2.is_readonly(), subdir2))
 565
 566             d.addCallback(lambda ign: n.set_uri(u"mutable_ro", other_file_uri, other_file_uri,
 567                                                 metadata={"no-write": True}))
 568             d.addCallback(lambda ign: n.get(u"mutable_ro"))
 569             d.addCallback(lambda mutable_ro: self.failUnless(mutable_ro.is_readonly(), mutable_ro))
 570
 571             d.addCallback(lambda ign: n.create_subdirectory(u"subdir_ro", metadata={"no-write": True}))
 572             d.addCallback(lambda ign: n.get(u"subdir_ro"))
 573             d.addCallback(lambda subdir_ro: self.failUnless(subdir_ro.is_readonly(), subdir_ro))
 574
 575             return d
 576
 577         d.addCallback(_then)
 578
 579         d.addErrback(self.explain_error)
 580         return d
 581
 582
 583     def _do_initial_children_test(self, mdmf=False):
 584         c = self.g.clients[0]
 585         nm = c.nodemaker
 586
 587         kids = {one_nfd: (nm.create_from_cap(one_uri), {}),
 588                 u"two": (nm.create_from_cap(setup_py_uri),
 589                          {"metakey": "metavalue"}),
 590                 u"mut": (nm.create_from_cap(mut_write_uri, mut_read_uri), {}),
 591                 u"mdmf": (nm.create_from_cap(mdmf_write_uri, mdmf_read_uri), {}),
 592                 u"fut": (nm.create_from_cap(future_write_uri, future_read_uri), {}),
 593                 u"fro": (nm.create_from_cap(None, future_read_uri), {}),
 594                 u"fut-unic": (nm.create_from_cap(future_nonascii_write_uri, future_nonascii_read_uri), {}),
 595                 u"fro-unic": (nm.create_from_cap(None, future_nonascii_read_uri), {}),
 596                 u"empty_litdir": (nm.create_from_cap(empty_litdir_uri), {}),
 597                 u"tiny_litdir": (nm.create_from_cap(tiny_litdir_uri), {}),
 598                 }
 599         if mdmf:
 600             d = c.create_dirnode(kids, version=MDMF_VERSION)
 601         else:
 602             d = c.create_dirnode(kids)
 603
 604         def _created(dn):
 605             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
 606             backing_node = dn._node
 607             if mdmf:
 608                 self.failUnlessEqual(backing_node.get_version(),
 609                                      MDMF_VERSION)
 610             else:
 611                 self.failUnlessEqual(backing_node.get_version(),
 612                                      SDMF_VERSION)
 613             self.failUnless(dn.is_mutable())
 614             self.failIf(dn.is_readonly())
 615             self.failIf(dn.is_unknown())
 616             self.failIf(dn.is_allowed_in_immutable_directory())
 617             dn.raise_error()
 618             rep = str(dn)
 619             self.failUnless("RW-MUT" in rep)
 620             return dn.list()
 621         d.addCallback(_created)
 622
 623         def _check_kids(children):
 624             self.failUnlessReallyEqual(set(children.keys()),
 625                                        set([one_nfc, u"two", u"mut", u"mdmf", u"fut", u"fro",
 626                                         u"fut-unic", u"fro-unic", u"empty_litdir", u"tiny_litdir"]))
 627             one_node, one_metadata = children[one_nfc]
 628             two_node, two_metadata = children[u"two"]
 629             mut_node, mut_metadata = children[u"mut"]
 630             mdmf_node, mdmf_metadata = children[u"mdmf"]
 631             fut_node, fut_metadata = children[u"fut"]
 632             fro_node, fro_metadata = children[u"fro"]
 633             futna_node, futna_metadata = children[u"fut-unic"]
 634             frona_node, frona_metadata = children[u"fro-unic"]
 635             emptylit_node, emptylit_metadata = children[u"empty_litdir"]
 636             tinylit_node, tinylit_metadata = children[u"tiny_litdir"]
 637
 638             self.failUnlessReallyEqual(one_node.get_size(), 3)
 639             self.failUnlessReallyEqual(one_node.get_uri(), one_uri)
 640             self.failUnlessReallyEqual(one_node.get_readonly_uri(), one_uri)
 641             self.failUnless(isinstance(one_metadata, dict), one_metadata)
 642
 643             self.failUnlessReallyEqual(two_node.get_size(), 14861)
 644             self.failUnlessReallyEqual(two_node.get_uri(), setup_py_uri)
 645             self.failUnlessReallyEqual(two_node.get_readonly_uri(), setup_py_uri)
 646             self.failUnlessEqual(two_metadata["metakey"], "metavalue")
 647
 648             self.failUnlessReallyEqual(mut_node.get_uri(), mut_write_uri)
 649             self.failUnlessReallyEqual(mut_node.get_readonly_uri(), mut_read_uri)
 650             self.failUnless(isinstance(mut_metadata, dict), mut_metadata)
 651
 652             self.failUnlessReallyEqual(mdmf_node.get_uri(), mdmf_write_uri)
 653             self.failUnlessReallyEqual(mdmf_node.get_readonly_uri(), mdmf_read_uri)
 654             self.failUnless(isinstance(mdmf_metadata, dict), mdmf_metadata)
 655
 656             self.failUnless(fut_node.is_unknown())
 657             self.failUnlessReallyEqual(fut_node.get_uri(), future_write_uri)
 658             self.failUnlessReallyEqual(fut_node.get_readonly_uri(), "ro." + future_read_uri)
 659             self.failUnless(isinstance(fut_metadata, dict), fut_metadata)
 660
 661             self.failUnless(futna_node.is_unknown())
 662             self.failUnlessReallyEqual(futna_node.get_uri(), future_nonascii_write_uri)
 663             self.failUnlessReallyEqual(futna_node.get_readonly_uri(), "ro." + future_nonascii_read_uri)
 664             self.failUnless(isinstance(futna_metadata, dict), futna_metadata)
 665
 666             self.failUnless(fro_node.is_unknown())
 667             self.failUnlessReallyEqual(fro_node.get_uri(), "ro." + future_read_uri)
 668             self.failUnlessReallyEqual(fut_node.get_readonly_uri(), "ro." + future_read_uri)
 669             self.failUnless(isinstance(fro_metadata, dict), fro_metadata)
 670
 671             self.failUnless(frona_node.is_unknown())
 672             self.failUnlessReallyEqual(frona_node.get_uri(), "ro." + future_nonascii_read_uri)
 673             self.failUnlessReallyEqual(futna_node.get_readonly_uri(), "ro." + future_nonascii_read_uri)
 674             self.failUnless(isinstance(frona_metadata, dict), frona_metadata)
 675
 676             self.failIf(emptylit_node.is_unknown())
 677             self.failUnlessReallyEqual(emptylit_node.get_storage_index(), None)
 678             self.failIf(tinylit_node.is_unknown())
 679             self.failUnlessReallyEqual(tinylit_node.get_storage_index(), None)
 680
 681             d2 = defer.succeed(None)
 682             d2.addCallback(lambda ignored: emptylit_node.list())
 683             d2.addCallback(lambda children: self.failUnlessEqual(children, {}))
 684             d2.addCallback(lambda ignored: tinylit_node.list())
 685             d2.addCallback(lambda children: self.failUnlessReallyEqual(set(children.keys()),
 686                                                                        set([u"short"])))
 687             d2.addCallback(lambda ignored: tinylit_node.list())
 688             d2.addCallback(lambda children: children[u"short"][0].read(MemAccum()))
 689             d2.addCallback(lambda accum: self.failUnlessReallyEqual(accum.data, "The end."))
 690             return d2
 691         d.addCallback(_check_kids)
 692
 693         d.addCallback(lambda ign: nm.create_new_mutable_directory(kids))
 694         d.addCallback(lambda dn: dn.list())
 695         d.addCallback(_check_kids)
 696
 697         bad_future_node = UnknownNode(future_write_uri, None)
 698         bad_kids1 = {one_nfd: (bad_future_node, {})}
 699         # This should fail because we don't know how to diminish the future_write_uri
 700         # cap (given in a write slot and not prefixed with "ro." or "imm.") to a readcap.
 701         d.addCallback(lambda ign:
 702                       self.shouldFail(MustNotBeUnknownRWError, "bad_kids1",
 703                                       "cannot attach unknown",
 704                                       nm.create_new_mutable_directory,
 705                                       bad_kids1))
 706         bad_kids2 = {one_nfd: (nm.create_from_cap(one_uri), None)}
 707         d.addCallback(lambda ign:
 708                       self.shouldFail(AssertionError, "bad_kids2",
 709                                       "requires metadata to be a dict",
 710                                       nm.create_new_mutable_directory,
 711                                       bad_kids2))
 712         return d
 713
 714     def _do_basic_test(self, mdmf=False):
 715         c = self.g.clients[0]
 716         d = None
 717         if mdmf:
 718             d = c.create_dirnode(version=MDMF_VERSION)
 719         else:
 720             d = c.create_dirnode()
 721         def _done(res):
 722             self.failUnless(isinstance(res, dirnode.DirectoryNode))
 723             self.failUnless(res.is_mutable())
 724             self.failIf(res.is_readonly())
 725             self.failIf(res.is_unknown())
 726             self.failIf(res.is_allowed_in_immutable_directory())
 727             res.raise_error()
 728             rep = str(res)
 729             self.failUnless("RW-MUT" in rep)
 730         d.addCallback(_done)
 731         return d
 732
 733     def test_basic(self):
 734         self.basedir = "dirnode/Dirnode/test_basic"
 735         self.set_up_grid()
 736         return self._do_basic_test()
 737
 738     def test_basic_mdmf(self):
 739         self.basedir = "dirnode/Dirnode/test_basic_mdmf"
 740         self.set_up_grid()
 741         return self._do_basic_test(mdmf=True)
 742
 743     def test_initial_children(self):
 744         self.basedir = "dirnode/Dirnode/test_initial_children"
 745         self.set_up_grid()
 746         return self._do_initial_children_test()
 747
 748     def test_immutable(self):
 749         self.basedir = "dirnode/Dirnode/test_immutable"
 750         self.set_up_grid()
 751         c = self.g.clients[0]
 752         nm = c.nodemaker
 753
 754         kids = {one_nfd: (nm.create_from_cap(one_uri), {}),
 755                 u"two": (nm.create_from_cap(setup_py_uri),
 756                          {"metakey": "metavalue"}),
 757                 u"fut": (nm.create_from_cap(None, future_read_uri), {}),
 758                 u"futna": (nm.create_from_cap(None, future_nonascii_read_uri), {}),
 759                 u"empty_litdir": (nm.create_from_cap(empty_litdir_uri), {}),
 760                 u"tiny_litdir": (nm.create_from_cap(tiny_litdir_uri), {}),
 761                 }
 762         d = c.create_immutable_dirnode(kids)
 763
 764         def _created(dn):
 765             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
 766             self.failIf(dn.is_mutable())
 767             self.failUnless(dn.is_readonly())
 768             self.failIf(dn.is_unknown())
 769             self.failUnless(dn.is_allowed_in_immutable_directory())
 770             dn.raise_error()
 771             rep = str(dn)
 772             self.failUnless("RO-IMM" in rep)
 773             cap = dn.get_cap()
 774             self.failUnlessIn("CHK", cap.to_string())
 775             self.cap = cap
 776             return dn.list()
 777         d.addCallback(_created)
 778
 779         def _check_kids(children):
 780             self.failUnlessReallyEqual(set(children.keys()),
 781                                        set([one_nfc, u"two", u"fut", u"futna", u"empty_litdir", u"tiny_litdir"]))
 782             one_node, one_metadata = children[one_nfc]
 783             two_node, two_metadata = children[u"two"]
 784             fut_node, fut_metadata = children[u"fut"]
 785             futna_node, futna_metadata = children[u"futna"]
 786             emptylit_node, emptylit_metadata = children[u"empty_litdir"]
 787             tinylit_node, tinylit_metadata = children[u"tiny_litdir"]
 788
 789             self.failUnlessReallyEqual(one_node.get_size(), 3)
 790             self.failUnlessReallyEqual(one_node.get_uri(), one_uri)
 791             self.failUnlessReallyEqual(one_node.get_readonly_uri(), one_uri)
 792             self.failUnless(isinstance(one_metadata, dict), one_metadata)
 793
 794             self.failUnlessReallyEqual(two_node.get_size(), 14861)
 795             self.failUnlessReallyEqual(two_node.get_uri(), setup_py_uri)
 796             self.failUnlessReallyEqual(two_node.get_readonly_uri(), setup_py_uri)
 797             self.failUnlessEqual(two_metadata["metakey"], "metavalue")
 798
 799             self.failUnless(fut_node.is_unknown())
 800             self.failUnlessReallyEqual(fut_node.get_uri(), "imm." + future_read_uri)
 801             self.failUnlessReallyEqual(fut_node.get_readonly_uri(), "imm." + future_read_uri)
 802             self.failUnless(isinstance(fut_metadata, dict), fut_metadata)
 803
 804             self.failUnless(futna_node.is_unknown())
 805             self.failUnlessReallyEqual(futna_node.get_uri(), "imm." + future_nonascii_read_uri)
 806             self.failUnlessReallyEqual(futna_node.get_readonly_uri(), "imm." + future_nonascii_read_uri)
 807             self.failUnless(isinstance(futna_metadata, dict), futna_metadata)
 808
 809             self.failIf(emptylit_node.is_unknown())
 810             self.failUnlessReallyEqual(emptylit_node.get_storage_index(), None)
 811             self.failIf(tinylit_node.is_unknown())
 812             self.failUnlessReallyEqual(tinylit_node.get_storage_index(), None)
 813
 814             d2 = defer.succeed(None)
 815             d2.addCallback(lambda ignored: emptylit_node.list())
 816             d2.addCallback(lambda children: self.failUnlessEqual(children, {}))
 817             d2.addCallback(lambda ignored: tinylit_node.list())
 818             d2.addCallback(lambda children: self.failUnlessReallyEqual(set(children.keys()),
 819                                                                        set([u"short"])))
 820             d2.addCallback(lambda ignored: tinylit_node.list())
 821             d2.addCallback(lambda children: children[u"short"][0].read(MemAccum()))
 822             d2.addCallback(lambda accum: self.failUnlessReallyEqual(accum.data, "The end."))
 823             return d2
 824
 825         d.addCallback(_check_kids)
 826
 827         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
 828         d.addCallback(lambda dn: dn.list())
 829         d.addCallback(_check_kids)
 830
 831         bad_future_node1 = UnknownNode(future_write_uri, None)
 832         bad_kids1 = {one_nfd: (bad_future_node1, {})}
 833         d.addCallback(lambda ign:
 834                       self.shouldFail(MustNotBeUnknownRWError, "bad_kids1",
 835                                       "cannot attach unknown",
 836                                       c.create_immutable_dirnode,
 837                                       bad_kids1))
 838         bad_future_node2 = UnknownNode(future_write_uri, future_read_uri)
 839         bad_kids2 = {one_nfd: (bad_future_node2, {})}
 840         d.addCallback(lambda ign:
 841                       self.shouldFail(MustBeDeepImmutableError, "bad_kids2",
 842                                       "is not allowed in an immutable directory",
 843                                       c.create_immutable_dirnode,
 844                                       bad_kids2))
 845         bad_kids3 = {one_nfd: (nm.create_from_cap(one_uri), None)}
 846         d.addCallback(lambda ign:
 847                       self.shouldFail(AssertionError, "bad_kids3",
 848                                       "requires metadata to be a dict",
 849                                       c.create_immutable_dirnode,
 850                                       bad_kids3))
 851         bad_kids4 = {one_nfd: (nm.create_from_cap(mut_write_uri), {})}
 852         d.addCallback(lambda ign:
 853                       self.shouldFail(MustBeDeepImmutableError, "bad_kids4",
 854                                       "is not allowed in an immutable directory",
 855                                       c.create_immutable_dirnode,
 856                                       bad_kids4))
 857         bad_kids5 = {one_nfd: (nm.create_from_cap(mut_read_uri), {})}
 858         d.addCallback(lambda ign:
 859                       self.shouldFail(MustBeDeepImmutableError, "bad_kids5",
 860                                       "is not allowed in an immutable directory",
 861                                       c.create_immutable_dirnode,
 862                                       bad_kids5))
 863         bad_kids6 = {one_nfd: (nm.create_from_cap(mdmf_write_uri), {})}
 864         d.addCallback(lambda ign:
 865                       self.shouldFail(MustBeDeepImmutableError, "bad_kids6",
 866                                       "is not allowed in an immutable directory",
 867                                       c.create_immutable_dirnode,
 868                                       bad_kids6))
 869         bad_kids7 = {one_nfd: (nm.create_from_cap(mdmf_read_uri), {})}
 870         d.addCallback(lambda ign:
 871                       self.shouldFail(MustBeDeepImmutableError, "bad_kids7",
 872                                       "is not allowed in an immutable directory",
 873                                       c.create_immutable_dirnode,
 874                                       bad_kids7))
 875         d.addCallback(lambda ign: c.create_immutable_dirnode({}))
 876         def _created_empty(dn):
 877             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
 878             self.failIf(dn.is_mutable())
 879             self.failUnless(dn.is_readonly())
 880             self.failIf(dn.is_unknown())
 881             self.failUnless(dn.is_allowed_in_immutable_directory())
 882             dn.raise_error()
 883             rep = str(dn)
 884             self.failUnless("RO-IMM" in rep)
 885             cap = dn.get_cap()
 886             self.failUnlessIn("LIT", cap.to_string())
 887             self.failUnlessReallyEqual(cap.to_string(), "URI:DIR2-LIT:")
 888             self.cap = cap
 889             return dn.list()
 890         d.addCallback(_created_empty)
 891         d.addCallback(lambda kids: self.failUnlessEqual(kids, {}))
 892         smallkids = {u"o": (nm.create_from_cap(one_uri), {})}
 893         d.addCallback(lambda ign: c.create_immutable_dirnode(smallkids))
 894         def _created_small(dn):
 895             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
 896             self.failIf(dn.is_mutable())
 897             self.failUnless(dn.is_readonly())
 898             self.failIf(dn.is_unknown())
 899             self.failUnless(dn.is_allowed_in_immutable_directory())
 900             dn.raise_error()
 901             rep = str(dn)
 902             self.failUnless("RO-IMM" in rep)
 903             cap = dn.get_cap()
 904             self.failUnlessIn("LIT", cap.to_string())
 905             self.failUnlessReallyEqual(cap.to_string(),
 906                                        "URI:DIR2-LIT:gi4tumj2n4wdcmz2kvjesosmjfkdu3rvpbtwwlbqhiwdeot3puwcy")
 907             self.cap = cap
 908             return dn.list()
 909         d.addCallback(_created_small)
 910         d.addCallback(lambda kids: self.failUnlessReallyEqual(kids.keys(), [u"o"]))
 911
 912         # now test n.create_subdirectory(mutable=False)
 913         d.addCallback(lambda ign: c.create_dirnode())
 914         def _made_parent(n):
 915             d = n.create_subdirectory(u"subdir", kids, mutable=False)
 916             d.addCallback(lambda sd: sd.list())
 917             d.addCallback(_check_kids)
 918             d.addCallback(lambda ign: n.list())
 919             d.addCallback(lambda children:
 920                           self.failUnlessReallyEqual(children.keys(), [u"subdir"]))
 921             d.addCallback(lambda ign: n.get(u"subdir"))
 922             d.addCallback(lambda sd: sd.list())
 923             d.addCallback(_check_kids)
 924             d.addCallback(lambda ign: n.get(u"subdir"))
 925             d.addCallback(lambda sd: self.failIf(sd.is_mutable()))
 926             bad_kids8 = {one_nfd: (nm.create_from_cap(mut_write_uri), {})}
 927             d.addCallback(lambda ign:
 928                           self.shouldFail(MustBeDeepImmutableError, "bad_kids8",
 929                                           "is not allowed in an immutable directory",
 930                                           n.create_subdirectory,
 931                                           u"sub2", bad_kids8, mutable=False))
 932             bad_kids9 = {one_nfd: (nm.create_from_cap(mdmf_write_uri), {})}
 933             d.addCallback(lambda ign:
 934                           self.shouldFail(MustBeDeepImmutableError, "bad_kids9",
 935                                           "is not allowed in an immutable directory",
 936                                           n.create_subdirectory,
 937                                           u"sub2", bad_kids9, mutable=False))
 938             return d
 939         d.addCallback(_made_parent)
 940         return d
 941
 942     def test_directory_representation(self):
 943         self.basedir = "dirnode/Dirnode/test_directory_representation"
 944         self.set_up_grid()
 945         c = self.g.clients[0]
 946         nm = c.nodemaker
 947
 948         # This test checks that any trailing spaces in URIs are retained in the
 949         # encoded directory, but stripped when we get them out of the directory.
 950         # See ticket #925 for why we want that.
 951         # It also tests that we store child names as UTF-8 NFC, and normalize
 952         # them again when retrieving them.
 953
 954         stripped_write_uri = "lafs://from_the_future\t"
 955         stripped_read_uri = "lafs://readonly_from_the_future\t"
 956         spacedout_write_uri = stripped_write_uri + "  "
 957         spacedout_read_uri = stripped_read_uri + "  "
 958
 959         child = nm.create_from_cap(spacedout_write_uri, spacedout_read_uri)
 960         self.failUnlessReallyEqual(child.get_write_uri(), spacedout_write_uri)
 961         self.failUnlessReallyEqual(child.get_readonly_uri(), "ro." + spacedout_read_uri)
 962
 963         child_dottedi = u"ch\u0131\u0307ld"
 964
 965         kids_in   = {child_dottedi: (child, {}), one_nfd: (child, {})}
 966         kids_out  = {child_dottedi: (child, {}), one_nfc: (child, {})}
 967         kids_norm = {u"child":      (child, {}), one_nfc: (child, {})}
 968         d = c.create_dirnode(kids_in)
 969
 970         def _created(dn):
 971             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
 972             self.failUnless(dn.is_mutable())
 973             self.failIf(dn.is_readonly())
 974             dn.raise_error()
 975             self.cap = dn.get_cap()
 976             self.rootnode = dn
 977             return dn._node.download_best_version()
 978         d.addCallback(_created)
 979
 980         def _check_data(data):
 981             # Decode the netstring representation of the directory to check that the
 982             # spaces are retained when the URIs are stored, and that the names are stored
 983             # as NFC.
 984             position = 0
 985             numkids = 0
 986             while position < len(data):
 987                 entries, position = split_netstring(data, 1, position)
 988                 entry = entries[0]
 989                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
 990                 name = name_utf8.decode("utf-8")
 991                 rw_uri = self.rootnode._decrypt_rwcapdata(rwcapdata)
 992                 self.failUnlessIn(name, kids_out)
 993                 (expected_child, ign) = kids_out[name]
 994                 self.failUnlessReallyEqual(rw_uri, expected_child.get_write_uri())
 995                 self.failUnlessReallyEqual("ro." + ro_uri, expected_child.get_readonly_uri())
 996                 numkids += 1
 997
 998             self.failUnlessReallyEqual(numkids, len(kids_out))
 999             return self.rootnode
1000         d.addCallback(_check_data)
1001
1002         # Mock up a hypothetical future version of Unicode that adds a canonical equivalence
1003         # between dotless-i + dot-above, and 'i'. That would actually be prohibited by the
1004         # stability rules, but similar additions involving currently-unassigned characters
1005         # would not be.
1006         old_normalize = unicodedata.normalize
1007         def future_normalize(form, s):
1008             assert form == 'NFC', form
1009             return old_normalize(form, s).replace(u"\u0131\u0307", u"i")
1010
1011         def _list(node):
1012             unicodedata.normalize = future_normalize
1013             d2 = node.list()
1014             def _undo_mock(res):
1015                 unicodedata.normalize = old_normalize
1016                 return res
1017             d2.addBoth(_undo_mock)
1018             return d2
1019         d.addCallback(_list)
1020
1021         def _check_kids(children):
1022             # Now when we use the real directory listing code, the trailing spaces
1023             # should have been stripped (and "ro." should have been prepended to the
1024             # ro_uri, since it's unknown). Also the dotless-i + dot-above should have been
1025             # normalized to 'i'.
1026
1027             self.failUnlessReallyEqual(set(children.keys()), set(kids_norm.keys()))
1028             child_node, child_metadata = children[u"child"]
1029
1030             self.failUnlessReallyEqual(child_node.get_write_uri(), stripped_write_uri)
1031             self.failUnlessReallyEqual(child_node.get_readonly_uri(), "ro." + stripped_read_uri)
1032         d.addCallback(_check_kids)
1033
1034         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
1035         d.addCallback(_list)
1036         d.addCallback(_check_kids)  # again with dirnode recreated from cap
1037         return d
1038
1039     def test_check(self):
1040         self.basedir = "dirnode/Dirnode/test_check"
1041         self.set_up_grid()
1042         c = self.g.clients[0]
1043         d = c.create_dirnode()
1044         d.addCallback(lambda dn: dn.check(Monitor()))
1045         def _done(res):
1046             self.failUnless(res.is_healthy())
1047         d.addCallback(_done)
1048         return d
1049
1050     def _test_deepcheck_create(self, version=SDMF_VERSION):
1051         # create a small tree with a loop, and some non-directories
1052         #  root/
1053         #  root/subdir/
1054         #  root/subdir/file1
1055         #  root/subdir/link -> root
1056         #  root/rodir
1057         c = self.g.clients[0]
1058         d = c.create_dirnode(version=version)
1059         def _created_root(rootnode):
1060             self._rootnode = rootnode
1061             self.failUnlessEqual(rootnode._node.get_version(), version)
1062             return rootnode.create_subdirectory(u"subdir")
1063         d.addCallback(_created_root)
1064         def _created_subdir(subdir):
1065             self._subdir = subdir
1066             d = subdir.add_file(u"file1", upload.Data("data"*100, None))
1067             d.addCallback(lambda res: subdir.set_node(u"link", self._rootnode))
1068             d.addCallback(lambda res: c.create_dirnode())
1069             d.addCallback(lambda dn:
1070                           self._rootnode.set_uri(u"rodir",
1071                                                  dn.get_uri(),
1072                                                  dn.get_readonly_uri()))
1073             return d
1074         d.addCallback(_created_subdir)
1075         def _done(res):
1076             return self._rootnode
1077         d.addCallback(_done)
1078         return d
1079
1080     def test_deepcheck(self):
1081         self.basedir = "dirnode/Dirnode/test_deepcheck"
1082         self.set_up_grid()
1083         d = self._test_deepcheck_create()
1084         d.addCallback(lambda rootnode: rootnode.start_deep_check().when_done())
1085         def _check_results(r):
1086             self.failUnless(IDeepCheckResults.providedBy(r))
1087             c = r.get_counters()
1088             self.failUnlessReallyEqual(c,
1089                                        {"count-objects-checked": 4,
1090                                         "count-objects-healthy": 4,
1091                                         "count-objects-unhealthy": 0,
1092                                         "count-objects-unrecoverable": 0,
1093                                         "count-corrupt-shares": 0,
1094                                         })
1095             self.failIf(r.get_corrupt_shares())
1096             self.failUnlessReallyEqual(len(r.get_all_results()), 4)
1097         d.addCallback(_check_results)
1098         return d
1099
1100     def test_deepcheck_cachemisses(self):
1101         self.basedir = "dirnode/Dirnode/test_mdmf_cachemisses"
1102         self.set_up_grid()
1103         d = self._test_deepcheck_create()
1104         # Clear the counters and set the rootnode
1105         d.addCallback(lambda rootnode:
1106                       not [ss._clear_counters() for ss
1107                            in self.g.wrappers_by_id.values()] or rootnode)
1108         d.addCallback(lambda rootnode: rootnode.start_deep_check().when_done())
1109         def _check(ign):
1110             count = sum([ss.counter_by_methname['slot_readv']
1111                          for ss in self.g.wrappers_by_id.values()])
1112             self.failIf(count > 60, 'Expected only 60 cache misses,'
1113                                     'unfortunately there were %d' % (count,))
1114         d.addCallback(_check)
1115         return d
1116
1117     def test_deepcheck_mdmf(self):
1118         self.basedir = "dirnode/Dirnode/test_deepcheck_mdmf"
1119         self.set_up_grid()
1120         d = self._test_deepcheck_create(MDMF_VERSION)
1121         d.addCallback(lambda rootnode: rootnode.start_deep_check().when_done())
1122         def _check_results(r):
1123             self.failUnless(IDeepCheckResults.providedBy(r))
1124             c = r.get_counters()
1125             self.failUnlessReallyEqual(c,
1126                                        {"count-objects-checked": 4,
1127                                         "count-objects-healthy": 4,
1128                                         "count-objects-unhealthy": 0,
1129                                         "count-objects-unrecoverable": 0,
1130                                         "count-corrupt-shares": 0,
1131                                         })
1132             self.failIf(r.get_corrupt_shares())
1133             self.failUnlessReallyEqual(len(r.get_all_results()), 4)
1134         d.addCallback(_check_results)
1135         return d
1136
1137     def test_deepcheck_and_repair(self):
1138         self.basedir = "dirnode/Dirnode/test_deepcheck_and_repair"
1139         self.set_up_grid()
1140         d = self._test_deepcheck_create()
1141         d.addCallback(lambda rootnode:
1142                       rootnode.start_deep_check_and_repair().when_done())
1143         def _check_results(r):
1144             self.failUnless(IDeepCheckAndRepairResults.providedBy(r))
1145             c = r.get_counters()
1146             self.failUnlessReallyEqual(c,
1147                                        {"count-objects-checked": 4,
1148                                         "count-objects-healthy-pre-repair": 4,
1149                                         "count-objects-unhealthy-pre-repair": 0,
1150                                         "count-objects-unrecoverable-pre-repair": 0,
1151                                         "count-corrupt-shares-pre-repair": 0,
1152                                         "count-objects-healthy-post-repair": 4,
1153                                         "count-objects-unhealthy-post-repair": 0,
1154                                         "count-objects-unrecoverable-post-repair": 0,
1155                                         "count-corrupt-shares-post-repair": 0,
1156                                         "count-repairs-attempted": 0,
1157                                         "count-repairs-successful": 0,
1158                                         "count-repairs-unsuccessful": 0,
1159                                         })
1160             self.failIf(r.get_corrupt_shares())
1161             self.failIf(r.get_remaining_corrupt_shares())
1162             self.failUnlessReallyEqual(len(r.get_all_results()), 4)
1163         d.addCallback(_check_results)
1164         return d
1165
1166     def test_deepcheck_and_repair_mdmf(self):
1167         self.basedir = "dirnode/Dirnode/test_deepcheck_and_repair_mdmf"
1168         self.set_up_grid()
1169         d = self._test_deepcheck_create(version=MDMF_VERSION)
1170         d.addCallback(lambda rootnode:
1171                       rootnode.start_deep_check_and_repair().when_done())
1172         def _check_results(r):
1173             self.failUnless(IDeepCheckAndRepairResults.providedBy(r))
1174             c = r.get_counters()
1175             self.failUnlessReallyEqual(c,
1176                                        {"count-objects-checked": 4,
1177                                         "count-objects-healthy-pre-repair": 4,
1178                                         "count-objects-unhealthy-pre-repair": 0,
1179                                         "count-objects-unrecoverable-pre-repair": 0,
1180                                         "count-corrupt-shares-pre-repair": 0,
1181                                         "count-objects-healthy-post-repair": 4,
1182                                         "count-objects-unhealthy-post-repair": 0,
1183                                         "count-objects-unrecoverable-post-repair": 0,
1184                                         "count-corrupt-shares-post-repair": 0,
1185                                         "count-repairs-attempted": 0,
1186                                         "count-repairs-successful": 0,
1187                                         "count-repairs-unsuccessful": 0,
1188                                         })
1189             self.failIf(r.get_corrupt_shares())
1190             self.failIf(r.get_remaining_corrupt_shares())
1191             self.failUnlessReallyEqual(len(r.get_all_results()), 4)
1192         d.addCallback(_check_results)
1193         return d
1194
1195     def _mark_file_bad(self, rootnode):
1196         self.delete_shares_numbered(rootnode.get_uri(), [0])
1197         return rootnode
1198
1199     def test_deepcheck_problems(self):
1200         self.basedir = "dirnode/Dirnode/test_deepcheck_problems"
1201         self.set_up_grid()
1202         d = self._test_deepcheck_create()
1203         d.addCallback(lambda rootnode: self._mark_file_bad(rootnode))
1204         d.addCallback(lambda rootnode: rootnode.start_deep_check().when_done())
1205         def _check_results(r):
1206             c = r.get_counters()
1207             self.failUnlessReallyEqual(c,
1208                                        {"count-objects-checked": 4,
1209                                         "count-objects-healthy": 3,
1210                                         "count-objects-unhealthy": 1,
1211                                         "count-objects-unrecoverable": 0,
1212                                         "count-corrupt-shares": 0,
1213                                         })
1214             #self.failUnlessReallyEqual(len(r.get_problems()), 1) # TODO
1215         d.addCallback(_check_results)
1216         return d
1217
1218     def test_deepcheck_problems_mdmf(self):
1219         self.basedir = "dirnode/Dirnode/test_deepcheck_problems_mdmf"
1220         self.set_up_grid()
1221         d = self._test_deepcheck_create(version=MDMF_VERSION)
1222         d.addCallback(lambda rootnode: self._mark_file_bad(rootnode))
1223         d.addCallback(lambda rootnode: rootnode.start_deep_check().when_done())
1224         def _check_results(r):
1225             c = r.get_counters()
1226             self.failUnlessReallyEqual(c,
1227                                        {"count-objects-checked": 4,
1228                                         "count-objects-healthy": 3,
1229                                         "count-objects-unhealthy": 1,
1230                                         "count-objects-unrecoverable": 0,
1231                                         "count-corrupt-shares": 0,
1232                                         })
1233             #self.failUnlessReallyEqual(len(r.get_problems()), 1) # TODO
1234         d.addCallback(_check_results)
1235         return d
1236
1237     def _do_readonly_test(self, version=SDMF_VERSION):
1238         c = self.g.clients[0]
1239         nm = c.nodemaker
1240         filecap = make_chk_file_uri(1234)
1241         filenode = nm.create_from_cap(filecap)
1242         uploadable = upload.Data("some data", convergence="some convergence string")
1243
1244         d = c.create_dirnode(version=version)
1245         def _created(rw_dn):
1246             backing_node = rw_dn._node
1247             self.failUnlessEqual(backing_node.get_version(), version)
1248             d2 = rw_dn.set_uri(u"child", filecap, filecap)
1249             d2.addCallback(lambda res: rw_dn)
1250             return d2
1251         d.addCallback(_created)
1252
1253         def _ready(rw_dn):
1254             ro_uri = rw_dn.get_readonly_uri()
1255             ro_dn = c.create_node_from_uri(ro_uri)
1256             self.failUnless(ro_dn.is_readonly())
1257             self.failUnless(ro_dn.is_mutable())
1258             self.failIf(ro_dn.is_unknown())
1259             self.failIf(ro_dn.is_allowed_in_immutable_directory())
1260             ro_dn.raise_error()
1261
1262             self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
1263                             ro_dn.set_uri, u"newchild", filecap, filecap)
1264             self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
1265                             ro_dn.set_node, u"newchild", filenode)
1266             self.shouldFail(dirnode.NotWriteableError, "set_nodes ro", None,
1267                             ro_dn.set_nodes, { u"newchild": (filenode, None) })
1268             self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
1269                             ro_dn.add_file, u"newchild", uploadable)
1270             self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
1271                             ro_dn.delete, u"child")
1272             self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
1273                             ro_dn.create_subdirectory, u"newchild")
1274             self.shouldFail(dirnode.NotWriteableError, "set_metadata_for ro", None,
1275                             ro_dn.set_metadata_for, u"child", {})
1276             self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
1277                             ro_dn.move_child_to, u"child", rw_dn)
1278             self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
1279                             rw_dn.move_child_to, u"child", ro_dn)
1280             return ro_dn.list()
1281         d.addCallback(_ready)
1282         def _listed(children):
1283             self.failUnless(u"child" in children)
1284         d.addCallback(_listed)
1285         return d
1286
1287     def test_readonly(self):
1288         self.basedir = "dirnode/Dirnode/test_readonly"
1289         self.set_up_grid()
1290         return self._do_readonly_test()
1291
1292     def test_readonly_mdmf(self):
1293         self.basedir = "dirnode/Dirnode/test_readonly_mdmf"
1294         self.set_up_grid()
1295         return self._do_readonly_test(version=MDMF_VERSION)
1296
1297     def failUnlessGreaterThan(self, a, b):
1298         self.failUnless(a > b, "%r should be > %r" % (a, b))
1299
1300     def failUnlessGreaterOrEqualThan(self, a, b):
1301         self.failUnless(a >= b, "%r should be >= %r" % (a, b))
1302
1303     def test_create(self):
1304         self.basedir = "dirnode/Dirnode/test_create"
1305         self.set_up_grid()
1306         return self._do_create_test()
1307
1308     def test_update_metadata(self):
1309         (t1, t2, t3) = (626644800.0, 634745640.0, 892226160.0)
1310
1311         md1 = dirnode.update_metadata({"ctime": t1}, {}, t2)
1312         self.failUnlessEqual(md1, {"tahoe":{"linkcrtime": t1, "linkmotime": t2}})
1313
1314         md2 = dirnode.update_metadata(md1, {"key": "value", "tahoe": {"bad": "mojo"}}, t3)
1315         self.failUnlessEqual(md2, {"key": "value",
1316                                    "tahoe":{"linkcrtime": t1, "linkmotime": t3}})
1317
1318         md3 = dirnode.update_metadata({}, None, t3)
1319         self.failUnlessEqual(md3, {"tahoe":{"linkcrtime": t3, "linkmotime": t3}})
1320
1321         md4 = dirnode.update_metadata({}, {"bool": True, "number": 42}, t1)
1322         self.failUnlessEqual(md4, {"bool": True, "number": 42,
1323                                    "tahoe":{"linkcrtime": t1, "linkmotime": t1}})
1324
1325     def _do_create_subdirectory_test(self, version=SDMF_VERSION):
1326         c = self.g.clients[0]
1327         nm = c.nodemaker
1328
1329         d = c.create_dirnode(version=version)
1330         def _then(n):
1331             # /
1332             self.rootnode = n
1333             fake_file_uri = make_mutable_file_uri()
1334             other_file_uri = make_mutable_file_uri()
1335             md = {"metakey": "metavalue"}
1336             kids = {u"kid1": (nm.create_from_cap(fake_file_uri), {}),
1337                     u"kid2": (nm.create_from_cap(other_file_uri), md),
1338                     }
1339             d = n.create_subdirectory(u"subdir", kids,
1340                                       mutable_version=version)
1341             def _check(sub):
1342                 d = n.get_child_at_path(u"subdir")
1343                 d.addCallback(lambda sub2: self.failUnlessReallyEqual(sub2.get_uri(),
1344                                                                       sub.get_uri()))
1345                 d.addCallback(lambda ign: sub.list())
1346                 return d
1347             d.addCallback(_check)
1348             def _check_kids(kids2):
1349                 self.failUnlessEqual(set(kids.keys()), set(kids2.keys()))
1350                 self.failUnlessEqual(kids2[u"kid2"][1]["metakey"], "metavalue")
1351             d.addCallback(_check_kids)
1352             return d
1353         d.addCallback(_then)
1354         return d
1355
1356     def test_create_subdirectory(self):
1357         self.basedir = "dirnode/Dirnode/test_create_subdirectory"
1358         self.set_up_grid()
1359         return self._do_create_subdirectory_test()
1360
1361     def test_create_subdirectory_mdmf(self):
1362         self.basedir = "dirnode/Dirnode/test_create_subdirectory_mdmf"
1363         self.set_up_grid()
1364         return self._do_create_subdirectory_test(version=MDMF_VERSION)
1365
1366     def test_create_mdmf(self):
1367         self.basedir = "dirnode/Dirnode/test_mdmf"
1368         self.set_up_grid()
1369         return self._do_create_test(mdmf=True)
1370
1371     def test_mdmf_initial_children(self):
1372         self.basedir = "dirnode/Dirnode/test_mdmf"
1373         self.set_up_grid()
1374         return self._do_initial_children_test(mdmf=True)
1375
1376 class MinimalFakeMutableFile:
1377     def get_writekey(self):
1378         return "writekey"
1379
1380 class Packing(testutil.ReallyEqualMixin, unittest.TestCase):
1381     # This is a base32-encoded representation of the directory tree
1382     # root/file1
1383     # root/file2
1384     # root/file3
1385     # as represented after being fed to _pack_contents.
1386     # We have it here so we can decode it, feed it to
1387     # _unpack_contents, and verify that _unpack_contents
1388     # works correctly.
1389
1390     known_tree = "GM4TOORVHJTGS3DFGEWDSNJ2KVJESOSDJBFTU33MPB2GS3LZNVYG6N3GGI3WU5TIORTXC3DOMJ2G4NB2MVWXUZDONBVTE5LNGRZWK2LYN55GY23XGNYXQMTOMZUWU5TENN4DG23ZG5UTO2L2NQ2DO6LFMRWDMZJWGRQTUMZ2GEYDUMJQFQYTIMZ22XZKZORX5XS7CAQCSK3URR6QOHISHRCMGER5LRFSZRNAS5ZSALCS6TWFQAE754IVOIKJVK73WZPP3VUUEDTX3WHTBBZ5YX3CEKHCPG3ZWQLYA4QM6LDRCF7TJQYWLIZHKGN5ROA3AUZPXESBNLQQ6JTC2DBJU2D47IZJTLR3PKZ4RVF57XLPWY7FX7SZV3T6IJ3ORFW37FXUPGOE3ROPFNUX5DCGMAQJ3PGGULBRGM3TU6ZCMN2GS3LFEI5CAMJSGQ3DMNRTHA4TOLRUGI3TKNRWGEWCAITUMFUG6ZJCHIQHWITMNFXGW3LPORUW2ZJCHIQDCMRUGY3DMMZYHE3S4NBSG42TMNRRFQQCE3DJNZVWG4TUNFWWKIR2EAYTENBWGY3DGOBZG4XDIMRXGU3DMML5FQQCE3LUNFWWKIR2EAYTENBWGY3DGOBZG4XDIMRXGU3DMML5FQWDGOJRHI2TUZTJNRSTELBZGQ5FKUSJHJBUQSZ2MFYGKZ3SOBSWQ43IO52WO23CNAZWU3DUGVSWSNTIOE5DK33POVTW4ZLNMNWDK6DHPA2GS2THNF2W25DEN5VGY2LQNFRGG5DKNNRHO5TZPFTWI6LNMRYGQ2LCGJTHM4J2GM5DCMB2GQWDCNBSHKVVQBGRYMACKJ27CVQ6O6B4QPR72RFVTGOZUI76XUSWAX73JRV5PYRHMIFYZIA25MXDPGUGML6M2NMRSG4YD4W4K37ZDYSXHMJ3IUVT4F64YTQQVBJFFFOUC7J7LAB2VFCL5UKKGMR2D3F4EPOYC7UYWQZNR5KXHBSNXLCNBX2SNF22DCXJIHSMEKWEWOG5XCJEVVZ7UW5IB6I64XXQSJ34B5CAYZGZIIMR6LBRGMZTU6ZCMN2GS3LFEI5CAMJSGQ3DMNRTHA4TOLRUGMYDEMJYFQQCE5DBNBXWKIR2EB5SE3DJNZVW233UNFWWKIR2EAYTENBWGY3DGOBZG4XDIMZQGIYTQLBAEJWGS3TLMNZHI2LNMURDUIBRGI2DMNRWGM4DSNZOGQZTAMRRHB6SYIBCNV2GS3LFEI5CAMJSGQ3DMNRTHA4TOLRUGMYDEMJYPUWCYMZZGU5DKOTGNFWGKMZMHE2DUVKSJE5EGSCLHJRW25DDPBYTO2DXPB3GM6DBNYZTI6LJMV3DM2LWNB4TU4LWMNSWW3LKORXWK5DEMN3TI23NNE3WEM3SORRGY5THPA3TKNBUMNZG453BOF2GSZLXMVWWI3DJOFZW623RHIZTUMJQHI2SYMJUGI5BOSHWDPG3WKPAVXCF3XMKA7QVIWPRMWJHDTQHD27AHDCPJWDQENQ5H5ZZILTXQNIXXCIW4LKQABU2GCFRG5FHQN7CHD7HF4EKNRZFIV2ZYQIBM7IQU7F4RGB3XCX3FREPBKQ7UCICHVWPCYFGA6OLH3J45LXQ6GWWICJ3PGWJNLZ7PCRNLAPNYUGU6BENS7OXMBEOOFRIZV3PF2FFWZ5WHDPKXERYP7GNHKRMGEZTOOT3EJRXI2LNMURDUIBRGI2DMNRWGM4DSNZOGQZTGNRSGY4SYIBCORQWQ33FEI5CA6ZCNRUW423NN52GS3LFEI5CAMJSGQ3DMNRTHA4TOLRUGMZTMMRWHEWCAITMNFXGWY3SORUW2ZJCHIQDCMRUGY3DMMZYHE3S4NBTGM3DENRZPUWCAITNORUW2ZJCHIQDCMRUGY3DMMZYHE3S4NBTGM3DENRZPUWCY==="
1391
1392     def test_unpack_and_pack_behavior(self):
1393         known_tree = b32decode(self.known_tree)
1394         nodemaker = NodeMaker(None, None, None,
1395                               None, None,
1396                               {"k": 3, "n": 10}, None, None)
1397         write_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
1398         filenode = nodemaker.create_from_cap(write_uri)
1399         node = dirnode.DirectoryNode(filenode, nodemaker, None)
1400         children = node._unpack_contents(known_tree)
1401         self._check_children(children)
1402
1403         packed_children = node._pack_contents(children)
1404         children = node._unpack_contents(packed_children)
1405         self._check_children(children)
1406
1407     def _check_children(self, children):
1408         # Are all the expected child nodes there?
1409         self.failUnless(children.has_key(u'file1'))
1410         self.failUnless(children.has_key(u'file2'))
1411         self.failUnless(children.has_key(u'file3'))
1412
1413         # Are the metadata for child 3 right?
1414         file3_rocap = "URI:CHK:cmtcxq7hwxvfxan34yiev6ivhy:qvcekmjtoetdcw4kmi7b3rtblvgx7544crnwaqtiewemdliqsokq:3:10:5"
1415         file3_rwcap = "URI:CHK:cmtcxq7hwxvfxan34yiev6ivhy:qvcekmjtoetdcw4kmi7b3rtblvgx7544crnwaqtiewemdliqsokq:3:10:5"
1416         file3_metadata = {'ctime': 1246663897.4336269, 'tahoe': {'linkmotime': 1246663897.4336269, 'linkcrtime': 1246663897.4336269}, 'mtime': 1246663897.4336269}
1417         self.failUnlessEqual(file3_metadata, children[u'file3'][1])
1418         self.failUnlessReallyEqual(file3_rocap,
1419                                    children[u'file3'][0].get_readonly_uri())
1420         self.failUnlessReallyEqual(file3_rwcap,
1421                                    children[u'file3'][0].get_uri())
1422
1423         # Are the metadata for child 2 right?
1424         file2_rocap = "URI:CHK:apegrpehshwugkbh3jlt5ei6hq:5oougnemcl5xgx4ijgiumtdojlipibctjkbwvyygdymdphib2fvq:3:10:4"
1425         file2_rwcap = "URI:CHK:apegrpehshwugkbh3jlt5ei6hq:5oougnemcl5xgx4ijgiumtdojlipibctjkbwvyygdymdphib2fvq:3:10:4"
1426         file2_metadata = {'ctime': 1246663897.430218, 'tahoe': {'linkmotime': 1246663897.430218, 'linkcrtime': 1246663897.430218}, 'mtime': 1246663897.430218}
1427         self.failUnlessEqual(file2_metadata, children[u'file2'][1])
1428         self.failUnlessReallyEqual(file2_rocap,
1429                                    children[u'file2'][0].get_readonly_uri())
1430         self.failUnlessReallyEqual(file2_rwcap,
1431                                    children[u'file2'][0].get_uri())
1432
1433         # Are the metadata for child 1 right?
1434         file1_rocap = "URI:CHK:olxtimympo7f27jvhtgqlnbtn4:emzdnhk2um4seixozlkw3qx2nfijvdkx3ky7i7izl47yedl6e64a:3:10:10"
1435         file1_rwcap = "URI:CHK:olxtimympo7f27jvhtgqlnbtn4:emzdnhk2um4seixozlkw3qx2nfijvdkx3ky7i7izl47yedl6e64a:3:10:10"
1436         file1_metadata = {'ctime': 1246663897.4275661, 'tahoe': {'linkmotime': 1246663897.4275661, 'linkcrtime': 1246663897.4275661}, 'mtime': 1246663897.4275661}
1437         self.failUnlessEqual(file1_metadata, children[u'file1'][1])
1438         self.failUnlessReallyEqual(file1_rocap,
1439                                    children[u'file1'][0].get_readonly_uri())
1440         self.failUnlessReallyEqual(file1_rwcap,
1441                                    children[u'file1'][0].get_uri())
1442
1443     def _make_kids(self, nm, which):
1444         caps = {"imm": "URI:CHK:n7r3m6wmomelk4sep3kw5cvduq:os7ijw5c3maek7pg65e5254k2fzjflavtpejjyhshpsxuqzhcwwq:3:20:14861",
1445                 "lit": "URI:LIT:n5xgk", # LIT for "one"
1446                 "write": "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq",
1447                 "read": "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q",
1448                 "dirwrite": "URI:DIR2:n6x24zd3seu725yluj75q5boaa:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq",
1449                 "dirread":  "URI:DIR2-RO:b7sr5qsifnicca7cbk3rhrhbvq:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq",
1450                 }
1451         kids = {}
1452         for name in which:
1453             kids[unicode(name)] = (nm.create_from_cap(caps[name]), {})
1454         return kids
1455
1456     def test_deep_immutable(self):
1457         nm = NodeMaker(None, None, None, None, None, {"k": 3, "n": 10}, None, None)
1458         fn = MinimalFakeMutableFile()
1459
1460         kids = self._make_kids(nm, ["imm", "lit", "write", "read",
1461                                     "dirwrite", "dirread"])
1462         packed = dirnode.pack_children(kids, fn.get_writekey(), deep_immutable=False)
1463         self.failUnlessIn("lit", packed)
1464
1465         kids = self._make_kids(nm, ["imm", "lit"])
1466         packed = dirnode.pack_children(kids, fn.get_writekey(), deep_immutable=True)
1467         self.failUnlessIn("lit", packed)
1468
1469         kids = self._make_kids(nm, ["imm", "lit", "write"])
1470         self.failUnlessRaises(dirnode.MustBeDeepImmutableError,
1471                               dirnode.pack_children,
1472                               kids, fn.get_writekey(), deep_immutable=True)
1473
1474         # read-only is not enough: all children must be immutable
1475         kids = self._make_kids(nm, ["imm", "lit", "read"])
1476         self.failUnlessRaises(dirnode.MustBeDeepImmutableError,
1477                               dirnode.pack_children,
1478                               kids, fn.get_writekey(), deep_immutable=True)
1479
1480         kids = self._make_kids(nm, ["imm", "lit", "dirwrite"])
1481         self.failUnlessRaises(dirnode.MustBeDeepImmutableError,
1482                               dirnode.pack_children,
1483                               kids, fn.get_writekey(), deep_immutable=True)
1484
1485         kids = self._make_kids(nm, ["imm", "lit", "dirread"])
1486         self.failUnlessRaises(dirnode.MustBeDeepImmutableError,
1487                               dirnode.pack_children,
1488                               kids, fn.get_writekey(), deep_immutable=True)
1489
1490 class FakeMutableFile:
1491     implements(IMutableFileNode)
1492     counter = 0
1493     def __init__(self, initial_contents=""):
1494         data = self._get_initial_contents(initial_contents)
1495         self.data = data.read(data.get_size())
1496         self.data = "".join(self.data)
1497
1498         counter = FakeMutableFile.counter
1499         FakeMutableFile.counter += 1
1500         writekey = hashutil.ssk_writekey_hash(str(counter))
1501         fingerprint = hashutil.ssk_pubkey_fingerprint_hash(str(counter))
1502         self.uri = uri.WriteableSSKFileURI(writekey, fingerprint)
1503
1504     def _get_initial_contents(self, contents):
1505         if isinstance(contents, str):
1506             return contents
1507         if contents is None:
1508             return ""
1509         assert callable(contents), "%s should be callable, not %s" % \
1510                (contents, type(contents))
1511         return contents(self)
1512
1513     def get_cap(self):
1514         return self.uri
1515
1516     def get_uri(self):
1517         return self.uri.to_string()
1518
1519     def get_write_uri(self):
1520         return self.uri.to_string()
1521
1522     def download_best_version(self):
1523         return defer.succeed(self.data)
1524
1525     def get_writekey(self):
1526         return "writekey"
1527
1528     def is_readonly(self):
1529         return False
1530
1531     def is_mutable(self):
1532         return True
1533
1534     def is_unknown(self):
1535         return False
1536
1537     def is_allowed_in_immutable_directory(self):
1538         return False
1539
1540     def raise_error(self):
1541         pass
1542
1543     def modify(self, modifier):
1544         data = modifier(self.data, None, True)
1545         self.data = data
1546         return defer.succeed(None)
1547
1548 class FakeNodeMaker(NodeMaker):
1549     def create_mutable_file(self, contents="", keysize=None, version=None):
1550         return defer.succeed(FakeMutableFile(contents))
1551
1552 class FakeClient2(Client):
1553     def __init__(self):
1554         self.nodemaker = FakeNodeMaker(None, None, None,
1555                                        None, None,
1556                                        {"k":3,"n":10}, None, None)
1557     def create_node_from_uri(self, rwcap, rocap):
1558         return self.nodemaker.create_from_cap(rwcap, rocap)
1559
1560 class Dirnode2(testutil.ReallyEqualMixin, testutil.ShouldFailMixin, unittest.TestCase):
1561     def setUp(self):
1562         client = FakeClient2()
1563         self.nodemaker = client.nodemaker
1564
1565     def test_from_future(self):
1566         # Create a mutable directory that contains unknown URI types, and make sure
1567         # we tolerate them properly.
1568         d = self.nodemaker.create_new_mutable_directory()
1569         future_write_uri = u"x-tahoe-crazy://I_am_from_the_future_rw_\u263A".encode('utf-8')
1570         future_read_uri = u"x-tahoe-crazy-readonly://I_am_from_the_future_ro_\u263A".encode('utf-8')
1571         future_imm_uri = u"x-tahoe-crazy-immutable://I_am_from_the_future_imm_\u263A".encode('utf-8')
1572         future_node = UnknownNode(future_write_uri, future_read_uri)
1573         def _then(n):
1574             self._node = n
1575             return n.set_node(u"future", future_node)
1576         d.addCallback(_then)
1577
1578         # We should be prohibited from adding an unknown URI to a directory
1579         # just in the rw_uri slot, since we don't know how to diminish the cap
1580         # to a readcap (for the ro_uri slot).
1581         d.addCallback(lambda ign:
1582              self.shouldFail(MustNotBeUnknownRWError,
1583                              "copy unknown",
1584                              "cannot attach unknown rw cap as child",
1585                              self._node.set_uri, u"add",
1586                              future_write_uri, None))
1587
1588         # However, we should be able to add both rw_uri and ro_uri as a pair of
1589         # unknown URIs.
1590         d.addCallback(lambda ign: self._node.set_uri(u"add-pair",
1591                                                      future_write_uri, future_read_uri))
1592
1593         # and to add an URI prefixed with "ro." or "imm." when it is given in a
1594         # write slot (or URL parameter).
1595         d.addCallback(lambda ign: self._node.set_uri(u"add-ro",
1596                                                      "ro." + future_read_uri, None))
1597         d.addCallback(lambda ign: self._node.set_uri(u"add-imm",
1598                                                      "imm." + future_imm_uri, None))
1599
1600         d.addCallback(lambda ign: self._node.list())
1601         def _check(children):
1602             self.failUnlessReallyEqual(len(children), 4)
1603             (fn, metadata) = children[u"future"]
1604             self.failUnless(isinstance(fn, UnknownNode), fn)
1605             self.failUnlessReallyEqual(fn.get_uri(), future_write_uri)
1606             self.failUnlessReallyEqual(fn.get_write_uri(), future_write_uri)
1607             self.failUnlessReallyEqual(fn.get_readonly_uri(), "ro." + future_read_uri)
1608
1609             (fn2, metadata2) = children[u"add-pair"]
1610             self.failUnless(isinstance(fn2, UnknownNode), fn2)
1611             self.failUnlessReallyEqual(fn2.get_uri(), future_write_uri)
1612             self.failUnlessReallyEqual(fn2.get_write_uri(), future_write_uri)
1613             self.failUnlessReallyEqual(fn2.get_readonly_uri(), "ro." + future_read_uri)
1614
1615             (fn3, metadata3) = children[u"add-ro"]
1616             self.failUnless(isinstance(fn3, UnknownNode), fn3)
1617             self.failUnlessReallyEqual(fn3.get_uri(), "ro." + future_read_uri)
1618             self.failUnlessReallyEqual(fn3.get_write_uri(), None)
1619             self.failUnlessReallyEqual(fn3.get_readonly_uri(), "ro." + future_read_uri)
1620
1621             (fn4, metadata4) = children[u"add-imm"]
1622             self.failUnless(isinstance(fn4, UnknownNode), fn4)
1623             self.failUnlessReallyEqual(fn4.get_uri(), "imm." + future_imm_uri)
1624             self.failUnlessReallyEqual(fn4.get_write_uri(), None)
1625             self.failUnlessReallyEqual(fn4.get_readonly_uri(), "imm." + future_imm_uri)
1626
1627             # We should also be allowed to copy the "future" UnknownNode, because
1628             # it contains all the information that was in the original directory
1629             # (readcap and writecap), so we're preserving everything.
1630             return self._node.set_node(u"copy", fn)
1631         d.addCallback(_check)
1632
1633         d.addCallback(lambda ign: self._node.list())
1634         def _check2(children):
1635             self.failUnlessReallyEqual(len(children), 5)
1636             (fn, metadata) = children[u"copy"]
1637             self.failUnless(isinstance(fn, UnknownNode), fn)
1638             self.failUnlessReallyEqual(fn.get_uri(), future_write_uri)
1639             self.failUnlessReallyEqual(fn.get_write_uri(), future_write_uri)
1640             self.failUnlessReallyEqual(fn.get_readonly_uri(), "ro." + future_read_uri)
1641         d.addCallback(_check2)
1642         return d
1643
1644     def test_unknown_strip_prefix_for_ro(self):
1645         self.failUnlessReallyEqual(strip_prefix_for_ro("foo",     False), "foo")
1646         self.failUnlessReallyEqual(strip_prefix_for_ro("ro.foo",  False), "foo")
1647         self.failUnlessReallyEqual(strip_prefix_for_ro("imm.foo", False), "imm.foo")
1648         self.failUnlessReallyEqual(strip_prefix_for_ro("foo",     True),  "foo")
1649         self.failUnlessReallyEqual(strip_prefix_for_ro("ro.foo",  True),  "foo")
1650         self.failUnlessReallyEqual(strip_prefix_for_ro("imm.foo", True),  "foo")
1651
1652     def test_unknownnode(self):
1653         lit_uri = one_uri
1654
1655         # This does not attempt to be exhaustive.
1656         no_no        = [# Opaque node, but not an error.
1657                         ( 0, UnknownNode(None, None)),
1658                         ( 1, UnknownNode(None, None, deep_immutable=True)),
1659                        ]
1660         unknown_rw   = [# These are errors because we're only given a rw_uri, and we can't
1661                         # diminish it.
1662                         ( 2, UnknownNode("foo", None)),
1663                         ( 3, UnknownNode("foo", None, deep_immutable=True)),
1664                         ( 4, UnknownNode("ro.foo", None, deep_immutable=True)),
1665                         ( 5, UnknownNode("ro." + mut_read_uri, None, deep_immutable=True)),
1666                         ( 5.1, UnknownNode("ro." + mdmf_read_uri, None, deep_immutable=True)),
1667                         ( 6, UnknownNode("URI:SSK-RO:foo", None, deep_immutable=True)),
1668                         ( 7, UnknownNode("URI:SSK:foo", None)),
1669                        ]
1670         must_be_ro   = [# These are errors because a readonly constraint is not met.
1671                         ( 8, UnknownNode("ro." + mut_write_uri, None)),
1672                         ( 8.1, UnknownNode("ro." + mdmf_write_uri, None)),
1673                         ( 9, UnknownNode(None, "ro." + mut_write_uri)),
1674                         ( 9.1, UnknownNode(None, "ro." + mdmf_write_uri)),
1675                        ]
1676         must_be_imm  = [# These are errors because an immutable constraint is not met.
1677                         (10, UnknownNode(None, "ro.URI:SSK-RO:foo", deep_immutable=True)),
1678                         (11, UnknownNode(None, "imm.URI:SSK:foo")),
1679                         (12, UnknownNode(None, "imm.URI:SSK-RO:foo")),
1680                         (13, UnknownNode("bar", "ro.foo", deep_immutable=True)),
1681                         (14, UnknownNode("bar", "imm.foo", deep_immutable=True)),
1682                         (15, UnknownNode("bar", "imm." + lit_uri, deep_immutable=True)),
1683                         (16, UnknownNode("imm." + mut_write_uri, None)),
1684                         (16.1, UnknownNode("imm." + mdmf_write_uri, None)),
1685                         (17, UnknownNode("imm." + mut_read_uri, None)),
1686                         (17.1, UnknownNode("imm." + mdmf_read_uri, None)),
1687                         (18, UnknownNode("bar", "imm.foo")),
1688                        ]
1689         bad_uri      = [# These are errors because the URI is bad once we've stripped the prefix.
1690                         (19, UnknownNode("ro.URI:SSK-RO:foo", None)),
1691                         (20, UnknownNode("imm.URI:CHK:foo", None, deep_immutable=True)),
1692                         (21, UnknownNode(None, "URI:CHK:foo")),
1693                         (22, UnknownNode(None, "URI:CHK:foo", deep_immutable=True)),
1694                        ]
1695         ro_prefixed  = [# These are valid, and the readcap should end up with a ro. prefix.
1696                         (23, UnknownNode(None, "foo")),
1697                         (24, UnknownNode(None, "ro.foo")),
1698                         (25, UnknownNode(None, "ro." + lit_uri)),
1699                         (26, UnknownNode("bar", "foo")),
1700                         (27, UnknownNode("bar", "ro.foo")),
1701                         (28, UnknownNode("bar", "ro." + lit_uri)),
1702                         (29, UnknownNode("ro.foo", None)),
1703                         (30, UnknownNode("ro." + lit_uri, None)),
1704                        ]
1705         imm_prefixed = [# These are valid, and the readcap should end up with an imm. prefix.
1706                         (31, UnknownNode(None, "foo", deep_immutable=True)),
1707                         (32, UnknownNode(None, "ro.foo", deep_immutable=True)),
1708                         (33, UnknownNode(None, "imm.foo")),
1709                         (34, UnknownNode(None, "imm.foo", deep_immutable=True)),
1710                         (35, UnknownNode("imm." + lit_uri, None)),
1711                         (36, UnknownNode("imm." + lit_uri, None, deep_immutable=True)),
1712                         (37, UnknownNode(None, "imm." + lit_uri)),
1713                         (38, UnknownNode(None, "imm." + lit_uri, deep_immutable=True)),
1714                        ]
1715         error = unknown_rw + must_be_ro + must_be_imm + bad_uri
1716         ok = ro_prefixed + imm_prefixed
1717
1718         for (i, n) in no_no + error + ok:
1719             self.failUnless(n.is_unknown(), i)
1720
1721         for (i, n) in no_no + error:
1722             self.failUnless(n.get_uri() is None, i)
1723             self.failUnless(n.get_write_uri() is None, i)
1724             self.failUnless(n.get_readonly_uri() is None, i)
1725
1726         for (i, n) in no_no + ok:
1727             n.raise_error()
1728
1729         for (i, n) in unknown_rw:
1730             self.failUnlessRaises(MustNotBeUnknownRWError, lambda n=n: n.raise_error())
1731
1732         for (i, n) in must_be_ro:
1733             self.failUnlessRaises(MustBeReadonlyError, lambda n=n: n.raise_error())
1734
1735         for (i, n) in must_be_imm:
1736             self.failUnlessRaises(MustBeDeepImmutableError, lambda n=n: n.raise_error())
1737
1738         for (i, n) in bad_uri:
1739             self.failUnlessRaises(uri.BadURIError, lambda n=n: n.raise_error())
1740
1741         for (i, n) in ok:
1742             self.failIf(n.get_readonly_uri() is None, i)
1743
1744         for (i, n) in ro_prefixed:
1745             self.failUnless(n.get_readonly_uri().startswith("ro."), i)
1746
1747         for (i, n) in imm_prefixed:
1748             self.failUnless(n.get_readonly_uri().startswith("imm."), i)
1749
1750
1751
1752 class DeepStats(testutil.ReallyEqualMixin, unittest.TestCase):
1753     timeout = 240 # It takes longer than 120 seconds on Francois's arm box.
1754     def test_stats(self):
1755         ds = dirnode.DeepStats(None)
1756         ds.add("count-files")
1757         ds.add("size-immutable-files", 123)
1758         ds.histogram("size-files-histogram", 123)
1759         ds.max("largest-directory", 444)
1760
1761         s = ds.get_results()
1762         self.failUnlessReallyEqual(s["count-files"], 1)
1763         self.failUnlessReallyEqual(s["size-immutable-files"], 123)
1764         self.failUnlessReallyEqual(s["largest-directory"], 444)
1765         self.failUnlessReallyEqual(s["count-literal-files"], 0)
1766
1767         ds.add("count-files")
1768         ds.add("size-immutable-files", 321)
1769         ds.histogram("size-files-histogram", 321)
1770         ds.max("largest-directory", 2)
1771
1772         s = ds.get_results()
1773         self.failUnlessReallyEqual(s["count-files"], 2)
1774         self.failUnlessReallyEqual(s["size-immutable-files"], 444)
1775         self.failUnlessReallyEqual(s["largest-directory"], 444)
1776         self.failUnlessReallyEqual(s["count-literal-files"], 0)
1777         self.failUnlessReallyEqual(s["size-files-histogram"],
1778                                    [ (101, 316, 1), (317, 1000, 1) ])
1779
1780         ds = dirnode.DeepStats(None)
1781         for i in range(1, 1100):
1782             ds.histogram("size-files-histogram", i)
1783         ds.histogram("size-files-histogram", 4*1000*1000*1000*1000) # 4TB
1784         s = ds.get_results()
1785         self.failUnlessReallyEqual(s["size-files-histogram"],
1786                                    [ (1, 3, 3),
1787                                      (4, 10, 7),
1788                                      (11, 31, 21),
1789                                      (32, 100, 69),
1790                                      (101, 316, 216),
1791                                      (317, 1000, 684),
1792                                      (1001, 3162, 99),
1793                                      (3162277660169L, 10000000000000L, 1),
1794                                      ])
1795
1796 class UCWEingMutableFileNode(MutableFileNode):
1797     please_ucwe_after_next_upload = False
1798
1799     def _upload(self, new_contents, servermap):
1800         d = MutableFileNode._upload(self, new_contents, servermap)
1801         def _ucwe(res):
1802             if self.please_ucwe_after_next_upload:
1803                 self.please_ucwe_after_next_upload = False
1804                 raise UncoordinatedWriteError()
1805             return res
1806         d.addCallback(_ucwe)
1807         return d
1808
1809 class UCWEingNodeMaker(NodeMaker):
1810     def _create_mutable(self, cap):
1811         n = UCWEingMutableFileNode(self.storage_broker, self.secret_holder,
1812                                    self.default_encoding_parameters,
1813                                    self.history)
1814         return n.init_from_cap(cap)
1815
1816
1817 class Deleter(GridTestMixin, testutil.ReallyEqualMixin, unittest.TestCase):
1818     timeout = 3600 # It takes longer than 433 seconds on Zandr's ARM box.
1819     def test_retry(self):
1820         # ticket #550, a dirnode.delete which experiences an
1821         # UncoordinatedWriteError will fail with an incorrect "you're
1822         # deleting something which isn't there" NoSuchChildError exception.
1823
1824         # to trigger this, we start by creating a directory with a single
1825         # file in it. Then we create a special dirnode that uses a modified
1826         # MutableFileNode which will raise UncoordinatedWriteError once on
1827         # demand. We then call dirnode.delete, which ought to retry and
1828         # succeed.
1829
1830         self.basedir = self.mktemp()
1831         self.set_up_grid()
1832         c0 = self.g.clients[0]
1833         d = c0.create_dirnode()
1834         small = upload.Data("Small enough for a LIT", None)
1835         def _created_dir(dn):
1836             self.root = dn
1837             self.root_uri = dn.get_uri()
1838             return dn.add_file(u"file", small)
1839         d.addCallback(_created_dir)
1840         def _do_delete(ignored):
1841             nm = UCWEingNodeMaker(c0.storage_broker, c0._secret_holder,
1842                                   c0.get_history(), c0.getServiceNamed("uploader"),
1843                                   c0.terminator,
1844                                   c0.get_encoding_parameters(),
1845                                   c0.mutable_file_default,
1846                                   c0._key_generator)
1847             n = nm.create_from_cap(self.root_uri)
1848             assert n._node.please_ucwe_after_next_upload == False
1849             n._node.please_ucwe_after_next_upload = True
1850             # This should succeed, not raise an exception
1851             return n.delete(u"file")
1852         d.addCallback(_do_delete)
1853
1854         return d
1855
1856 class Adder(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin):
1857
1858     def test_overwrite(self):
1859         # note: This functionality could be tested without actually creating
1860         # several RSA keys. It would be faster without the GridTestMixin: use
1861         # dn.set_node(nodemaker.create_from_cap(make_chk_file_uri())) instead
1862         # of dn.add_file, and use a special NodeMaker that creates fake
1863         # mutable files.
1864         self.basedir = "dirnode/Adder/test_overwrite"
1865         self.set_up_grid()
1866         c = self.g.clients[0]
1867         fileuri = make_chk_file_uri(1234)
1868         filenode = c.nodemaker.create_from_cap(fileuri)
1869         d = c.create_dirnode()
1870
1871         def _create_directory_tree(root_node):
1872             # Build
1873             # root/file1
1874             # root/file2
1875             # root/dir1
1876             d = root_node.add_file(u'file1', upload.Data("Important Things",
1877                 None))
1878             d.addCallback(lambda res:
1879                 root_node.add_file(u'file2', upload.Data("Sekrit Codes", None)))
1880             d.addCallback(lambda res:
1881                 root_node.create_subdirectory(u"dir1"))
1882             d.addCallback(lambda res: root_node)
1883             return d
1884
1885         d.addCallback(_create_directory_tree)
1886
1887         def _test_adder(root_node):
1888             d = root_node.set_node(u'file1', filenode)
1889             # We've overwritten file1. Let's try it with a directory
1890             d.addCallback(lambda res:
1891                 root_node.create_subdirectory(u'dir2'))
1892             d.addCallback(lambda res:
1893                 root_node.set_node(u'dir2', filenode))
1894             # We try overwriting a file with a child while also specifying
1895             # overwrite=False. We should receive an ExistingChildError
1896             # when we do this.
1897             d.addCallback(lambda res:
1898                 self.shouldFail(ExistingChildError, "set_node",
1899                                 "child 'file1' already exists",
1900                                 root_node.set_node, u"file1",
1901                                 filenode, overwrite=False))
1902             # If we try with a directory, we should see the same thing
1903             d.addCallback(lambda res:
1904                 self.shouldFail(ExistingChildError, "set_node",
1905                                 "child 'dir1' already exists",
1906                                 root_node.set_node, u'dir1', filenode,
1907                                 overwrite=False))
1908             d.addCallback(lambda res:
1909                 root_node.set_node(u'file1', filenode,
1910                                    overwrite="only-files"))
1911             d.addCallback(lambda res:
1912                 self.shouldFail(ExistingChildError, "set_node",
1913                                 "child 'dir1' already exists",
1914                                 root_node.set_node, u'dir1', filenode,
1915                                 overwrite="only-files"))
1916             return d
1917
1918         d.addCallback(_test_adder)
1919         return d