src/allmydata/test/test_dirnode.py

   1 import time
   2 import unicodedata
   3 from zope.interface import implements
   4 from twisted.trial import unittest
   5 from twisted.internet import defer
   6 from twisted.internet.interfaces import IConsumer
   7 from allmydata import uri, dirnode
   8 from allmydata.client import Client
   9 from allmydata.immutable import upload
  10 from allmydata.interfaces import IImmutableFileNode, IMutableFileNode, \
  11      ExistingChildError, NoSuchChildError, MustNotBeUnknownRWError, \
  12      MustBeDeepImmutableError, MustBeReadonlyError, \
  13      IDeepCheckResults, IDeepCheckAndRepairResults, \
  14      MDMF_VERSION, SDMF_VERSION
  15 from allmydata.mutable.filenode import MutableFileNode
  16 from allmydata.mutable.common import UncoordinatedWriteError
  17 from allmydata.util import hashutil, base32
  18 from allmydata.util.netstring import split_netstring
  19 from allmydata.monitor import Monitor
  20 from allmydata.test.common import make_chk_file_uri, make_mutable_file_uri, \
  21      ErrorMixin
  22 from allmydata.test.no_network import GridTestMixin
  23 from allmydata.unknown import UnknownNode, strip_prefix_for_ro
  24 from allmydata.nodemaker import NodeMaker
  25 from base64 import b32decode
  26 import allmydata.test.common_util as testutil
  27
  28 class MemAccum:
  29     implements(IConsumer)
  30     def registerProducer(self, producer, streaming):
  31         self.producer = producer
  32         self.producer.resumeProducing()
  33         pass
  34     def unregisterProducer(self):
  35         pass
  36     def write(self, data):
  37         assert not hasattr(self, 'data')
  38         self.data = data
  39         self.producer.resumeProducing()
  40
  41 setup_py_uri = "URI:CHK:n7r3m6wmomelk4sep3kw5cvduq:os7ijw5c3maek7pg65e5254k2fzjflavtpejjyhshpsxuqzhcwwq:3:20:14861"
  42 one_uri = "URI:LIT:n5xgk" # LIT for "one"
  43 mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
  44 empty_litdir_uri = "URI:DIR2-LIT:"
  45 tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
  46 mut_read_uri = "URI:SSK-RO:jf6wkflosyvntwxqcdo7a54jvm:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
  47 future_write_uri = "x-tahoe-crazy://I_am_from_the_future."
  48 future_read_uri = "x-tahoe-crazy-readonly://I_am_from_the_future."
  49 future_nonascii_write_uri = u"x-tahoe-even-more-crazy://I_am_from_the_future_rw_\u263A".encode('utf-8')
  50 future_nonascii_read_uri = u"x-tahoe-even-more-crazy-readonly://I_am_from_the_future_ro_\u263A".encode('utf-8')
  51
  52 # 'o' 'n' 'e-macron'
  53 one_nfc = u"on\u0113"
  54 one_nfd = u"one\u0304"
  55
  56 class Dirnode(GridTestMixin, unittest.TestCase,
  57               testutil.ReallyEqualMixin, testutil.ShouldFailMixin, testutil.StallMixin, ErrorMixin):
  58     timeout = 480 # It occasionally takes longer than 240 seconds on Francois's arm box.
  59
  60     def _do_create_test(self, mdmf=False):
  61         c = self.g.clients[0]
  62
  63         self.expected_manifest = []
  64         self.expected_verifycaps = set()
  65         self.expected_storage_indexes = set()
  66
  67         d = None
  68         if mdmf:
  69             d = c.create_dirnode(version=MDMF_VERSION)
  70         else:
  71             d = c.create_dirnode()
  72         def _then(n):
  73             # /
  74             self.rootnode = n
  75             backing_node = n._node
  76             if mdmf:
  77                 self.failUnlessEqual(backing_node.get_version(),
  78                                      MDMF_VERSION)
  79             else:
  80                 self.failUnlessEqual(backing_node.get_version(),
  81                                      SDMF_VERSION)
  82             self.failUnless(n.is_mutable())
  83             u = n.get_uri()
  84             self.failUnless(u)
  85             cap_formats = []
  86             if mdmf:
  87                 cap_formats = ["URI:DIR2-MDMF:",
  88                                "URI:DIR2-MDMF-RO:",
  89                                "URI:DIR2-MDMF-Verifier:"]
  90             else:
  91                 cap_formats = ["URI:DIR2:",
  92                                "URI:DIR2-RO",
  93                                "URI:DIR2-Verifier:"]
  94             rw, ro, v = cap_formats
  95             self.failUnless(u.startswith(rw), u)
  96             u_ro = n.get_readonly_uri()
  97             self.failUnless(u_ro.startswith(ro), u_ro)
  98             u_v = n.get_verify_cap().to_string()
  99             self.failUnless(u_v.startswith(v), u_v)
 100             u_r = n.get_repair_cap().to_string()
 101             self.failUnlessReallyEqual(u_r, u)
 102             self.expected_manifest.append( ((), u) )
 103             self.expected_verifycaps.add(u_v)
 104             si = n.get_storage_index()
 105             self.expected_storage_indexes.add(base32.b2a(si))
 106             expected_si = n._uri.get_storage_index()
 107             self.failUnlessReallyEqual(si, expected_si)
 108
 109             d = n.list()
 110             d.addCallback(lambda res: self.failUnlessEqual(res, {}))
 111             d.addCallback(lambda res: n.has_child(u"missing"))
 112             d.addCallback(lambda res: self.failIf(res))
 113
 114             fake_file_uri = make_mutable_file_uri()
 115             other_file_uri = make_mutable_file_uri()
 116             m = c.nodemaker.create_from_cap(fake_file_uri)
 117             ffu_v = m.get_verify_cap().to_string()
 118             self.expected_manifest.append( ((u"child",) , m.get_uri()) )
 119             self.expected_verifycaps.add(ffu_v)
 120             self.expected_storage_indexes.add(base32.b2a(m.get_storage_index()))
 121             d.addCallback(lambda res: n.set_uri(u"child",
 122                                                 fake_file_uri, fake_file_uri))
 123             d.addCallback(lambda res:
 124                           self.shouldFail(ExistingChildError, "set_uri-no",
 125                                           "child 'child' already exists",
 126                                           n.set_uri, u"child",
 127                                           other_file_uri, other_file_uri,
 128                                           overwrite=False))
 129             # /
 130             # /child = mutable
 131
 132             d.addCallback(lambda res: n.create_subdirectory(u"subdir"))
 133
 134             # /
 135             # /child = mutable
 136             # /subdir = directory
 137             def _created(subdir):
 138                 self.failUnless(isinstance(subdir, dirnode.DirectoryNode))
 139                 self.subdir = subdir
 140                 new_v = subdir.get_verify_cap().to_string()
 141                 assert isinstance(new_v, str)
 142                 self.expected_manifest.append( ((u"subdir",), subdir.get_uri()) )
 143                 self.expected_verifycaps.add(new_v)
 144                 si = subdir.get_storage_index()
 145                 self.expected_storage_indexes.add(base32.b2a(si))
 146             d.addCallback(_created)
 147
 148             d.addCallback(lambda res:
 149                           self.shouldFail(ExistingChildError, "mkdir-no",
 150                                           "child 'subdir' already exists",
 151                                           n.create_subdirectory, u"subdir",
 152                                           overwrite=False))
 153
 154             d.addCallback(lambda res: n.list())
 155             d.addCallback(lambda children:
 156                           self.failUnlessReallyEqual(set(children.keys()),
 157                                                      set([u"child", u"subdir"])))
 158
 159             d.addCallback(lambda res: n.start_deep_stats().when_done())
 160             def _check_deepstats(stats):
 161                 self.failUnless(isinstance(stats, dict))
 162                 expected = {"count-immutable-files": 0,
 163                             "count-mutable-files": 1,
 164                             "count-literal-files": 0,
 165                             "count-files": 1,
 166                             "count-directories": 2,
 167                             "size-immutable-files": 0,
 168                             "size-literal-files": 0,
 169                             #"size-directories": 616, # varies
 170                             #"largest-directory": 616,
 171                             "largest-directory-children": 2,
 172                             "largest-immutable-file": 0,
 173                             }
 174                 for k,v in expected.iteritems():
 175                     self.failUnlessReallyEqual(stats[k], v,
 176                                                "stats[%s] was %s, not %s" %
 177                                                (k, stats[k], v))
 178                 self.failUnless(stats["size-directories"] > 500,
 179                                 stats["size-directories"])
 180                 self.failUnless(stats["largest-directory"] > 500,
 181                                 stats["largest-directory"])
 182                 self.failUnlessReallyEqual(stats["size-files-histogram"], [])
 183             d.addCallback(_check_deepstats)
 184
 185             d.addCallback(lambda res: n.build_manifest().when_done())
 186             def _check_manifest(res):
 187                 manifest = res["manifest"]
 188                 self.failUnlessReallyEqual(sorted(manifest),
 189                                            sorted(self.expected_manifest))
 190                 stats = res["stats"]
 191                 _check_deepstats(stats)
 192                 self.failUnlessReallyEqual(self.expected_verifycaps,
 193                                            res["verifycaps"])
 194                 self.failUnlessReallyEqual(self.expected_storage_indexes,
 195                                            res["storage-index"])
 196             d.addCallback(_check_manifest)
 197
 198             def _add_subsubdir(res):
 199                 return self.subdir.create_subdirectory(u"subsubdir")
 200             d.addCallback(_add_subsubdir)
 201             # /
 202             # /child = mutable
 203             # /subdir = directory
 204             # /subdir/subsubdir = directory
 205             d.addCallback(lambda res: n.get_child_at_path(u"subdir/subsubdir"))
 206             d.addCallback(lambda subsubdir:
 207                           self.failUnless(isinstance(subsubdir,
 208                                                      dirnode.DirectoryNode)))
 209             d.addCallback(lambda res: n.get_child_at_path(u""))
 210             d.addCallback(lambda res: self.failUnlessReallyEqual(res.get_uri(),
 211                                                                  n.get_uri()))
 212
 213             d.addCallback(lambda res: n.get_metadata_for(u"child"))
 214             d.addCallback(lambda metadata:
 215                           self.failUnlessEqual(set(metadata.keys()),
 216                                                set(["tahoe"])))
 217
 218             d.addCallback(lambda res:
 219                           self.shouldFail(NoSuchChildError, "gcamap-no",
 220                                           "nope",
 221                                           n.get_child_and_metadata_at_path,
 222                                           u"subdir/nope"))
 223             d.addCallback(lambda res:
 224                           n.get_child_and_metadata_at_path(u""))
 225             def _check_child_and_metadata1(res):
 226                 child, metadata = res
 227                 self.failUnless(isinstance(child, dirnode.DirectoryNode))
 228                 # edge-metadata needs at least one path segment
 229                 self.failUnlessEqual(set(metadata.keys()), set([]))
 230             d.addCallback(_check_child_and_metadata1)
 231             d.addCallback(lambda res:
 232                           n.get_child_and_metadata_at_path(u"child"))
 233
 234             def _check_child_and_metadata2(res):
 235                 child, metadata = res
 236                 self.failUnlessReallyEqual(child.get_uri(),
 237                                            fake_file_uri)
 238                 self.failUnlessEqual(set(metadata.keys()), set(["tahoe"]))
 239             d.addCallback(_check_child_and_metadata2)
 240
 241             d.addCallback(lambda res:
 242                           n.get_child_and_metadata_at_path(u"subdir/subsubdir"))
 243             def _check_child_and_metadata3(res):
 244                 child, metadata = res
 245                 self.failUnless(isinstance(child, dirnode.DirectoryNode))
 246                 self.failUnlessEqual(set(metadata.keys()), set(["tahoe"]))
 247             d.addCallback(_check_child_and_metadata3)
 248
 249             # set_uri + metadata
 250             # it should be possible to add a child without any metadata
 251             d.addCallback(lambda res: n.set_uri(u"c2",
 252                                                 fake_file_uri, fake_file_uri,
 253                                                 {}))
 254             d.addCallback(lambda res: n.get_metadata_for(u"c2"))
 255             d.addCallback(lambda metadata:
 256                           self.failUnlessEqual(set(metadata.keys()), set(["tahoe"])))
 257
 258             # You can't override the link timestamps.
 259             d.addCallback(lambda res: n.set_uri(u"c2",
 260                                                 fake_file_uri, fake_file_uri,
 261                                                 { 'tahoe': {'linkcrtime': "bogus"}}))
 262             d.addCallback(lambda res: n.get_metadata_for(u"c2"))
 263             def _has_good_linkcrtime(metadata):
 264                 self.failUnless(metadata.has_key('tahoe'))
 265                 self.failUnless(metadata['tahoe'].has_key('linkcrtime'))
 266                 self.failIfEqual(metadata['tahoe']['linkcrtime'], 'bogus')
 267             d.addCallback(_has_good_linkcrtime)
 268
 269             # if we don't set any defaults, the child should get timestamps
 270             d.addCallback(lambda res: n.set_uri(u"c3",
 271                                                 fake_file_uri, fake_file_uri))
 272             d.addCallback(lambda res: n.get_metadata_for(u"c3"))
 273             d.addCallback(lambda metadata:
 274                           self.failUnlessEqual(set(metadata.keys()), set(["tahoe"])))
 275
 276             # we can also add specific metadata at set_uri() time
 277             d.addCallback(lambda res: n.set_uri(u"c4",
 278                                                 fake_file_uri, fake_file_uri,
 279                                                 {"key": "value"}))
 280             d.addCallback(lambda res: n.get_metadata_for(u"c4"))
 281             d.addCallback(lambda metadata:
 282                               self.failUnless((set(metadata.keys()) == set(["key", "tahoe"])) and
 283                                               (metadata['key'] == "value"), metadata))
 284
 285             d.addCallback(lambda res: n.delete(u"c2"))
 286             d.addCallback(lambda res: n.delete(u"c3"))
 287             d.addCallback(lambda res: n.delete(u"c4"))
 288
 289             # set_node + metadata
 290             # it should be possible to add a child without any metadata except for timestamps
 291             d.addCallback(lambda res: n.set_node(u"d2", n, {}))
 292             d.addCallback(lambda res: c.create_dirnode())
 293             d.addCallback(lambda n2:
 294                           self.shouldFail(ExistingChildError, "set_node-no",
 295                                           "child 'd2' already exists",
 296                                           n.set_node, u"d2", n2,
 297                                           overwrite=False))
 298             d.addCallback(lambda res: n.get_metadata_for(u"d2"))
 299             d.addCallback(lambda metadata:
 300                           self.failUnlessEqual(set(metadata.keys()), set(["tahoe"])))
 301
 302             # if we don't set any defaults, the child should get timestamps
 303             d.addCallback(lambda res: n.set_node(u"d3", n))
 304             d.addCallback(lambda res: n.get_metadata_for(u"d3"))
 305             d.addCallback(lambda metadata:
 306                           self.failUnlessEqual(set(metadata.keys()), set(["tahoe"])))
 307
 308             # we can also add specific metadata at set_node() time
 309             d.addCallback(lambda res: n.set_node(u"d4", n,
 310                                                 {"key": "value"}))
 311             d.addCallback(lambda res: n.get_metadata_for(u"d4"))
 312             d.addCallback(lambda metadata:
 313                           self.failUnless((set(metadata.keys()) == set(["key", "tahoe"])) and
 314                                           (metadata["key"] == "value"), metadata))
 315
 316             d.addCallback(lambda res: n.delete(u"d2"))
 317             d.addCallback(lambda res: n.delete(u"d3"))
 318             d.addCallback(lambda res: n.delete(u"d4"))
 319
 320             # metadata through set_children()
 321             d.addCallback(lambda res:
 322                           n.set_children({
 323                               u"e1": (fake_file_uri, fake_file_uri),
 324                               u"e2": (fake_file_uri, fake_file_uri, {}),
 325                               u"e3": (fake_file_uri, fake_file_uri,
 326                                       {"key": "value"}),
 327                               }))
 328             d.addCallback(lambda n2: self.failUnlessIdentical(n2, n))
 329             d.addCallback(lambda res:
 330                           self.shouldFail(ExistingChildError, "set_children-no",
 331                                           "child 'e1' already exists",
 332                                           n.set_children,
 333                                           { u"e1": (other_file_uri,
 334                                                     other_file_uri),
 335                                             u"new": (other_file_uri,
 336                                                      other_file_uri),
 337                                             },
 338                                           overwrite=False))
 339             # and 'new' should not have been created
 340             d.addCallback(lambda res: n.list())
 341             d.addCallback(lambda children: self.failIf(u"new" in children))
 342             d.addCallback(lambda res: n.get_metadata_for(u"e1"))
 343             d.addCallback(lambda metadata:
 344                           self.failUnlessEqual(set(metadata.keys()), set(["tahoe"])))
 345             d.addCallback(lambda res: n.get_metadata_for(u"e2"))
 346             d.addCallback(lambda metadata:
 347                           self.failUnlessEqual(set(metadata.keys()), set(["tahoe"])))
 348             d.addCallback(lambda res: n.get_metadata_for(u"e3"))
 349             d.addCallback(lambda metadata:
 350                           self.failUnless((set(metadata.keys()) == set(["key", "tahoe"])) and
 351                                           (metadata["key"] == "value"), metadata))
 352
 353             d.addCallback(lambda res: n.delete(u"e1"))
 354             d.addCallback(lambda res: n.delete(u"e2"))
 355             d.addCallback(lambda res: n.delete(u"e3"))
 356
 357             # metadata through set_nodes()
 358             d.addCallback(lambda res:
 359                           n.set_nodes({ u"f1": (n, None),
 360                                         u"f2": (n, {}),
 361                                         u"f3": (n, {"key": "value"}),
 362                                         }))
 363             d.addCallback(lambda n2: self.failUnlessIdentical(n2, n))
 364             d.addCallback(lambda res:
 365                           self.shouldFail(ExistingChildError, "set_nodes-no",
 366                                           "child 'f1' already exists",
 367                                           n.set_nodes, { u"f1": (n, None),
 368                                                          u"new": (n, None), },
 369                                           overwrite=False))
 370             # and 'new' should not have been created
 371             d.addCallback(lambda res: n.list())
 372             d.addCallback(lambda children: self.failIf(u"new" in children))
 373             d.addCallback(lambda res: n.get_metadata_for(u"f1"))
 374             d.addCallback(lambda metadata:
 375                           self.failUnlessEqual(set(metadata.keys()), set(["tahoe"])))
 376             d.addCallback(lambda res: n.get_metadata_for(u"f2"))
 377             d.addCallback(lambda metadata:
 378                           self.failUnlessEqual(set(metadata.keys()), set(["tahoe"])))
 379             d.addCallback(lambda res: n.get_metadata_for(u"f3"))
 380             d.addCallback(lambda metadata:
 381                           self.failUnless((set(metadata.keys()) == set(["key", "tahoe"])) and
 382                                           (metadata["key"] == "value"), metadata))
 383
 384             d.addCallback(lambda res: n.delete(u"f1"))
 385             d.addCallback(lambda res: n.delete(u"f2"))
 386             d.addCallback(lambda res: n.delete(u"f3"))
 387
 388
 389             d.addCallback(lambda res:
 390                           n.set_metadata_for(u"child",
 391                                              {"tags": ["web2.0-compatible"], "tahoe": {"bad": "mojo"}}))
 392             d.addCallback(lambda n1: n1.get_metadata_for(u"child"))
 393             d.addCallback(lambda metadata:
 394                           self.failUnless((set(metadata.keys()) == set(["tags", "tahoe"])) and
 395                                           metadata["tags"] == ["web2.0-compatible"] and
 396                                           "bad" not in metadata["tahoe"], metadata))
 397
 398             d.addCallback(lambda res:
 399                           self.shouldFail(NoSuchChildError, "set_metadata_for-nosuch", "",
 400                                           n.set_metadata_for, u"nosuch", {}))
 401
 402
 403             def _start(res):
 404                 self._start_timestamp = time.time()
 405             d.addCallback(_start)
 406             # simplejson-1.7.1 (as shipped on Ubuntu 'gutsy') rounds all
 407             # floats to hundredeths (it uses str(num) instead of repr(num)).
 408             # simplejson-1.7.3 does not have this bug. To prevent this bug
 409             # from causing the test to fail, stall for more than a few
 410             # hundrededths of a second.
 411             d.addCallback(self.stall, 0.1)
 412             d.addCallback(lambda res: n.add_file(u"timestamps",
 413                                                  upload.Data("stamp me", convergence="some convergence string")))
 414             d.addCallback(self.stall, 0.1)
 415             def _stop(res):
 416                 self._stop_timestamp = time.time()
 417             d.addCallback(_stop)
 418
 419             d.addCallback(lambda res: n.get_metadata_for(u"timestamps"))
 420             def _check_timestamp1(metadata):
 421                 self.failUnlessEqual(set(metadata.keys()), set(["tahoe"]))
 422                 tahoe_md = metadata["tahoe"]
 423                 self.failUnlessEqual(set(tahoe_md.keys()), set(["linkcrtime", "linkmotime"]))
 424
 425                 self.failUnlessGreaterOrEqualThan(tahoe_md["linkcrtime"],
 426                                                   self._start_timestamp)
 427                 self.failUnlessGreaterOrEqualThan(self._stop_timestamp,
 428                                                   tahoe_md["linkcrtime"])
 429                 self.failUnlessGreaterOrEqualThan(tahoe_md["linkmotime"],
 430                                                   self._start_timestamp)
 431                 self.failUnlessGreaterOrEqualThan(self._stop_timestamp,
 432                                                   tahoe_md["linkmotime"])
 433                 # Our current timestamp rules say that replacing an existing
 434                 # child should preserve the 'linkcrtime' but update the
 435                 # 'linkmotime'
 436                 self._old_linkcrtime = tahoe_md["linkcrtime"]
 437                 self._old_linkmotime = tahoe_md["linkmotime"]
 438             d.addCallback(_check_timestamp1)
 439             d.addCallback(self.stall, 2.0) # accomodate low-res timestamps
 440             d.addCallback(lambda res: n.set_node(u"timestamps", n))
 441             d.addCallback(lambda res: n.get_metadata_for(u"timestamps"))
 442             def _check_timestamp2(metadata):
 443                 self.failUnlessIn("tahoe", metadata)
 444                 tahoe_md = metadata["tahoe"]
 445                 self.failUnlessEqual(set(tahoe_md.keys()), set(["linkcrtime", "linkmotime"]))
 446
 447                 self.failUnlessReallyEqual(tahoe_md["linkcrtime"], self._old_linkcrtime)
 448                 self.failUnlessGreaterThan(tahoe_md["linkmotime"], self._old_linkmotime)
 449                 return n.delete(u"timestamps")
 450             d.addCallback(_check_timestamp2)
 451
 452             d.addCallback(lambda res: n.delete(u"subdir"))
 453             d.addCallback(lambda old_child:
 454                           self.failUnlessReallyEqual(old_child.get_uri(),
 455                                                      self.subdir.get_uri()))
 456
 457             d.addCallback(lambda res: n.list())
 458             d.addCallback(lambda children:
 459                           self.failUnlessReallyEqual(set(children.keys()),
 460                                                      set([u"child"])))
 461
 462             uploadable1 = upload.Data("some data", convergence="converge")
 463             d.addCallback(lambda res: n.add_file(u"newfile", uploadable1))
 464             d.addCallback(lambda newnode:
 465                           self.failUnless(IImmutableFileNode.providedBy(newnode)))
 466             uploadable2 = upload.Data("some data", convergence="stuff")
 467             d.addCallback(lambda res:
 468                           self.shouldFail(ExistingChildError, "add_file-no",
 469                                           "child 'newfile' already exists",
 470                                           n.add_file, u"newfile",
 471                                           uploadable2,
 472                                           overwrite=False))
 473             d.addCallback(lambda res: n.list())
 474             d.addCallback(lambda children:
 475                           self.failUnlessReallyEqual(set(children.keys()),
 476                                                      set([u"child", u"newfile"])))
 477             d.addCallback(lambda res: n.get_metadata_for(u"newfile"))
 478             d.addCallback(lambda metadata:
 479                           self.failUnlessEqual(set(metadata.keys()), set(["tahoe"])))
 480
 481             uploadable3 = upload.Data("some data", convergence="converge")
 482             d.addCallback(lambda res: n.add_file(u"newfile-metadata",
 483                                                  uploadable3,
 484                                                  {"key": "value"}))
 485             d.addCallback(lambda newnode:
 486                           self.failUnless(IImmutableFileNode.providedBy(newnode)))
 487             d.addCallback(lambda res: n.get_metadata_for(u"newfile-metadata"))
 488             d.addCallback(lambda metadata:
 489                               self.failUnless((set(metadata.keys()) == set(["key", "tahoe"])) and
 490                                               (metadata['key'] == "value"), metadata))
 491             d.addCallback(lambda res: n.delete(u"newfile-metadata"))
 492
 493             d.addCallback(lambda res: n.create_subdirectory(u"subdir2"))
 494             def _created2(subdir2):
 495                 self.subdir2 = subdir2
 496                 # put something in the way, to make sure it gets overwritten
 497                 return subdir2.add_file(u"child", upload.Data("overwrite me",
 498                                                               "converge"))
 499             d.addCallback(_created2)
 500
 501             d.addCallback(lambda res:
 502                           n.move_child_to(u"child", self.subdir2))
 503             d.addCallback(lambda res: n.list())
 504             d.addCallback(lambda children:
 505                           self.failUnlessReallyEqual(set(children.keys()),
 506                                                      set([u"newfile", u"subdir2"])))
 507             d.addCallback(lambda res: self.subdir2.list())
 508             d.addCallback(lambda children:
 509                           self.failUnlessReallyEqual(set(children.keys()),
 510                                                      set([u"child"])))
 511             d.addCallback(lambda res: self.subdir2.get(u"child"))
 512             d.addCallback(lambda child:
 513                           self.failUnlessReallyEqual(child.get_uri(),
 514                                                      fake_file_uri))
 515
 516             # move it back, using new_child_name=
 517             d.addCallback(lambda res:
 518                           self.subdir2.move_child_to(u"child", n, u"newchild"))
 519             d.addCallback(lambda res: n.list())
 520             d.addCallback(lambda children:
 521                           self.failUnlessReallyEqual(set(children.keys()),
 522                                                      set([u"newchild", u"newfile",
 523                                                           u"subdir2"])))
 524             d.addCallback(lambda res: self.subdir2.list())
 525             d.addCallback(lambda children:
 526                           self.failUnlessReallyEqual(set(children.keys()), set([])))
 527
 528             # now make sure that we honor overwrite=False
 529             d.addCallback(lambda res:
 530                           self.subdir2.set_uri(u"newchild",
 531                                                other_file_uri, other_file_uri))
 532
 533             d.addCallback(lambda res:
 534                           self.shouldFail(ExistingChildError, "move_child_to-no",
 535                                           "child 'newchild' already exists",
 536                                           n.move_child_to, u"newchild",
 537                                           self.subdir2,
 538                                           overwrite=False))
 539             d.addCallback(lambda res: self.subdir2.get(u"newchild"))
 540             d.addCallback(lambda child:
 541                           self.failUnlessReallyEqual(child.get_uri(),
 542                                                      other_file_uri))
 543
 544
 545             # Setting the no-write field should diminish a mutable cap to read-only
 546             # (for both files and directories).
 547
 548             d.addCallback(lambda ign: n.set_uri(u"mutable", other_file_uri, other_file_uri))
 549             d.addCallback(lambda ign: n.get(u"mutable"))
 550             d.addCallback(lambda mutable: self.failIf(mutable.is_readonly(), mutable))
 551             d.addCallback(lambda ign: n.set_metadata_for(u"mutable", {"no-write": True}))
 552             d.addCallback(lambda ign: n.get(u"mutable"))
 553             d.addCallback(lambda mutable: self.failUnless(mutable.is_readonly(), mutable))
 554             d.addCallback(lambda ign: n.set_metadata_for(u"mutable", {"no-write": True}))
 555             d.addCallback(lambda ign: n.get(u"mutable"))
 556             d.addCallback(lambda mutable: self.failUnless(mutable.is_readonly(), mutable))
 557
 558             d.addCallback(lambda ign: n.get(u"subdir2"))
 559             d.addCallback(lambda subdir2: self.failIf(subdir2.is_readonly()))
 560             d.addCallback(lambda ign: n.set_metadata_for(u"subdir2", {"no-write": True}))
 561             d.addCallback(lambda ign: n.get(u"subdir2"))
 562             d.addCallback(lambda subdir2: self.failUnless(subdir2.is_readonly(), subdir2))
 563
 564             d.addCallback(lambda ign: n.set_uri(u"mutable_ro", other_file_uri, other_file_uri,
 565                                                 metadata={"no-write": True}))
 566             d.addCallback(lambda ign: n.get(u"mutable_ro"))
 567             d.addCallback(lambda mutable_ro: self.failUnless(mutable_ro.is_readonly(), mutable_ro))
 568
 569             d.addCallback(lambda ign: n.create_subdirectory(u"subdir_ro", metadata={"no-write": True}))
 570             d.addCallback(lambda ign: n.get(u"subdir_ro"))
 571             d.addCallback(lambda subdir_ro: self.failUnless(subdir_ro.is_readonly(), subdir_ro))
 572
 573             return d
 574
 575         d.addCallback(_then)
 576
 577         d.addErrback(self.explain_error)
 578         return d
 579
 580
 581     def _do_initial_children_test(self, mdmf=False):
 582         c = self.g.clients[0]
 583         nm = c.nodemaker
 584
 585         kids = {one_nfd: (nm.create_from_cap(one_uri), {}),
 586                 u"two": (nm.create_from_cap(setup_py_uri),
 587                          {"metakey": "metavalue"}),
 588                 u"mut": (nm.create_from_cap(mut_write_uri, mut_read_uri), {}),
 589                 u"fut": (nm.create_from_cap(future_write_uri, future_read_uri), {}),
 590                 u"fro": (nm.create_from_cap(None, future_read_uri), {}),
 591                 u"fut-unic": (nm.create_from_cap(future_nonascii_write_uri, future_nonascii_read_uri), {}),
 592                 u"fro-unic": (nm.create_from_cap(None, future_nonascii_read_uri), {}),
 593                 u"empty_litdir": (nm.create_from_cap(empty_litdir_uri), {}),
 594                 u"tiny_litdir": (nm.create_from_cap(tiny_litdir_uri), {}),
 595                 }
 596         if mdmf:
 597             d = c.create_dirnode(kids, version=MDMF_VERSION)
 598         else:
 599             d = c.create_dirnode(kids)
 600
 601         def _created(dn):
 602             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
 603             backing_node = dn._node
 604             if mdmf:
 605                 self.failUnlessEqual(backing_node.get_version(),
 606                                      MDMF_VERSION)
 607             else:
 608                 self.failUnlessEqual(backing_node.get_version(),
 609                                      SDMF_VERSION)
 610             self.failUnless(dn.is_mutable())
 611             self.failIf(dn.is_readonly())
 612             self.failIf(dn.is_unknown())
 613             self.failIf(dn.is_allowed_in_immutable_directory())
 614             dn.raise_error()
 615             rep = str(dn)
 616             self.failUnless("RW-MUT" in rep)
 617             return dn.list()
 618         d.addCallback(_created)
 619
 620         def _check_kids(children):
 621             self.failUnlessReallyEqual(set(children.keys()),
 622                                        set([one_nfc, u"two", u"mut", u"fut", u"fro",
 623                                         u"fut-unic", u"fro-unic", u"empty_litdir", u"tiny_litdir"]))
 624             one_node, one_metadata = children[one_nfc]
 625             two_node, two_metadata = children[u"two"]
 626             mut_node, mut_metadata = children[u"mut"]
 627             fut_node, fut_metadata = children[u"fut"]
 628             fro_node, fro_metadata = children[u"fro"]
 629             futna_node, futna_metadata = children[u"fut-unic"]
 630             frona_node, frona_metadata = children[u"fro-unic"]
 631             emptylit_node, emptylit_metadata = children[u"empty_litdir"]
 632             tinylit_node, tinylit_metadata = children[u"tiny_litdir"]
 633
 634             self.failUnlessReallyEqual(one_node.get_size(), 3)
 635             self.failUnlessReallyEqual(one_node.get_uri(), one_uri)
 636             self.failUnlessReallyEqual(one_node.get_readonly_uri(), one_uri)
 637             self.failUnless(isinstance(one_metadata, dict), one_metadata)
 638
 639             self.failUnlessReallyEqual(two_node.get_size(), 14861)
 640             self.failUnlessReallyEqual(two_node.get_uri(), setup_py_uri)
 641             self.failUnlessReallyEqual(two_node.get_readonly_uri(), setup_py_uri)
 642             self.failUnlessEqual(two_metadata["metakey"], "metavalue")
 643
 644             self.failUnlessReallyEqual(mut_node.get_uri(), mut_write_uri)
 645             self.failUnlessReallyEqual(mut_node.get_readonly_uri(), mut_read_uri)
 646             self.failUnless(isinstance(mut_metadata, dict), mut_metadata)
 647
 648             self.failUnless(fut_node.is_unknown())
 649             self.failUnlessReallyEqual(fut_node.get_uri(), future_write_uri)
 650             self.failUnlessReallyEqual(fut_node.get_readonly_uri(), "ro." + future_read_uri)
 651             self.failUnless(isinstance(fut_metadata, dict), fut_metadata)
 652
 653             self.failUnless(futna_node.is_unknown())
 654             self.failUnlessReallyEqual(futna_node.get_uri(), future_nonascii_write_uri)
 655             self.failUnlessReallyEqual(futna_node.get_readonly_uri(), "ro." + future_nonascii_read_uri)
 656             self.failUnless(isinstance(futna_metadata, dict), futna_metadata)
 657
 658             self.failUnless(fro_node.is_unknown())
 659             self.failUnlessReallyEqual(fro_node.get_uri(), "ro." + future_read_uri)
 660             self.failUnlessReallyEqual(fut_node.get_readonly_uri(), "ro." + future_read_uri)
 661             self.failUnless(isinstance(fro_metadata, dict), fro_metadata)
 662
 663             self.failUnless(frona_node.is_unknown())
 664             self.failUnlessReallyEqual(frona_node.get_uri(), "ro." + future_nonascii_read_uri)
 665             self.failUnlessReallyEqual(futna_node.get_readonly_uri(), "ro." + future_nonascii_read_uri)
 666             self.failUnless(isinstance(frona_metadata, dict), frona_metadata)
 667
 668             self.failIf(emptylit_node.is_unknown())
 669             self.failUnlessReallyEqual(emptylit_node.get_storage_index(), None)
 670             self.failIf(tinylit_node.is_unknown())
 671             self.failUnlessReallyEqual(tinylit_node.get_storage_index(), None)
 672
 673             d2 = defer.succeed(None)
 674             d2.addCallback(lambda ignored: emptylit_node.list())
 675             d2.addCallback(lambda children: self.failUnlessEqual(children, {}))
 676             d2.addCallback(lambda ignored: tinylit_node.list())
 677             d2.addCallback(lambda children: self.failUnlessReallyEqual(set(children.keys()),
 678                                                                        set([u"short"])))
 679             d2.addCallback(lambda ignored: tinylit_node.list())
 680             d2.addCallback(lambda children: children[u"short"][0].read(MemAccum()))
 681             d2.addCallback(lambda accum: self.failUnlessReallyEqual(accum.data, "The end."))
 682             return d2
 683         d.addCallback(_check_kids)
 684
 685         d.addCallback(lambda ign: nm.create_new_mutable_directory(kids))
 686         d.addCallback(lambda dn: dn.list())
 687         d.addCallback(_check_kids)
 688
 689         bad_future_node = UnknownNode(future_write_uri, None)
 690         bad_kids1 = {one_nfd: (bad_future_node, {})}
 691         # This should fail because we don't know how to diminish the future_write_uri
 692         # cap (given in a write slot and not prefixed with "ro." or "imm.") to a readcap.
 693         d.addCallback(lambda ign:
 694                       self.shouldFail(MustNotBeUnknownRWError, "bad_kids1",
 695                                       "cannot attach unknown",
 696                                       nm.create_new_mutable_directory,
 697                                       bad_kids1))
 698         bad_kids2 = {one_nfd: (nm.create_from_cap(one_uri), None)}
 699         d.addCallback(lambda ign:
 700                       self.shouldFail(AssertionError, "bad_kids2",
 701                                       "requires metadata to be a dict",
 702                                       nm.create_new_mutable_directory,
 703                                       bad_kids2))
 704         return d
 705
 706     def _do_basic_test(self, mdmf=False):
 707         c = self.g.clients[0]
 708         d = None
 709         if mdmf:
 710             d = c.create_dirnode(version=MDMF_VERSION)
 711         else:
 712             d = c.create_dirnode()
 713         def _done(res):
 714             self.failUnless(isinstance(res, dirnode.DirectoryNode))
 715             self.failUnless(res.is_mutable())
 716             self.failIf(res.is_readonly())
 717             self.failIf(res.is_unknown())
 718             self.failIf(res.is_allowed_in_immutable_directory())
 719             res.raise_error()
 720             rep = str(res)
 721             self.failUnless("RW-MUT" in rep)
 722         d.addCallback(_done)
 723         return d
 724
 725     def test_basic(self):
 726         self.basedir = "dirnode/Dirnode/test_basic"
 727         self.set_up_grid()
 728         return self._do_basic_test()
 729
 730     def test_basic_mdmf(self):
 731         self.basedir = "dirnode/Dirnode/test_basic_mdmf"
 732         self.set_up_grid()
 733         return self._do_basic_test(mdmf=True)
 734
 735     def test_initial_children(self):
 736         self.basedir = "dirnode/Dirnode/test_initial_children"
 737         self.set_up_grid()
 738         return self._do_initial_children_test()
 739
 740     def test_immutable(self):
 741         self.basedir = "dirnode/Dirnode/test_immutable"
 742         self.set_up_grid()
 743         c = self.g.clients[0]
 744         nm = c.nodemaker
 745
 746         kids = {one_nfd: (nm.create_from_cap(one_uri), {}),
 747                 u"two": (nm.create_from_cap(setup_py_uri),
 748                          {"metakey": "metavalue"}),
 749                 u"fut": (nm.create_from_cap(None, future_read_uri), {}),
 750                 u"futna": (nm.create_from_cap(None, future_nonascii_read_uri), {}),
 751                 u"empty_litdir": (nm.create_from_cap(empty_litdir_uri), {}),
 752                 u"tiny_litdir": (nm.create_from_cap(tiny_litdir_uri), {}),
 753                 }
 754         d = c.create_immutable_dirnode(kids)
 755
 756         def _created(dn):
 757             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
 758             self.failIf(dn.is_mutable())
 759             self.failUnless(dn.is_readonly())
 760             self.failIf(dn.is_unknown())
 761             self.failUnless(dn.is_allowed_in_immutable_directory())
 762             dn.raise_error()
 763             rep = str(dn)
 764             self.failUnless("RO-IMM" in rep)
 765             cap = dn.get_cap()
 766             self.failUnlessIn("CHK", cap.to_string())
 767             self.cap = cap
 768             return dn.list()
 769         d.addCallback(_created)
 770
 771         def _check_kids(children):
 772             self.failUnlessReallyEqual(set(children.keys()),
 773                                        set([one_nfc, u"two", u"fut", u"futna", u"empty_litdir", u"tiny_litdir"]))
 774             one_node, one_metadata = children[one_nfc]
 775             two_node, two_metadata = children[u"two"]
 776             fut_node, fut_metadata = children[u"fut"]
 777             futna_node, futna_metadata = children[u"futna"]
 778             emptylit_node, emptylit_metadata = children[u"empty_litdir"]
 779             tinylit_node, tinylit_metadata = children[u"tiny_litdir"]
 780
 781             self.failUnlessReallyEqual(one_node.get_size(), 3)
 782             self.failUnlessReallyEqual(one_node.get_uri(), one_uri)
 783             self.failUnlessReallyEqual(one_node.get_readonly_uri(), one_uri)
 784             self.failUnless(isinstance(one_metadata, dict), one_metadata)
 785
 786             self.failUnlessReallyEqual(two_node.get_size(), 14861)
 787             self.failUnlessReallyEqual(two_node.get_uri(), setup_py_uri)
 788             self.failUnlessReallyEqual(two_node.get_readonly_uri(), setup_py_uri)
 789             self.failUnlessEqual(two_metadata["metakey"], "metavalue")
 790
 791             self.failUnless(fut_node.is_unknown())
 792             self.failUnlessReallyEqual(fut_node.get_uri(), "imm." + future_read_uri)
 793             self.failUnlessReallyEqual(fut_node.get_readonly_uri(), "imm." + future_read_uri)
 794             self.failUnless(isinstance(fut_metadata, dict), fut_metadata)
 795
 796             self.failUnless(futna_node.is_unknown())
 797             self.failUnlessReallyEqual(futna_node.get_uri(), "imm." + future_nonascii_read_uri)
 798             self.failUnlessReallyEqual(futna_node.get_readonly_uri(), "imm." + future_nonascii_read_uri)
 799             self.failUnless(isinstance(futna_metadata, dict), futna_metadata)
 800
 801             self.failIf(emptylit_node.is_unknown())
 802             self.failUnlessReallyEqual(emptylit_node.get_storage_index(), None)
 803             self.failIf(tinylit_node.is_unknown())
 804             self.failUnlessReallyEqual(tinylit_node.get_storage_index(), None)
 805
 806             d2 = defer.succeed(None)
 807             d2.addCallback(lambda ignored: emptylit_node.list())
 808             d2.addCallback(lambda children: self.failUnlessEqual(children, {}))
 809             d2.addCallback(lambda ignored: tinylit_node.list())
 810             d2.addCallback(lambda children: self.failUnlessReallyEqual(set(children.keys()),
 811                                                                        set([u"short"])))
 812             d2.addCallback(lambda ignored: tinylit_node.list())
 813             d2.addCallback(lambda children: children[u"short"][0].read(MemAccum()))
 814             d2.addCallback(lambda accum: self.failUnlessReallyEqual(accum.data, "The end."))
 815             return d2
 816
 817         d.addCallback(_check_kids)
 818
 819         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
 820         d.addCallback(lambda dn: dn.list())
 821         d.addCallback(_check_kids)
 822
 823         bad_future_node1 = UnknownNode(future_write_uri, None)
 824         bad_kids1 = {one_nfd: (bad_future_node1, {})}
 825         d.addCallback(lambda ign:
 826                       self.shouldFail(MustNotBeUnknownRWError, "bad_kids1",
 827                                       "cannot attach unknown",
 828                                       c.create_immutable_dirnode,
 829                                       bad_kids1))
 830         bad_future_node2 = UnknownNode(future_write_uri, future_read_uri)
 831         bad_kids2 = {one_nfd: (bad_future_node2, {})}
 832         d.addCallback(lambda ign:
 833                       self.shouldFail(MustBeDeepImmutableError, "bad_kids2",
 834                                       "is not allowed in an immutable directory",
 835                                       c.create_immutable_dirnode,
 836                                       bad_kids2))
 837         bad_kids3 = {one_nfd: (nm.create_from_cap(one_uri), None)}
 838         d.addCallback(lambda ign:
 839                       self.shouldFail(AssertionError, "bad_kids3",
 840                                       "requires metadata to be a dict",
 841                                       c.create_immutable_dirnode,
 842                                       bad_kids3))
 843         bad_kids4 = {one_nfd: (nm.create_from_cap(mut_write_uri), {})}
 844         d.addCallback(lambda ign:
 845                       self.shouldFail(MustBeDeepImmutableError, "bad_kids4",
 846                                       "is not allowed in an immutable directory",
 847                                       c.create_immutable_dirnode,
 848                                       bad_kids4))
 849         bad_kids5 = {one_nfd: (nm.create_from_cap(mut_read_uri), {})}
 850         d.addCallback(lambda ign:
 851                       self.shouldFail(MustBeDeepImmutableError, "bad_kids5",
 852                                       "is not allowed in an immutable directory",
 853                                       c.create_immutable_dirnode,
 854                                       bad_kids5))
 855         d.addCallback(lambda ign: c.create_immutable_dirnode({}))
 856         def _created_empty(dn):
 857             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
 858             self.failIf(dn.is_mutable())
 859             self.failUnless(dn.is_readonly())
 860             self.failIf(dn.is_unknown())
 861             self.failUnless(dn.is_allowed_in_immutable_directory())
 862             dn.raise_error()
 863             rep = str(dn)
 864             self.failUnless("RO-IMM" in rep)
 865             cap = dn.get_cap()
 866             self.failUnlessIn("LIT", cap.to_string())
 867             self.failUnlessReallyEqual(cap.to_string(), "URI:DIR2-LIT:")
 868             self.cap = cap
 869             return dn.list()
 870         d.addCallback(_created_empty)
 871         d.addCallback(lambda kids: self.failUnlessEqual(kids, {}))
 872         smallkids = {u"o": (nm.create_from_cap(one_uri), {})}
 873         d.addCallback(lambda ign: c.create_immutable_dirnode(smallkids))
 874         def _created_small(dn):
 875             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
 876             self.failIf(dn.is_mutable())
 877             self.failUnless(dn.is_readonly())
 878             self.failIf(dn.is_unknown())
 879             self.failUnless(dn.is_allowed_in_immutable_directory())
 880             dn.raise_error()
 881             rep = str(dn)
 882             self.failUnless("RO-IMM" in rep)
 883             cap = dn.get_cap()
 884             self.failUnlessIn("LIT", cap.to_string())
 885             self.failUnlessReallyEqual(cap.to_string(),
 886                                        "URI:DIR2-LIT:gi4tumj2n4wdcmz2kvjesosmjfkdu3rvpbtwwlbqhiwdeot3puwcy")
 887             self.cap = cap
 888             return dn.list()
 889         d.addCallback(_created_small)
 890         d.addCallback(lambda kids: self.failUnlessReallyEqual(kids.keys(), [u"o"]))
 891
 892         # now test n.create_subdirectory(mutable=False)
 893         d.addCallback(lambda ign: c.create_dirnode())
 894         def _made_parent(n):
 895             d = n.create_subdirectory(u"subdir", kids, mutable=False)
 896             d.addCallback(lambda sd: sd.list())
 897             d.addCallback(_check_kids)
 898             d.addCallback(lambda ign: n.list())
 899             d.addCallback(lambda children:
 900                           self.failUnlessReallyEqual(children.keys(), [u"subdir"]))
 901             d.addCallback(lambda ign: n.get(u"subdir"))
 902             d.addCallback(lambda sd: sd.list())
 903             d.addCallback(_check_kids)
 904             d.addCallback(lambda ign: n.get(u"subdir"))
 905             d.addCallback(lambda sd: self.failIf(sd.is_mutable()))
 906             bad_kids = {one_nfd: (nm.create_from_cap(mut_write_uri), {})}
 907             d.addCallback(lambda ign:
 908                           self.shouldFail(MustBeDeepImmutableError, "YZ",
 909                                           "is not allowed in an immutable directory",
 910                                           n.create_subdirectory,
 911                                           u"sub2", bad_kids, mutable=False))
 912             return d
 913         d.addCallback(_made_parent)
 914         return d
 915
 916     def test_directory_representation(self):
 917         self.basedir = "dirnode/Dirnode/test_directory_representation"
 918         self.set_up_grid()
 919         c = self.g.clients[0]
 920         nm = c.nodemaker
 921
 922         # This test checks that any trailing spaces in URIs are retained in the
 923         # encoded directory, but stripped when we get them out of the directory.
 924         # See ticket #925 for why we want that.
 925         # It also tests that we store child names as UTF-8 NFC, and normalize
 926         # them again when retrieving them.
 927
 928         stripped_write_uri = "lafs://from_the_future\t"
 929         stripped_read_uri = "lafs://readonly_from_the_future\t"
 930         spacedout_write_uri = stripped_write_uri + "  "
 931         spacedout_read_uri = stripped_read_uri + "  "
 932
 933         child = nm.create_from_cap(spacedout_write_uri, spacedout_read_uri)
 934         self.failUnlessReallyEqual(child.get_write_uri(), spacedout_write_uri)
 935         self.failUnlessReallyEqual(child.get_readonly_uri(), "ro." + spacedout_read_uri)
 936
 937         child_dottedi = u"ch\u0131\u0307ld"
 938
 939         kids_in   = {child_dottedi: (child, {}), one_nfd: (child, {})}
 940         kids_out  = {child_dottedi: (child, {}), one_nfc: (child, {})}
 941         kids_norm = {u"child":      (child, {}), one_nfc: (child, {})}
 942         d = c.create_dirnode(kids_in)
 943
 944         def _created(dn):
 945             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
 946             self.failUnless(dn.is_mutable())
 947             self.failIf(dn.is_readonly())
 948             dn.raise_error()
 949             self.cap = dn.get_cap()
 950             self.rootnode = dn
 951             return dn._node.download_best_version()
 952         d.addCallback(_created)
 953
 954         def _check_data(data):
 955             # Decode the netstring representation of the directory to check that the
 956             # spaces are retained when the URIs are stored, and that the names are stored
 957             # as NFC.
 958             position = 0
 959             numkids = 0
 960             while position < len(data):
 961                 entries, position = split_netstring(data, 1, position)
 962                 entry = entries[0]
 963                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
 964                 name = name_utf8.decode("utf-8")
 965                 rw_uri = self.rootnode._decrypt_rwcapdata(rwcapdata)
 966                 self.failUnlessIn(name, kids_out)
 967                 (expected_child, ign) = kids_out[name]
 968                 self.failUnlessReallyEqual(rw_uri, expected_child.get_write_uri())
 969                 self.failUnlessReallyEqual("ro." + ro_uri, expected_child.get_readonly_uri())
 970                 numkids += 1
 971
 972             self.failUnlessReallyEqual(numkids, len(kids_out))
 973             return self.rootnode
 974         d.addCallback(_check_data)
 975
 976         # Mock up a hypothetical future version of Unicode that adds a canonical equivalence
 977         # between dotless-i + dot-above, and 'i'. That would actually be prohibited by the
 978         # stability rules, but similar additions involving currently-unassigned characters
 979         # would not be.
 980         old_normalize = unicodedata.normalize
 981         def future_normalize(form, s):
 982             assert form == 'NFC', form
 983             return old_normalize(form, s).replace(u"\u0131\u0307", u"i")
 984
 985         def _list(node):
 986             unicodedata.normalize = future_normalize
 987             d2 = node.list()
 988             def _undo_mock(res):
 989                 unicodedata.normalize = old_normalize
 990                 return res
 991             d2.addBoth(_undo_mock)
 992             return d2
 993         d.addCallback(_list)
 994
 995         def _check_kids(children):
 996             # Now when we use the real directory listing code, the trailing spaces
 997             # should have been stripped (and "ro." should have been prepended to the
 998             # ro_uri, since it's unknown). Also the dotless-i + dot-above should have been
 999             # normalized to 'i'.
1000
1001             self.failUnlessReallyEqual(set(children.keys()), set(kids_norm.keys()))
1002             child_node, child_metadata = children[u"child"]
1003
1004             self.failUnlessReallyEqual(child_node.get_write_uri(), stripped_write_uri)
1005             self.failUnlessReallyEqual(child_node.get_readonly_uri(), "ro." + stripped_read_uri)
1006         d.addCallback(_check_kids)
1007
1008         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
1009         d.addCallback(_list)
1010         d.addCallback(_check_kids)  # again with dirnode recreated from cap
1011         return d
1012
1013     def test_check(self):
1014         self.basedir = "dirnode/Dirnode/test_check"
1015         self.set_up_grid()
1016         c = self.g.clients[0]
1017         d = c.create_dirnode()
1018         d.addCallback(lambda dn: dn.check(Monitor()))
1019         def _done(res):
1020             self.failUnless(res.is_healthy())
1021         d.addCallback(_done)
1022         return d
1023
1024     def _test_deepcheck_create(self, version=SDMF_VERSION):
1025         # create a small tree with a loop, and some non-directories
1026         #  root/
1027         #  root/subdir/
1028         #  root/subdir/file1
1029         #  root/subdir/link -> root
1030         #  root/rodir
1031         c = self.g.clients[0]
1032         d = c.create_dirnode(version=version)
1033         def _created_root(rootnode):
1034             self._rootnode = rootnode
1035             self.failUnlessEqual(rootnode._node.get_version(), version)
1036             return rootnode.create_subdirectory(u"subdir")
1037         d.addCallback(_created_root)
1038         def _created_subdir(subdir):
1039             self._subdir = subdir
1040             d = subdir.add_file(u"file1", upload.Data("data"*100, None))
1041             d.addCallback(lambda res: subdir.set_node(u"link", self._rootnode))
1042             d.addCallback(lambda res: c.create_dirnode())
1043             d.addCallback(lambda dn:
1044                           self._rootnode.set_uri(u"rodir",
1045                                                  dn.get_uri(),
1046                                                  dn.get_readonly_uri()))
1047             return d
1048         d.addCallback(_created_subdir)
1049         def _done(res):
1050             return self._rootnode
1051         d.addCallback(_done)
1052         return d
1053
1054     def test_deepcheck(self):
1055         self.basedir = "dirnode/Dirnode/test_deepcheck"
1056         self.set_up_grid()
1057         d = self._test_deepcheck_create()
1058         d.addCallback(lambda rootnode: rootnode.start_deep_check().when_done())
1059         def _check_results(r):
1060             self.failUnless(IDeepCheckResults.providedBy(r))
1061             c = r.get_counters()
1062             self.failUnlessReallyEqual(c,
1063                                        {"count-objects-checked": 4,
1064                                         "count-objects-healthy": 4,
1065                                         "count-objects-unhealthy": 0,
1066                                         "count-objects-unrecoverable": 0,
1067                                         "count-corrupt-shares": 0,
1068                                         })
1069             self.failIf(r.get_corrupt_shares())
1070             self.failUnlessReallyEqual(len(r.get_all_results()), 4)
1071         d.addCallback(_check_results)
1072         return d
1073
1074     def test_deepcheck_mdmf(self):
1075         self.basedir = "dirnode/Dirnode/test_deepcheck_mdmf"
1076         self.set_up_grid()
1077         d = self._test_deepcheck_create(MDMF_VERSION)
1078         d.addCallback(lambda rootnode: rootnode.start_deep_check().when_done())
1079         def _check_results(r):
1080             self.failUnless(IDeepCheckResults.providedBy(r))
1081             c = r.get_counters()
1082             self.failUnlessReallyEqual(c,
1083                                        {"count-objects-checked": 4,
1084                                         "count-objects-healthy": 4,
1085                                         "count-objects-unhealthy": 0,
1086                                         "count-objects-unrecoverable": 0,
1087                                         "count-corrupt-shares": 0,
1088                                         })
1089             self.failIf(r.get_corrupt_shares())
1090             self.failUnlessReallyEqual(len(r.get_all_results()), 4)
1091         d.addCallback(_check_results)
1092         return d
1093
1094     def test_deepcheck_and_repair(self):
1095         self.basedir = "dirnode/Dirnode/test_deepcheck_and_repair"
1096         self.set_up_grid()
1097         d = self._test_deepcheck_create()
1098         d.addCallback(lambda rootnode:
1099                       rootnode.start_deep_check_and_repair().when_done())
1100         def _check_results(r):
1101             self.failUnless(IDeepCheckAndRepairResults.providedBy(r))
1102             c = r.get_counters()
1103             self.failUnlessReallyEqual(c,
1104                                        {"count-objects-checked": 4,
1105                                         "count-objects-healthy-pre-repair": 4,
1106                                         "count-objects-unhealthy-pre-repair": 0,
1107                                         "count-objects-unrecoverable-pre-repair": 0,
1108                                         "count-corrupt-shares-pre-repair": 0,
1109                                         "count-objects-healthy-post-repair": 4,
1110                                         "count-objects-unhealthy-post-repair": 0,
1111                                         "count-objects-unrecoverable-post-repair": 0,
1112                                         "count-corrupt-shares-post-repair": 0,
1113                                         "count-repairs-attempted": 0,
1114                                         "count-repairs-successful": 0,
1115                                         "count-repairs-unsuccessful": 0,
1116                                         })
1117             self.failIf(r.get_corrupt_shares())
1118             self.failIf(r.get_remaining_corrupt_shares())
1119             self.failUnlessReallyEqual(len(r.get_all_results()), 4)
1120         d.addCallback(_check_results)
1121         return d
1122
1123     def test_deepcheck_and_repair_mdmf(self):
1124         self.basedir = "dirnode/Dirnode/test_deepcheck_and_repair_mdmf"
1125         self.set_up_grid()
1126         d = self._test_deepcheck_create(version=MDMF_VERSION)
1127         d.addCallback(lambda rootnode:
1128                       rootnode.start_deep_check_and_repair().when_done())
1129         def _check_results(r):
1130             self.failUnless(IDeepCheckAndRepairResults.providedBy(r))
1131             c = r.get_counters()
1132             self.failUnlessReallyEqual(c,
1133                                        {"count-objects-checked": 4,
1134                                         "count-objects-healthy-pre-repair": 4,
1135                                         "count-objects-unhealthy-pre-repair": 0,
1136                                         "count-objects-unrecoverable-pre-repair": 0,
1137                                         "count-corrupt-shares-pre-repair": 0,
1138                                         "count-objects-healthy-post-repair": 4,
1139                                         "count-objects-unhealthy-post-repair": 0,
1140                                         "count-objects-unrecoverable-post-repair": 0,
1141                                         "count-corrupt-shares-post-repair": 0,
1142                                         "count-repairs-attempted": 0,
1143                                         "count-repairs-successful": 0,
1144                                         "count-repairs-unsuccessful": 0,
1145                                         })
1146             self.failIf(r.get_corrupt_shares())
1147             self.failIf(r.get_remaining_corrupt_shares())
1148             self.failUnlessReallyEqual(len(r.get_all_results()), 4)
1149         d.addCallback(_check_results)
1150         return d
1151
1152     def _mark_file_bad(self, rootnode):
1153         self.delete_shares_numbered(rootnode.get_uri(), [0])
1154         return rootnode
1155
1156     def test_deepcheck_problems(self):
1157         self.basedir = "dirnode/Dirnode/test_deepcheck_problems"
1158         self.set_up_grid()
1159         d = self._test_deepcheck_create()
1160         d.addCallback(lambda rootnode: self._mark_file_bad(rootnode))
1161         d.addCallback(lambda rootnode: rootnode.start_deep_check().when_done())
1162         def _check_results(r):
1163             c = r.get_counters()
1164             self.failUnlessReallyEqual(c,
1165                                        {"count-objects-checked": 4,
1166                                         "count-objects-healthy": 3,
1167                                         "count-objects-unhealthy": 1,
1168                                         "count-objects-unrecoverable": 0,
1169                                         "count-corrupt-shares": 0,
1170                                         })
1171             #self.failUnlessReallyEqual(len(r.get_problems()), 1) # TODO
1172         d.addCallback(_check_results)
1173         return d
1174
1175     def test_deepcheck_problems_mdmf(self):
1176         self.basedir = "dirnode/Dirnode/test_deepcheck_problems_mdmf"
1177         self.set_up_grid()
1178         d = self._test_deepcheck_create(version=MDMF_VERSION)
1179         d.addCallback(lambda rootnode: self._mark_file_bad(rootnode))
1180         d.addCallback(lambda rootnode: rootnode.start_deep_check().when_done())
1181         def _check_results(r):
1182             c = r.get_counters()
1183             self.failUnlessReallyEqual(c,
1184                                        {"count-objects-checked": 4,
1185                                         "count-objects-healthy": 3,
1186                                         "count-objects-unhealthy": 1,
1187                                         "count-objects-unrecoverable": 0,
1188                                         "count-corrupt-shares": 0,
1189                                         })
1190             #self.failUnlessReallyEqual(len(r.get_problems()), 1) # TODO
1191         d.addCallback(_check_results)
1192         return d
1193
1194     def _do_readonly_test(self, version=SDMF_VERSION):
1195         c = self.g.clients[0]
1196         nm = c.nodemaker
1197         filecap = make_chk_file_uri(1234)
1198         filenode = nm.create_from_cap(filecap)
1199         uploadable = upload.Data("some data", convergence="some convergence string")
1200
1201         d = c.create_dirnode(version=version)
1202         def _created(rw_dn):
1203             backing_node = rw_dn._node
1204             self.failUnlessEqual(backing_node.get_version(), version)
1205             d2 = rw_dn.set_uri(u"child", filecap, filecap)
1206             d2.addCallback(lambda res: rw_dn)
1207             return d2
1208         d.addCallback(_created)
1209
1210         def _ready(rw_dn):
1211             ro_uri = rw_dn.get_readonly_uri()
1212             ro_dn = c.create_node_from_uri(ro_uri)
1213             self.failUnless(ro_dn.is_readonly())
1214             self.failUnless(ro_dn.is_mutable())
1215             self.failIf(ro_dn.is_unknown())
1216             self.failIf(ro_dn.is_allowed_in_immutable_directory())
1217             ro_dn.raise_error()
1218
1219             self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
1220                             ro_dn.set_uri, u"newchild", filecap, filecap)
1221             self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
1222                             ro_dn.set_node, u"newchild", filenode)
1223             self.shouldFail(dirnode.NotWriteableError, "set_nodes ro", None,
1224                             ro_dn.set_nodes, { u"newchild": (filenode, None) })
1225             self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
1226                             ro_dn.add_file, u"newchild", uploadable)
1227             self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
1228                             ro_dn.delete, u"child")
1229             self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
1230                             ro_dn.create_subdirectory, u"newchild")
1231             self.shouldFail(dirnode.NotWriteableError, "set_metadata_for ro", None,
1232                             ro_dn.set_metadata_for, u"child", {})
1233             self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
1234                             ro_dn.move_child_to, u"child", rw_dn)
1235             self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None,
1236                             rw_dn.move_child_to, u"child", ro_dn)
1237             return ro_dn.list()
1238         d.addCallback(_ready)
1239         def _listed(children):
1240             self.failUnless(u"child" in children)
1241         d.addCallback(_listed)
1242         return d
1243
1244     def test_readonly(self):
1245         self.basedir = "dirnode/Dirnode/test_readonly"
1246         self.set_up_grid()
1247         return self._do_readonly_test()
1248
1249     def test_readonly_mdmf(self):
1250         self.basedir = "dirnode/Dirnode/test_readonly_mdmf"
1251         self.set_up_grid()
1252         return self._do_readonly_test(version=MDMF_VERSION)
1253
1254     def failUnlessGreaterThan(self, a, b):
1255         self.failUnless(a > b, "%r should be > %r" % (a, b))
1256
1257     def failUnlessGreaterOrEqualThan(self, a, b):
1258         self.failUnless(a >= b, "%r should be >= %r" % (a, b))
1259
1260     def test_create(self):
1261         self.basedir = "dirnode/Dirnode/test_create"
1262         self.set_up_grid()
1263         return self._do_create_test()
1264
1265     def test_update_metadata(self):
1266         (t1, t2, t3) = (626644800.0, 634745640.0, 892226160.0)
1267
1268         md1 = dirnode.update_metadata({"ctime": t1}, {}, t2)
1269         self.failUnlessEqual(md1, {"tahoe":{"linkcrtime": t1, "linkmotime": t2}})
1270
1271         md2 = dirnode.update_metadata(md1, {"key": "value", "tahoe": {"bad": "mojo"}}, t3)
1272         self.failUnlessEqual(md2, {"key": "value",
1273                                    "tahoe":{"linkcrtime": t1, "linkmotime": t3}})
1274
1275         md3 = dirnode.update_metadata({}, None, t3)
1276         self.failUnlessEqual(md3, {"tahoe":{"linkcrtime": t3, "linkmotime": t3}})
1277
1278         md4 = dirnode.update_metadata({}, {"bool": True, "number": 42}, t1)
1279         self.failUnlessEqual(md4, {"bool": True, "number": 42,
1280                                    "tahoe":{"linkcrtime": t1, "linkmotime": t1}})
1281
1282     def _do_create_subdirectory_test(self, version=SDMF_VERSION):
1283         c = self.g.clients[0]
1284         nm = c.nodemaker
1285
1286         d = c.create_dirnode(version=version)
1287         def _then(n):
1288             # /
1289             self.rootnode = n
1290             fake_file_uri = make_mutable_file_uri()
1291             other_file_uri = make_mutable_file_uri()
1292             md = {"metakey": "metavalue"}
1293             kids = {u"kid1": (nm.create_from_cap(fake_file_uri), {}),
1294                     u"kid2": (nm.create_from_cap(other_file_uri), md),
1295                     }
1296             d = n.create_subdirectory(u"subdir", kids,
1297                                       mutable_version=version)
1298             def _check(sub):
1299                 d = n.get_child_at_path(u"subdir")
1300                 d.addCallback(lambda sub2: self.failUnlessReallyEqual(sub2.get_uri(),
1301                                                                       sub.get_uri()))
1302                 d.addCallback(lambda ign: sub.list())
1303                 return d
1304             d.addCallback(_check)
1305             def _check_kids(kids2):
1306                 self.failUnlessEqual(set(kids.keys()), set(kids2.keys()))
1307                 self.failUnlessEqual(kids2[u"kid2"][1]["metakey"], "metavalue")
1308             d.addCallback(_check_kids)
1309             return d
1310         d.addCallback(_then)
1311         return d
1312
1313     def test_create_subdirectory(self):
1314         self.basedir = "dirnode/Dirnode/test_create_subdirectory"
1315         self.set_up_grid()
1316         return self._do_create_subdirectory_test()
1317
1318     def test_create_subdirectory_mdmf(self):
1319         self.basedir = "dirnode/Dirnode/test_create_subdirectory_mdmf"
1320         self.set_up_grid()
1321         return self._do_create_subdirectory_test(version=MDMF_VERSION)
1322
1323     def test_create_mdmf(self):
1324         self.basedir = "dirnode/Dirnode/test_mdmf"
1325         self.set_up_grid()
1326         return self._do_create_test(mdmf=True)
1327
1328     def test_mdmf_initial_children(self):
1329         self.basedir = "dirnode/Dirnode/test_mdmf"
1330         self.set_up_grid()
1331         return self._do_initial_children_test(mdmf=True)
1332
1333 class MinimalFakeMutableFile:
1334     def get_writekey(self):
1335         return "writekey"
1336
1337 class Packing(testutil.ReallyEqualMixin, unittest.TestCase):
1338     # This is a base32-encoded representation of the directory tree
1339     # root/file1
1340     # root/file2
1341     # root/file3
1342     # as represented after being fed to _pack_contents.
1343     # We have it here so we can decode it, feed it to
1344     # _unpack_contents, and verify that _unpack_contents
1345     # works correctly.
1346
1347     known_tree = "GM4TOORVHJTGS3DFGEWDSNJ2KVJESOSDJBFTU33MPB2GS3LZNVYG6N3GGI3WU5TIORTXC3DOMJ2G4NB2MVWXUZDONBVTE5LNGRZWK2LYN55GY23XGNYXQMTOMZUWU5TENN4DG23ZG5UTO2L2NQ2DO6LFMRWDMZJWGRQTUMZ2GEYDUMJQFQYTIMZ22XZKZORX5XS7CAQCSK3URR6QOHISHRCMGER5LRFSZRNAS5ZSALCS6TWFQAE754IVOIKJVK73WZPP3VUUEDTX3WHTBBZ5YX3CEKHCPG3ZWQLYA4QM6LDRCF7TJQYWLIZHKGN5ROA3AUZPXESBNLQQ6JTC2DBJU2D47IZJTLR3PKZ4RVF57XLPWY7FX7SZV3T6IJ3ORFW37FXUPGOE3ROPFNUX5DCGMAQJ3PGGULBRGM3TU6ZCMN2GS3LFEI5CAMJSGQ3DMNRTHA4TOLRUGI3TKNRWGEWCAITUMFUG6ZJCHIQHWITMNFXGW3LPORUW2ZJCHIQDCMRUGY3DMMZYHE3S4NBSG42TMNRRFQQCE3DJNZVWG4TUNFWWKIR2EAYTENBWGY3DGOBZG4XDIMRXGU3DMML5FQQCE3LUNFWWKIR2EAYTENBWGY3DGOBZG4XDIMRXGU3DMML5FQWDGOJRHI2TUZTJNRSTELBZGQ5FKUSJHJBUQSZ2MFYGKZ3SOBSWQ43IO52WO23CNAZWU3DUGVSWSNTIOE5DK33POVTW4ZLNMNWDK6DHPA2GS2THNF2W25DEN5VGY2LQNFRGG5DKNNRHO5TZPFTWI6LNMRYGQ2LCGJTHM4J2GM5DCMB2GQWDCNBSHKVVQBGRYMACKJ27CVQ6O6B4QPR72RFVTGOZUI76XUSWAX73JRV5PYRHMIFYZIA25MXDPGUGML6M2NMRSG4YD4W4K37ZDYSXHMJ3IUVT4F64YTQQVBJFFFOUC7J7LAB2VFCL5UKKGMR2D3F4EPOYC7UYWQZNR5KXHBSNXLCNBX2SNF22DCXJIHSMEKWEWOG5XCJEVVZ7UW5IB6I64XXQSJ34B5CAYZGZIIMR6LBRGMZTU6ZCMN2GS3LFEI5CAMJSGQ3DMNRTHA4TOLRUGMYDEMJYFQQCE5DBNBXWKIR2EB5SE3DJNZVW233UNFWWKIR2EAYTENBWGY3DGOBZG4XDIMZQGIYTQLBAEJWGS3TLMNZHI2LNMURDUIBRGI2DMNRWGM4DSNZOGQZTAMRRHB6SYIBCNV2GS3LFEI5CAMJSGQ3DMNRTHA4TOLRUGMYDEMJYPUWCYMZZGU5DKOTGNFWGKMZMHE2DUVKSJE5EGSCLHJRW25DDPBYTO2DXPB3GM6DBNYZTI6LJMV3DM2LWNB4TU4LWMNSWW3LKORXWK5DEMN3TI23NNE3WEM3SORRGY5THPA3TKNBUMNZG453BOF2GSZLXMVWWI3DJOFZW623RHIZTUMJQHI2SYMJUGI5BOSHWDPG3WKPAVXCF3XMKA7QVIWPRMWJHDTQHD27AHDCPJWDQENQ5H5ZZILTXQNIXXCIW4LKQABU2GCFRG5FHQN7CHD7HF4EKNRZFIV2ZYQIBM7IQU7F4RGB3XCX3FREPBKQ7UCICHVWPCYFGA6OLH3J45LXQ6GWWICJ3PGWJNLZ7PCRNLAPNYUGU6BENS7OXMBEOOFRIZV3PF2FFWZ5WHDPKXERYP7GNHKRMGEZTOOT3EJRXI2LNMURDUIBRGI2DMNRWGM4DSNZOGQZTGNRSGY4SYIBCORQWQ33FEI5CA6ZCNRUW423NN52GS3LFEI5CAMJSGQ3DMNRTHA4TOLRUGMZTMMRWHEWCAITMNFXGWY3SORUW2ZJCHIQDCMRUGY3DMMZYHE3S4NBTGM3DENRZPUWCAITNORUW2ZJCHIQDCMRUGY3DMMZYHE3S4NBTGM3DENRZPUWCY==="
1348
1349     def test_unpack_and_pack_behavior(self):
1350         known_tree = b32decode(self.known_tree)
1351         nodemaker = NodeMaker(None, None, None,
1352                               None, None,
1353                               {"k": 3, "n": 10}, None)
1354         write_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
1355         filenode = nodemaker.create_from_cap(write_uri)
1356         node = dirnode.DirectoryNode(filenode, nodemaker, None)
1357         children = node._unpack_contents(known_tree)
1358         self._check_children(children)
1359
1360         packed_children = node._pack_contents(children)
1361         children = node._unpack_contents(packed_children)
1362         self._check_children(children)
1363
1364     def _check_children(self, children):
1365         # Are all the expected child nodes there?
1366         self.failUnless(children.has_key(u'file1'))
1367         self.failUnless(children.has_key(u'file2'))
1368         self.failUnless(children.has_key(u'file3'))
1369
1370         # Are the metadata for child 3 right?
1371         file3_rocap = "URI:CHK:cmtcxq7hwxvfxan34yiev6ivhy:qvcekmjtoetdcw4kmi7b3rtblvgx7544crnwaqtiewemdliqsokq:3:10:5"
1372         file3_rwcap = "URI:CHK:cmtcxq7hwxvfxan34yiev6ivhy:qvcekmjtoetdcw4kmi7b3rtblvgx7544crnwaqtiewemdliqsokq:3:10:5"
1373         file3_metadata = {'ctime': 1246663897.4336269, 'tahoe': {'linkmotime': 1246663897.4336269, 'linkcrtime': 1246663897.4336269}, 'mtime': 1246663897.4336269}
1374         self.failUnlessEqual(file3_metadata, children[u'file3'][1])
1375         self.failUnlessReallyEqual(file3_rocap,
1376                                    children[u'file3'][0].get_readonly_uri())
1377         self.failUnlessReallyEqual(file3_rwcap,
1378                                    children[u'file3'][0].get_uri())
1379
1380         # Are the metadata for child 2 right?
1381         file2_rocap = "URI:CHK:apegrpehshwugkbh3jlt5ei6hq:5oougnemcl5xgx4ijgiumtdojlipibctjkbwvyygdymdphib2fvq:3:10:4"
1382         file2_rwcap = "URI:CHK:apegrpehshwugkbh3jlt5ei6hq:5oougnemcl5xgx4ijgiumtdojlipibctjkbwvyygdymdphib2fvq:3:10:4"
1383         file2_metadata = {'ctime': 1246663897.430218, 'tahoe': {'linkmotime': 1246663897.430218, 'linkcrtime': 1246663897.430218}, 'mtime': 1246663897.430218}
1384         self.failUnlessEqual(file2_metadata, children[u'file2'][1])
1385         self.failUnlessReallyEqual(file2_rocap,
1386                                    children[u'file2'][0].get_readonly_uri())
1387         self.failUnlessReallyEqual(file2_rwcap,
1388                                    children[u'file2'][0].get_uri())
1389
1390         # Are the metadata for child 1 right?
1391         file1_rocap = "URI:CHK:olxtimympo7f27jvhtgqlnbtn4:emzdnhk2um4seixozlkw3qx2nfijvdkx3ky7i7izl47yedl6e64a:3:10:10"
1392         file1_rwcap = "URI:CHK:olxtimympo7f27jvhtgqlnbtn4:emzdnhk2um4seixozlkw3qx2nfijvdkx3ky7i7izl47yedl6e64a:3:10:10"
1393         file1_metadata = {'ctime': 1246663897.4275661, 'tahoe': {'linkmotime': 1246663897.4275661, 'linkcrtime': 1246663897.4275661}, 'mtime': 1246663897.4275661}
1394         self.failUnlessEqual(file1_metadata, children[u'file1'][1])
1395         self.failUnlessReallyEqual(file1_rocap,
1396                                    children[u'file1'][0].get_readonly_uri())
1397         self.failUnlessReallyEqual(file1_rwcap,
1398                                    children[u'file1'][0].get_uri())
1399
1400     def _make_kids(self, nm, which):
1401         caps = {"imm": "URI:CHK:n7r3m6wmomelk4sep3kw5cvduq:os7ijw5c3maek7pg65e5254k2fzjflavtpejjyhshpsxuqzhcwwq:3:20:14861",
1402                 "lit": "URI:LIT:n5xgk", # LIT for "one"
1403                 "write": "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq",
1404                 "read": "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q",
1405                 "dirwrite": "URI:DIR2:n6x24zd3seu725yluj75q5boaa:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq",
1406                 "dirread":  "URI:DIR2-RO:b7sr5qsifnicca7cbk3rhrhbvq:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq",
1407                 }
1408         kids = {}
1409         for name in which:
1410             kids[unicode(name)] = (nm.create_from_cap(caps[name]), {})
1411         return kids
1412
1413     def test_deep_immutable(self):
1414         nm = NodeMaker(None, None, None, None, None, {"k": 3, "n": 10}, None)
1415         fn = MinimalFakeMutableFile()
1416
1417         kids = self._make_kids(nm, ["imm", "lit", "write", "read",
1418                                     "dirwrite", "dirread"])
1419         packed = dirnode.pack_children(kids, fn.get_writekey(), deep_immutable=False)
1420         self.failUnlessIn("lit", packed)
1421
1422         kids = self._make_kids(nm, ["imm", "lit"])
1423         packed = dirnode.pack_children(kids, fn.get_writekey(), deep_immutable=True)
1424         self.failUnlessIn("lit", packed)
1425
1426         kids = self._make_kids(nm, ["imm", "lit", "write"])
1427         self.failUnlessRaises(dirnode.MustBeDeepImmutableError,
1428                               dirnode.pack_children,
1429                               kids, fn.get_writekey(), deep_immutable=True)
1430
1431         # read-only is not enough: all children must be immutable
1432         kids = self._make_kids(nm, ["imm", "lit", "read"])
1433         self.failUnlessRaises(dirnode.MustBeDeepImmutableError,
1434                               dirnode.pack_children,
1435                               kids, fn.get_writekey(), deep_immutable=True)
1436
1437         kids = self._make_kids(nm, ["imm", "lit", "dirwrite"])
1438         self.failUnlessRaises(dirnode.MustBeDeepImmutableError,
1439                               dirnode.pack_children,
1440                               kids, fn.get_writekey(), deep_immutable=True)
1441
1442         kids = self._make_kids(nm, ["imm", "lit", "dirread"])
1443         self.failUnlessRaises(dirnode.MustBeDeepImmutableError,
1444                               dirnode.pack_children,
1445                               kids, fn.get_writekey(), deep_immutable=True)
1446
1447 class FakeMutableFile:
1448     implements(IMutableFileNode)
1449     counter = 0
1450     def __init__(self, initial_contents=""):
1451         data = self._get_initial_contents(initial_contents)
1452         self.data = data.read(data.get_size())
1453         self.data = "".join(self.data)
1454
1455         counter = FakeMutableFile.counter
1456         FakeMutableFile.counter += 1
1457         writekey = hashutil.ssk_writekey_hash(str(counter))
1458         fingerprint = hashutil.ssk_pubkey_fingerprint_hash(str(counter))
1459         self.uri = uri.WriteableSSKFileURI(writekey, fingerprint)
1460
1461     def _get_initial_contents(self, contents):
1462         if isinstance(contents, str):
1463             return contents
1464         if contents is None:
1465             return ""
1466         assert callable(contents), "%s should be callable, not %s" % \
1467                (contents, type(contents))
1468         return contents(self)
1469
1470     def get_cap(self):
1471         return self.uri
1472
1473     def get_uri(self):
1474         return self.uri.to_string()
1475
1476     def get_write_uri(self):
1477         return self.uri.to_string()
1478
1479     def download_best_version(self):
1480         return defer.succeed(self.data)
1481
1482     def get_writekey(self):
1483         return "writekey"
1484
1485     def is_readonly(self):
1486         return False
1487
1488     def is_mutable(self):
1489         return True
1490
1491     def is_unknown(self):
1492         return False
1493
1494     def is_allowed_in_immutable_directory(self):
1495         return False
1496
1497     def raise_error(self):
1498         pass
1499
1500     def modify(self, modifier):
1501         data = modifier(self.data, None, True)
1502         self.data = data
1503         return defer.succeed(None)
1504
1505 class FakeNodeMaker(NodeMaker):
1506     def create_mutable_file(self, contents="", keysize=None, version=None):
1507         return defer.succeed(FakeMutableFile(contents))
1508
1509 class FakeClient2(Client):
1510     def __init__(self):
1511         self.nodemaker = FakeNodeMaker(None, None, None,
1512                                        None, None,
1513                                        {"k":3,"n":10}, None)
1514     def create_node_from_uri(self, rwcap, rocap):
1515         return self.nodemaker.create_from_cap(rwcap, rocap)
1516
1517 class Dirnode2(testutil.ReallyEqualMixin, testutil.ShouldFailMixin, unittest.TestCase):
1518     def setUp(self):
1519         client = FakeClient2()
1520         self.nodemaker = client.nodemaker
1521
1522     def test_from_future(self):
1523         # Create a mutable directory that contains unknown URI types, and make sure
1524         # we tolerate them properly.
1525         d = self.nodemaker.create_new_mutable_directory()
1526         future_write_uri = u"x-tahoe-crazy://I_am_from_the_future_rw_\u263A".encode('utf-8')
1527         future_read_uri = u"x-tahoe-crazy-readonly://I_am_from_the_future_ro_\u263A".encode('utf-8')
1528         future_imm_uri = u"x-tahoe-crazy-immutable://I_am_from_the_future_imm_\u263A".encode('utf-8')
1529         future_node = UnknownNode(future_write_uri, future_read_uri)
1530         def _then(n):
1531             self._node = n
1532             return n.set_node(u"future", future_node)
1533         d.addCallback(_then)
1534
1535         # We should be prohibited from adding an unknown URI to a directory
1536         # just in the rw_uri slot, since we don't know how to diminish the cap
1537         # to a readcap (for the ro_uri slot).
1538         d.addCallback(lambda ign:
1539              self.shouldFail(MustNotBeUnknownRWError,
1540                              "copy unknown",
1541                              "cannot attach unknown rw cap as child",
1542                              self._node.set_uri, u"add",
1543                              future_write_uri, None))
1544
1545         # However, we should be able to add both rw_uri and ro_uri as a pair of
1546         # unknown URIs.
1547         d.addCallback(lambda ign: self._node.set_uri(u"add-pair",
1548                                                      future_write_uri, future_read_uri))
1549
1550         # and to add an URI prefixed with "ro." or "imm." when it is given in a
1551         # write slot (or URL parameter).
1552         d.addCallback(lambda ign: self._node.set_uri(u"add-ro",
1553                                                      "ro." + future_read_uri, None))
1554         d.addCallback(lambda ign: self._node.set_uri(u"add-imm",
1555                                                      "imm." + future_imm_uri, None))
1556
1557         d.addCallback(lambda ign: self._node.list())
1558         def _check(children):
1559             self.failUnlessReallyEqual(len(children), 4)
1560             (fn, metadata) = children[u"future"]
1561             self.failUnless(isinstance(fn, UnknownNode), fn)
1562             self.failUnlessReallyEqual(fn.get_uri(), future_write_uri)
1563             self.failUnlessReallyEqual(fn.get_write_uri(), future_write_uri)
1564             self.failUnlessReallyEqual(fn.get_readonly_uri(), "ro." + future_read_uri)
1565
1566             (fn2, metadata2) = children[u"add-pair"]
1567             self.failUnless(isinstance(fn2, UnknownNode), fn2)
1568             self.failUnlessReallyEqual(fn2.get_uri(), future_write_uri)
1569             self.failUnlessReallyEqual(fn2.get_write_uri(), future_write_uri)
1570             self.failUnlessReallyEqual(fn2.get_readonly_uri(), "ro." + future_read_uri)
1571
1572             (fn3, metadata3) = children[u"add-ro"]
1573             self.failUnless(isinstance(fn3, UnknownNode), fn3)
1574             self.failUnlessReallyEqual(fn3.get_uri(), "ro." + future_read_uri)
1575             self.failUnlessReallyEqual(fn3.get_write_uri(), None)
1576             self.failUnlessReallyEqual(fn3.get_readonly_uri(), "ro." + future_read_uri)
1577
1578             (fn4, metadata4) = children[u"add-imm"]
1579             self.failUnless(isinstance(fn4, UnknownNode), fn4)
1580             self.failUnlessReallyEqual(fn4.get_uri(), "imm." + future_imm_uri)
1581             self.failUnlessReallyEqual(fn4.get_write_uri(), None)
1582             self.failUnlessReallyEqual(fn4.get_readonly_uri(), "imm." + future_imm_uri)
1583
1584             # We should also be allowed to copy the "future" UnknownNode, because
1585             # it contains all the information that was in the original directory
1586             # (readcap and writecap), so we're preserving everything.
1587             return self._node.set_node(u"copy", fn)
1588         d.addCallback(_check)
1589
1590         d.addCallback(lambda ign: self._node.list())
1591         def _check2(children):
1592             self.failUnlessReallyEqual(len(children), 5)
1593             (fn, metadata) = children[u"copy"]
1594             self.failUnless(isinstance(fn, UnknownNode), fn)
1595             self.failUnlessReallyEqual(fn.get_uri(), future_write_uri)
1596             self.failUnlessReallyEqual(fn.get_write_uri(), future_write_uri)
1597             self.failUnlessReallyEqual(fn.get_readonly_uri(), "ro." + future_read_uri)
1598         d.addCallback(_check2)
1599         return d
1600
1601     def test_unknown_strip_prefix_for_ro(self):
1602         self.failUnlessReallyEqual(strip_prefix_for_ro("foo",     False), "foo")
1603         self.failUnlessReallyEqual(strip_prefix_for_ro("ro.foo",  False), "foo")
1604         self.failUnlessReallyEqual(strip_prefix_for_ro("imm.foo", False), "imm.foo")
1605         self.failUnlessReallyEqual(strip_prefix_for_ro("foo",     True),  "foo")
1606         self.failUnlessReallyEqual(strip_prefix_for_ro("ro.foo",  True),  "foo")
1607         self.failUnlessReallyEqual(strip_prefix_for_ro("imm.foo", True),  "foo")
1608
1609     def test_unknownnode(self):
1610         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
1611         mut_read_uri = "URI:SSK-RO:jf6wkflosyvntwxqcdo7a54jvm:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
1612         lit_uri = "URI:LIT:n5xgk"
1613
1614         # This does not attempt to be exhaustive.
1615         no_no        = [# Opaque node, but not an error.
1616                         ( 0, UnknownNode(None, None)),
1617                         ( 1, UnknownNode(None, None, deep_immutable=True)),
1618                        ]
1619         unknown_rw   = [# These are errors because we're only given a rw_uri, and we can't
1620                         # diminish it.
1621                         ( 2, UnknownNode("foo", None)),
1622                         ( 3, UnknownNode("foo", None, deep_immutable=True)),
1623                         ( 4, UnknownNode("ro.foo", None, deep_immutable=True)),
1624                         ( 5, UnknownNode("ro." + mut_read_uri, None, deep_immutable=True)),
1625                         ( 6, UnknownNode("URI:SSK-RO:foo", None, deep_immutable=True)),
1626                         ( 7, UnknownNode("URI:SSK:foo", None)),
1627                        ]
1628         must_be_ro   = [# These are errors because a readonly constraint is not met.
1629                         ( 8, UnknownNode("ro." + mut_write_uri, None)),
1630                         ( 9, UnknownNode(None, "ro." + mut_write_uri)),
1631                        ]
1632         must_be_imm  = [# These are errors because an immutable constraint is not met.
1633                         (10, UnknownNode(None, "ro.URI:SSK-RO:foo", deep_immutable=True)),
1634                         (11, UnknownNode(None, "imm.URI:SSK:foo")),
1635                         (12, UnknownNode(None, "imm.URI:SSK-RO:foo")),
1636                         (13, UnknownNode("bar", "ro.foo", deep_immutable=True)),
1637                         (14, UnknownNode("bar", "imm.foo", deep_immutable=True)),
1638                         (15, UnknownNode("bar", "imm." + lit_uri, deep_immutable=True)),
1639                         (16, UnknownNode("imm." + mut_write_uri, None)),
1640                         (17, UnknownNode("imm." + mut_read_uri, None)),
1641                         (18, UnknownNode("bar", "imm.foo")),
1642                        ]
1643         bad_uri      = [# These are errors because the URI is bad once we've stripped the prefix.
1644                         (19, UnknownNode("ro.URI:SSK-RO:foo", None)),
1645                         (20, UnknownNode("imm.URI:CHK:foo", None, deep_immutable=True)),
1646                         (21, UnknownNode(None, "URI:CHK:foo")),
1647                         (22, UnknownNode(None, "URI:CHK:foo", deep_immutable=True)),
1648                        ]
1649         ro_prefixed  = [# These are valid, and the readcap should end up with a ro. prefix.
1650                         (23, UnknownNode(None, "foo")),
1651                         (24, UnknownNode(None, "ro.foo")),
1652                         (25, UnknownNode(None, "ro." + lit_uri)),
1653                         (26, UnknownNode("bar", "foo")),
1654                         (27, UnknownNode("bar", "ro.foo")),
1655                         (28, UnknownNode("bar", "ro." + lit_uri)),
1656                         (29, UnknownNode("ro.foo", None)),
1657                         (30, UnknownNode("ro." + lit_uri, None)),
1658                        ]
1659         imm_prefixed = [# These are valid, and the readcap should end up with an imm. prefix.
1660                         (31, UnknownNode(None, "foo", deep_immutable=True)),
1661                         (32, UnknownNode(None, "ro.foo", deep_immutable=True)),
1662                         (33, UnknownNode(None, "imm.foo")),
1663                         (34, UnknownNode(None, "imm.foo", deep_immutable=True)),
1664                         (35, UnknownNode("imm." + lit_uri, None)),
1665                         (36, UnknownNode("imm." + lit_uri, None, deep_immutable=True)),
1666                         (37, UnknownNode(None, "imm." + lit_uri)),
1667                         (38, UnknownNode(None, "imm." + lit_uri, deep_immutable=True)),
1668                        ]
1669         error = unknown_rw + must_be_ro + must_be_imm + bad_uri
1670         ok = ro_prefixed + imm_prefixed
1671
1672         for (i, n) in no_no + error + ok:
1673             self.failUnless(n.is_unknown(), i)
1674
1675         for (i, n) in no_no + error:
1676             self.failUnless(n.get_uri() is None, i)
1677             self.failUnless(n.get_write_uri() is None, i)
1678             self.failUnless(n.get_readonly_uri() is None, i)
1679
1680         for (i, n) in no_no + ok:
1681             n.raise_error()
1682
1683         for (i, n) in unknown_rw:
1684             self.failUnlessRaises(MustNotBeUnknownRWError, lambda: n.raise_error())
1685
1686         for (i, n) in must_be_ro:
1687             self.failUnlessRaises(MustBeReadonlyError, lambda: n.raise_error())
1688
1689         for (i, n) in must_be_imm:
1690             self.failUnlessRaises(MustBeDeepImmutableError, lambda: n.raise_error())
1691
1692         for (i, n) in bad_uri:
1693             self.failUnlessRaises(uri.BadURIError, lambda: n.raise_error())
1694
1695         for (i, n) in ok:
1696             self.failIf(n.get_readonly_uri() is None, i)
1697
1698         for (i, n) in ro_prefixed:
1699             self.failUnless(n.get_readonly_uri().startswith("ro."), i)
1700
1701         for (i, n) in imm_prefixed:
1702             self.failUnless(n.get_readonly_uri().startswith("imm."), i)
1703
1704
1705
1706 class DeepStats(testutil.ReallyEqualMixin, unittest.TestCase):
1707     timeout = 240 # It takes longer than 120 seconds on Francois's arm box.
1708     def test_stats(self):
1709         ds = dirnode.DeepStats(None)
1710         ds.add("count-files")
1711         ds.add("size-immutable-files", 123)
1712         ds.histogram("size-files-histogram", 123)
1713         ds.max("largest-directory", 444)
1714
1715         s = ds.get_results()
1716         self.failUnlessReallyEqual(s["count-files"], 1)
1717         self.failUnlessReallyEqual(s["size-immutable-files"], 123)
1718         self.failUnlessReallyEqual(s["largest-directory"], 444)
1719         self.failUnlessReallyEqual(s["count-literal-files"], 0)
1720
1721         ds.add("count-files")
1722         ds.add("size-immutable-files", 321)
1723         ds.histogram("size-files-histogram", 321)
1724         ds.max("largest-directory", 2)
1725
1726         s = ds.get_results()
1727         self.failUnlessReallyEqual(s["count-files"], 2)
1728         self.failUnlessReallyEqual(s["size-immutable-files"], 444)
1729         self.failUnlessReallyEqual(s["largest-directory"], 444)
1730         self.failUnlessReallyEqual(s["count-literal-files"], 0)
1731         self.failUnlessReallyEqual(s["size-files-histogram"],
1732                                    [ (101, 316, 1), (317, 1000, 1) ])
1733
1734         ds = dirnode.DeepStats(None)
1735         for i in range(1, 1100):
1736             ds.histogram("size-files-histogram", i)
1737         ds.histogram("size-files-histogram", 4*1000*1000*1000*1000) # 4TB
1738         s = ds.get_results()
1739         self.failUnlessReallyEqual(s["size-files-histogram"],
1740                                    [ (1, 3, 3),
1741                                      (4, 10, 7),
1742                                      (11, 31, 21),
1743                                      (32, 100, 69),
1744                                      (101, 316, 216),
1745                                      (317, 1000, 684),
1746                                      (1001, 3162, 99),
1747                                      (3162277660169L, 10000000000000L, 1),
1748                                      ])
1749
1750 class UCWEingMutableFileNode(MutableFileNode):
1751     please_ucwe_after_next_upload = False
1752
1753     def _upload(self, new_contents, servermap):
1754         d = MutableFileNode._upload(self, new_contents, servermap)
1755         def _ucwe(res):
1756             if self.please_ucwe_after_next_upload:
1757                 self.please_ucwe_after_next_upload = False
1758                 raise UncoordinatedWriteError()
1759             return res
1760         d.addCallback(_ucwe)
1761         return d
1762
1763 class UCWEingNodeMaker(NodeMaker):
1764     def _create_mutable(self, cap):
1765         n = UCWEingMutableFileNode(self.storage_broker, self.secret_holder,
1766                                    self.default_encoding_parameters,
1767                                    self.history)
1768         return n.init_from_cap(cap)
1769
1770
1771 class Deleter(GridTestMixin, testutil.ReallyEqualMixin, unittest.TestCase):
1772     timeout = 3600 # It takes longer than 433 seconds on Zandr's ARM box.
1773     def test_retry(self):
1774         # ticket #550, a dirnode.delete which experiences an
1775         # UncoordinatedWriteError will fail with an incorrect "you're
1776         # deleting something which isn't there" NoSuchChildError exception.
1777
1778         # to trigger this, we start by creating a directory with a single
1779         # file in it. Then we create a special dirnode that uses a modified
1780         # MutableFileNode which will raise UncoordinatedWriteError once on
1781         # demand. We then call dirnode.delete, which ought to retry and
1782         # succeed.
1783
1784         self.basedir = self.mktemp()
1785         self.set_up_grid()
1786         c0 = self.g.clients[0]
1787         d = c0.create_dirnode()
1788         small = upload.Data("Small enough for a LIT", None)
1789         def _created_dir(dn):
1790             self.root = dn
1791             self.root_uri = dn.get_uri()
1792             return dn.add_file(u"file", small)
1793         d.addCallback(_created_dir)
1794         def _do_delete(ignored):
1795             nm = UCWEingNodeMaker(c0.storage_broker, c0._secret_holder,
1796                                   c0.get_history(), c0.getServiceNamed("uploader"),
1797                                   c0.terminator,
1798                                   c0.get_encoding_parameters(),
1799                                   c0._key_generator)
1800             n = nm.create_from_cap(self.root_uri)
1801             assert n._node.please_ucwe_after_next_upload == False
1802             n._node.please_ucwe_after_next_upload = True
1803             # This should succeed, not raise an exception
1804             return n.delete(u"file")
1805         d.addCallback(_do_delete)
1806
1807         return d
1808
1809 class Adder(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin):
1810
1811     def test_overwrite(self):
1812         # note: This functionality could be tested without actually creating
1813         # several RSA keys. It would be faster without the GridTestMixin: use
1814         # dn.set_node(nodemaker.create_from_cap(make_chk_file_uri())) instead
1815         # of dn.add_file, and use a special NodeMaker that creates fake
1816         # mutable files.
1817         self.basedir = "dirnode/Adder/test_overwrite"
1818         self.set_up_grid()
1819         c = self.g.clients[0]
1820         fileuri = make_chk_file_uri(1234)
1821         filenode = c.nodemaker.create_from_cap(fileuri)
1822         d = c.create_dirnode()
1823
1824         def _create_directory_tree(root_node):
1825             # Build
1826             # root/file1
1827             # root/file2
1828             # root/dir1
1829             d = root_node.add_file(u'file1', upload.Data("Important Things",
1830                 None))
1831             d.addCallback(lambda res:
1832                 root_node.add_file(u'file2', upload.Data("Sekrit Codes", None)))
1833             d.addCallback(lambda res:
1834                 root_node.create_subdirectory(u"dir1"))
1835             d.addCallback(lambda res: root_node)
1836             return d
1837
1838         d.addCallback(_create_directory_tree)
1839
1840         def _test_adder(root_node):
1841             d = root_node.set_node(u'file1', filenode)
1842             # We've overwritten file1. Let's try it with a directory
1843             d.addCallback(lambda res:
1844                 root_node.create_subdirectory(u'dir2'))
1845             d.addCallback(lambda res:
1846                 root_node.set_node(u'dir2', filenode))
1847             # We try overwriting a file with a child while also specifying
1848             # overwrite=False. We should receive an ExistingChildError
1849             # when we do this.
1850             d.addCallback(lambda res:
1851                 self.shouldFail(ExistingChildError, "set_node",
1852                                 "child 'file1' already exists",
1853                                 root_node.set_node, u"file1",
1854                                 filenode, overwrite=False))
1855             # If we try with a directory, we should see the same thing
1856             d.addCallback(lambda res:
1857                 self.shouldFail(ExistingChildError, "set_node",
1858                                 "child 'dir1' already exists",
1859                                 root_node.set_node, u'dir1', filenode,
1860                                 overwrite=False))
1861             d.addCallback(lambda res:
1862                 root_node.set_node(u'file1', filenode,
1863                                    overwrite="only-files"))
1864             d.addCallback(lambda res:
1865                 self.shouldFail(ExistingChildError, "set_node",
1866                                 "child 'dir1' already exists",
1867                                 root_node.set_node, u'dir1', filenode,
1868                                 overwrite="only-files"))
1869             return d
1870
1871         d.addCallback(_test_adder)
1872         return d