src/allmydata/test/test_dirnode.py

   1
   2 import time
   3 from zope.interface import implements
   4 from twisted.trial import unittest
   5 from twisted.internet import defer
   6 from allmydata import uri, dirnode
   7 from allmydata.immutable import upload
   8 from allmydata.interfaces import IURI, IClient, IMutableFileNode, \
   9      INewDirectoryURI, IReadonlyNewDirectoryURI, IFileNode, \
  10      ExistingChildError, NoSuchChildError, \
  11      IDeepCheckResults, IDeepCheckAndRepairResults
  12 from allmydata.mutable.node import MutableFileNode
  13 from allmydata.mutable.common import UncoordinatedWriteError
  14 from allmydata.util import hashutil, base32
  15 from allmydata.monitor import Monitor
  16 from allmydata.test.common import make_chk_file_uri, make_mutable_file_uri, \
  17      FakeDirectoryNode, create_chk_filenode, ErrorMixin, SystemTestMixin
  18 from allmydata.checker_results import CheckerResults, CheckAndRepairResults
  19 import common_util as testutil
  20
  21 # to test dirnode.py, we want to construct a tree of real DirectoryNodes that
  22 # contain pointers to fake files. We start with a fake MutableFileNode that
  23 # stores all of its data in a static table.
  24
  25 class Marker:
  26     implements(IFileNode, IMutableFileNode) # sure, why not
  27     def __init__(self, nodeuri):
  28         if not isinstance(nodeuri, str):
  29             nodeuri = nodeuri.to_string()
  30         self.nodeuri = nodeuri
  31         si = hashutil.tagged_hash("tag1", nodeuri)[:16]
  32         self.storage_index = si
  33         fp = hashutil.tagged_hash("tag2", nodeuri)
  34         self.verifieruri = uri.SSKVerifierURI(storage_index=si, fingerprint=fp)
  35     def get_uri(self):
  36         return self.nodeuri
  37     def get_readonly_uri(self):
  38         return self.nodeuri
  39     def get_verifier(self):
  40         return self.verifieruri
  41     def get_storage_index(self):
  42         return self.storage_index
  43
  44     def check(self, monitor, verify=False):
  45         r = CheckerResults("", None)
  46         r.set_healthy(True)
  47         r.set_recoverable(True)
  48         return defer.succeed(r)
  49
  50     def check_and_repair(self, monitor, verify=False):
  51         d = self.check(verify)
  52         def _got(cr):
  53             r = CheckAndRepairResults(None)
  54             r.pre_repair_results = r.post_repair_results = cr
  55             return r
  56         d.addCallback(_got)
  57         return d
  58
  59 # dirnode requires three methods from the client: upload(),
  60 # create_node_from_uri(), and create_empty_dirnode(). Of these, upload() is
  61 # only used by the convenience composite method add_file().
  62
  63 class FakeClient:
  64     implements(IClient)
  65
  66     def upload(self, uploadable):
  67         d = uploadable.get_size()
  68         d.addCallback(lambda size: uploadable.read(size))
  69         def _got_data(datav):
  70             data = "".join(datav)
  71             n = create_chk_filenode(self, data)
  72             results = upload.UploadResults()
  73             results.uri = n.get_uri()
  74             return results
  75         d.addCallback(_got_data)
  76         return d
  77
  78     def create_node_from_uri(self, u):
  79         u = IURI(u)
  80         if (INewDirectoryURI.providedBy(u)
  81             or IReadonlyNewDirectoryURI.providedBy(u)):
  82             return FakeDirectoryNode(self).init_from_uri(u)
  83         return Marker(u.to_string())
  84
  85     def create_empty_dirnode(self):
  86         n = FakeDirectoryNode(self)
  87         d = n.create()
  88         d.addCallback(lambda res: n)
  89         return d
  90
  91
  92 class Dirnode(unittest.TestCase,
  93               testutil.ShouldFailMixin, testutil.StallMixin, ErrorMixin):
  94     def setUp(self):
  95         self.client = FakeClient()
  96
  97     def test_basic(self):
  98         d = self.client.create_empty_dirnode()
  99         def _done(res):
 100             self.failUnless(isinstance(res, FakeDirectoryNode))
 101             rep = str(res)
 102             self.failUnless("RW" in rep)
 103         d.addCallback(_done)
 104         return d
 105
 106     def test_corrupt(self):
 107         d = self.client.create_empty_dirnode()
 108         def _created(dn):
 109             u = make_mutable_file_uri()
 110             d = dn.set_uri(u"child", u, {})
 111             d.addCallback(lambda res: dn.list())
 112             def _check1(children):
 113                 self.failUnless(u"child" in children)
 114             d.addCallback(_check1)
 115             d.addCallback(lambda res:
 116                           self.shouldFail(NoSuchChildError, "get bogus", None,
 117                                           dn.get, u"bogus"))
 118             def _corrupt(res):
 119                 filenode = dn._node
 120                 si = IURI(filenode.get_uri()).storage_index
 121                 old_contents = filenode.all_contents[si]
 122                 # we happen to know that the writecap is encrypted near the
 123                 # end of the string. Flip one of its bits and make sure we
 124                 # detect the corruption.
 125                 new_contents = testutil.flip_bit(old_contents, -10)
 126                 # TODO: also test flipping bits in the other portions
 127                 filenode.all_contents[si] = new_contents
 128             d.addCallback(_corrupt)
 129             def _check2(res):
 130                 self.shouldFail(hashutil.IntegrityCheckError, "corrupt",
 131                                 "HMAC does not match, crypttext is corrupted",
 132                                 dn.list)
 133             d.addCallback(_check2)
 134             return d
 135         d.addCallback(_created)
 136         return d
 137
 138     def test_check(self):
 139         d = self.client.create_empty_dirnode()
 140         d.addCallback(lambda dn: dn.check(Monitor()))
 141         def _done(res):
 142             self.failUnless(res.is_healthy())
 143         d.addCallback(_done)
 144         return d
 145
 146     def _test_deepcheck_create(self):
 147         # create a small tree with a loop, and some non-directories
 148         #  root/
 149         #  root/subdir/
 150         #  root/subdir/file1
 151         #  root/subdir/link -> root
 152         d = self.client.create_empty_dirnode()
 153         def _created_root(rootnode):
 154             self._rootnode = rootnode
 155             return rootnode.create_empty_directory(u"subdir")
 156         d.addCallback(_created_root)
 157         def _created_subdir(subdir):
 158             self._subdir = subdir
 159             d = subdir.add_file(u"file1", upload.Data("data", None))
 160             d.addCallback(lambda res: subdir.set_node(u"link", self._rootnode))
 161             return d
 162         d.addCallback(_created_subdir)
 163         def _done(res):
 164             return self._rootnode
 165         d.addCallback(_done)
 166         return d
 167
 168     def test_deepcheck(self):
 169         d = self._test_deepcheck_create()
 170         d.addCallback(lambda rootnode: rootnode.start_deep_check().when_done())
 171         def _check_results(r):
 172             self.failUnless(IDeepCheckResults.providedBy(r))
 173             c = r.get_counters()
 174             self.failUnlessEqual(c,
 175                                  {"count-objects-checked": 3,
 176                                   "count-objects-healthy": 3,
 177                                   "count-objects-unhealthy": 0,
 178                                   "count-objects-unrecoverable": 0,
 179                                   "count-corrupt-shares": 0,
 180                                   })
 181             self.failIf(r.get_corrupt_shares())
 182             self.failUnlessEqual(len(r.get_all_results()), 3)
 183         d.addCallback(_check_results)
 184         return d
 185
 186     def test_deepcheck_and_repair(self):
 187         d = self._test_deepcheck_create()
 188         d.addCallback(lambda rootnode:
 189                       rootnode.start_deep_check_and_repair().when_done())
 190         def _check_results(r):
 191             self.failUnless(IDeepCheckAndRepairResults.providedBy(r))
 192             c = r.get_counters()
 193             self.failUnlessEqual(c,
 194                                  {"count-objects-checked": 3,
 195                                   "count-objects-healthy-pre-repair": 3,
 196                                   "count-objects-unhealthy-pre-repair": 0,
 197                                   "count-objects-unrecoverable-pre-repair": 0,
 198                                   "count-corrupt-shares-pre-repair": 0,
 199                                   "count-objects-healthy-post-repair": 3,
 200                                   "count-objects-unhealthy-post-repair": 0,
 201                                   "count-objects-unrecoverable-post-repair": 0,
 202                                   "count-corrupt-shares-post-repair": 0,
 203                                   "count-repairs-attempted": 0,
 204                                   "count-repairs-successful": 0,
 205                                   "count-repairs-unsuccessful": 0,
 206                                   })
 207             self.failIf(r.get_corrupt_shares())
 208             self.failIf(r.get_remaining_corrupt_shares())
 209             self.failUnlessEqual(len(r.get_all_results()), 3)
 210         d.addCallback(_check_results)
 211         return d
 212
 213     def _mark_file_bad(self, rootnode):
 214         si = IURI(rootnode.get_uri())._filenode_uri.storage_index
 215         rootnode._node.bad_shares[si] = "unhealthy"
 216         return rootnode
 217
 218     def test_deepcheck_problems(self):
 219         d = self._test_deepcheck_create()
 220         d.addCallback(lambda rootnode: self._mark_file_bad(rootnode))
 221         d.addCallback(lambda rootnode: rootnode.start_deep_check().when_done())
 222         def _check_results(r):
 223             c = r.get_counters()
 224             self.failUnlessEqual(c,
 225                                  {"count-objects-checked": 3,
 226                                   "count-objects-healthy": 2,
 227                                   "count-objects-unhealthy": 1,
 228                                   "count-objects-unrecoverable": 0,
 229                                   "count-corrupt-shares": 0,
 230                                   })
 231             #self.failUnlessEqual(len(r.get_problems()), 1) # TODO
 232         d.addCallback(_check_results)
 233         return d
 234
 235     def test_readonly(self):
 236         fileuri = make_chk_file_uri(1234)
 237         filenode = self.client.create_node_from_uri(fileuri)
 238         uploadable = upload.Data("some data", convergence="some convergence string")
 239
 240         d = self.client.create_empty_dirnode()
 241         def _created(rw_dn):
 242             d2 = rw_dn.set_uri(u"child", fileuri)
 243             d2.addCallback(lambda res: rw_dn)
 244             return d2
 245         d.addCallback(_created)
 246
 247         def _ready(rw_dn):
 248             ro_uri = rw_dn.get_readonly_uri()
 249             ro_dn = self.client.create_node_from_uri(ro_uri)
 250             self.failUnless(ro_dn.is_readonly())
 251             self.failUnless(ro_dn.is_mutable())
 252
 253             self.shouldFail(dirnode.NotMutableError, "set_uri ro", None,
 254                             ro_dn.set_uri, u"newchild", fileuri)
 255             self.shouldFail(dirnode.NotMutableError, "set_uri ro", None,
 256                             ro_dn.set_node, u"newchild", filenode)
 257             self.shouldFail(dirnode.NotMutableError, "set_nodes ro", None,
 258                             ro_dn.set_nodes, [ (u"newchild", filenode) ])
 259             self.shouldFail(dirnode.NotMutableError, "set_uri ro", None,
 260                             ro_dn.add_file, u"newchild", uploadable)
 261             self.shouldFail(dirnode.NotMutableError, "set_uri ro", None,
 262                             ro_dn.delete, u"child")
 263             self.shouldFail(dirnode.NotMutableError, "set_uri ro", None,
 264                             ro_dn.create_empty_directory, u"newchild")
 265             self.shouldFail(dirnode.NotMutableError, "set_metadata_for ro", None,
 266                             ro_dn.set_metadata_for, u"child", {})
 267             self.shouldFail(dirnode.NotMutableError, "set_uri ro", None,
 268                             ro_dn.move_child_to, u"child", rw_dn)
 269             self.shouldFail(dirnode.NotMutableError, "set_uri ro", None,
 270                             rw_dn.move_child_to, u"child", ro_dn)
 271             return ro_dn.list()
 272         d.addCallback(_ready)
 273         def _listed(children):
 274             self.failUnless(u"child" in children)
 275         d.addCallback(_listed)
 276         return d
 277
 278     def failUnlessGreaterThan(self, a, b):
 279         self.failUnless(a > b, "%r should be > %r" % (a, b))
 280
 281     def failUnlessGreaterOrEqualThan(self, a, b):
 282         self.failUnless(a >= b, "%r should be >= %r" % (a, b))
 283
 284     def test_create(self):
 285         self.expected_manifest = []
 286         self.expected_verifycaps = set()
 287         self.expected_storage_indexes = set()
 288
 289         d = self.client.create_empty_dirnode()
 290         def _then(n):
 291             # /
 292             self.failUnless(n.is_mutable())
 293             u = n.get_uri()
 294             self.failUnless(u)
 295             self.failUnless(u.startswith("URI:DIR2:"), u)
 296             u_ro = n.get_readonly_uri()
 297             self.failUnless(u_ro.startswith("URI:DIR2-RO:"), u_ro)
 298             u_v = n.get_verifier().to_string()
 299             self.failUnless(u_v.startswith("URI:DIR2-Verifier:"), u_v)
 300             self.expected_manifest.append( ((), u) )
 301             self.expected_verifycaps.add(u_v)
 302             si = n.get_storage_index()
 303             self.expected_storage_indexes.add(base32.b2a(si))
 304             expected_si = n._uri._filenode_uri.storage_index
 305             self.failUnlessEqual(si, expected_si)
 306
 307             d = n.list()
 308             d.addCallback(lambda res: self.failUnlessEqual(res, {}))
 309             d.addCallback(lambda res: n.has_child(u"missing"))
 310             d.addCallback(lambda res: self.failIf(res))
 311             fake_file_uri = make_mutable_file_uri()
 312             other_file_uri = make_mutable_file_uri()
 313             m = Marker(fake_file_uri)
 314             ffu_v = m.get_verifier().to_string()
 315             self.expected_manifest.append( ((u"child",) , m.get_uri()) )
 316             self.expected_verifycaps.add(ffu_v)
 317             self.expected_storage_indexes.add(base32.b2a(m.get_storage_index()))
 318             d.addCallback(lambda res: n.set_uri(u"child", fake_file_uri))
 319             d.addCallback(lambda res:
 320                           self.shouldFail(ExistingChildError, "set_uri-no",
 321                                           "child 'child' already exists",
 322                                           n.set_uri, u"child", other_file_uri,
 323                                           overwrite=False))
 324             # /
 325             # /child = mutable
 326
 327             d.addCallback(lambda res: n.create_empty_directory(u"subdir"))
 328
 329             # /
 330             # /child = mutable
 331             # /subdir = directory
 332             def _created(subdir):
 333                 self.failUnless(isinstance(subdir, FakeDirectoryNode))
 334                 self.subdir = subdir
 335                 new_v = subdir.get_verifier().to_string()
 336                 assert isinstance(new_v, str)
 337                 self.expected_manifest.append( ((u"subdir",), subdir.get_uri()) )
 338                 self.expected_verifycaps.add(new_v)
 339                 si = subdir.get_storage_index()
 340                 self.expected_storage_indexes.add(base32.b2a(si))
 341             d.addCallback(_created)
 342
 343             d.addCallback(lambda res:
 344                           self.shouldFail(ExistingChildError, "mkdir-no",
 345                                           "child 'subdir' already exists",
 346                                           n.create_empty_directory, u"subdir",
 347                                           overwrite=False))
 348
 349             d.addCallback(lambda res: n.list())
 350             d.addCallback(lambda children:
 351                           self.failUnlessEqual(sorted(children.keys()),
 352                                                sorted([u"child", u"subdir"])))
 353
 354             d.addCallback(lambda res: n.start_deep_stats().when_done())
 355             def _check_deepstats(stats):
 356                 self.failUnless(isinstance(stats, dict))
 357                 expected = {"count-immutable-files": 0,
 358                             "count-mutable-files": 1,
 359                             "count-literal-files": 0,
 360                             "count-files": 1,
 361                             "count-directories": 2,
 362                             "size-immutable-files": 0,
 363                             "size-literal-files": 0,
 364                             #"size-directories": 616, # varies
 365                             #"largest-directory": 616,
 366                             "largest-directory-children": 2,
 367                             "largest-immutable-file": 0,
 368                             }
 369                 for k,v in expected.iteritems():
 370                     self.failUnlessEqual(stats[k], v,
 371                                          "stats[%s] was %s, not %s" %
 372                                          (k, stats[k], v))
 373                 self.failUnless(stats["size-directories"] > 500,
 374                                 stats["size-directories"])
 375                 self.failUnless(stats["largest-directory"] > 500,
 376                                 stats["largest-directory"])
 377                 self.failUnlessEqual(stats["size-files-histogram"], [])
 378             d.addCallback(_check_deepstats)
 379
 380             d.addCallback(lambda res: n.build_manifest().when_done())
 381             def _check_manifest(res):
 382                 manifest = res["manifest"]
 383                 self.failUnlessEqual(sorted(manifest),
 384                                      sorted(self.expected_manifest))
 385                 stats = res["stats"]
 386                 _check_deepstats(stats)
 387                 self.failUnlessEqual(self.expected_verifycaps,
 388                                      res["verifycaps"])
 389                 self.failUnlessEqual(self.expected_storage_indexes,
 390                                      res["storage-index"])
 391             d.addCallback(_check_manifest)
 392
 393             def _add_subsubdir(res):
 394                 return self.subdir.create_empty_directory(u"subsubdir")
 395             d.addCallback(_add_subsubdir)
 396             # /
 397             # /child = mutable
 398             # /subdir = directory
 399             # /subdir/subsubdir = directory
 400             d.addCallback(lambda res: n.get_child_at_path(u"subdir/subsubdir"))
 401             d.addCallback(lambda subsubdir:
 402                           self.failUnless(isinstance(subsubdir,
 403                                                      FakeDirectoryNode)))
 404             d.addCallback(lambda res: n.get_child_at_path(u""))
 405             d.addCallback(lambda res: self.failUnlessEqual(res.get_uri(),
 406                                                            n.get_uri()))
 407
 408             d.addCallback(lambda res: n.get_metadata_for(u"child"))
 409             d.addCallback(lambda metadata:
 410                           self.failUnlessEqual(sorted(metadata.keys()),
 411                                                ["ctime", "mtime"]))
 412
 413             d.addCallback(lambda res:
 414                           self.shouldFail(NoSuchChildError, "gcamap-no",
 415                                           "nope",
 416                                           n.get_child_and_metadata_at_path,
 417                                           u"subdir/nope"))
 418             d.addCallback(lambda res:
 419                           n.get_child_and_metadata_at_path(u""))
 420             def _check_child_and_metadata1(res):
 421                 child, metadata = res
 422                 self.failUnless(isinstance(child, FakeDirectoryNode))
 423                 # edge-metadata needs at least one path segment
 424                 self.failUnlessEqual(sorted(metadata.keys()), [])
 425             d.addCallback(_check_child_and_metadata1)
 426             d.addCallback(lambda res:
 427                           n.get_child_and_metadata_at_path(u"child"))
 428
 429             def _check_child_and_metadata2(res):
 430                 child, metadata = res
 431                 self.failUnlessEqual(child.get_uri(),
 432                                      fake_file_uri.to_string())
 433                 self.failUnlessEqual(sorted(metadata.keys()),
 434                                      ["ctime", "mtime"])
 435             d.addCallback(_check_child_and_metadata2)
 436
 437             d.addCallback(lambda res:
 438                           n.get_child_and_metadata_at_path(u"subdir/subsubdir"))
 439             def _check_child_and_metadata3(res):
 440                 child, metadata = res
 441                 self.failUnless(isinstance(child, FakeDirectoryNode))
 442                 self.failUnlessEqual(sorted(metadata.keys()),
 443                                      ["ctime", "mtime"])
 444             d.addCallback(_check_child_and_metadata3)
 445
 446             # set_uri + metadata
 447             # it should be possible to add a child without any metadata
 448             d.addCallback(lambda res: n.set_uri(u"c2", fake_file_uri, {}))
 449             d.addCallback(lambda res: n.get_metadata_for(u"c2"))
 450             d.addCallback(lambda metadata: self.failUnlessEqual(metadata, {}))
 451
 452             # if we don't set any defaults, the child should get timestamps
 453             d.addCallback(lambda res: n.set_uri(u"c3", fake_file_uri))
 454             d.addCallback(lambda res: n.get_metadata_for(u"c3"))
 455             d.addCallback(lambda metadata:
 456                           self.failUnlessEqual(sorted(metadata.keys()),
 457                                                ["ctime", "mtime"]))
 458
 459             # or we can add specific metadata at set_uri() time, which
 460             # overrides the timestamps
 461             d.addCallback(lambda res: n.set_uri(u"c4", fake_file_uri,
 462                                                 {"key": "value"}))
 463             d.addCallback(lambda res: n.get_metadata_for(u"c4"))
 464             d.addCallback(lambda metadata:
 465                           self.failUnlessEqual(metadata, {"key": "value"}))
 466
 467             d.addCallback(lambda res: n.delete(u"c2"))
 468             d.addCallback(lambda res: n.delete(u"c3"))
 469             d.addCallback(lambda res: n.delete(u"c4"))
 470
 471             # set_node + metadata
 472             # it should be possible to add a child without any metadata
 473             d.addCallback(lambda res: n.set_node(u"d2", n, {}))
 474             d.addCallback(lambda res: self.client.create_empty_dirnode())
 475             d.addCallback(lambda n2:
 476                           self.shouldFail(ExistingChildError, "set_node-no",
 477                                           "child 'd2' already exists",
 478                                           n.set_node, u"d2", n2,
 479                                           overwrite=False))
 480             d.addCallback(lambda res: n.get_metadata_for(u"d2"))
 481             d.addCallback(lambda metadata: self.failUnlessEqual(metadata, {}))
 482
 483             # if we don't set any defaults, the child should get timestamps
 484             d.addCallback(lambda res: n.set_node(u"d3", n))
 485             d.addCallback(lambda res: n.get_metadata_for(u"d3"))
 486             d.addCallback(lambda metadata:
 487                           self.failUnlessEqual(sorted(metadata.keys()),
 488                                                ["ctime", "mtime"]))
 489
 490             # or we can add specific metadata at set_node() time, which
 491             # overrides the timestamps
 492             d.addCallback(lambda res: n.set_node(u"d4", n,
 493                                                 {"key": "value"}))
 494             d.addCallback(lambda res: n.get_metadata_for(u"d4"))
 495             d.addCallback(lambda metadata:
 496                           self.failUnlessEqual(metadata, {"key": "value"}))
 497
 498             d.addCallback(lambda res: n.delete(u"d2"))
 499             d.addCallback(lambda res: n.delete(u"d3"))
 500             d.addCallback(lambda res: n.delete(u"d4"))
 501
 502             # metadata through set_children()
 503             d.addCallback(lambda res: n.set_children([ (u"e1", fake_file_uri),
 504                                                    (u"e2", fake_file_uri, {}),
 505                                                    (u"e3", fake_file_uri,
 506                                                     {"key": "value"}),
 507                                                    ]))
 508             d.addCallback(lambda res:
 509                           self.shouldFail(ExistingChildError, "set_children-no",
 510                                           "child 'e1' already exists",
 511                                           n.set_children,
 512                                           [ (u"e1", other_file_uri),
 513                                             (u"new", other_file_uri), ],
 514                                           overwrite=False))
 515             # and 'new' should not have been created
 516             d.addCallback(lambda res: n.list())
 517             d.addCallback(lambda children: self.failIf(u"new" in children))
 518             d.addCallback(lambda res: n.get_metadata_for(u"e1"))
 519             d.addCallback(lambda metadata:
 520                           self.failUnlessEqual(sorted(metadata.keys()),
 521                                                ["ctime", "mtime"]))
 522             d.addCallback(lambda res: n.get_metadata_for(u"e2"))
 523             d.addCallback(lambda metadata: self.failUnlessEqual(metadata, {}))
 524             d.addCallback(lambda res: n.get_metadata_for(u"e3"))
 525             d.addCallback(lambda metadata:
 526                           self.failUnlessEqual(metadata, {"key": "value"}))
 527
 528             d.addCallback(lambda res: n.delete(u"e1"))
 529             d.addCallback(lambda res: n.delete(u"e2"))
 530             d.addCallback(lambda res: n.delete(u"e3"))
 531
 532             # metadata through set_nodes()
 533             d.addCallback(lambda res: n.set_nodes([ (u"f1", n),
 534                                                     (u"f2", n, {}),
 535                                                     (u"f3", n,
 536                                                      {"key": "value"}),
 537                                                     ]))
 538             d.addCallback(lambda res:
 539                           self.shouldFail(ExistingChildError, "set_nodes-no",
 540                                           "child 'f1' already exists",
 541                                           n.set_nodes,
 542                                           [ (u"f1", n),
 543                                             (u"new", n), ],
 544                                           overwrite=False))
 545             # and 'new' should not have been created
 546             d.addCallback(lambda res: n.list())
 547             d.addCallback(lambda children: self.failIf(u"new" in children))
 548             d.addCallback(lambda res: n.get_metadata_for(u"f1"))
 549             d.addCallback(lambda metadata:
 550                           self.failUnlessEqual(sorted(metadata.keys()),
 551                                                ["ctime", "mtime"]))
 552             d.addCallback(lambda res: n.get_metadata_for(u"f2"))
 553             d.addCallback(lambda metadata: self.failUnlessEqual(metadata, {}))
 554             d.addCallback(lambda res: n.get_metadata_for(u"f3"))
 555             d.addCallback(lambda metadata:
 556                           self.failUnlessEqual(metadata, {"key": "value"}))
 557
 558             d.addCallback(lambda res: n.delete(u"f1"))
 559             d.addCallback(lambda res: n.delete(u"f2"))
 560             d.addCallback(lambda res: n.delete(u"f3"))
 561
 562
 563             d.addCallback(lambda res:
 564                           n.set_metadata_for(u"child",
 565                                              {"tags": ["web2.0-compatible"]}))
 566             d.addCallback(lambda n1: n1.get_metadata_for(u"child"))
 567             d.addCallback(lambda metadata:
 568                           self.failUnlessEqual(metadata,
 569                                                {"tags": ["web2.0-compatible"]}))
 570
 571             def _start(res):
 572                 self._start_timestamp = time.time()
 573             d.addCallback(_start)
 574             # simplejson-1.7.1 (as shipped on Ubuntu 'gutsy') rounds all
 575             # floats to hundredeths (it uses str(num) instead of repr(num)).
 576             # simplejson-1.7.3 does not have this bug. To prevent this bug
 577             # from causing the test to fail, stall for more than a few
 578             # hundrededths of a second.
 579             d.addCallback(self.stall, 0.1)
 580             d.addCallback(lambda res: n.add_file(u"timestamps",
 581                                                  upload.Data("stamp me", convergence="some convergence string")))
 582             d.addCallback(self.stall, 0.1)
 583             def _stop(res):
 584                 self._stop_timestamp = time.time()
 585             d.addCallback(_stop)
 586
 587             d.addCallback(lambda res: n.get_metadata_for(u"timestamps"))
 588             def _check_timestamp1(metadata):
 589                 self.failUnless("ctime" in metadata)
 590                 self.failUnless("mtime" in metadata)
 591                 self.failUnlessGreaterOrEqualThan(metadata["ctime"],
 592                                                   self._start_timestamp)
 593                 self.failUnlessGreaterOrEqualThan(self._stop_timestamp,
 594                                                   metadata["ctime"])
 595                 self.failUnlessGreaterOrEqualThan(metadata["mtime"],
 596                                                   self._start_timestamp)
 597                 self.failUnlessGreaterOrEqualThan(self._stop_timestamp,
 598                                                   metadata["mtime"])
 599                 # Our current timestamp rules say that replacing an existing
 600                 # child should preserve the 'ctime' but update the mtime
 601                 self._old_ctime = metadata["ctime"]
 602                 self._old_mtime = metadata["mtime"]
 603             d.addCallback(_check_timestamp1)
 604             d.addCallback(self.stall, 2.0) # accomodate low-res timestamps
 605             d.addCallback(lambda res: n.set_node(u"timestamps", n))
 606             d.addCallback(lambda res: n.get_metadata_for(u"timestamps"))
 607             def _check_timestamp2(metadata):
 608                 self.failUnlessEqual(metadata["ctime"], self._old_ctime,
 609                                      "%s != %s" % (metadata["ctime"],
 610                                                    self._old_ctime))
 611                 self.failUnlessGreaterThan(metadata["mtime"], self._old_mtime)
 612                 return n.delete(u"timestamps")
 613             d.addCallback(_check_timestamp2)
 614
 615             # also make sure we can add/update timestamps on a
 616             # previously-existing child that didn't have any, since there are
 617             # a lot of 0.7.0-generated edges around out there
 618             d.addCallback(lambda res: n.set_node(u"no_timestamps", n, {}))
 619             d.addCallback(lambda res: n.set_node(u"no_timestamps", n))
 620             d.addCallback(lambda res: n.get_metadata_for(u"no_timestamps"))
 621             d.addCallback(lambda metadata:
 622                           self.failUnlessEqual(sorted(metadata.keys()),
 623                                                ["ctime", "mtime"]))
 624             d.addCallback(lambda res: n.delete(u"no_timestamps"))
 625
 626             d.addCallback(lambda res: n.delete(u"subdir"))
 627             d.addCallback(lambda old_child:
 628                           self.failUnlessEqual(old_child.get_uri(),
 629                                                self.subdir.get_uri()))
 630
 631             d.addCallback(lambda res: n.list())
 632             d.addCallback(lambda children:
 633                           self.failUnlessEqual(sorted(children.keys()),
 634                                                sorted([u"child"])))
 635
 636             uploadable = upload.Data("some data", convergence="some convergence string")
 637             d.addCallback(lambda res: n.add_file(u"newfile", uploadable))
 638             d.addCallback(lambda newnode:
 639                           self.failUnless(IFileNode.providedBy(newnode)))
 640             other_uploadable = upload.Data("some data", convergence="stuff")
 641             d.addCallback(lambda res:
 642                           self.shouldFail(ExistingChildError, "add_file-no",
 643                                           "child 'newfile' already exists",
 644                                           n.add_file, u"newfile",
 645                                           other_uploadable,
 646                                           overwrite=False))
 647             d.addCallback(lambda res: n.list())
 648             d.addCallback(lambda children:
 649                           self.failUnlessEqual(sorted(children.keys()),
 650                                                sorted([u"child", u"newfile"])))
 651             d.addCallback(lambda res: n.get_metadata_for(u"newfile"))
 652             d.addCallback(lambda metadata:
 653                           self.failUnlessEqual(sorted(metadata.keys()),
 654                                                ["ctime", "mtime"]))
 655
 656             d.addCallback(lambda res: n.add_file(u"newfile-metadata",
 657                                                  uploadable,
 658                                                  {"key": "value"}))
 659             d.addCallback(lambda newnode:
 660                           self.failUnless(IFileNode.providedBy(newnode)))
 661             d.addCallback(lambda res: n.get_metadata_for(u"newfile-metadata"))
 662             d.addCallback(lambda metadata:
 663                           self.failUnlessEqual(metadata, {"key": "value"}))
 664             d.addCallback(lambda res: n.delete(u"newfile-metadata"))
 665
 666             d.addCallback(lambda res: n.create_empty_directory(u"subdir2"))
 667             def _created2(subdir2):
 668                 self.subdir2 = subdir2
 669                 # put something in the way, to make sure it gets overwritten
 670                 return subdir2.add_file(u"child", upload.Data("overwrite me",
 671                                                               "converge"))
 672             d.addCallback(_created2)
 673
 674             d.addCallback(lambda res:
 675                           n.move_child_to(u"child", self.subdir2))
 676             d.addCallback(lambda res: n.list())
 677             d.addCallback(lambda children:
 678                           self.failUnlessEqual(sorted(children.keys()),
 679                                                sorted([u"newfile", u"subdir2"])))
 680             d.addCallback(lambda res: self.subdir2.list())
 681             d.addCallback(lambda children:
 682                           self.failUnlessEqual(sorted(children.keys()),
 683                                                sorted([u"child"])))
 684             d.addCallback(lambda res: self.subdir2.get(u"child"))
 685             d.addCallback(lambda child:
 686                           self.failUnlessEqual(child.get_uri(),
 687                                                fake_file_uri.to_string()))
 688
 689             # move it back, using new_child_name=
 690             d.addCallback(lambda res:
 691                           self.subdir2.move_child_to(u"child", n, u"newchild"))
 692             d.addCallback(lambda res: n.list())
 693             d.addCallback(lambda children:
 694                           self.failUnlessEqual(sorted(children.keys()),
 695                                                sorted([u"newchild", u"newfile",
 696                                                        u"subdir2"])))
 697             d.addCallback(lambda res: self.subdir2.list())
 698             d.addCallback(lambda children:
 699                           self.failUnlessEqual(sorted(children.keys()), []))
 700
 701             # now make sure that we honor overwrite=False
 702             d.addCallback(lambda res:
 703                           self.subdir2.set_uri(u"newchild", other_file_uri))
 704
 705             d.addCallback(lambda res:
 706                           self.shouldFail(ExistingChildError, "move_child_to-no",
 707                                           "child 'newchild' already exists",
 708                                           n.move_child_to, u"newchild",
 709                                           self.subdir2,
 710                                           overwrite=False))
 711             d.addCallback(lambda res: self.subdir2.get(u"newchild"))
 712             d.addCallback(lambda child:
 713                           self.failUnlessEqual(child.get_uri(),
 714                                                other_file_uri.to_string()))
 715
 716             return d
 717
 718         d.addCallback(_then)
 719
 720         d.addErrback(self.explain_error)
 721         return d
 722
 723 class DeepStats(unittest.TestCase):
 724     def test_stats(self):
 725         ds = dirnode.DeepStats(None)
 726         ds.add("count-files")
 727         ds.add("size-immutable-files", 123)
 728         ds.histogram("size-files-histogram", 123)
 729         ds.max("largest-directory", 444)
 730
 731         s = ds.get_results()
 732         self.failUnlessEqual(s["count-files"], 1)
 733         self.failUnlessEqual(s["size-immutable-files"], 123)
 734         self.failUnlessEqual(s["largest-directory"], 444)
 735         self.failUnlessEqual(s["count-literal-files"], 0)
 736
 737         ds.add("count-files")
 738         ds.add("size-immutable-files", 321)
 739         ds.histogram("size-files-histogram", 321)
 740         ds.max("largest-directory", 2)
 741
 742         s = ds.get_results()
 743         self.failUnlessEqual(s["count-files"], 2)
 744         self.failUnlessEqual(s["size-immutable-files"], 444)
 745         self.failUnlessEqual(s["largest-directory"], 444)
 746         self.failUnlessEqual(s["count-literal-files"], 0)
 747         self.failUnlessEqual(s["size-files-histogram"],
 748                              [ (101, 316, 1), (317, 1000, 1) ])
 749
 750         ds = dirnode.DeepStats(None)
 751         for i in range(1, 1100):
 752             ds.histogram("size-files-histogram", i)
 753         ds.histogram("size-files-histogram", 4*1000*1000*1000*1000) # 4TB
 754         s = ds.get_results()
 755         self.failUnlessEqual(s["size-files-histogram"],
 756                              [ (1, 3, 3),
 757                                (4, 10, 7),
 758                                (11, 31, 21),
 759                                (32, 100, 69),
 760                                (101, 316, 216),
 761                                (317, 1000, 684),
 762                                (1001, 3162, 99),
 763                                (3162277660169L, 10000000000000L, 1),
 764                                ])
 765
 766 class UCWEingMutableFileNode(MutableFileNode):
 767     please_ucwe_after_next_upload = False
 768
 769     def _upload(self, new_contents, servermap):
 770         d = MutableFileNode._upload(self, new_contents, servermap)
 771         def _ucwe(res):
 772             raise UncoordinatedWriteError()
 773         d.addCallback(_ucwe)
 774         return d
 775 class UCWEingNewDirectoryNode(dirnode.NewDirectoryNode):
 776     filenode_class = UCWEingMutableFileNode
 777
 778
 779 class Deleter(SystemTestMixin, unittest.TestCase):
 780     def test_retry(self):
 781         # ticket #550, a dirnode.delete which experiences an
 782         # UncoordinatedWriteError will fail with an incorrect "you're
 783         # deleting something which isn't there" NoSuchChildError exception.
 784
 785         # to trigger this, we start by creating a directory with a single
 786         # file in it. Then we create a special dirnode that uses a modified
 787         # MutableFileNode which will raise UncoordinatedWriteError once on
 788         # demand. We then call dirnode.delete, which ought to retry and
 789         # succeed.
 790
 791         self.basedir = self.mktemp()
 792         d = self.set_up_nodes()
 793         d.addCallback(lambda ignored: self.clients[0].create_empty_dirnode())
 794         small = upload.Data("Small enough for a LIT", None)
 795         def _created_dir(dn):
 796             self.root = dn
 797             self.root_uri = dn.get_uri()
 798             return dn.add_file(u"file", small)
 799         d.addCallback(_created_dir)
 800         def _do_delete(ignored):
 801             n = UCWEingNewDirectoryNode(self.clients[0]).init_from_uri(self.root_uri)
 802             # This should succeed, not raise an exception
 803             return n.delete(u"file")
 804         d.addCallback(_do_delete)
 805
 806         return d
 807