src/allmydata/test/test_dirnode.py

   1
   2 import time
   3 from zope.interface import implements
   4 from twisted.trial import unittest
   5 from allmydata import uri, dirnode, upload
   6 from allmydata.interfaces import IURI, IClient, IMutableFileNode, \
   7      INewDirectoryURI, IReadonlyNewDirectoryURI, IFileNode
   8 from allmydata.util import hashutil, testutil
   9 from allmydata.test.common import make_chk_file_uri, make_mutable_file_uri, \
  10      FakeDirectoryNode, create_chk_filenode
  11
  12 # to test dirnode.py, we want to construct a tree of real DirectoryNodes that
  13 # contain pointers to fake files. We start with a fake MutableFileNode that
  14 # stores all of its data in a static table.
  15
  16 class Marker:
  17     implements(IFileNode, IMutableFileNode) # sure, why not
  18     def __init__(self, nodeuri):
  19         if not isinstance(nodeuri, str):
  20             nodeuri = nodeuri.to_string()
  21         self.nodeuri = nodeuri
  22         si = hashutil.tagged_hash("tag1", nodeuri)[:16]
  23         fp = hashutil.tagged_hash("tag2", nodeuri)
  24         self.verifieruri = uri.SSKVerifierURI(storage_index=si,
  25                                               fingerprint=fp).to_string()
  26     def get_uri(self):
  27         return self.nodeuri
  28     def get_readonly_uri(self):
  29         return self.nodeuri
  30     def get_verifier(self):
  31         return self.verifieruri
  32
  33 # dirnode requires three methods from the client: upload(),
  34 # create_node_from_uri(), and create_empty_dirnode(). Of these, upload() is
  35 # only used by the convenience composite method add_file().
  36
  37 class FakeClient:
  38     implements(IClient)
  39
  40     def upload(self, uploadable):
  41         d = uploadable.get_size()
  42         d.addCallback(lambda size: uploadable.read(size))
  43         def _got_data(datav):
  44             data = "".join(datav)
  45             n = create_chk_filenode(self, data)
  46             results = upload.UploadResults()
  47             results.uri = n.get_uri()
  48             return results
  49         d.addCallback(_got_data)
  50         return d
  51
  52     def create_node_from_uri(self, u):
  53         u = IURI(u)
  54         if (INewDirectoryURI.providedBy(u)
  55             or IReadonlyNewDirectoryURI.providedBy(u)):
  56             return FakeDirectoryNode(self).init_from_uri(u)
  57         return Marker(u.to_string())
  58
  59     def create_empty_dirnode(self):
  60         n = FakeDirectoryNode(self)
  61         d = n.create()
  62         d.addCallback(lambda res: n)
  63         return d
  64
  65
  66 class Dirnode(unittest.TestCase, testutil.ShouldFailMixin, testutil.StallMixin):
  67     def setUp(self):
  68         self.client = FakeClient()
  69
  70     def test_basic(self):
  71         d = self.client.create_empty_dirnode()
  72         def _done(res):
  73             self.failUnless(isinstance(res, FakeDirectoryNode))
  74             rep = str(res)
  75             self.failUnless("RW" in rep)
  76         d.addCallback(_done)
  77         return d
  78
  79     def test_corrupt(self):
  80         d = self.client.create_empty_dirnode()
  81         def _created(dn):
  82             u = make_mutable_file_uri()
  83             d = dn.set_uri(u"child", u, {})
  84             d.addCallback(lambda res: dn.list())
  85             def _check1(children):
  86                 self.failUnless(u"child" in children)
  87             d.addCallback(_check1)
  88             d.addCallback(lambda res:
  89                           self.shouldFail(KeyError, "get bogus", None,
  90                                           dn.get, u"bogus"))
  91             def _corrupt(res):
  92                 filenode = dn._node
  93                 si = IURI(filenode.get_uri()).storage_index
  94                 old_contents = filenode.all_contents[si]
  95                 # we happen to know that the writecap is encrypted near the
  96                 # end of the string. Flip one of its bits and make sure we
  97                 # detect the corruption.
  98                 new_contents = testutil.flip_bit(old_contents, -10)
  99                 # TODO: also test flipping bits in the other portions
 100                 filenode.all_contents[si] = new_contents
 101             d.addCallback(_corrupt)
 102             def _check2(res):
 103                 self.shouldFail(hashutil.IntegrityCheckError, "corrupt",
 104                                 "HMAC does not match, crypttext is corrupted",
 105                                 dn.list)
 106             d.addCallback(_check2)
 107             return d
 108         d.addCallback(_created)
 109         return d
 110
 111     def test_check(self):
 112         d = self.client.create_empty_dirnode()
 113         d.addCallback(lambda dn: dn.check())
 114         def _done(res):
 115             pass
 116         d.addCallback(_done)
 117         return d
 118
 119     def test_readonly(self):
 120         fileuri = make_chk_file_uri(1234)
 121         filenode = self.client.create_node_from_uri(fileuri)
 122         uploadable = upload.Data("some data", convergence="some convergence string")
 123
 124         d = self.client.create_empty_dirnode()
 125         def _created(rw_dn):
 126             d2 = rw_dn.set_uri(u"child", fileuri)
 127             d2.addCallback(lambda res: rw_dn)
 128             return d2
 129         d.addCallback(_created)
 130
 131         def _ready(rw_dn):
 132             ro_uri = rw_dn.get_readonly_uri()
 133             ro_dn = self.client.create_node_from_uri(ro_uri)
 134             self.failUnless(ro_dn.is_readonly())
 135             self.failUnless(ro_dn.is_mutable())
 136
 137             self.shouldFail(dirnode.NotMutableError, "set_uri ro", None,
 138                             ro_dn.set_uri, u"newchild", fileuri)
 139             self.shouldFail(dirnode.NotMutableError, "set_uri ro", None,
 140                             ro_dn.set_node, u"newchild", filenode)
 141             self.shouldFail(dirnode.NotMutableError, "set_uri ro", None,
 142                             ro_dn.add_file, u"newchild", uploadable)
 143             self.shouldFail(dirnode.NotMutableError, "set_uri ro", None,
 144                             ro_dn.delete, u"child")
 145             self.shouldFail(dirnode.NotMutableError, "set_uri ro", None,
 146                             ro_dn.create_empty_directory, u"newchild")
 147             self.shouldFail(dirnode.NotMutableError, "set_uri ro", None,
 148                             ro_dn.move_child_to, u"child", rw_dn)
 149             self.shouldFail(dirnode.NotMutableError, "set_uri ro", None,
 150                             rw_dn.move_child_to, u"child", ro_dn)
 151             return ro_dn.list()
 152         d.addCallback(_ready)
 153         def _listed(children):
 154             self.failUnless(u"child" in children)
 155         d.addCallback(_listed)
 156         return d
 157
 158     def failUnlessGreaterThan(self, a, b):
 159         self.failUnless(a > b, "%r should be > %r" % (a, b))
 160
 161     def failUnlessGreaterOrEqualThan(self, a, b):
 162         self.failUnless(a >= b, "%r should be >= %r" % (a, b))
 163
 164     def test_create(self):
 165         self.expected_manifest = []
 166
 167         d = self.client.create_empty_dirnode()
 168         def _then(n):
 169             self.failUnless(n.is_mutable())
 170             u = n.get_uri()
 171             self.failUnless(u)
 172             self.failUnless(u.startswith("URI:DIR2:"), u)
 173             u_ro = n.get_readonly_uri()
 174             self.failUnless(u_ro.startswith("URI:DIR2-RO:"), u_ro)
 175             u_v = n.get_verifier()
 176             self.failUnless(u_v.startswith("URI:DIR2-Verifier:"), u_v)
 177             self.expected_manifest.append(u_v)
 178
 179             d = n.list()
 180             d.addCallback(lambda res: self.failUnlessEqual(res, {}))
 181             d.addCallback(lambda res: n.has_child(u"missing"))
 182             d.addCallback(lambda res: self.failIf(res))
 183             fake_file_uri = make_mutable_file_uri()
 184             m = Marker(fake_file_uri)
 185             ffu_v = m.get_verifier()
 186             assert isinstance(ffu_v, str)
 187             self.expected_manifest.append(ffu_v)
 188             d.addCallback(lambda res: n.set_uri(u"child", fake_file_uri))
 189
 190             d.addCallback(lambda res: n.create_empty_directory(u"subdir"))
 191             def _created(subdir):
 192                 self.failUnless(isinstance(subdir, FakeDirectoryNode))
 193                 self.subdir = subdir
 194                 new_v = subdir.get_verifier()
 195                 assert isinstance(new_v, str)
 196                 self.expected_manifest.append(new_v)
 197             d.addCallback(_created)
 198
 199             d.addCallback(lambda res: n.list())
 200             d.addCallback(lambda children:
 201                           self.failUnlessEqual(sorted(children.keys()),
 202                                                sorted([u"child", u"subdir"])))
 203
 204             d.addCallback(lambda res: n.build_manifest())
 205             def _check_manifest(manifest):
 206                 self.failUnlessEqual(sorted(manifest),
 207                                      sorted(self.expected_manifest))
 208             d.addCallback(_check_manifest)
 209
 210             d.addCallback(lambda res: n.deep_stats())
 211             def _check_deepstats(stats):
 212                 self.failUnless(isinstance(stats, dict))
 213                 expected = {"count-immutable-files": 0,
 214                             "count-mutable-files": 1,
 215                             "count-literal-files": 0,
 216                             "count-files": 1,
 217                             "count-directories": 2,
 218                             "size-immutable-files": 0,
 219                             "size-literal-files": 0,
 220                             #"size-directories": 616, # varies
 221                             #"largest-directory": 616,
 222                             "largest-directory-children": 2,
 223                             "largest-immutable-file": 0,
 224                             }
 225                 for k,v in expected.iteritems():
 226                     self.failUnlessEqual(stats[k], v,
 227                                          "stats[%s] was %s, not %s" %
 228                                          (k, stats[k], v))
 229                 self.failUnless(stats["size-directories"] > 600,
 230                                 stats["size-directories"])
 231                 self.failUnless(stats["largest-directory"] > 600,
 232                                 stats["largest-directory"])
 233             d.addCallback(_check_deepstats)
 234
 235             def _add_subsubdir(res):
 236                 return self.subdir.create_empty_directory(u"subsubdir")
 237             d.addCallback(_add_subsubdir)
 238             d.addCallback(lambda res: n.get_child_at_path(u"subdir/subsubdir"))
 239             d.addCallback(lambda subsubdir:
 240                           self.failUnless(isinstance(subsubdir,
 241                                                      FakeDirectoryNode)))
 242             d.addCallback(lambda res: n.get_child_at_path(u""))
 243             d.addCallback(lambda res: self.failUnlessEqual(res.get_uri(),
 244                                                            n.get_uri()))
 245
 246             d.addCallback(lambda res: n.get_metadata_for(u"child"))
 247             d.addCallback(lambda metadata:
 248                           self.failUnlessEqual(sorted(metadata.keys()),
 249                                                ["ctime", "mtime"]))
 250
 251             # set_uri + metadata
 252             # it should be possible to add a child without any metadata
 253             d.addCallback(lambda res: n.set_uri(u"c2", fake_file_uri, {}))
 254             d.addCallback(lambda res: n.get_metadata_for(u"c2"))
 255             d.addCallback(lambda metadata: self.failUnlessEqual(metadata, {}))
 256
 257             # if we don't set any defaults, the child should get timestamps
 258             d.addCallback(lambda res: n.set_uri(u"c3", fake_file_uri))
 259             d.addCallback(lambda res: n.get_metadata_for(u"c3"))
 260             d.addCallback(lambda metadata:
 261                           self.failUnlessEqual(sorted(metadata.keys()),
 262                                                ["ctime", "mtime"]))
 263
 264             # or we can add specific metadata at set_uri() time, which
 265             # overrides the timestamps
 266             d.addCallback(lambda res: n.set_uri(u"c4", fake_file_uri,
 267                                                 {"key": "value"}))
 268             d.addCallback(lambda res: n.get_metadata_for(u"c4"))
 269             d.addCallback(lambda metadata:
 270                           self.failUnlessEqual(metadata, {"key": "value"}))
 271
 272             d.addCallback(lambda res: n.delete(u"c2"))
 273             d.addCallback(lambda res: n.delete(u"c3"))
 274             d.addCallback(lambda res: n.delete(u"c4"))
 275
 276             # set_node + metadata
 277             # it should be possible to add a child without any metadata
 278             d.addCallback(lambda res: n.set_node(u"d2", n, {}))
 279             d.addCallback(lambda res: n.get_metadata_for(u"d2"))
 280             d.addCallback(lambda metadata: self.failUnlessEqual(metadata, {}))
 281
 282             # if we don't set any defaults, the child should get timestamps
 283             d.addCallback(lambda res: n.set_node(u"d3", n))
 284             d.addCallback(lambda res: n.get_metadata_for(u"d3"))
 285             d.addCallback(lambda metadata:
 286                           self.failUnlessEqual(sorted(metadata.keys()),
 287                                                ["ctime", "mtime"]))
 288
 289             # or we can add specific metadata at set_node() time, which
 290             # overrides the timestamps
 291             d.addCallback(lambda res: n.set_node(u"d4", n,
 292                                                 {"key": "value"}))
 293             d.addCallback(lambda res: n.get_metadata_for(u"d4"))
 294             d.addCallback(lambda metadata:
 295                           self.failUnlessEqual(metadata, {"key": "value"}))
 296
 297             d.addCallback(lambda res: n.delete(u"d2"))
 298             d.addCallback(lambda res: n.delete(u"d3"))
 299             d.addCallback(lambda res: n.delete(u"d4"))
 300
 301             # metadata through set_children()
 302             d.addCallback(lambda res: n.set_children([ (u"e1", fake_file_uri),
 303                                                    (u"e2", fake_file_uri, {}),
 304                                                    (u"e3", fake_file_uri,
 305                                                     {"key": "value"}),
 306                                                    ]))
 307             d.addCallback(lambda res: n.get_metadata_for(u"e1"))
 308             d.addCallback(lambda metadata:
 309                           self.failUnlessEqual(sorted(metadata.keys()),
 310                                                ["ctime", "mtime"]))
 311             d.addCallback(lambda res: n.get_metadata_for(u"e2"))
 312             d.addCallback(lambda metadata: self.failUnlessEqual(metadata, {}))
 313             d.addCallback(lambda res: n.get_metadata_for(u"e3"))
 314             d.addCallback(lambda metadata:
 315                           self.failUnlessEqual(metadata, {"key": "value"}))
 316
 317             d.addCallback(lambda res: n.delete(u"e1"))
 318             d.addCallback(lambda res: n.delete(u"e2"))
 319             d.addCallback(lambda res: n.delete(u"e3"))
 320
 321             # metadata through set_nodes()
 322             d.addCallback(lambda res: n.set_nodes([ (u"f1", n),
 323                                                     (u"f2", n, {}),
 324                                                     (u"f3", n,
 325                                                      {"key": "value"}),
 326                                                     ]))
 327             d.addCallback(lambda res: n.get_metadata_for(u"f1"))
 328             d.addCallback(lambda metadata:
 329                           self.failUnlessEqual(sorted(metadata.keys()),
 330                                                ["ctime", "mtime"]))
 331             d.addCallback(lambda res: n.get_metadata_for(u"f2"))
 332             d.addCallback(lambda metadata: self.failUnlessEqual(metadata, {}))
 333             d.addCallback(lambda res: n.get_metadata_for(u"f3"))
 334             d.addCallback(lambda metadata:
 335                           self.failUnlessEqual(metadata, {"key": "value"}))
 336
 337             d.addCallback(lambda res: n.delete(u"f1"))
 338             d.addCallback(lambda res: n.delete(u"f2"))
 339             d.addCallback(lambda res: n.delete(u"f3"))
 340
 341
 342             d.addCallback(lambda res:
 343                           n.set_metadata_for(u"child",
 344                                              {"tags": ["web2.0-compatible"]}))
 345             d.addCallback(lambda n1: n1.get_metadata_for(u"child"))
 346             d.addCallback(lambda metadata:
 347                           self.failUnlessEqual(metadata,
 348                                                {"tags": ["web2.0-compatible"]}))
 349
 350             def _start(res):
 351                 self._start_timestamp = time.time()
 352             d.addCallback(_start)
 353             # simplejson-1.7.1 (as shipped on Ubuntu 'gutsy') rounds all
 354             # floats to hundredeths (it uses str(num) instead of repr(num)).
 355             # simplejson-1.7.3 does not have this bug. To prevent this bug
 356             # from causing the test to fail, stall for more than a few
 357             # hundrededths of a second.
 358             d.addCallback(self.stall, 0.1)
 359             d.addCallback(lambda res: n.add_file(u"timestamps",
 360                                                  upload.Data("stamp me", convergence="some convergence string")))
 361             d.addCallback(self.stall, 0.1)
 362             def _stop(res):
 363                 self._stop_timestamp = time.time()
 364             d.addCallback(_stop)
 365
 366             d.addCallback(lambda res: n.get_metadata_for(u"timestamps"))
 367             def _check_timestamp1(metadata):
 368                 self.failUnless("ctime" in metadata)
 369                 self.failUnless("mtime" in metadata)
 370                 self.failUnlessGreaterOrEqualThan(metadata["ctime"],
 371                                                   self._start_timestamp)
 372                 self.failUnlessGreaterOrEqualThan(self._stop_timestamp,
 373                                                   metadata["ctime"])
 374                 self.failUnlessGreaterOrEqualThan(metadata["mtime"],
 375                                                   self._start_timestamp)
 376                 self.failUnlessGreaterOrEqualThan(self._stop_timestamp,
 377                                                   metadata["mtime"])
 378                 # Our current timestamp rules say that replacing an existing
 379                 # child should preserve the 'ctime' but update the mtime
 380                 self._old_ctime = metadata["ctime"]
 381                 self._old_mtime = metadata["mtime"]
 382             d.addCallback(_check_timestamp1)
 383             d.addCallback(self.stall, 2.0) # accomodate low-res timestamps
 384             d.addCallback(lambda res: n.set_node(u"timestamps", n))
 385             d.addCallback(lambda res: n.get_metadata_for(u"timestamps"))
 386             def _check_timestamp2(metadata):
 387                 self.failUnlessEqual(metadata["ctime"], self._old_ctime,
 388                                      "%s != %s" % (metadata["ctime"],
 389                                                    self._old_ctime))
 390                 self.failUnlessGreaterThan(metadata["mtime"], self._old_mtime)
 391                 return n.delete(u"timestamps")
 392             d.addCallback(_check_timestamp2)
 393
 394             # also make sure we can add/update timestamps on a
 395             # previously-existing child that didn't have any, since there are
 396             # a lot of 0.7.0-generated edges around out there
 397             d.addCallback(lambda res: n.set_node(u"no_timestamps", n, {}))
 398             d.addCallback(lambda res: n.set_node(u"no_timestamps", n))
 399             d.addCallback(lambda res: n.get_metadata_for(u"no_timestamps"))
 400             d.addCallback(lambda metadata:
 401                           self.failUnlessEqual(sorted(metadata.keys()),
 402                                                ["ctime", "mtime"]))
 403             d.addCallback(lambda res: n.delete(u"no_timestamps"))
 404
 405             d.addCallback(lambda res: n.delete(u"subdir"))
 406             d.addCallback(lambda old_child:
 407                           self.failUnlessEqual(old_child.get_uri(),
 408                                                self.subdir.get_uri()))
 409
 410             d.addCallback(lambda res: n.list())
 411             d.addCallback(lambda children:
 412                           self.failUnlessEqual(sorted(children.keys()),
 413                                                sorted([u"child"])))
 414
 415             uploadable = upload.Data("some data", convergence="some convergence string")
 416             d.addCallback(lambda res: n.add_file(u"newfile", uploadable))
 417             d.addCallback(lambda newnode:
 418                           self.failUnless(IFileNode.providedBy(newnode)))
 419             d.addCallback(lambda res: n.list())
 420             d.addCallback(lambda children:
 421                           self.failUnlessEqual(sorted(children.keys()),
 422                                                sorted([u"child", u"newfile"])))
 423             d.addCallback(lambda res: n.get_metadata_for(u"newfile"))
 424             d.addCallback(lambda metadata:
 425                           self.failUnlessEqual(sorted(metadata.keys()),
 426                                                ["ctime", "mtime"]))
 427
 428             uploadable = upload.Data("some data", convergence="some convergence string")
 429             d.addCallback(lambda res: n.add_file(u"newfile-metadata",
 430                                                  uploadable,
 431                                                  {"key": "value"}))
 432             d.addCallback(lambda newnode:
 433                           self.failUnless(IFileNode.providedBy(newnode)))
 434             d.addCallback(lambda res: n.get_metadata_for(u"newfile-metadata"))
 435             d.addCallback(lambda metadata:
 436                           self.failUnlessEqual(metadata, {"key": "value"}))
 437             d.addCallback(lambda res: n.delete(u"newfile-metadata"))
 438
 439             d.addCallback(lambda res: n.create_empty_directory(u"subdir2"))
 440             def _created2(subdir2):
 441                 self.subdir2 = subdir2
 442             d.addCallback(_created2)
 443
 444             d.addCallback(lambda res:
 445                           n.move_child_to(u"child", self.subdir2))
 446             d.addCallback(lambda res: n.list())
 447             d.addCallback(lambda children:
 448                           self.failUnlessEqual(sorted(children.keys()),
 449                                                sorted([u"newfile", u"subdir2"])))
 450             d.addCallback(lambda res: self.subdir2.list())
 451             d.addCallback(lambda children:
 452                           self.failUnlessEqual(sorted(children.keys()),
 453                                                sorted([u"child"])))
 454
 455             return d
 456
 457         d.addCallback(_then)
 458
 459         return d
 460
 461
 462 netstring = hashutil.netstring
 463 split_netstring = dirnode.split_netstring
 464
 465 class Netstring(unittest.TestCase):
 466     def test_split(self):
 467         a = netstring("hello") + netstring("world")
 468         self.failUnlessEqual(split_netstring(a, 2), ("hello", "world"))
 469         self.failUnlessEqual(split_netstring(a, 2, False), ("hello", "world"))
 470         self.failUnlessEqual(split_netstring(a, 2, True),
 471                              ("hello", "world", ""))
 472         self.failUnlessRaises(ValueError, split_netstring, a, 3)
 473         self.failUnlessRaises(ValueError, split_netstring, a+" extra", 2)
 474         self.failUnlessRaises(ValueError, split_netstring, a+" extra", 2, False)
 475
 476     def test_extra(self):
 477         a = netstring("hello")
 478         self.failUnlessEqual(split_netstring(a, 1, True), ("hello", ""))
 479         b = netstring("hello") + "extra stuff"
 480         self.failUnlessEqual(split_netstring(b, 1, True),
 481                              ("hello", "extra stuff"))
 482
 483     def test_nested(self):
 484         a = netstring("hello") + netstring("world") + "extra stuff"
 485         b = netstring("a") + netstring("is") + netstring(a) + netstring(".")
 486         top = split_netstring(b, 4)
 487         self.failUnlessEqual(len(top), 4)
 488         self.failUnlessEqual(top[0], "a")
 489         self.failUnlessEqual(top[1], "is")
 490         self.failUnlessEqual(top[2], a)
 491         self.failUnlessEqual(top[3], ".")
 492         self.failUnlessRaises(ValueError, split_netstring, a, 2)
 493         self.failUnlessRaises(ValueError, split_netstring, a, 2, False)
 494         bottom = split_netstring(a, 2, True)
 495         self.failUnlessEqual(bottom, ("hello", "world", "extra stuff"))
 496