From 0f5ef5184d126286163ab01ec35ca47df596559c Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Tue, 4 Dec 2007 14:32:04 -0700 Subject: [PATCH] test_dirnode.py: obtain full coverage of dirnode.py --- src/allmydata/dirnode.py | 7 +- src/allmydata/interfaces.py | 27 +++ src/allmydata/test/test_dirnode.py | 337 +++++++++++++++++++++++++++++ src/allmydata/test/test_mutable.py | 141 +----------- src/allmydata/util/testutil.py | 28 +++ 5 files changed, 398 insertions(+), 142 deletions(-) create mode 100644 src/allmydata/test/test_dirnode.py diff --git a/src/allmydata/dirnode.py b/src/allmydata/dirnode.py index 9b5e630d..0659d687 100644 --- a/src/allmydata/dirnode.py +++ b/src/allmydata/dirnode.py @@ -6,8 +6,7 @@ from twisted.internet import defer import simplejson from allmydata.mutable import NotMutableError from allmydata.interfaces import IMutableFileNode, IDirectoryNode,\ - IURI, IFileNode, \ - IVerifierURI + IURI, IFileNode, IMutableFileURI, IVerifierURI from allmydata.util import hashutil from allmydata.util.hashutil import netstring from allmydata.uri import NewDirectoryURI @@ -67,7 +66,7 @@ class NewDirectoryNode: d.addCallback(self._filenode_created) return d def _filenode_created(self, res): - self._uri = NewDirectoryURI(self._node._uri) + self._uri = NewDirectoryURI(IMutableFileURI(self._node.get_uri())) return self def _read(self): @@ -159,7 +158,7 @@ class NewDirectoryNode: def check(self): """Perform a file check. See IChecker.check for details.""" - pass # TODO + return defer.succeed(None) # TODO def list(self): """I return a Deferred that fires with a dictionary mapping child diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 1ee51f55..bea99976 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -1135,6 +1135,33 @@ class IChecker(Interface): might need to back away from this in the future. """ +class IClient(Interface): + def upload(uploadable, wait_for_numpeers): + """Upload some data into a CHK, get back the URI string for it. + @param uploadable: something that implements IUploadable + @param wait_for_numpeers: don't upload anything until we have at least + this many peers connected + @return: a Deferred that fires with the (string) URI for this file. + """ + def create_empty_dirnode(wait_for_numpeers): + """Create a new dirnode, empty and unattached. + @param wait_for_numpeers: don't create anything until we have at least + this many peers connected. + @return: a Deferred that fires with the new IDirectoryNode instance. + """ + def create_node_from_uri(uri): + """Create a new IFilesystemNode instance from the uri, synchronously. + @param uri: a string or IURI-providing instance. This could be for a + LiteralFileNode, a CHK file node, a mutable file node, or + a directory node + @return: an instance that provides IFilesystemNode (or more usefully one + of its subclasses). File-specifying URIs will result in + IFileNode or IMutableFileNode -providing instances, like + FileNode, LiteralFileNode, or MutableFileNode. + Directory-specifying URIs will result in + IDirectoryNode-providing instances, like NewDirectoryNode. + """ + class NotCapableError(Exception): """You have tried to write to a read-only node.""" diff --git a/src/allmydata/test/test_dirnode.py b/src/allmydata/test/test_dirnode.py new file mode 100644 index 00000000..76c5a804 --- /dev/null +++ b/src/allmydata/test/test_dirnode.py @@ -0,0 +1,337 @@ + +import os +from zope.interface import implements +from twisted.trial import unittest +from twisted.internet import defer +from allmydata import uri, dirnode, upload +from allmydata.interfaces import IURI, IClient, IMutableFileNode, \ + INewDirectoryURI, IReadonlyNewDirectoryURI, IFileNode +from allmydata.util import hashutil, testutil + +# to test dirnode.py, we want to construct a tree of real DirectoryNodes that +# contain pointers to fake files. We start with a fake MutableFileNode that +# stores all of its data in a static table. + +def make_chk_file_uri(size): + return uri.CHKFileURI(key=os.urandom(16), + uri_extension_hash=os.urandom(32), + needed_shares=3, + total_shares=10, + size=size) + +def make_mutable_file_uri(): + return uri.WriteableSSKFileURI(writekey=os.urandom(16), + fingerprint=os.urandom(32)) +def make_verifier_uri(): + return uri.SSKVerifierURI(storage_index=os.urandom(16), + fingerprint=os.urandom(32)) + +class FakeMutableFileNode: + implements(IMutableFileNode) + all_contents = {} + def __init__(self, client): + self.client = client + self.my_uri = make_mutable_file_uri() + self.storage_index = self.my_uri.storage_index + def create(self, initial_contents, wait_for_numpeers=None): + self.all_contents[self.storage_index] = initial_contents + return defer.succeed(self) + def init_from_uri(self, myuri): + self.my_uri = IURI(myuri) + self.storage_index = self.my_uri.storage_index + return self + def get_uri(self): + return self.my_uri + def is_readonly(self): + return self.my_uri.is_readonly() + def is_mutable(self): + return self.my_uri.is_mutable() + def download_to_data(self): + return defer.succeed(self.all_contents[self.storage_index]) + def get_writekey(self): + return "\x00"*16 + + def replace(self, new_contents, wait_for_numpeers=None): + self.all_contents[self.storage_index] = new_contents + return defer.succeed(None) + +class MyDirectoryNode(dirnode.NewDirectoryNode): + filenode_class = FakeMutableFileNode + +class Marker: + implements(IFileNode, IMutableFileNode) # sure, why not + def __init__(self, nodeuri): + if not isinstance(nodeuri, str): + nodeuri = nodeuri.to_string() + self.nodeuri = nodeuri + si = hashutil.tagged_hash("tag1", nodeuri) + fp = hashutil.tagged_hash("tag2", nodeuri) + self.verifieruri = uri.SSKVerifierURI(storage_index=si, + fingerprint=fp).to_string() + def get_uri(self): + return self.nodeuri + def get_readonly_uri(self): + return self.nodeuri + def get_verifier(self): + return self.verifieruri + +# dirnode requires three methods from the client: upload(), +# create_node_from_uri(), and create_empty_dirnode(). Of these, upload() is +# only used by the convenience composite method add_file(). + +class FakeClient: + implements(IClient) + chk_contents = {} + + def upload(self, uploadable, wait_for_numpeers): + d = uploadable.get_size() + d.addCallback(lambda size: uploadable.read(size)) + def _got_data(datav): + data = "".join(datav) + u = make_chk_file_uri(len(data)) + self.chk_contents[u] = data + return u + d.addCallback(_got_data) + return d + + def create_node_from_uri(self, u): + u = IURI(u) + if (INewDirectoryURI.providedBy(u) + or IReadonlyNewDirectoryURI.providedBy(u)): + return MyDirectoryNode(self).init_from_uri(u) + return Marker(u.to_string()) + + def create_empty_dirnode(self, wait_for_numpeers): + n = MyDirectoryNode(self) + d = n.create(wait_for_numpeers) + d.addCallback(lambda res: n) + return d + + +class Dirnode(unittest.TestCase, testutil.ShouldFailMixin): + def setUp(self): + self.client = FakeClient() + + def test_basic(self): + d = self.client.create_empty_dirnode(0) + def _done(res): + self.failUnless(isinstance(res, MyDirectoryNode)) + rep = str(res) + self.failUnless("RW" in rep) + d.addCallback(_done) + return d + + def test_corrupt(self): + d = self.client.create_empty_dirnode(0) + def _created(dn): + u = make_mutable_file_uri() + d = dn.set_uri("child", u) + d.addCallback(lambda res: dn.list()) + def _check1(children): + self.failUnless("child" in children) + d.addCallback(_check1) + d.addCallback(lambda res: + self.shouldFail(KeyError, "get bogus", None, + dn.get, "bogus")) + def _corrupt(res): + filenode = dn._node + si = IURI(filenode.get_uri()).storage_index + old_contents = filenode.all_contents[si] + # we happen to know that the writecap is encrypted near the + # end of the string. Flip one of its bits and make sure we + # detect the corruption. + new_contents = testutil.flip_bit(old_contents, -10) + filenode.all_contents[si] = new_contents + d.addCallback(_corrupt) + def _check2(res): + self.shouldFail(hashutil.IntegrityCheckError, "corrupt", + "HMAC does not match, crypttext is corrupted", + dn.list) + d.addCallback(_check2) + return d + d.addCallback(_created) + return d + + def test_check(self): + d = self.client.create_empty_dirnode(0) + d.addCallback(lambda dn: dn.check()) + def _done(res): + pass + d.addCallback(_done) + return d + + def test_readonly(self): + fileuri = make_chk_file_uri(1234) + filenode = self.client.create_node_from_uri(fileuri) + uploadable = upload.Data("some data") + + d = self.client.create_empty_dirnode(0) + def _created(rw_dn): + d2 = rw_dn.set_uri("child", fileuri) + d2.addCallback(lambda res: rw_dn) + return d2 + d.addCallback(_created) + + def _ready(rw_dn): + ro_uri = rw_dn.get_readonly_uri() + ro_dn = self.client.create_node_from_uri(ro_uri) + self.failUnless(ro_dn.is_readonly()) + self.failUnless(ro_dn.is_mutable()) + + self.shouldFail(dirnode.NotMutableError, "set_uri ro", None, + ro_dn.set_uri, "newchild", fileuri) + self.shouldFail(dirnode.NotMutableError, "set_uri ro", None, + ro_dn.set_node, "newchild", filenode) + self.shouldFail(dirnode.NotMutableError, "set_uri ro", None, + ro_dn.add_file, "newchild", uploadable) + self.shouldFail(dirnode.NotMutableError, "set_uri ro", None, + ro_dn.delete, "child") + self.shouldFail(dirnode.NotMutableError, "set_uri ro", None, + ro_dn.create_empty_directory, "newchild") + self.shouldFail(dirnode.NotMutableError, "set_uri ro", None, + ro_dn.move_child_to, "child", rw_dn) + self.shouldFail(dirnode.NotMutableError, "set_uri ro", None, + rw_dn.move_child_to, "child", ro_dn) + return ro_dn.list() + d.addCallback(_ready) + def _listed(children): + self.failUnless("child" in children) + d.addCallback(_listed) + return d + + def test_create(self): + self.expected_manifest = [] + + d = self.client.create_empty_dirnode(wait_for_numpeers=1) + def _then(n): + self.failUnless(n.is_mutable()) + u = n.get_uri() + self.failUnless(u) + self.failUnless(u.startswith("URI:DIR2:"), u) + u_ro = n.get_readonly_uri() + self.failUnless(u_ro.startswith("URI:DIR2-RO:"), u_ro) + u_v = n.get_verifier() + self.failUnless(u_v.startswith("URI:DIR2-Verifier:"), u_v) + self.expected_manifest.append(u_v) + + d = n.list() + d.addCallback(lambda res: self.failUnlessEqual(res, {})) + d.addCallback(lambda res: n.has_child("missing")) + d.addCallback(lambda res: self.failIf(res)) + fake_file_uri = make_mutable_file_uri() + m = Marker(fake_file_uri) + ffu_v = m.get_verifier() + assert isinstance(ffu_v, str) + self.expected_manifest.append(ffu_v) + d.addCallback(lambda res: n.set_uri("child", fake_file_uri)) + + d.addCallback(lambda res: n.create_empty_directory("subdir", wait_for_numpeers=1)) + def _created(subdir): + self.failUnless(isinstance(subdir, MyDirectoryNode)) + self.subdir = subdir + new_v = subdir.get_verifier() + assert isinstance(new_v, str) + self.expected_manifest.append(new_v) + d.addCallback(_created) + + d.addCallback(lambda res: n.list()) + d.addCallback(lambda children: + self.failUnlessEqual(sorted(children.keys()), + sorted(["child", "subdir"]))) + + d.addCallback(lambda res: n.build_manifest()) + def _check_manifest(manifest): + self.failUnlessEqual(sorted(manifest), + sorted(self.expected_manifest)) + d.addCallback(_check_manifest) + + def _add_subsubdir(res): + return self.subdir.create_empty_directory("subsubdir", wait_for_numpeers=1) + d.addCallback(_add_subsubdir) + d.addCallback(lambda res: n.get_child_at_path("subdir/subsubdir")) + d.addCallback(lambda subsubdir: + self.failUnless(isinstance(subsubdir, + MyDirectoryNode))) + d.addCallback(lambda res: n.get_child_at_path("")) + d.addCallback(lambda res: self.failUnlessEqual(res.get_uri(), + n.get_uri())) + + d.addCallback(lambda res: n.get_metadata_for("child")) + d.addCallback(lambda metadata: self.failUnlessEqual(metadata, {})) + + d.addCallback(lambda res: n.delete("subdir")) + d.addCallback(lambda old_child: + self.failUnlessEqual(old_child.get_uri(), + self.subdir.get_uri())) + + d.addCallback(lambda res: n.list()) + d.addCallback(lambda children: + self.failUnlessEqual(sorted(children.keys()), + sorted(["child"]))) + + uploadable = upload.Data("some data") + d.addCallback(lambda res: n.add_file("newfile", uploadable)) + d.addCallback(lambda newnode: + self.failUnless(IFileNode.providedBy(newnode))) + d.addCallback(lambda res: n.list()) + d.addCallback(lambda children: + self.failUnlessEqual(sorted(children.keys()), + sorted(["child", "newfile"]))) + + d.addCallback(lambda res: n.create_empty_directory("subdir2")) + def _created2(subdir2): + self.subdir2 = subdir2 + d.addCallback(_created2) + + d.addCallback(lambda res: + n.move_child_to("child", self.subdir2)) + d.addCallback(lambda res: n.list()) + d.addCallback(lambda children: + self.failUnlessEqual(sorted(children.keys()), + sorted(["newfile", "subdir2"]))) + d.addCallback(lambda res: self.subdir2.list()) + d.addCallback(lambda children: + self.failUnlessEqual(sorted(children.keys()), + sorted(["child"]))) + + return d + + d.addCallback(_then) + + return d + +netstring = hashutil.netstring +split_netstring = dirnode.split_netstring + +class Netstring(unittest.TestCase): + def test_split(self): + a = netstring("hello") + netstring("world") + self.failUnlessEqual(split_netstring(a, 2), ("hello", "world")) + self.failUnlessEqual(split_netstring(a, 2, False), ("hello", "world")) + self.failUnlessEqual(split_netstring(a, 2, True), + ("hello", "world", "")) + self.failUnlessRaises(ValueError, split_netstring, a, 3) + self.failUnlessRaises(ValueError, split_netstring, a+" extra", 2) + self.failUnlessRaises(ValueError, split_netstring, a+" extra", 2, False) + + def test_extra(self): + a = netstring("hello") + self.failUnlessEqual(split_netstring(a, 1, True), ("hello", "")) + b = netstring("hello") + "extra stuff" + self.failUnlessEqual(split_netstring(b, 1, True), + ("hello", "extra stuff")) + + def test_nested(self): + a = netstring("hello") + netstring("world") + "extra stuff" + b = netstring("a") + netstring("is") + netstring(a) + netstring(".") + top = split_netstring(b, 4) + self.failUnlessEqual(len(top), 4) + self.failUnlessEqual(top[0], "a") + self.failUnlessEqual(top[1], "is") + self.failUnlessEqual(top[2], a) + self.failUnlessEqual(top[3], ".") + self.failUnlessRaises(ValueError, split_netstring, a, 2) + self.failUnlessRaises(ValueError, split_netstring, a, 2, False) + bottom = split_netstring(a, 2, True) + self.failUnlessEqual(bottom, ("hello", "world", "extra stuff")) + diff --git a/src/allmydata/test/test_mutable.py b/src/allmydata/test/test_mutable.py index 7d923334..4f66afa0 100644 --- a/src/allmydata/test/test_mutable.py +++ b/src/allmydata/test/test_mutable.py @@ -3,46 +3,14 @@ import itertools, struct from twisted.trial import unittest from twisted.internet import defer from twisted.python import failure, log -from allmydata import mutable, uri, dirnode, upload -from allmydata.dirnode import split_netstring -from allmydata.util.hashutil import netstring, tagged_hash +from allmydata import mutable, uri, dirnode +from allmydata.util.hashutil import tagged_hash from allmydata.encode import NotEnoughPeersError from allmydata.interfaces import IURI, INewDirectoryURI, \ - IMutableFileURI, IFileNode, IUploadable, IFileURI + IMutableFileURI, IUploadable, IFileURI from allmydata.filenode import LiteralFileNode import sha -class Netstring(unittest.TestCase): - def test_split(self): - a = netstring("hello") + netstring("world") - self.failUnlessEqual(split_netstring(a, 2), ("hello", "world")) - self.failUnlessEqual(split_netstring(a, 2, False), ("hello", "world")) - self.failUnlessEqual(split_netstring(a, 2, True), - ("hello", "world", "")) - self.failUnlessRaises(ValueError, split_netstring, a+" extra", 2) - self.failUnlessRaises(ValueError, split_netstring, a+" extra", 2, False) - - def test_extra(self): - a = netstring("hello") - self.failUnlessEqual(split_netstring(a, 1, True), ("hello", "")) - b = netstring("hello") + "extra stuff" - self.failUnlessEqual(split_netstring(b, 1, True), - ("hello", "extra stuff")) - - def test_nested(self): - a = netstring("hello") + netstring("world") + "extra stuff" - b = netstring("a") + netstring("is") + netstring(a) + netstring(".") - top = split_netstring(b, 4) - self.failUnlessEqual(len(top), 4) - self.failUnlessEqual(top[0], "a") - self.failUnlessEqual(top[1], "is") - self.failUnlessEqual(top[2], a) - self.failUnlessEqual(top[3], ".") - self.failUnlessRaises(ValueError, split_netstring, a, 2) - self.failUnlessRaises(ValueError, split_netstring, a, 2, False) - bottom = split_netstring(a, 2, True) - self.failUnlessEqual(bottom, ("hello", "world", "extra stuff")) - class FakeFilenode(mutable.MutableFileNode): counter = itertools.count(1) all_contents = {} @@ -441,106 +409,3 @@ class FakePrivKey: return "PRIVKEY-%d" % self.count def sign(self, data): return "SIGN(%s)" % data - -class Dirnode(unittest.TestCase): - def setUp(self): - self.client = FakeClient() - - def test_create(self): - self.expected_manifest = [] - - d = self.client.create_empty_dirnode(wait_for_numpeers=1) - def _then(n): - self.failUnless(n.is_mutable()) - u = n.get_uri() - self.failUnless(u) - self.failUnless(u.startswith("URI:DIR2:"), u) - u_ro = n.get_readonly_uri() - self.failUnless(u_ro.startswith("URI:DIR2-RO:"), u_ro) - u_v = n.get_verifier() - self.failUnless(u_v.startswith("URI:DIR2-Verifier:"), u_v) - self.expected_manifest.append(u_v) - - d = n.list() - d.addCallback(lambda res: self.failUnlessEqual(res, {})) - d.addCallback(lambda res: n.has_child("missing")) - d.addCallback(lambda res: self.failIf(res)) - fake_file_uri = uri.WriteableSSKFileURI("a"*16,"b"*32) - ffu_v = fake_file_uri.get_verifier().to_string() - self.expected_manifest.append(ffu_v) - d.addCallback(lambda res: n.set_uri("child", fake_file_uri)) - - d.addCallback(lambda res: n.create_empty_directory("subdir", wait_for_numpeers=1)) - def _created(subdir): - self.failUnless(isinstance(subdir, FakeNewDirectoryNode)) - self.subdir = subdir - new_v = subdir.get_verifier() - self.expected_manifest.append(new_v) - d.addCallback(_created) - - d.addCallback(lambda res: n.list()) - d.addCallback(lambda children: - self.failUnlessEqual(sorted(children.keys()), - sorted(["child", "subdir"]))) - - d.addCallback(lambda res: n.build_manifest()) - def _check_manifest(manifest): - self.failUnlessEqual(sorted(manifest), - sorted(self.expected_manifest)) - d.addCallback(_check_manifest) - - def _add_subsubdir(res): - return self.subdir.create_empty_directory("subsubdir", wait_for_numpeers=1) - d.addCallback(_add_subsubdir) - d.addCallback(lambda res: n.get_child_at_path("subdir/subsubdir")) - d.addCallback(lambda subsubdir: - self.failUnless(isinstance(subsubdir, - FakeNewDirectoryNode))) - d.addCallback(lambda res: n.get_child_at_path("")) - d.addCallback(lambda res: self.failUnlessEqual(res.get_uri(), - n.get_uri())) - - d.addCallback(lambda res: n.get_metadata_for("child")) - d.addCallback(lambda metadata: self.failUnlessEqual(metadata, {})) - - d.addCallback(lambda res: n.delete("subdir")) - d.addCallback(lambda old_child: - self.failUnlessEqual(old_child.get_uri(), - self.subdir.get_uri())) - - d.addCallback(lambda res: n.list()) - d.addCallback(lambda children: - self.failUnlessEqual(sorted(children.keys()), - sorted(["child"]))) - - uploadable = upload.Data("some data") - d.addCallback(lambda res: n.add_file("newfile", uploadable)) - d.addCallback(lambda newnode: - self.failUnless(IFileNode.providedBy(newnode))) - d.addCallback(lambda res: n.list()) - d.addCallback(lambda children: - self.failUnlessEqual(sorted(children.keys()), - sorted(["child", "newfile"]))) - - d.addCallback(lambda res: n.create_empty_directory("subdir2")) - def _created2(subdir2): - self.subdir2 = subdir2 - d.addCallback(_created2) - - d.addCallback(lambda res: - n.move_child_to("child", self.subdir2)) - d.addCallback(lambda res: n.list()) - d.addCallback(lambda children: - self.failUnlessEqual(sorted(children.keys()), - sorted(["newfile", "subdir2"]))) - d.addCallback(lambda res: self.subdir2.list()) - d.addCallback(lambda children: - self.failUnlessEqual(sorted(children.keys()), - sorted(["child"]))) - - return d - - d.addCallback(_then) - - return d - diff --git a/src/allmydata/util/testutil.py b/src/allmydata/util/testutil.py index 370102f7..ece4609d 100644 --- a/src/allmydata/util/testutil.py +++ b/src/allmydata/util/testutil.py @@ -1,6 +1,16 @@ import os, signal, time from twisted.internet import reactor, defer +from twisted.python import failure + + +def flip_bit(good, which): + # flip the low-order bit of good[which] + if which == -1: + pieces = good[:which], good[-1:], "" + else: + pieces = good[:which], good[which:which+1], good[which+1:] + return pieces[0] + chr(ord(pieces[1]) ^ 0x01) + pieces[2] class SignalMixin: # This class is necessary for any code which wants to use Processes @@ -37,6 +47,24 @@ class PollMixin: reactor.callLater(pollinterval, d.callback, None) return d +class ShouldFailMixin: + + def shouldFail(self, expected_failure, which, substring, callable, *args, **kwargs): + assert substring is None or isinstance(substring, str) + d = defer.maybeDeferred(callable, *args, **kwargs) + def done(res): + if isinstance(res, failure.Failure): + res.trap(expected_failure) + if substring: + self.failUnless(substring in str(res), + "substring '%s' not in '%s'" + % (substring, str(res))) + else: + self.fail("%s was supposed to raise %s, not get '%s'" % + (which, expected_failure, res)) + d.addBoth(done) + return d + class TestMixin(SignalMixin): def setUp(self, repeatable=False): -- 2.45.2