From: Brian Warner Date: Thu, 12 Nov 2009 00:22:33 +0000 (-0800) Subject: nodemaker: implement immutable directories (internal interface), for #607 X-Git-Tag: trac-4100~2 X-Git-Url: https://git.rkrishnan.org/components/com_hotproperty/simplejson/encoder.py.html?a=commitdiff_plain;h=5fe713fc52dc331b92266d34af8d03b201e45c47;p=tahoe-lafs%2Ftahoe-lafs.git nodemaker: implement immutable directories (internal interface), for #607 * nodemaker.create_from_cap() now handles DIR2-CHK and DIR2-LIT * client.create_immutable_dirnode() is used to create them * no webapi yet --- diff --git a/src/allmydata/client.py b/src/allmydata/client.py index be6eb06e..93e1eb4e 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -460,6 +460,9 @@ class Client(node.Node, pollmixin.PollMixin): def create_dirnode(self, initial_children={}): d = self.nodemaker.create_new_mutable_directory(initial_children) return d + def create_immutable_dirnode(self, children): + return self.nodemaker.create_immutable_directory(children, + self.convergence) def create_mutable_file(self, contents=None, keysize=None): return self.nodemaker.create_mutable_file(contents, keysize) diff --git a/src/allmydata/dirnode.py b/src/allmydata/dirnode.py index 73578653..c8ad5db2 100644 --- a/src/allmydata/dirnode.py +++ b/src/allmydata/dirnode.py @@ -18,8 +18,7 @@ from allmydata.monitor import Monitor from allmydata.util import hashutil, mathutil, base32, log from allmydata.util.assertutil import precondition from allmydata.util.netstring import netstring, split_netstring -from allmydata.uri import DirectoryURI, ReadonlyDirectoryURI, \ - LiteralFileURI, from_string +from allmydata.uri import LiteralFileURI, from_string, wrap_dirnode_cap from pycryptopp.cipher.aes import AES from allmydata.util.dictutil import AuxValueDict @@ -125,8 +124,11 @@ class Adder: def _encrypt_rwcap(filenode, rwcap): assert isinstance(rwcap, str) + writekey = filenode.get_writekey() + if not writekey: + return "" salt = hashutil.mutable_rwcap_salt_hash(rwcap) - key = hashutil.mutable_rwcap_key_hash(salt, filenode.get_writekey()) + key = hashutil.mutable_rwcap_key_hash(salt, writekey) cryptor = AES(key) crypttext = cryptor.process(rwcap) mac = hashutil.hmac(key, salt + crypttext) @@ -188,20 +190,23 @@ class DirectoryNode: def __init__(self, filenode, nodemaker, uploader): self._node = filenode filenode_cap = filenode.get_cap() - if filenode_cap.is_readonly(): - self._uri = ReadonlyDirectoryURI(filenode_cap) - else: - self._uri = DirectoryURI(filenode_cap) + self._uri = wrap_dirnode_cap(filenode_cap) self._nodemaker = nodemaker self._uploader = uploader self._most_recent_size = None def __repr__(self): - return "<%s %s %s>" % (self.__class__.__name__, self.is_readonly() and "RO" or "RW", hasattr(self, '_uri') and self._uri.abbrev()) + return "<%s %s-%s %s>" % (self.__class__.__name__, + self.is_readonly() and "RO" or "RW", + self.is_mutable() and "MUT" or "IMM", + hasattr(self, '_uri') and self._uri.abbrev()) def get_size(self): # return the size of our backing mutable file, in bytes, if we've # fetched it. + if not self._node.is_mutable(): + # TODO?: consider using IMutableFileNode.providedBy(self._node) + return self._node.get_size() return self._most_recent_size def _set_size(self, data): @@ -209,8 +214,12 @@ class DirectoryNode: return data def _read(self): - d = self._node.download_best_version() - d.addCallback(self._set_size) + if self._node.is_mutable(): + # use the IMutableFileNode API. + d = self._node.download_best_version() + d.addCallback(self._set_size) + else: + d = self._node.download_to_data() d.addCallback(self._unpack_contents) return d diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 8546d944..ece332c3 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -1638,7 +1638,7 @@ class IDownloadResults(Interface): class IUploader(Interface): def upload(uploadable): """Upload the file. 'uploadable' must impement IUploadable. This - returns a Deferred which fires with an UploadResults instance, from + returns a Deferred which fires with an IUploadResults instance, from which the URI of the file can be obtained as results.uri .""" def upload_ssk(write_capability, new_version, uploadable): diff --git a/src/allmydata/nodemaker.py b/src/allmydata/nodemaker.py index cb00394a..63cc18ec 100644 --- a/src/allmydata/nodemaker.py +++ b/src/allmydata/nodemaker.py @@ -3,11 +3,16 @@ from zope.interface import implements from allmydata.util.assertutil import precondition from allmydata.interfaces import INodeMaker from allmydata.immutable.filenode import FileNode, LiteralFileNode +from allmydata.immutable.upload import Data from allmydata.mutable.filenode import MutableFileNode from allmydata.dirnode import DirectoryNode, pack_children from allmydata.unknown import UnknownNode from allmydata import uri +class DummyImmutableFileNode: + def get_writekey(self): + return None + class NodeMaker: implements(INodeMaker) @@ -67,7 +72,10 @@ class NodeMaker: return self._create_immutable(cap) if isinstance(cap, (uri.ReadonlySSKFileURI, uri.WriteableSSKFileURI)): return self._create_mutable(cap) - if isinstance(cap, (uri.ReadonlyDirectoryURI, uri.DirectoryURI)): + if isinstance(cap, (uri.DirectoryURI, + uri.ReadonlyDirectoryURI, + uri.ImmutableDirectoryURI, + uri.LiteralDirectoryURI)): filenode = self._create_from_cap(cap.get_filenode_cap()) return self._create_dirnode(filenode) return None @@ -92,3 +100,22 @@ class NodeMaker: pack_children(n, initial_children)) d.addCallback(self._create_dirnode) return d + + def create_immutable_directory(self, children, convergence): + for (name, (node, metadata)) in children.iteritems(): + precondition(not isinstance(node, UnknownNode), + "create_immutable_directory does not accept UnknownNode", node) + precondition(isinstance(metadata, dict), + "create_immutable_directory requires metadata to be a dict, not None", metadata) + precondition(not node.is_mutable(), + "create_immutable_directory requires immutable children", node) + n = DummyImmutableFileNode() # writekey=None + packed = pack_children(n, children) + uploadable = Data(packed, convergence) + d = self.uploader.upload(uploadable, history=self.history) + def _uploaded(results): + filecap = self.create_from_cap(results.uri) + return filecap + d.addCallback(_uploaded) + d.addCallback(self._create_dirnode) + return d diff --git a/src/allmydata/test/test_dirnode.py b/src/allmydata/test/test_dirnode.py index 2136ee6e..8d1f1689 100644 --- a/src/allmydata/test/test_dirnode.py +++ b/src/allmydata/test/test_dirnode.py @@ -32,7 +32,7 @@ class Dirnode(GridTestMixin, unittest.TestCase, def _done(res): self.failUnless(isinstance(res, dirnode.DirectoryNode)) rep = str(res) - self.failUnless("RW" in rep) + self.failUnless("RW-MUT" in rep) d.addCallback(_done) return d @@ -51,7 +51,7 @@ class Dirnode(GridTestMixin, unittest.TestCase, def _created(dn): self.failUnless(isinstance(dn, dirnode.DirectoryNode)) rep = str(dn) - self.failUnless("RW" in rep) + self.failUnless("RW-MUT" in rep) return dn.list() d.addCallback(_created) def _check_kids(children): @@ -83,6 +83,102 @@ class Dirnode(GridTestMixin, unittest.TestCase, bad_kids2)) return d + def test_immutable(self): + self.basedir = "dirnode/Dirnode/test_immutable" + self.set_up_grid() + c = self.g.clients[0] + nm = c.nodemaker + setup_py_uri = "URI:CHK:n7r3m6wmomelk4sep3kw5cvduq:os7ijw5c3maek7pg65e5254k2fzjflavtpejjyhshpsxuqzhcwwq:3:20:14861" + one_uri = "URI:LIT:n5xgk" # LIT for "one" + mut_readcap = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q" + mut_writecap = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq" + kids = {u"one": (nm.create_from_cap(one_uri), {}), + u"two": (nm.create_from_cap(setup_py_uri), + {"metakey": "metavalue"}), + } + d = c.create_immutable_dirnode(kids) + def _created(dn): + self.failUnless(isinstance(dn, dirnode.DirectoryNode)) + self.failIf(dn.is_mutable()) + self.failUnless(dn.is_readonly()) + rep = str(dn) + self.failUnless("RO-IMM" in rep) + cap = dn.get_cap() + self.failUnlessIn("CHK", cap.to_string()) + self.cap = cap + return dn.list() + d.addCallback(_created) + def _check_kids(children): + self.failUnlessEqual(sorted(children.keys()), [u"one", u"two"]) + one_node, one_metadata = children[u"one"] + two_node, two_metadata = children[u"two"] + self.failUnlessEqual(one_node.get_size(), 3) + self.failUnlessEqual(two_node.get_size(), 14861) + self.failUnless(isinstance(one_metadata, dict), one_metadata) + self.failUnlessEqual(two_metadata["metakey"], "metavalue") + d.addCallback(_check_kids) + d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string())) + d.addCallback(lambda dn: dn.list()) + d.addCallback(_check_kids) + future_writecap = "x-tahoe-crazy://I_am_from_the_future." + future_readcap = "x-tahoe-crazy-readonly://I_am_from_the_future." + future_node = UnknownNode(future_writecap, future_readcap) + bad_kids1 = {u"one": (future_node, {})} + d.addCallback(lambda ign: + self.shouldFail(AssertionError, "bad_kids1", + "does not accept UnknownNode", + c.create_immutable_dirnode, + bad_kids1)) + bad_kids2 = {u"one": (nm.create_from_cap(one_uri), None)} + d.addCallback(lambda ign: + self.shouldFail(AssertionError, "bad_kids2", + "requires metadata to be a dict", + c.create_immutable_dirnode, + bad_kids2)) + bad_kids3 = {u"one": (nm.create_from_cap(mut_writecap), {})} + d.addCallback(lambda ign: + self.shouldFail(AssertionError, "bad_kids3", + "create_immutable_directory requires immutable children", + c.create_immutable_dirnode, + bad_kids3)) + bad_kids4 = {u"one": (nm.create_from_cap(mut_readcap), {})} + d.addCallback(lambda ign: + self.shouldFail(AssertionError, "bad_kids4", + "create_immutable_directory requires immutable children", + c.create_immutable_dirnode, + bad_kids4)) + d.addCallback(lambda ign: c.create_immutable_dirnode({})) + def _created_empty(dn): + self.failUnless(isinstance(dn, dirnode.DirectoryNode)) + self.failIf(dn.is_mutable()) + self.failUnless(dn.is_readonly()) + rep = str(dn) + self.failUnless("RO-IMM" in rep) + cap = dn.get_cap() + self.failUnlessIn("LIT", cap.to_string()) + self.failUnlessEqual(cap.to_string(), "URI:DIR2-LIT:") + self.cap = cap + return dn.list() + d.addCallback(_created_empty) + d.addCallback(lambda kids: self.failUnlessEqual(kids, {})) + smallkids = {u"o": (nm.create_from_cap(one_uri), {})} + d.addCallback(lambda ign: c.create_immutable_dirnode(smallkids)) + def _created_small(dn): + self.failUnless(isinstance(dn, dirnode.DirectoryNode)) + self.failIf(dn.is_mutable()) + self.failUnless(dn.is_readonly()) + rep = str(dn) + self.failUnless("RO-IMM" in rep) + cap = dn.get_cap() + self.failUnlessIn("LIT", cap.to_string()) + self.failUnlessEqual(cap.to_string(), + "URI:DIR2-LIT:gi4tumj2n4wdcmz2kvjesosmjfkdu3rvpbtwwlbqhiwdeot3puwcy") + self.cap = cap + return dn.list() + d.addCallback(_created_small) + d.addCallback(lambda kids: self.failUnlessEqual(kids.keys(), [u"o"])) + return d + def test_check(self): self.basedir = "dirnode/Dirnode/test_check" self.set_up_grid() diff --git a/src/allmydata/test/test_uri.py b/src/allmydata/test/test_uri.py index efdbf22e..9a2143ef 100644 --- a/src/allmydata/test/test_uri.py +++ b/src/allmydata/test/test_uri.py @@ -371,7 +371,8 @@ class NewDirnode(unittest.TestCase): self.failUnless(str(u2_verifier)) def test_literal(self): - u1 = uri.LiteralDirectoryURI("data") + u0 = uri.LiteralFileURI("data") + u1 = uri.LiteralDirectoryURI(u0) self.failUnless(str(u1)) u1s = u1.to_string() self.failUnlessEqual(u1.to_string(), "URI:DIR2-LIT:mrqxiyi") diff --git a/src/allmydata/uri.py b/src/allmydata/uri.py index dce0115f..83d80f5d 100644 --- a/src/allmydata/uri.py +++ b/src/allmydata/uri.py @@ -166,6 +166,7 @@ class LiteralFileURI(_BaseURI): def __init__(self, data=None): if data is not None: + assert isinstance(data, str) self.data = data @classmethod @@ -454,13 +455,20 @@ class LiteralDirectoryURI(_ImmutableDirectoryBaseURI): BASE_STRING_RE=re.compile('^'+BASE_STRING) BASE_HUMAN_RE=re.compile('^'+OPTIONALHTTPLEAD+'URI'+SEP+'DIR2-LIT'+SEP) INNER_URI_CLASS=LiteralFileURI - def __init__(self, data=None): - filenode_uri = LiteralFileURI(data) - _ImmutableDirectoryBaseURI.__init__(self, filenode_uri) def get_verify_cap(self): # LIT caps have no verifier, since they aren't distributed return None +def wrap_dirnode_cap(filecap): + if isinstance(filecap, WriteableSSKFileURI): + return DirectoryURI(filecap) + if isinstance(filecap, ReadonlySSKFileURI): + return ReadonlyDirectoryURI(filecap) + if isinstance(filecap, CHKFileURI): + return ImmutableDirectoryURI(filecap) + if isinstance(filecap, LiteralFileURI): + return LiteralDirectoryURI(filecap) + assert False, "cannot wrap a dirnode around %s" % filecap.__class__ class DirectoryURIVerifier(_DirectoryBaseURI): implements(IVerifierURI)