From: Brian Warner Date: Thu, 1 Nov 2007 22:15:29 +0000 (-0700) Subject: mutable: first pass at dirnodes, filenodes, new URIs. Some test coverage. X-Git-Tag: allmydata-tahoe-0.7.0~332 X-Git-Url: https://git.rkrishnan.org/module-simplejson.decoder.html?a=commitdiff_plain;h=1d8a4cdfe7d4247331afff4d5688820fd88419ef;p=tahoe-lafs%2Ftahoe-lafs.git mutable: first pass at dirnodes, filenodes, new URIs. Some test coverage. The URI typenames need revision, and only a few dirnode methods are implemented. Filenodes are non-functional, but URI/key-management is in place. There are a lot of classes with names like "NewDirectoryNode" that will need to be rename once we decide what (if any) backwards compatibility want to retain. --- diff --git a/src/allmydata/client.py b/src/allmydata/client.py index c9e7eeb7..59c7f4d0 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -207,3 +207,29 @@ class Client(node.Node, Referenceable, testutil.PollMixin): d.addCallback(lambda res: None) return d + + def create_empty_dirnode(self): + from allmydata.mutable import NewDirectoryNode + n = NewDirectoryNode(self) + d = n.create() + d.addCallback(lambda res: n) + return d + + def create_dirnode_from_uri(self, u): + from allmydata.mutable import NewDirectoryNode + return NewDirectoryNode(self).init_from_uri(u) + + def create_mutable_file(self, contents=""): + from allmydata.mutable import MutableFileNode + n = MutableFileNode(self) + d = n.create(contents) + d.addCallback(lambda res: n) + return d + + def create_mutable_file_from_uri(self, u): + from allmydata.mutable import MutableFileNode + return MutableFileNode(self).init_from_uri(u) + + def create_file_from_uri(self, u): + from allmydata.mutable import FileNode + return FileNode(u, self) diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 6cedc8bf..8aec8a88 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -427,6 +427,12 @@ class IFileURI(Interface): def get_size(): """Return the length (in bytes) of the file that I represent.""" +class IMutableFileURI(Interface): + """I am a URI which represents a mutable filenode.""" + pass +class INewDirectoryURI(Interface): + pass + class IFileNode(Interface): def download(target): @@ -453,6 +459,45 @@ class IFileNode(Interface): def check(): """Perform a file check. See IChecker.check for details.""" +class IMutableFileNode(Interface): + def download_to_data(): + """Download the file's contents. Return a Deferred that fires with + those contents. If there are multiple retrievable versions in the + grid (because you failed to avoid simultaneous writes, see + docs/mutable.txt), this will return the first version that it can + reconstruct, and will silently ignore the others. In the future, a + more advanced API will signal and provide access to the multiple + heads.""" + def replace(newdata): + """Replace the old contents with the new data. Returns a Deferred + that fires (with None) when the operation is complete. + + If the node detects that there are multiple outstanding versions of + the file, this will raise ConsistencyError, and may leave the + distributed file in an unusual state (the node will try to ensure + that at least one version of the file remains retrievable, but it may + or may not be the one you just tried to upload). You should respond + to this by downloading the current contents of the file and retrying + the replace() operation. + """ + + def get_uri(): + pass + def get_verifier(): + pass + def check(): + pass + + def get_writekey(): + """Return this filenode's writekey, or None if the node does not have + write-capability. This may be used to assist with data structures + that need to make certain data available only to writers, such as the + read-write child caps in dirnodes. The recommended process is to have + reader-visible data be submitted to the filenode in the clear (where + it will be encrypted by the filenode using the readkey), but encrypt + writer-visible data using this writekey. + """ + class IDirectoryNode(Interface): def is_mutable(): """Return True if this directory is mutable, False if it is read-only. diff --git a/src/allmydata/mutable.py b/src/allmydata/mutable.py new file mode 100644 index 00000000..78c0e050 --- /dev/null +++ b/src/allmydata/mutable.py @@ -0,0 +1,313 @@ + +import os +from zope.interface import implements +from twisted.internet import defer +import simplejson +from allmydata.interfaces import IMutableFileNode, IDirectoryNode,\ + IMutableFileURI, INewDirectoryURI, IURI, IFileNode, NotMutableError +from allmydata.util import hashutil +from allmydata.util.hashutil import netstring +from allmydata.dirnode import IntegrityCheckError +from allmydata.uri import WriteableSSKFileURI, NewDirectoryURI +from allmydata.Crypto.Cipher import AES + +class MutableFileNode: + implements(IMutableFileNode) + + def __init__(self, client): + self._client = client + self._pubkey = None # filled in upon first read + self._privkey = None # filled in if we're mutable + self._sharemap = {} # known shares, shnum-to-nodeid + + def init_from_uri(self, myuri): + self._uri = IMutableFileURI(myuri) + return self + + def create(self, initial_contents): + """Call this when the filenode is first created. This will generate + the keys, generate the initial shares, allocate shares, and upload + the initial contents. Returns a Deferred that fires (with the + MutableFileNode instance you should use) when it completes. + """ + self._privkey = "very private" + self._pubkey = "public" + self._writekey = hashutil.ssk_writekey_hash(self._privkey) + self._fingerprint = hashutil.ssk_pubkey_fingerprint_hash(self._pubkey) + self._uri = WriteableSSKFileURI(self._writekey, self._fingerprint) + d = defer.succeed(None) + return d + + + def get_uri(self): + return self._uri.to_string() + + def is_mutable(self): + return self._uri.is_mutable() + + def __hash__(self): + return hash((self.__class__, self.uri)) + def __cmp__(self, them): + if cmp(type(self), type(them)): + return cmp(type(self), type(them)) + if cmp(self.__class__, them.__class__): + return cmp(self.__class__, them.__class__) + return cmp(self.uri, them.uri) + + def get_verifier(self): + return IMutableFileURI(self.uri).get_verifier() + + def check(self): + verifier = self.get_verifier() + return self._client.getServiceNamed("checker").check(verifier) + + def download(self, target): + #downloader = self._client.getServiceNamed("downloader") + #return downloader.download(self.uri, target) + raise NotImplementedError + + def download_to_data(self): + #downloader = self._client.getServiceNamed("downloader") + #return downloader.download_to_data(self.uri) + return defer.succeed("this isn't going to fool you, is it") + + def replace(self, newdata): + return defer.succeed(None) + +# use client.create_mutable_file() to make one of these + +def split_netstring(data, numstrings, allow_leftover=False): + """like string.split(), but extracts netstrings. If allow_leftover=False, + returns numstrings elements, and throws ValueError if there was leftover + data. If allow_leftover=True, returns numstrings+1 elements, in which the + last element is the leftover data (possibly an empty string)""" + elements = [] + assert numstrings >= 0 + while data: + colon = data.index(":") + length = int(data[:colon]) + string = data[colon+1:colon+1+length] + assert len(string) == length + elements.append(string) + assert data[colon+1+length] == "," + data = data[colon+1+length+1:] + if len(elements) == numstrings: + break + if len(elements) < numstrings: + raise ValueError("ran out of netstrings") + if allow_leftover: + return tuple(elements + [data]) + if data: + raise ValueError("leftover data in netstrings") + return tuple(elements) + +class NewDirectoryNode: + implements(IDirectoryNode) + filenode_class = MutableFileNode + + def __init__(self, client): + self._client = client + def init_from_uri(self, myuri): + u = INewDirectoryURI(myuri) + self._uri = u + self._node = self.filenode_class(self._client) + self._node.init_from_uri(u.get_filenode_uri()) + return self + + def create(self): + # first we create a MutableFileNode with empty_contents, then use its + # URI to create our own. + self._node = self.filenode_class(self._client) + empty_contents = self._pack_contents({}) + d = self._node.create(empty_contents) + d.addCallback(self._filenode_created) + return d + def _filenode_created(self, res): + self._uri = NewDirectoryURI(self._node._uri) + return None + + def _read(self): + d = self._node.download_to_data() + d.addCallback(self._unpack_contents) + return d + + def _encrypt_rwcap(self, rwcap): + assert isinstance(rwcap, str) + IV = os.urandom(16) + key = hashutil.mutable_rwcap_key_hash(IV, self._node.writekey) + counterstart = "\x00"*16 + cryptor = AES.new(key=key, mode=AES.MODE_CTR, counterstart=counterstart) + crypttext = cryptor.encrypt(rwcap) + mac = hashutil.hmac(key, IV + crypttext) + assert len(mac) == 32 + return IV + crypttext + mac + + def _decrypt_rwcapdata(self, encwrcap): + IV = encwrcap[:16] + crypttext = encwrcap[16:-32] + mac = encwrcap[-32:] + key = hashutil.mutable_rwcap_key_hash(IV, self._node.writekey) + if mac != hashutil.hmac(key, IV+crypttext): + raise IntegrityCheckError("HMAC does not match, crypttext is corrupted") + counterstart = "\x00"*16 + cryptor = AES.new(key=key, mode=AES.MODE_CTR, counterstart=counterstart) + plaintext = cryptor.decrypt(crypttext) + return plaintext + + def _create_node(self, child_uri): + u = IURI(child_uri) + if INewDirectoryURI.providedBy(u): + return self._client.create_dirnode_from_uri(u) + if IFileNode.providedBy(u): + return self._client.create_file_from_uri(u) + if IMutableFileURI.providedBy(u): + return self._client.create_mutable_file_from_uri(u) + raise TypeError("cannot handle URI") + + def _unpack_contents(self, data): + # the directory is serialized as a list of netstrings, one per child. + # Each child is serialized as a list of four netstrings: (name, + # rocap, rwcap, metadata), in which the name,rocap,metadata are in + # cleartext. The rwcap is formatted as: + # pack("16ss32s", iv, AES(H(writekey+iv), plaintextrwcap), mac) + assert isinstance(data, str) + # an empty directory is serialized as an empty string + if data == "": + return {} + mutable = self.is_mutable() + children = {} + while len(data) > 0: + entry, data = split_netstring(data, 1, True) + name, rocap, rwcapdata, metadata_s = split_netstring(entry, 4) + if mutable: + rwcap = self._decrypt_rwcapdata(rwcapdata) + child = self._create_node(rwcap) + else: + child = self._create_node(rocap) + metadata = simplejson.loads(metadata_s) + assert isinstance(metadata, dict) + children[name] = (child, metadata) + return children + + def _pack_contents(self, children): + # expects children in the same format as _unpack_contents + assert isinstance(children, dict) + entries = [] + for name in sorted(children.keys()): + child, metadata = children[name] + assert (IFileNode.providedBy(child) + or IMutableFileNode.providedBy(child) + or IDirectoryNode.providedBy(child)) + assert isinstance(metadata, dict) + rwcap = child.get_uri() # might be RO if the child is not mutable + rocap = child.get_readonly() + entry = "".join([netstring(name), + netstring(rocap), + netstring(self._encrypt_rwcap(rwcap)), + netstring(simplejson.dumps(metadata))]) + entries.append(netstring(entry)) + return "".join(entries) + + def is_mutable(self): + return self._node.is_mutable() + + def get_uri(self): + return self._uri.to_string() + + def get_immutable_uri(self): + return self._uri.get_readonly().to_string() + + def get_verifier(self): + return self._uri.get_verifier().to_string() + + def check(self): + """Perform a file check. See IChecker.check for details.""" + pass + + def list(self): + """I return a Deferred that fires with a dictionary mapping child + name to an IFileNode or IDirectoryNode.""" + return self._read() + + def has_child(self, name): + """I return a Deferred that fires with a boolean, True if there + exists a child of the given name, False if not.""" + d = self._read() + d.addCallback(lambda children: children.has_key(name)) + return d + + def get(self, name): + """I return a Deferred that fires with a specific named child node, + either an IFileNode or an IDirectoryNode.""" + d = self._read() + d.addCallback(lambda children: children[name]) + return d + + def get_child_at_path(self, path): + """Transform a child path into an IDirectoryNode or IFileNode. + + I perform a recursive series of 'get' operations to find the named + descendant node. I return a Deferred that fires with the node, or + errbacks with IndexError if the node could not be found. + + The path can be either a single string (slash-separated) or a list of + path-name elements. + """ + + def set_uri(self, name, child_uri, metadata={}): + """I add a child (by URI) at the specific name. I return a Deferred + that fires when the operation finishes. I will replace any existing + child of the same name. + + The child_uri could be for a file, or for a directory (either + read-write or read-only, using a URI that came from get_uri() ). + + If this directory node is read-only, the Deferred will errback with a + NotMutableError.""" + return self.set_node(name, self._create_node(child_uri), metadata) + + def set_node(self, name, child, metadata={}): + """I add a child at the specific name. I return a Deferred that fires + when the operation finishes. This Deferred will fire with the child + node that was just added. I will replace any existing child of the + same name. + + If this directory node is read-only, the Deferred will errback with a + NotMutableError.""" + if self._node.is_readonly(): + return defer.fail(NotMutableError()) + d = self._read() + def _add(children): + children[name] = (child, metadata) + new_contents = self._pack_contents(children) + return self._node.replace(new_contents) + d.addCallback(_add) + d.addCallback(lambda res: None) + return d + + def add_file(self, name, uploadable): + """I upload a file (using the given IUploadable), then attach the + resulting FileNode to the directory at the given name. I return a + Deferred that fires (with the IFileNode of the uploaded file) when + the operation completes.""" + + def delete(self, name): + """I remove the child at the specific name. I return a Deferred that + fires when the operation finishes.""" + + def create_empty_directory(self, name): + """I create and attach an empty directory at the given name. I return + a Deferred that fires when the operation finishes.""" + + def move_child_to(self, current_child_name, new_parent, new_child_name=None): + """I take one of my children and move them to a new parent. The child + is referenced by name. On the new parent, the child will live under + 'new_child_name', which defaults to 'current_child_name'. I return a + Deferred that fires when the operation finishes.""" + + def build_manifest(self): + """Return a frozenset of verifier-capability strings for all nodes + (directories and files) reachable from this one.""" + +# use client.create_dirnode() to make one of these + diff --git a/src/allmydata/test/test_mutable.py b/src/allmydata/test/test_mutable.py new file mode 100644 index 00000000..0ed5dee4 --- /dev/null +++ b/src/allmydata/test/test_mutable.py @@ -0,0 +1,147 @@ + +from twisted.trial import unittest +from twisted.internet import defer + +from allmydata import mutable, uri +from allmydata.mutable import split_netstring +from allmydata.util.hashutil import netstring + +class Netstring(unittest.TestCase): + def test_split(self): + a = netstring("hello") + netstring("world") + self.failUnlessEqual(split_netstring(a, 2), ("hello", "world")) + self.failUnlessEqual(split_netstring(a, 2, False), ("hello", "world")) + self.failUnlessEqual(split_netstring(a, 2, True), + ("hello", "world", "")) + self.failUnlessRaises(ValueError, split_netstring, a+" extra", 2) + self.failUnlessRaises(ValueError, split_netstring, a+" extra", 2, False) + + def test_extra(self): + a = netstring("hello") + self.failUnlessEqual(split_netstring(a, 1, True), ("hello", "")) + b = netstring("hello") + "extra stuff" + self.failUnlessEqual(split_netstring(b, 1, True), + ("hello", "extra stuff")) + + def test_nested(self): + a = netstring("hello") + netstring("world") + "extra stuff" + b = netstring("a") + netstring("is") + netstring(a) + netstring(".") + top = split_netstring(b, 4) + self.failUnlessEqual(len(top), 4) + self.failUnlessEqual(top[0], "a") + self.failUnlessEqual(top[1], "is") + self.failUnlessEqual(top[2], a) + self.failUnlessEqual(top[3], ".") + self.failUnlessRaises(ValueError, split_netstring, a, 2) + self.failUnlessRaises(ValueError, split_netstring, a, 2, False) + bottom = split_netstring(a, 2, True) + self.failUnlessEqual(bottom, ("hello", "world", "extra stuff")) + +class FakeFilenode(mutable.MutableFileNode): + def init_from_uri(self, myuri): + self._uri = myuri + self.writekey = myuri.writekey + return self + def create(self, initial_contents): + self.contents = initial_contents + self.init_from_uri(uri.WriteableSSKFileURI("key", "fingerprint")) + return defer.succeed(None) + def download_to_data(self): + return defer.succeed(self.contents) + def replace(self, newdata): + self.contents = newdata + return defer.succeed(None) + def is_readonly(self): + return False + def get_readonly(self): + return "fake readonly" + +class FakeNewDirectoryNode(mutable.NewDirectoryNode): + filenode_class = FakeFilenode + +class MyClient: + def __init__(self): + pass + + def create_empty_dirnode(self): + n = FakeNewDirectoryNode(self) + d = n.create() + d.addCallback(lambda res: n) + return d + + def create_dirnode_from_uri(self, u): + return FakeNewDirectoryNode(self).init_from_uri(u) + + def create_mutable_file(self, contents=""): + n = FakeFilenode(self) + d = n.create(contents) + d.addCallback(lambda res: n) + return d + def create_mutable_file_from_uri(self, u): + return FakeFilenode(self).init_from_uri(u) + + +class Filenode(unittest.TestCase): + def setUp(self): + self.client = MyClient() + + def test_create(self): + d = self.client.create_mutable_file() + def _created(n): + d = n.replace("contents 1") + d.addCallback(lambda res: self.failUnlessIdentical(res, None)) + d.addCallback(lambda res: n.download_to_data()) + d.addCallback(lambda res: self.failUnlessEqual(res, "contents 1")) + d.addCallback(lambda res: n.replace("contents 2")) + d.addCallback(lambda res: n.download_to_data()) + d.addCallback(lambda res: self.failUnlessEqual(res, "contents 2")) + return d + d.addCallback(_created) + return d + + def test_create_with_initial_contents(self): + d = self.client.create_mutable_file("contents 1") + def _created(n): + d = n.download_to_data() + d.addCallback(lambda res: self.failUnlessEqual(res, "contents 1")) + d.addCallback(lambda res: n.replace("contents 2")) + d.addCallback(lambda res: n.download_to_data()) + d.addCallback(lambda res: self.failUnlessEqual(res, "contents 2")) + return d + d.addCallback(_created) + return d + +class Dirnode(unittest.TestCase): + def setUp(self): + self.client = MyClient() + + def test_create(self): + d = self.client.create_empty_dirnode() + def _check(n): + self.failUnless(n.is_mutable()) + u = n.get_uri() + self.failUnless(u) + self.failUnless(u.startswith("URI:DIR2:"), u) + u_ro = n.get_immutable_uri() + self.failUnless(u_ro.startswith("URI:DIR2-RO:"), u_ro) + u_v = n.get_verifier() + self.failUnless(u_v.startswith("URI:DIR2-Verifier:"), u_v) + + d = n.list() + d.addCallback(lambda res: self.failUnlessEqual(res, {})) + d.addCallback(lambda res: n.has_child("missing")) + d.addCallback(lambda res: self.failIf(res)) + fake_file_uri = uri.WriteableSSKFileURI("a"*16,"b"*32) + d.addCallback(lambda res: n.set_uri("child", fake_file_uri)) + d.addCallback(lambda res: self.failUnlessEqual(res, None)) + d.addCallback(lambda res: n.list()) + def _check_list(children): + self.failUnless("child" in children) + d.addCallback(_check_list) + + return d + + d.addCallback(_check) + + return d + diff --git a/src/allmydata/test/test_uri.py b/src/allmydata/test/test_uri.py index e4bd4b24..af72fe17 100644 --- a/src/allmydata/test/test_uri.py +++ b/src/allmydata/test/test_uri.py @@ -2,7 +2,8 @@ from twisted.trial import unittest from allmydata import uri from allmydata.util import hashutil -from allmydata.interfaces import IURI, IFileURI, IDirnodeURI, DirnodeURI +from allmydata.interfaces import IURI, IFileURI, IDirnodeURI, DirnodeURI, \ + IMutableFileURI, IVerifierURI from foolscap.schema import Violation class Literal(unittest.TestCase): @@ -184,3 +185,108 @@ class Constraint(unittest.TestCase): fileURI = 'URI:CHK:f3mf6az85wpcai8ma4qayfmxuc:nnw518w5hu3t5oohwtp7ah9n81z9rfg6c1ywk33ia3m64o67nsgo:3:10:345834' self.failUnlessRaises(Violation, DirnodeURI.checkObject, fileURI, False) + +class Mutable(unittest.TestCase): + def test_pack(self): + writekey = "\x01" * 16 + fingerprint = "\x02" * 32 + + u = uri.WriteableSSKFileURI(writekey, fingerprint) + self.failUnlessEqual(u.writekey, writekey) + self.failUnlessEqual(u.fingerprint, fingerprint) + self.failIf(u.is_readonly()) + self.failUnless(u.is_mutable()) + self.failUnless(IURI.providedBy(u)) + self.failUnless(IMutableFileURI.providedBy(u)) + self.failIf(IDirnodeURI.providedBy(u)) + + u2 = uri.from_string(u.to_string()) + self.failUnlessEqual(u2.writekey, writekey) + self.failUnlessEqual(u2.fingerprint, fingerprint) + self.failIf(u2.is_readonly()) + self.failUnless(u2.is_mutable()) + self.failUnless(IURI.providedBy(u2)) + self.failUnless(IMutableFileURI.providedBy(u2)) + self.failIf(IDirnodeURI.providedBy(u2)) + + u3 = u2.get_readonly() + readkey = hashutil.ssk_readkey_hash(writekey) + self.failUnlessEqual(u3.fingerprint, fingerprint) + self.failUnlessEqual(u3.readkey, readkey) + self.failUnless(u3.is_readonly()) + self.failUnless(u3.is_mutable()) + self.failUnless(IURI.providedBy(u3)) + self.failUnless(IMutableFileURI.providedBy(u3)) + self.failIf(IDirnodeURI.providedBy(u3)) + + u4 = uri.ReadonlySSKFileURI(readkey, fingerprint) + self.failUnlessEqual(u4.fingerprint, fingerprint) + self.failUnlessEqual(u4.readkey, readkey) + self.failUnless(u4.is_readonly()) + self.failUnless(u4.is_mutable()) + self.failUnless(IURI.providedBy(u4)) + self.failUnless(IMutableFileURI.providedBy(u4)) + self.failIf(IDirnodeURI.providedBy(u4)) + + u5 = u4.get_verifier() + self.failUnless(IVerifierURI.providedBy(u5)) + self.failUnlessEqual(u5.storage_index, u.storage_index) + u6 = IVerifierURI(u5.to_string()) + self.failUnless(IVerifierURI.providedBy(u6)) + self.failUnlessEqual(u6.storage_index, u.storage_index) + u7 = u.get_verifier() + self.failUnless(IVerifierURI.providedBy(u7)) + self.failUnlessEqual(u7.storage_index, u.storage_index) + + +class NewDirnode(unittest.TestCase): + def test_pack(self): + writekey = "\x01" * 16 + fingerprint = "\x02" * 16 + + n = uri.WriteableSSKFileURI(writekey, fingerprint) + u1 = uri.NewDirectoryURI(n) + self.failIf(u1.is_readonly()) + self.failUnless(u1.is_mutable()) + self.failUnless(IURI.providedBy(u1)) + self.failIf(IFileURI.providedBy(u1)) + self.failUnless(IDirnodeURI.providedBy(u1)) + + u2 = uri.from_string(u1.to_string()) + self.failUnlessEqual(u1.to_string(), u2.to_string()) + self.failIf(u2.is_readonly()) + self.failUnless(u2.is_mutable()) + self.failUnless(IURI.providedBy(u2)) + self.failIf(IFileURI.providedBy(u2)) + self.failUnless(IDirnodeURI.providedBy(u2)) + + u3 = u2.get_readonly() + self.failUnless(u3.is_readonly()) + self.failUnless(u3.is_mutable()) + self.failUnless(IURI.providedBy(u3)) + self.failIf(IFileURI.providedBy(u3)) + self.failUnless(IDirnodeURI.providedBy(u3)) + u3n = u3._filenode_uri + self.failUnless(u3n.is_readonly()) + self.failUnless(u3n.is_mutable()) + + u4 = uri.ReadonlyNewDirectoryURI(u2._filenode_uri.get_readonly()) + self.failUnlessEqual(u4.to_string(), u3.to_string()) + self.failUnless(u4.is_readonly()) + self.failUnless(u4.is_mutable()) + self.failUnless(IURI.providedBy(u4)) + self.failIf(IFileURI.providedBy(u4)) + self.failUnless(IDirnodeURI.providedBy(u4)) + + verifiers = [u1.get_verifier(), u2.get_verifier(), + u3.get_verifier(), u4.get_verifier(), + IVerifierURI(u1.get_verifier().to_string()), + uri.NewDirectoryURIVerifier(n.get_verifier()), + uri.NewDirectoryURIVerifier(n.get_verifier().to_string()), + ] + for v in verifiers: + self.failUnless(IVerifierURI.providedBy(v)) + self.failUnlessEqual(v._filenode_uri, + u1.get_verifier()._filenode_uri) + + diff --git a/src/allmydata/uri.py b/src/allmydata/uri.py index 746ae6b2..127c066c 100644 --- a/src/allmydata/uri.py +++ b/src/allmydata/uri.py @@ -3,7 +3,8 @@ import re from zope.interface import implements from twisted.python.components import registerAdapter from allmydata.util import idlib, hashutil -from allmydata.interfaces import IURI, IDirnodeURI, IFileURI, IVerifierURI +from allmydata.interfaces import IURI, IDirnodeURI, IFileURI, IVerifierURI, \ + IMutableFileURI # the URI shall be an ascii representation of the file. It shall contain # enough information to retrieve and validate the contents. It shall be @@ -176,6 +177,186 @@ class LiteralFileURI(_BaseURI): def get_size(self): return len(self.data) +class WriteableSSKFileURI(_BaseURI): + implements(IURI, IMutableFileURI) + + def __init__(self, *args, **kwargs): + if not args and not kwargs: + return + self.populate(*args, **kwargs) + + def populate(self, writekey, fingerprint): + self.writekey = writekey + self.readkey = hashutil.ssk_readkey_hash(writekey) + self.storage_index = hashutil.ssk_storage_index_hash(self.readkey) + self.fingerprint = fingerprint + + def init_from_string(self, uri): + assert uri.startswith("URI:SSK:"), uri + (header_uri, header_ssk, writekey_s, fingerprint_s) = uri.split(":") + self.populate(idlib.a2b(writekey_s), idlib.a2b(fingerprint_s)) + return self + + def to_string(self): + assert isinstance(self.writekey, str) + assert isinstance(self.fingerprint, str) + return "URI:SSK:%s:%s" % (idlib.b2a(self.writekey), + idlib.b2a(self.fingerprint)) + + def is_readonly(self): + return False + def is_mutable(self): + return True + def get_readonly(self): + return ReadonlySSKFileURI(self.readkey, self.fingerprint) + def get_verifier(self): + return SSKVerifierURI(self.storage_index, self.fingerprint) + +class ReadonlySSKFileURI(_BaseURI): + implements(IURI, IMutableFileURI) + + def __init__(self, *args, **kwargs): + if not args and not kwargs: + return + self.populate(*args, **kwargs) + + def populate(self, readkey, fingerprint): + self.readkey = readkey + self.storage_index = hashutil.ssk_storage_index_hash(self.readkey) + self.fingerprint = fingerprint + + def init_from_string(self, uri): + assert uri.startswith("URI:SSK-RO:"), uri + (header_uri, header_ssk, readkey_s, fingerprint_s) = uri.split(":") + self.populate(idlib.a2b(readkey_s), idlib.a2b(fingerprint_s)) + return self + + def to_string(self): + assert isinstance(self.readkey, str) + assert isinstance(self.fingerprint, str) + return "URI:SSK-RO:%s:%s" % (idlib.b2a(self.readkey), + idlib.b2a(self.fingerprint)) + + def is_readonly(self): + return True + def is_mutable(self): + return True + def get_readonly(self): + return self + def get_verifier(self): + return SSKVerifierURI(self.storage_index, self.fingerprint) + +class SSKVerifierURI(_BaseURI): + implements(IVerifierURI) + + def __init__(self, *args, **kwargs): + if not args and not kwargs: + return + self.populate(*args, **kwargs) + + def populate(self, storage_index, fingerprint): + self.storage_index = storage_index + self.fingerprint = fingerprint + + def init_from_string(self, uri): + assert uri.startswith("URI:SSK-Verifier:"), uri + (header_uri, header_ssk, + storage_index_s, fingerprint_s) = uri.split(":") + self.populate(idlib.a2b(storage_index_s), idlib.a2b(fingerprint_s)) + return self + + def to_string(self): + assert isinstance(self.storage_index, str) + assert isinstance(self.fingerprint, str) + return "URI:SSK-Verifier:%s:%s" % (idlib.b2a(self.storage_index), + idlib.b2a(self.fingerprint)) + +class NewDirectoryURI(_BaseURI): + implements(IURI, IDirnodeURI) + + def __init__(self, filenode_uri=None): + if filenode_uri: + assert not filenode_uri.is_readonly() + self._filenode_uri = filenode_uri + + def init_from_string(self, uri): + assert uri.startswith("URI:DIR2:") + (header_uri, header_dir2, bits) = uri.split(":", 2) + fn = WriteableSSKFileURI() + fn.init_from_string("URI:SSK:" + bits) + self._filenode_uri = fn + return self + + def to_string(self): + assert isinstance(self._filenode_uri, WriteableSSKFileURI) + fn_u = self._filenode_uri.to_string() + (header_uri, header_ssk, bits) = fn_u.split(":", 2) + return "URI:DIR2:" + bits + + def is_readonly(self): + return False + def is_mutable(self): + return True + def get_readonly(self): + return ReadonlyNewDirectoryURI(self._filenode_uri.get_readonly()) + def get_verifier(self): + return NewDirectoryURIVerifier(self._filenode_uri.get_verifier()) + +class ReadonlyNewDirectoryURI(_BaseURI): + implements(IURI, IDirnodeURI) + + def __init__(self, filenode_uri=None): + if filenode_uri: + assert filenode_uri.is_readonly() + self._filenode_uri = filenode_uri + + def init_from_string(self, uri): + assert uri.startswith("URI:DIR2-RO:") + (header_uri, header_dir2, bits) = uri.split(":", 2) + fn = ReadonlySSKFileURI() + fn.init_from_string("URI:SSK-RO:" + bits) + self._filenode_uri = fn + return self + + def to_string(self): + assert isinstance(self._filenode_uri, ReadonlySSKFileURI) + fn_u = self._filenode_uri.to_string() + (header_uri, header_ssk, bits) = fn_u.split(":", 2) + return "URI:DIR2-RO:" + bits + + def is_readonly(self): + return True + def is_mutable(self): + return True + def get_readonly(self): + return self + def get_verifier(self): + return NewDirectoryURIVerifier(self._filenode_uri.get_verifier()) + +class NewDirectoryURIVerifier(_BaseURI): + implements(IVerifierURI) + + def __init__(self, filenode_uri=None): + if filenode_uri: + filenode_uri = IVerifierURI(filenode_uri) + self._filenode_uri = filenode_uri + + def init_from_string(self, uri): + assert uri.startswith("URI:DIR2-Verifier:") + (header_uri, header_dir2, bits) = uri.split(":", 2) + fn = SSKVerifierURI() + fn.init_from_string("URI:SSK-Verifier:" + bits) + self._filenode_uri = fn + return self + + def to_string(self): + assert isinstance(self._filenode_uri, SSKVerifierURI) + fn_u = self._filenode_uri.to_string() + (header_uri, header_ssk, bits) = fn_u.split(":", 2) + return "URI:DIR2-Verifier:" + bits + + + class DirnodeURI(_BaseURI): implements(IURI, IDirnodeURI) @@ -300,8 +481,20 @@ def from_string(s): return ReadOnlyDirnodeURI().init_from_string(s) elif s.startswith("URI:DIR-Verifier:"): return DirnodeVerifierURI().init_from_string(s) + elif s.startswith("URI:SSK:"): + return WriteableSSKFileURI().init_from_string(s) + elif s.startswith("URI:SSK-RO:"): + return ReadonlySSKFileURI().init_from_string(s) + elif s.startswith("URI:SSK-Verifier:"): + return SSKVerifierURI().init_from_string(s) + elif s.startswith("URI:DIR2:"): + return NewDirectoryURI().init_from_string(s) + elif s.startswith("URI:DIR2-RO:"): + return ReadonlyNewDirectoryURI().init_from_string(s) + elif s.startswith("URI:DIR2-Verifier:"): + return NewDirectoryURIVerifier().init_from_string(s) else: - raise TypeError("unknown URI type: %s.." % s[:10]) + raise TypeError("unknown URI type: %s.." % s[:12]) registerAdapter(from_string, str, IURI) @@ -319,6 +512,12 @@ def from_string_filenode(s): registerAdapter(from_string_filenode, str, IFileURI) +def from_string_mutable_filenode(s): + u = from_string(s) + assert IMutableFileURI.providedBy(u) + return u +registerAdapter(from_string_mutable_filenode, str, IMutableFileURI) + def from_string_verifier(s): u = from_string(s) assert IVerifierURI.providedBy(u) diff --git a/src/allmydata/util/hashutil.py b/src/allmydata/util/hashutil.py index 54caf0a4..3b14e668 100644 --- a/src/allmydata/util/hashutil.py +++ b/src/allmydata/util/hashutil.py @@ -115,3 +115,15 @@ def hmac(tag, data): h1 = SHA256.new(ikey + data).digest() h2 = SHA256.new(okey + h1).digest() return h2 + +def mutable_rwcap_key_hash(iv, writekey): + return tagged_pair_hash("allmydata_mutable_rwcap_key_v1", iv, writekey) +def ssk_writekey_hash(privkey): + return tagged_hash("allmydata_mutable_writekey_v1", privkey) +def ssk_pubkey_fingerprint_hash(pubkey): + return tagged_hash("allmydata_mutable_pubkey_v1", pubkey) + +def ssk_readkey_hash(writekey): + return tagged_hash("allmydata_mutable_readkey_v1", writekey) +def ssk_storage_index_hash(readkey): + return tagged_hash("allmydata_mutable_storage_index_v1", readkey)