From: Brian Warner Date: Wed, 27 Jun 2007 00:16:58 +0000 (-0700) Subject: merge vdrive.py and filetable.py into a single dirnode.py X-Git-Tag: allmydata-tahoe-0.4.0~24 X-Git-Url: https://git.rkrishnan.org/architecture.txt?a=commitdiff_plain;h=b11fa201915010bf7787425b61801e98f494b703;p=tahoe-lafs%2Ftahoe-lafs.git merge vdrive.py and filetable.py into a single dirnode.py --- diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 70a47add..c6e20ba8 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -3,7 +3,7 @@ import os, sha, stat, time from foolscap import Referenceable, SturdyRef from zope.interface import implements from allmydata.interfaces import RIClient, IDirectoryNode -from allmydata import node, vdrive, uri +from allmydata import node, uri from twisted.internet import defer, reactor from twisted.application.internet import TimerService @@ -16,6 +16,7 @@ from allmydata.download import Downloader from allmydata.webish import WebishServer from allmydata.control import ControlServer from allmydata.introducer import IntroducerClient +from allmydata.dirnode import create_directory_node, create_directory class Client(node.Node, Referenceable): implements(RIClient) @@ -123,7 +124,7 @@ class Client(node.Node, Referenceable): def _got_vdrive_uri(self, root_uri): furl, wk = uri.unpack_dirnode_uri(root_uri) self._vdrive_furl = furl - return vdrive.create_directory_node(self, root_uri) + return create_directory_node(self, root_uri) def _got_vdrive_rootnode(self, rootnode): self.log("got vdrive root") @@ -145,10 +146,10 @@ class Client(node.Node, Referenceable): f = open(MY_VDRIVE_URI_FILE, "r") my_vdrive_uri = f.read().strip() f.close() - return vdrive.create_directory_node(self, my_vdrive_uri) + return create_directory_node(self, my_vdrive_uri) except EnvironmentError: assert self._vdrive_furl - d = vdrive.create_directory(self, self._vdrive_furl) + d = create_directory(self, self._vdrive_furl) def _got_directory(dirnode): f = open(MY_VDRIVE_URI_FILE, "w") f.write(dirnode.get_uri() + "\n") diff --git a/src/allmydata/dirnode.py b/src/allmydata/dirnode.py new file mode 100644 index 00000000..8a519c51 --- /dev/null +++ b/src/allmydata/dirnode.py @@ -0,0 +1,382 @@ + +import os.path +from zope.interface import implements +from twisted.application import service +from twisted.internet import defer +from foolscap import Referenceable +from allmydata import uri +from allmydata.interfaces import RIVirtualDriveServer, IDirectoryNode, IFileNode +from allmydata.util import bencode, idlib, hashutil, fileutil +from allmydata.Crypto.Cipher import AES + +# VirtualDriveServer is the side that hosts directory nodes + +class BadWriteEnablerError(Exception): + pass +class ChildAlreadyPresentError(Exception): + pass + +class NoPublicRootError(Exception): + pass + +class VirtualDriveServer(service.MultiService, Referenceable): + implements(RIVirtualDriveServer) + name = "filetable" + + def __init__(self, basedir, offer_public_root=True): + service.MultiService.__init__(self) + self._basedir = os.path.abspath(basedir) + fileutil.make_dirs(self._basedir) + self._root = None + if offer_public_root: + rootfile = os.path.join(self._basedir, "root") + if not os.path.exists(rootfile): + write_key = hashutil.random_key() + (wk, we, rk, index) = \ + hashutil.generate_dirnode_keys_from_writekey(write_key) + self.create_directory(index, we) + f = open(rootfile, "wb") + f.write(wk) + f.close() + self._root = wk + else: + f = open(rootfile, "rb") + self._root = f.read() + + def set_furl(self, myfurl): + self._myfurl = myfurl + + def get_public_root_uri(self): + if self._root: + return uri.pack_dirnode_uri(self._myfurl, self._root) + raise NoPublicRootError + remote_get_public_root_uri = get_public_root_uri + + def create_directory(self, index, write_enabler): + data = [write_enabler, []] + self._write_to_file(index, data) + return index + remote_create_directory = create_directory + + # the file on disk consists of the write_enabler token and a list of + # (H(name), E(name), E(write), E(read)) tuples. + + def _read_from_file(self, index): + name = idlib.b2a(index) + data = open(os.path.join(self._basedir, name), "rb").read() + return bencode.bdecode(data) + + def _write_to_file(self, index, data): + name = idlib.b2a(index) + f = open(os.path.join(self._basedir, name), "wb") + f.write(bencode.bencode(data)) + f.close() + + + def get(self, index, key): + data = self._read_from_file(index) + for (H_key, E_key, E_write, E_read) in data[1]: + if H_key == key: + return (E_write, E_read) + raise IndexError("unable to find key %s" % idlib.b2a(key)) + remote_get = get + + def list(self, index): + data = self._read_from_file(index) + response = [ (E_key, E_write, E_read) + for (H_key, E_key, E_write, E_read) in data[1] ] + return response + remote_list = list + + def delete(self, index, write_enabler, key): + data = self._read_from_file(index) + if data[0] != write_enabler: + raise BadWriteEnablerError + for i,(H_key, E_key, E_write, E_read) in enumerate(data[1]): + if H_key == key: + del data[1][i] + self._write_to_file(index, data) + return + raise IndexError("unable to find key %s" % idlib.b2a(key)) + remote_delete = delete + + def set(self, index, write_enabler, key, name, write, read): + data = self._read_from_file(index) + if data[0] != write_enabler: + raise BadWriteEnablerError + # first, see if the key is already present + for i,(H_key, E_key, E_write, E_read) in enumerate(data[1]): + if H_key == key: + raise ChildAlreadyPresentError + # now just append the data + data[1].append( (key, name, write, read) ) + self._write_to_file(index, data) + remote_set = set + +# whereas ImmutableDirectoryNodes and their support mechanisms live on the +# client side + +class NotMutableError(Exception): + pass + + +def create_directory_node(client, diruri): + assert uri.is_dirnode_uri(diruri) + if uri.is_mutable_dirnode_uri(diruri): + dirnode_class = MutableDirectoryNode + else: + dirnode_class = ImmutableDirectoryNode + (furl, key) = uri.unpack_dirnode_uri(diruri) + d = client.tub.getReference(furl) + def _got(rref): + dirnode = dirnode_class(diruri, client, rref, key) + return dirnode + d.addCallback(_got) + return d + +IV_LENGTH = 14 +def encrypt(key, data): + IV = os.urandom(IV_LENGTH) + counterstart = IV + "\x00"*(16-IV_LENGTH) + assert len(counterstart) == 16, len(counterstart) + cryptor = AES.new(key=key, mode=AES.MODE_CTR, counterstart=counterstart) + crypttext = cryptor.encrypt(data) + mac = hashutil.hmac(key, IV + crypttext) + assert len(mac) == 32 + return IV + crypttext + mac + +class IntegrityCheckError(Exception): + pass + +def decrypt(key, data): + assert len(data) >= (32+IV_LENGTH), len(data) + IV, crypttext, mac = data[:IV_LENGTH], data[IV_LENGTH:-32], data[-32:] + if mac != hashutil.hmac(key, IV+crypttext): + raise IntegrityCheckError("HMAC does not match, crypttext is corrupted") + counterstart = IV + "\x00"*(16-IV_LENGTH) + assert len(counterstart) == 16, len(counterstart) + cryptor = AES.new(key=key, mode=AES.MODE_CTR, counterstart=counterstart) + plaintext = cryptor.decrypt(crypttext) + return plaintext + + +class ImmutableDirectoryNode: + implements(IDirectoryNode) + + def __init__(self, myuri, client, rref, readkey): + self._uri = myuri + self._client = client + self._tub = client.tub + self._rref = rref + self._readkey = readkey + self._writekey = None + self._write_enabler = None + self._index = hashutil.dir_index_hash(self._readkey) + self._mutable = False + + def dump(self): + return ["URI: %s" % self._uri, + "rk: %s" % idlib.b2a(self._readkey), + "index: %s" % idlib.b2a(self._index), + ] + + def is_mutable(self): + return self._mutable + + def get_uri(self): + return self._uri + + def get_immutable_uri(self): + # return the dirnode URI for a read-only form of this directory + if self._mutable: + return uri.make_immutable_dirnode_uri(self._uri) + else: + return self._uri + + def __hash__(self): + return hash((self.__class__, self._uri)) + def __cmp__(self, them): + if cmp(type(self), type(them)): + return cmp(type(self), type(them)) + if cmp(self.__class__, them.__class__): + return cmp(self.__class__, them.__class__) + return cmp(self._uri, them._uri) + + def _encrypt(self, key, data): + return encrypt(key, data) + + def _decrypt(self, key, data): + return decrypt(key, data) + + def _decrypt_child(self, E_write, E_read): + if E_write and self._writekey: + # we prefer read-write children when we can get them + return self._decrypt(self._writekey, E_write) + else: + return self._decrypt(self._readkey, E_read) + + def list(self): + d = self._rref.callRemote("list", self._index) + entries = {} + def _got(res): + dl = [] + for (E_name, E_write, E_read) in res: + name = self._decrypt(self._readkey, E_name) + child_uri = self._decrypt_child(E_write, E_read) + d2 = self._create_node(child_uri) + def _created(node, name): + entries[name] = node + d2.addCallback(_created, name) + dl.append(d2) + return defer.DeferredList(dl) + d.addCallback(_got) + d.addCallback(lambda res: entries) + return d + + def _hash_name(self, name): + return hashutil.dir_name_hash(self._readkey, name) + + def get(self, name): + H_name = self._hash_name(name) + d = self._rref.callRemote("get", self._index, H_name) + def _check_index_error(f): + f.trap(IndexError) + raise IndexError("get(index=%s): unable to find child named '%s'" + % (idlib.b2a(self._index), name)) + d.addErrback(_check_index_error) + d.addCallback(lambda (E_write, E_read): + self._decrypt_child(E_write, E_read)) + d.addCallback(self._create_node) + return d + + def _set(self, name, write_child, read_child): + if not self._mutable: + return defer.fail(NotMutableError()) + H_name = self._hash_name(name) + E_name = self._encrypt(self._readkey, name) + E_write = "" + if self._writekey and write_child: + E_write = self._encrypt(self._writekey, write_child) + E_read = self._encrypt(self._readkey, read_child) + d = self._rref.callRemote("set", self._index, self._write_enabler, + H_name, E_name, E_write, E_read) + return d + + def set_uri(self, name, child_uri): + write, read = self._split_uri(child_uri) + return self._set(name, write, read) + + def set_node(self, name, child): + d = self.set_uri(name, child.get_uri()) + d.addCallback(lambda res: child) + return d + + def delete(self, name): + if not self._mutable: + return defer.fail(NotMutableError()) + H_name = self._hash_name(name) + d = self._rref.callRemote("delete", self._index, self._write_enabler, + H_name) + return d + + def _create_node(self, child_uri): + if uri.is_dirnode_uri(child_uri): + return create_directory_node(self._client, child_uri) + else: + return defer.succeed(FileNode(child_uri, self._client)) + + def _split_uri(self, child_uri): + if uri.is_dirnode_uri(child_uri): + if uri.is_mutable_dirnode_uri(child_uri): + write = child_uri + read = uri.make_immutable_dirnode_uri(child_uri) + else: + write = None + read = child_uri + return (write, read) + return (None, child_uri) # file + + def create_empty_directory(self, name): + if not self._mutable: + return defer.fail(NotMutableError()) + child_writekey = hashutil.random_key() + my_furl, parent_writekey = uri.unpack_dirnode_uri(self._uri) + child_uri = uri.pack_dirnode_uri(my_furl, child_writekey) + child = MutableDirectoryNode(child_uri, self._client, self._rref, + child_writekey) + d = self._rref.callRemote("create_directory", + child._index, child._write_enabler) + d.addCallback(lambda index: self.set_node(name, child)) + return d + + def add_file(self, name, uploadable): + if not self._mutable: + return defer.fail(NotMutableError()) + uploader = self._client.getServiceNamed("uploader") + d = uploader.upload(uploadable) + d.addCallback(lambda uri: self.set_node(name, + FileNode(uri, self._client))) + return d + + def move_child_to(self, current_child_name, + new_parent, new_child_name=None): + if not (self._mutable and new_parent.is_mutable()): + return defer.fail(NotMutableError()) + if new_child_name is None: + new_child_name = current_child_name + d = self.get(current_child_name) + d.addCallback(lambda child: new_parent.set_node(new_child_name, child)) + d.addCallback(lambda child: self.delete(current_child_name)) + return d + +class MutableDirectoryNode(ImmutableDirectoryNode): + implements(IDirectoryNode) + + def __init__(self, myuri, client, rref, writekey): + readkey = hashutil.dir_read_key_hash(writekey) + ImmutableDirectoryNode.__init__(self, myuri, client, rref, readkey) + self._writekey = writekey + self._write_enabler = hashutil.dir_write_enabler_hash(writekey) + self._mutable = True + +def create_directory(client, furl): + write_key = hashutil.random_key() + (wk, we, rk, index) = \ + hashutil.generate_dirnode_keys_from_writekey(write_key) + myuri = uri.pack_dirnode_uri(furl, wk) + d = client.tub.getReference(furl) + def _got_vdrive_server(vdrive_server): + node = MutableDirectoryNode(myuri, client, vdrive_server, wk) + d2 = vdrive_server.callRemote("create_directory", index, we) + d2.addCallback(lambda res: node) + return d2 + d.addCallback(_got_vdrive_server) + return d + +class FileNode: + implements(IFileNode) + + def __init__(self, uri, client): + self.uri = uri + self._client = client + + def get_uri(self): + return self.uri + + def __hash__(self): + return hash((self.__class__, self.uri)) + def __cmp__(self, them): + if cmp(type(self), type(them)): + return cmp(type(self), type(them)) + if cmp(self.__class__, them.__class__): + return cmp(self.__class__, them.__class__) + return cmp(self.uri, them.uri) + + def download(self, target): + downloader = self._client.getServiceNamed("downloader") + return downloader.download(self.uri, target) + + def download_to_data(self): + downloader = self._client.getServiceNamed("downloader") + return downloader.download_to_data(self.uri) + diff --git a/src/allmydata/filetable.py b/src/allmydata/filetable.py deleted file mode 100644 index 99b8777f..00000000 --- a/src/allmydata/filetable.py +++ /dev/null @@ -1,110 +0,0 @@ - -import os -from zope.interface import implements -from twisted.application import service -from foolscap import Referenceable -from allmydata.interfaces import RIVirtualDriveServer -from allmydata.util import bencode, idlib, hashutil, fileutil -from allmydata import uri - -class BadWriteEnablerError(Exception): - pass -class ChildAlreadyPresentError(Exception): - pass - -class NoPublicRootError(Exception): - pass - -class VirtualDriveServer(service.MultiService, Referenceable): - implements(RIVirtualDriveServer) - name = "filetable" - - def __init__(self, basedir, offer_public_root=True): - service.MultiService.__init__(self) - self._basedir = os.path.abspath(basedir) - fileutil.make_dirs(self._basedir) - self._root = None - if offer_public_root: - rootfile = os.path.join(self._basedir, "root") - if not os.path.exists(rootfile): - write_key = hashutil.random_key() - (wk, we, rk, index) = \ - hashutil.generate_dirnode_keys_from_writekey(write_key) - self.create_directory(index, we) - f = open(rootfile, "wb") - f.write(wk) - f.close() - self._root = wk - else: - f = open(rootfile, "rb") - self._root = f.read() - - def set_furl(self, myfurl): - self._myfurl = myfurl - - def get_public_root_uri(self): - if self._root: - return uri.pack_dirnode_uri(self._myfurl, self._root) - raise NoPublicRootError - remote_get_public_root_uri = get_public_root_uri - - def create_directory(self, index, write_enabler): - data = [write_enabler, []] - self._write_to_file(index, data) - return index - remote_create_directory = create_directory - - # the file on disk consists of the write_enabler token and a list of - # (H(name), E(name), E(write), E(read)) tuples. - - def _read_from_file(self, index): - name = idlib.b2a(index) - data = open(os.path.join(self._basedir, name), "rb").read() - return bencode.bdecode(data) - - def _write_to_file(self, index, data): - name = idlib.b2a(index) - f = open(os.path.join(self._basedir, name), "wb") - f.write(bencode.bencode(data)) - f.close() - - - def get(self, index, key): - data = self._read_from_file(index) - for (H_key, E_key, E_write, E_read) in data[1]: - if H_key == key: - return (E_write, E_read) - raise IndexError("unable to find key %s" % idlib.b2a(key)) - remote_get = get - - def list(self, index): - data = self._read_from_file(index) - response = [ (E_key, E_write, E_read) - for (H_key, E_key, E_write, E_read) in data[1] ] - return response - remote_list = list - - def delete(self, index, write_enabler, key): - data = self._read_from_file(index) - if data[0] != write_enabler: - raise BadWriteEnablerError - for i,(H_key, E_key, E_write, E_read) in enumerate(data[1]): - if H_key == key: - del data[1][i] - self._write_to_file(index, data) - return - raise IndexError("unable to find key %s" % idlib.b2a(key)) - remote_delete = delete - - def set(self, index, write_enabler, key, name, write, read): - data = self._read_from_file(index) - if data[0] != write_enabler: - raise BadWriteEnablerError - # first, see if the key is already present - for i,(H_key, E_key, E_write, E_read) in enumerate(data[1]): - if H_key == key: - raise ChildAlreadyPresentError - # now just append the data - data[1].append( (key, name, write, read) ) - self._write_to_file(index, data) - remote_set = set diff --git a/src/allmydata/introducer_and_vdrive.py b/src/allmydata/introducer_and_vdrive.py index 2eeb1db2..65995be6 100644 --- a/src/allmydata/introducer_and_vdrive.py +++ b/src/allmydata/introducer_and_vdrive.py @@ -1,7 +1,7 @@ import os.path from allmydata import node -from allmydata.filetable import VirtualDriveServer +from allmydata.dirnode import VirtualDriveServer from allmydata.introducer import Introducer diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index 09b463b3..379baef7 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -373,7 +373,7 @@ def dump_root_dirnode(basedir, config, out=sys.stdout, err=sys.stderr): return 1 def dump_directory_node(basedir, config, out=sys.stdout, err=sys.stderr): - from allmydata import filetable, vdrive, uri + from allmydata import uri, dirnode from allmydata.util import hashutil, idlib dir_uri = config['uri'] verbose = config['verbose'] @@ -400,7 +400,7 @@ def dump_directory_node(basedir, config, out=sys.stdout, err=sys.stderr): print >>out - vds = filetable.VirtualDriveServer(os.path.join(basedir, "vdrive"), False) + vds = dirnode.VirtualDriveServer(os.path.join(basedir, "vdrive"), False) data = vds._read_from_file(index) if we: if we != data[0]: @@ -412,16 +412,16 @@ def dump_directory_node(basedir, config, out=sys.stdout, err=sys.stderr): print >>out, " E_key %s" % idlib.b2a(E_key) print >>out, " E_write %s" % idlib.b2a(E_write) print >>out, " E_read %s" % idlib.b2a(E_read) - key = vdrive.decrypt(rk, E_key) + key = dirnode.decrypt(rk, E_key) print >>out, " key %s" % key if hashutil.dir_name_hash(rk, key) != H_key: print >>out, " ERROR: H_key does not match" if wk and E_write: if len(E_write) < 14: print >>out, " ERROR: write data is short:", idlib.b2a(E_write) - write = vdrive.decrypt(wk, E_write) + write = dirnode.decrypt(wk, E_write) print >>out, " write: %s" % write - read = vdrive.decrypt(rk, E_read) + read = dirnode.decrypt(rk, E_read) print >>out, " read: %s" % read print >>out diff --git a/src/allmydata/test/test_dirnode.py b/src/allmydata/test/test_dirnode.py new file mode 100644 index 00000000..c1c734fd --- /dev/null +++ b/src/allmydata/test/test_dirnode.py @@ -0,0 +1,442 @@ + +from twisted.trial import unittest +from cStringIO import StringIO +from foolscap import eventual +from twisted.internet import defer +from twisted.python import failure +from allmydata import uri, dirnode +from allmydata.util import hashutil +from allmydata.interfaces import IDirectoryNode +from allmydata.scripts import runner +from allmydata.dirnode import VirtualDriveServer, \ + ChildAlreadyPresentError, BadWriteEnablerError, NoPublicRootError + +# test the host-side code + +class DirectoryNode(unittest.TestCase): + def test_vdrive_server(self): + basedir = "dirnode_host/DirectoryNode/test_vdrive_server" + vds = VirtualDriveServer(basedir) + vds.set_furl("myFURL") + + root_uri = vds.get_public_root_uri() + self.failUnless(uri.is_dirnode_uri(root_uri)) + self.failUnless(uri.is_mutable_dirnode_uri(root_uri)) + furl, key = uri.unpack_dirnode_uri(root_uri) + self.failUnlessEqual(furl, "myFURL") + self.failUnlessEqual(len(key), hashutil.KEYLEN) + + wk, we, rk, index = hashutil.generate_dirnode_keys_from_writekey(key) + empty_list = vds.list(index) + self.failUnlessEqual(empty_list, []) + + vds.set(index, we, "key1", "name1", "write1", "read1") + vds.set(index, we, "key2", "name2", "", "read2") + + self.failUnlessRaises(ChildAlreadyPresentError, + vds.set, + index, we, "key2", "name2", "write2", "read2") + + self.failUnlessRaises(BadWriteEnablerError, + vds.set, + index, "not the write enabler", + "key2", "name2", "write2", "read2") + + self.failUnlessEqual(vds.get(index, "key1"), + ("write1", "read1")) + self.failUnlessEqual(vds.get(index, "key2"), + ("", "read2")) + self.failUnlessRaises(IndexError, + vds.get, index, "key3") + + self.failUnlessEqual(sorted(vds.list(index)), + [ ("name1", "write1", "read1"), + ("name2", "", "read2"), + ]) + + self.failUnlessRaises(BadWriteEnablerError, + vds.delete, + index, "not the write enabler", "name1") + self.failUnlessEqual(sorted(vds.list(index)), + [ ("name1", "write1", "read1"), + ("name2", "", "read2"), + ]) + self.failUnlessRaises(IndexError, + vds.delete, + index, we, "key3") + + vds.delete(index, we, "key1") + self.failUnlessEqual(sorted(vds.list(index)), + [ ("name2", "", "read2"), + ]) + self.failUnlessRaises(IndexError, + vds.get, index, "key1") + self.failUnlessEqual(vds.get(index, "key2"), + ("", "read2")) + + + vds2 = VirtualDriveServer(basedir) + vds2.set_furl("myFURL") + root_uri2 = vds.get_public_root_uri() + self.failUnless(uri.is_mutable_dirnode_uri(root_uri2)) + furl2, key2 = uri.unpack_dirnode_uri(root_uri2) + (wk2, we2, rk2, index2) = \ + hashutil.generate_dirnode_keys_from_writekey(key2) + self.failUnlessEqual(sorted(vds2.list(index2)), + [ ("name2", "", "read2"), + ]) + + def test_no_root(self): + basedir = "dirnode_host/DirectoryNode/test_no_root" + vds = VirtualDriveServer(basedir, offer_public_root=False) + vds.set_furl("myFURL") + + self.failUnlessRaises(NoPublicRootError, + vds.get_public_root_uri) + + +# and the client-side too + +class LocalReference: + def __init__(self, target): + self.target = target + def callRemote(self, methname, *args, **kwargs): + def _call(ignored): + meth = getattr(self.target, methname) + return meth(*args, **kwargs) + d = eventual.fireEventually(None) + d.addCallback(_call) + return d + +class MyTub: + def __init__(self, vds, myfurl): + self.vds = vds + self.myfurl = myfurl + def getReference(self, furl): + assert furl == self.myfurl + return eventual.fireEventually(LocalReference(self.vds)) + +class MyClient: + def __init__(self, vds, myfurl): + self.tub = MyTub(vds, myfurl) + +class Test(unittest.TestCase): + def test_create_directory(self): + basedir = "vdrive/test_create_directory/vdrive" + vds = dirnode.VirtualDriveServer(basedir) + vds.set_furl("myFURL") + self.client = client = MyClient(vds, "myFURL") + d = dirnode.create_directory(client, "myFURL") + def _created(node): + self.failUnless(IDirectoryNode.providedBy(node)) + self.failUnless(node.is_mutable()) + d.addCallback(_created) + return d + + def test_one(self): + self.basedir = basedir = "vdrive/test_one/vdrive" + vds = dirnode.VirtualDriveServer(basedir) + vds.set_furl("myFURL") + root_uri = vds.get_public_root_uri() + + self.client = client = MyClient(vds, "myFURL") + d1 = dirnode.create_directory_node(client, root_uri) + d2 = dirnode.create_directory_node(client, root_uri) + d = defer.gatherResults( [d1,d2] ) + d.addCallback(self._test_one_1) + return d + + def _test_one_1(self, (rootnode1, rootnode2) ): + self.failUnlessEqual(rootnode1, rootnode2) + self.failIfEqual(rootnode1, "not") + + self.rootnode = rootnode = rootnode1 + self.failUnless(rootnode.is_mutable()) + self.readonly_uri = rootnode.get_immutable_uri() + d = dirnode.create_directory_node(self.client, self.readonly_uri) + d.addCallback(self._test_one_2) + return d + + def _test_one_2(self, ro_rootnode): + self.ro_rootnode = ro_rootnode + self.failIf(ro_rootnode.is_mutable()) + self.failUnlessEqual(ro_rootnode.get_immutable_uri(), + self.readonly_uri) + + rootnode = self.rootnode + + ignored = rootnode.dump() + + # root/ + d = rootnode.list() + def _listed(res): + self.failUnlessEqual(res, {}) + d.addCallback(_listed) + + file1 = uri.pack_uri("i"*32, "k"*16, "e"*32, 25, 100, 12345) + file2 = uri.pack_uri("i"*31 + "2", "k"*16, "e"*32, 25, 100, 12345) + file2_node = dirnode.FileNode(file2, None) + d.addCallback(lambda res: rootnode.set_uri("foo", file1)) + # root/ + # root/foo =file1 + + d.addCallback(lambda res: rootnode.list()) + def _listed2(res): + self.failUnlessEqual(res.keys(), ["foo"]) + file1_node = res["foo"] + self.failUnless(isinstance(file1_node, dirnode.FileNode)) + self.failUnlessEqual(file1_node.uri, file1) + d.addCallback(_listed2) + + d.addCallback(lambda res: rootnode.get("foo")) + def _got_foo(res): + self.failUnless(isinstance(res, dirnode.FileNode)) + self.failUnlessEqual(res.uri, file1) + d.addCallback(_got_foo) + + d.addCallback(lambda res: rootnode.get("missing")) + # this should raise an exception + d.addBoth(self.shouldFail, IndexError, "get('missing')", + "unable to find child named 'missing'") + + d.addCallback(lambda res: rootnode.create_empty_directory("bar")) + # root/ + # root/foo =file1 + # root/bar/ + + d.addCallback(lambda res: rootnode.list()) + d.addCallback(self.failUnlessKeysMatch, ["foo", "bar"]) + def _listed3(res): + self.failIfEqual(res["foo"], res["bar"]) + self.failIfEqual(res["bar"], res["foo"]) + self.failIfEqual(res["foo"], "not") + self.failIfEqual(res["bar"], self.rootnode) + self.failUnlessEqual(res["foo"], res["foo"]) + # make sure the objects can be used as dict keys + testdict = {res["foo"]: 1, res["bar"]: 2} + bar_node = res["bar"] + self.failUnless(isinstance(bar_node, dirnode.MutableDirectoryNode)) + self.bar_node = bar_node + bar_ro_uri = bar_node.get_immutable_uri() + return rootnode.set_uri("bar-ro", bar_ro_uri) + d.addCallback(_listed3) + # root/ + # root/foo =file1 + # root/bar/ + # root/bar-ro/ (read-only) + + d.addCallback(lambda res: rootnode.list()) + d.addCallback(self.failUnlessKeysMatch, ["foo", "bar", "bar-ro"]) + def _listed4(res): + self.failIf(res["bar-ro"].is_mutable()) + self.bar_node_readonly = res["bar-ro"] + + # add another file to bar/ + bar = res["bar"] + return bar.set_node("file2", file2_node) + d.addCallback(_listed4) + d.addCallback(self.failUnlessIdentical, file2_node) + # and a directory + d.addCallback(lambda res: self.bar_node.create_empty_directory("baz")) + # root/ + # root/foo =file1 + # root/bar/ + # root/bar/file2 =file2 + # root/bar/baz/ + # root/bar-ro/ (read-only) + # root/bar-ro/file2 =file2 + # root/bar-ro/baz/ + + d.addCallback(lambda res: self.bar_node.list()) + d.addCallback(self.failUnlessKeysMatch, ["file2", "baz"]) + d.addCallback(lambda res: + self.failUnless(res["baz"].is_mutable())) + + d.addCallback(lambda res: self.bar_node_readonly.list()) + d.addCallback(self.failUnlessKeysMatch, ["file2", "baz"]) + d.addCallback(lambda res: + self.failIf(res["baz"].is_mutable())) + + # try to add a file to bar-ro, should get exception + d.addCallback(lambda res: + self.bar_node_readonly.set_uri("file3", file2)) + d.addBoth(self.shouldFail, dirnode.NotMutableError, + "bar-ro.set('file3')") + + # try to delete a file from bar-ro, should get exception + d.addCallback(lambda res: self.bar_node_readonly.delete("file2")) + d.addBoth(self.shouldFail, dirnode.NotMutableError, + "bar-ro.delete('file2')") + + # try to mkdir in bar-ro, should get exception + d.addCallback(lambda res: + self.bar_node_readonly.create_empty_directory("boffo")) + d.addBoth(self.shouldFail, dirnode.NotMutableError, + "bar-ro.mkdir('boffo')") + + d.addCallback(lambda res: rootnode.delete("foo")) + # root/ + # root/bar/ + # root/bar/file2 =file2 + # root/bar/baz/ + # root/bar-ro/ (read-only) + # root/bar-ro/file2 =file2 + # root/bar-ro/baz/ + + d.addCallback(lambda res: rootnode.list()) + d.addCallback(self.failUnlessKeysMatch, ["bar", "bar-ro"]) + + d.addCallback(lambda res: + self.bar_node.move_child_to("file2", + self.rootnode, "file4")) + # root/ + # root/file4 = file4 + # root/bar/ + # root/bar/baz/ + # root/bar-ro/ (read-only) + # root/bar-ro/baz/ + + d.addCallback(lambda res: rootnode.list()) + d.addCallback(self.failUnlessKeysMatch, ["bar", "bar-ro", "file4"]) + d.addCallback(lambda res:self.bar_node.list()) + d.addCallback(self.failUnlessKeysMatch, ["baz"]) + d.addCallback(lambda res:self.bar_node_readonly.list()) + d.addCallback(self.failUnlessKeysMatch, ["baz"]) + + + d.addCallback(lambda res: + rootnode.move_child_to("file4", + self.bar_node_readonly, "boffo")) + d.addBoth(self.shouldFail, dirnode.NotMutableError, + "mv root/file4 root/bar-ro/boffo") + + d.addCallback(lambda res: rootnode.list()) + d.addCallback(self.failUnlessKeysMatch, ["bar", "bar-ro", "file4"]) + d.addCallback(lambda res:self.bar_node.list()) + d.addCallback(self.failUnlessKeysMatch, ["baz"]) + d.addCallback(lambda res:self.bar_node_readonly.list()) + d.addCallback(self.failUnlessKeysMatch, ["baz"]) + + + d.addCallback(lambda res: + rootnode.move_child_to("file4", self.bar_node)) + + d.addCallback(lambda res: rootnode.list()) + d.addCallback(self.failUnlessKeysMatch, ["bar", "bar-ro"]) + d.addCallback(lambda res:self.bar_node.list()) + d.addCallback(self.failUnlessKeysMatch, ["baz", "file4"]) + d.addCallback(lambda res:self.bar_node_readonly.list()) + d.addCallback(self.failUnlessKeysMatch, ["baz", "file4"]) + + d.addCallback(self._test_one_3) + return d + + def _test_one_3(self, res): + # now test some of the diag tools with the data we've created + out,err = StringIO(), StringIO() + rc = runner.runner(["dump-root-dirnode", "vdrive/test_one"], + stdout=out, stderr=err) + output = out.getvalue() + self.failUnless(output.startswith("URI:DIR:fakeFURL:")) + self.failUnlessEqual(rc, 0) + + out,err = StringIO(), StringIO() + rc = runner.runner(["dump-dirnode", + "--basedir", "vdrive/test_one", + "--verbose", + self.bar_node.get_uri()], + stdout=out, stderr=err) + output = out.getvalue() + #print output + self.failUnlessEqual(rc, 0) + self.failUnless("dirnode uri: URI:DIR:myFURL" in output) + self.failUnless("write_enabler" in output) + self.failIf("write_enabler: None" in output) + self.failUnless("key baz\n" in output) + self.failUnless(" write: URI:DIR:myFURL:" in output) + self.failUnless(" read: URI:DIR-RO:myFURL:" in output) + self.failUnless("key file4\n" in output) + self.failUnless("H_key " in output) + + out,err = StringIO(), StringIO() + rc = runner.runner(["dump-dirnode", + "--basedir", "vdrive/test_one", + # non-verbose + "--uri", self.bar_node.get_uri()], + stdout=out, stderr=err) + output = out.getvalue() + #print output + self.failUnlessEqual(rc, 0) + self.failUnless("dirnode uri: URI:DIR:myFURL" in output) + self.failUnless("write_enabler" in output) + self.failIf("write_enabler: None" in output) + self.failUnless("key baz\n" in output) + self.failUnless(" write: URI:DIR:myFURL:" in output) + self.failUnless(" read: URI:DIR-RO:myFURL:" in output) + self.failUnless("key file4\n" in output) + self.failIf("H_key " in output) + + out,err = StringIO(), StringIO() + rc = runner.runner(["dump-dirnode", + "--basedir", "vdrive/test_one", + "--verbose", + self.bar_node_readonly.get_uri()], + stdout=out, stderr=err) + output = out.getvalue() + #print output + self.failUnlessEqual(rc, 0) + self.failUnless("dirnode uri: URI:DIR-RO:myFURL" in output) + self.failUnless("write_enabler: None" in output) + self.failUnless("key baz\n" in output) + self.failIf(" write: URI:DIR:myFURL:" in output) + self.failUnless(" read: URI:DIR-RO:myFURL:" in output) + self.failUnless("key file4\n" in output) + + def shouldFail(self, res, expected_failure, which, substring=None): + if isinstance(res, failure.Failure): + res.trap(expected_failure) + if substring: + self.failUnless(substring in str(res), + "substring '%s' not in '%s'" + % (substring, str(res))) + else: + self.fail("%s was supposed to raise %s, not get '%s'" % + (which, expected_failure, res)) + + def failUnlessKeysMatch(self, res, expected_keys): + self.failUnlessEqual(sorted(res.keys()), + sorted(expected_keys)) + return res + +def flip_bit(data, offset): + if offset < 0: + offset = len(data) + offset + return data[:offset] + chr(ord(data[offset]) ^ 0x01) + data[offset+1:] + +class Encryption(unittest.TestCase): + def test_loopback(self): + key = "k" * 16 + data = "This is some plaintext data." + crypttext = dirnode.encrypt(key, data) + plaintext = dirnode.decrypt(key, crypttext) + self.failUnlessEqual(data, plaintext) + + def test_hmac(self): + key = "j" * 16 + data = "This is some more plaintext data." + crypttext = dirnode.encrypt(key, data) + # flip a bit in the IV + self.failUnlessRaises(dirnode.IntegrityCheckError, + dirnode.decrypt, + key, flip_bit(crypttext, 0)) + # flip a bit in the crypttext + self.failUnlessRaises(dirnode.IntegrityCheckError, + dirnode.decrypt, + key, flip_bit(crypttext, 16)) + # flip a bit in the HMAC + self.failUnlessRaises(dirnode.IntegrityCheckError, + dirnode.decrypt, + key, flip_bit(crypttext, -1)) + plaintext = dirnode.decrypt(key, crypttext) + self.failUnlessEqual(data, plaintext) + diff --git a/src/allmydata/test/test_filetable.py b/src/allmydata/test/test_filetable.py deleted file mode 100644 index 4886941b..00000000 --- a/src/allmydata/test/test_filetable.py +++ /dev/null @@ -1,86 +0,0 @@ - -from twisted.trial import unittest -from allmydata import filetable, uri -from allmydata.util import hashutil - - -class FileTable(unittest.TestCase): - def test_vdrive_server(self): - basedir = "filetable/FileTable/test_vdrive_server" - vds = filetable.VirtualDriveServer(basedir) - vds.set_furl("myFURL") - - root_uri = vds.get_public_root_uri() - self.failUnless(uri.is_dirnode_uri(root_uri)) - self.failUnless(uri.is_mutable_dirnode_uri(root_uri)) - furl, key = uri.unpack_dirnode_uri(root_uri) - self.failUnlessEqual(furl, "myFURL") - self.failUnlessEqual(len(key), hashutil.KEYLEN) - - wk, we, rk, index = hashutil.generate_dirnode_keys_from_writekey(key) - empty_list = vds.list(index) - self.failUnlessEqual(empty_list, []) - - vds.set(index, we, "key1", "name1", "write1", "read1") - vds.set(index, we, "key2", "name2", "", "read2") - - self.failUnlessRaises(filetable.ChildAlreadyPresentError, - vds.set, - index, we, "key2", "name2", "write2", "read2") - - self.failUnlessRaises(filetable.BadWriteEnablerError, - vds.set, - index, "not the write enabler", - "key2", "name2", "write2", "read2") - - self.failUnlessEqual(vds.get(index, "key1"), - ("write1", "read1")) - self.failUnlessEqual(vds.get(index, "key2"), - ("", "read2")) - self.failUnlessRaises(IndexError, - vds.get, index, "key3") - - self.failUnlessEqual(sorted(vds.list(index)), - [ ("name1", "write1", "read1"), - ("name2", "", "read2"), - ]) - - self.failUnlessRaises(filetable.BadWriteEnablerError, - vds.delete, - index, "not the write enabler", "name1") - self.failUnlessEqual(sorted(vds.list(index)), - [ ("name1", "write1", "read1"), - ("name2", "", "read2"), - ]) - self.failUnlessRaises(IndexError, - vds.delete, - index, we, "key3") - - vds.delete(index, we, "key1") - self.failUnlessEqual(sorted(vds.list(index)), - [ ("name2", "", "read2"), - ]) - self.failUnlessRaises(IndexError, - vds.get, index, "key1") - self.failUnlessEqual(vds.get(index, "key2"), - ("", "read2")) - - - vds2 = filetable.VirtualDriveServer(basedir) - vds2.set_furl("myFURL") - root_uri2 = vds.get_public_root_uri() - self.failUnless(uri.is_mutable_dirnode_uri(root_uri2)) - furl2, key2 = uri.unpack_dirnode_uri(root_uri2) - (wk2, we2, rk2, index2) = \ - hashutil.generate_dirnode_keys_from_writekey(key2) - self.failUnlessEqual(sorted(vds2.list(index2)), - [ ("name2", "", "read2"), - ]) - - def test_no_root(self): - basedir = "FileTable/test_no_root" - vds = filetable.VirtualDriveServer(basedir, offer_public_root=False) - vds.set_furl("myFURL") - - self.failUnlessRaises(filetable.NoPublicRootError, - vds.get_public_root_uri) diff --git a/src/allmydata/test/test_vdrive.py b/src/allmydata/test/test_vdrive.py deleted file mode 100644 index bfd9415a..00000000 --- a/src/allmydata/test/test_vdrive.py +++ /dev/null @@ -1,353 +0,0 @@ - -from cStringIO import StringIO -from twisted.trial import unittest -from twisted.internet import defer -from twisted.python import failure -from allmydata import vdrive, filetable, uri -from allmydata.interfaces import IDirectoryNode -from allmydata.scripts import runner -from foolscap import eventual - -class LocalReference: - def __init__(self, target): - self.target = target - def callRemote(self, methname, *args, **kwargs): - def _call(ignored): - meth = getattr(self.target, methname) - return meth(*args, **kwargs) - d = eventual.fireEventually(None) - d.addCallback(_call) - return d - -class MyTub: - def __init__(self, vds, myfurl): - self.vds = vds - self.myfurl = myfurl - def getReference(self, furl): - assert furl == self.myfurl - return eventual.fireEventually(LocalReference(self.vds)) - -class MyClient: - def __init__(self, vds, myfurl): - self.tub = MyTub(vds, myfurl) - -class Test(unittest.TestCase): - def test_create_directory(self): - basedir = "vdrive/test_create_directory/vdrive" - vds = filetable.VirtualDriveServer(basedir) - vds.set_furl("myFURL") - self.client = client = MyClient(vds, "myFURL") - d = vdrive.create_directory(client, "myFURL") - def _created(node): - self.failUnless(IDirectoryNode.providedBy(node)) - self.failUnless(node.is_mutable()) - d.addCallback(_created) - return d - - def test_one(self): - self.basedir = basedir = "vdrive/test_one/vdrive" - vds = filetable.VirtualDriveServer(basedir) - vds.set_furl("myFURL") - root_uri = vds.get_public_root_uri() - - self.client = client = MyClient(vds, "myFURL") - d1 = vdrive.create_directory_node(client, root_uri) - d2 = vdrive.create_directory_node(client, root_uri) - d = defer.gatherResults( [d1,d2] ) - d.addCallback(self._test_one_1) - return d - - def _test_one_1(self, (rootnode1, rootnode2) ): - self.failUnlessEqual(rootnode1, rootnode2) - self.failIfEqual(rootnode1, "not") - - self.rootnode = rootnode = rootnode1 - self.failUnless(rootnode.is_mutable()) - self.readonly_uri = rootnode.get_immutable_uri() - d = vdrive.create_directory_node(self.client, self.readonly_uri) - d.addCallback(self._test_one_2) - return d - - def _test_one_2(self, ro_rootnode): - self.ro_rootnode = ro_rootnode - self.failIf(ro_rootnode.is_mutable()) - self.failUnlessEqual(ro_rootnode.get_immutable_uri(), - self.readonly_uri) - - rootnode = self.rootnode - - ignored = rootnode.dump() - - # root/ - d = rootnode.list() - def _listed(res): - self.failUnlessEqual(res, {}) - d.addCallback(_listed) - - file1 = uri.pack_uri("i"*32, "k"*16, "e"*32, 25, 100, 12345) - file2 = uri.pack_uri("i"*31 + "2", "k"*16, "e"*32, 25, 100, 12345) - file2_node = vdrive.FileNode(file2, None) - d.addCallback(lambda res: rootnode.set_uri("foo", file1)) - # root/ - # root/foo =file1 - - d.addCallback(lambda res: rootnode.list()) - def _listed2(res): - self.failUnlessEqual(res.keys(), ["foo"]) - file1_node = res["foo"] - self.failUnless(isinstance(file1_node, vdrive.FileNode)) - self.failUnlessEqual(file1_node.uri, file1) - d.addCallback(_listed2) - - d.addCallback(lambda res: rootnode.get("foo")) - def _got_foo(res): - self.failUnless(isinstance(res, vdrive.FileNode)) - self.failUnlessEqual(res.uri, file1) - d.addCallback(_got_foo) - - d.addCallback(lambda res: rootnode.get("missing")) - # this should raise an exception - d.addBoth(self.shouldFail, IndexError, "get('missing')", - "unable to find child named 'missing'") - - d.addCallback(lambda res: rootnode.create_empty_directory("bar")) - # root/ - # root/foo =file1 - # root/bar/ - - d.addCallback(lambda res: rootnode.list()) - d.addCallback(self.failUnlessKeysMatch, ["foo", "bar"]) - def _listed3(res): - self.failIfEqual(res["foo"], res["bar"]) - self.failIfEqual(res["bar"], res["foo"]) - self.failIfEqual(res["foo"], "not") - self.failIfEqual(res["bar"], self.rootnode) - self.failUnlessEqual(res["foo"], res["foo"]) - # make sure the objects can be used as dict keys - testdict = {res["foo"]: 1, res["bar"]: 2} - bar_node = res["bar"] - self.failUnless(isinstance(bar_node, vdrive.MutableDirectoryNode)) - self.bar_node = bar_node - bar_ro_uri = bar_node.get_immutable_uri() - return rootnode.set_uri("bar-ro", bar_ro_uri) - d.addCallback(_listed3) - # root/ - # root/foo =file1 - # root/bar/ - # root/bar-ro/ (read-only) - - d.addCallback(lambda res: rootnode.list()) - d.addCallback(self.failUnlessKeysMatch, ["foo", "bar", "bar-ro"]) - def _listed4(res): - self.failIf(res["bar-ro"].is_mutable()) - self.bar_node_readonly = res["bar-ro"] - - # add another file to bar/ - bar = res["bar"] - return bar.set_node("file2", file2_node) - d.addCallback(_listed4) - d.addCallback(self.failUnlessIdentical, file2_node) - # and a directory - d.addCallback(lambda res: self.bar_node.create_empty_directory("baz")) - # root/ - # root/foo =file1 - # root/bar/ - # root/bar/file2 =file2 - # root/bar/baz/ - # root/bar-ro/ (read-only) - # root/bar-ro/file2 =file2 - # root/bar-ro/baz/ - - d.addCallback(lambda res: self.bar_node.list()) - d.addCallback(self.failUnlessKeysMatch, ["file2", "baz"]) - d.addCallback(lambda res: - self.failUnless(res["baz"].is_mutable())) - - d.addCallback(lambda res: self.bar_node_readonly.list()) - d.addCallback(self.failUnlessKeysMatch, ["file2", "baz"]) - d.addCallback(lambda res: - self.failIf(res["baz"].is_mutable())) - - # try to add a file to bar-ro, should get exception - d.addCallback(lambda res: - self.bar_node_readonly.set_uri("file3", file2)) - d.addBoth(self.shouldFail, vdrive.NotMutableError, - "bar-ro.set('file3')") - - # try to delete a file from bar-ro, should get exception - d.addCallback(lambda res: self.bar_node_readonly.delete("file2")) - d.addBoth(self.shouldFail, vdrive.NotMutableError, - "bar-ro.delete('file2')") - - # try to mkdir in bar-ro, should get exception - d.addCallback(lambda res: - self.bar_node_readonly.create_empty_directory("boffo")) - d.addBoth(self.shouldFail, vdrive.NotMutableError, - "bar-ro.mkdir('boffo')") - - d.addCallback(lambda res: rootnode.delete("foo")) - # root/ - # root/bar/ - # root/bar/file2 =file2 - # root/bar/baz/ - # root/bar-ro/ (read-only) - # root/bar-ro/file2 =file2 - # root/bar-ro/baz/ - - d.addCallback(lambda res: rootnode.list()) - d.addCallback(self.failUnlessKeysMatch, ["bar", "bar-ro"]) - - d.addCallback(lambda res: - self.bar_node.move_child_to("file2", - self.rootnode, "file4")) - # root/ - # root/file4 = file4 - # root/bar/ - # root/bar/baz/ - # root/bar-ro/ (read-only) - # root/bar-ro/baz/ - - d.addCallback(lambda res: rootnode.list()) - d.addCallback(self.failUnlessKeysMatch, ["bar", "bar-ro", "file4"]) - d.addCallback(lambda res:self.bar_node.list()) - d.addCallback(self.failUnlessKeysMatch, ["baz"]) - d.addCallback(lambda res:self.bar_node_readonly.list()) - d.addCallback(self.failUnlessKeysMatch, ["baz"]) - - - d.addCallback(lambda res: - rootnode.move_child_to("file4", - self.bar_node_readonly, "boffo")) - d.addBoth(self.shouldFail, vdrive.NotMutableError, - "mv root/file4 root/bar-ro/boffo") - - d.addCallback(lambda res: rootnode.list()) - d.addCallback(self.failUnlessKeysMatch, ["bar", "bar-ro", "file4"]) - d.addCallback(lambda res:self.bar_node.list()) - d.addCallback(self.failUnlessKeysMatch, ["baz"]) - d.addCallback(lambda res:self.bar_node_readonly.list()) - d.addCallback(self.failUnlessKeysMatch, ["baz"]) - - - d.addCallback(lambda res: - rootnode.move_child_to("file4", self.bar_node)) - - d.addCallback(lambda res: rootnode.list()) - d.addCallback(self.failUnlessKeysMatch, ["bar", "bar-ro"]) - d.addCallback(lambda res:self.bar_node.list()) - d.addCallback(self.failUnlessKeysMatch, ["baz", "file4"]) - d.addCallback(lambda res:self.bar_node_readonly.list()) - d.addCallback(self.failUnlessKeysMatch, ["baz", "file4"]) - - d.addCallback(self._test_one_3) - return d - - def _test_one_3(self, res): - # now test some of the diag tools with the data we've created - out,err = StringIO(), StringIO() - rc = runner.runner(["dump-root-dirnode", "vdrive/test_one"], - stdout=out, stderr=err) - output = out.getvalue() - self.failUnless(output.startswith("URI:DIR:fakeFURL:")) - self.failUnlessEqual(rc, 0) - - out,err = StringIO(), StringIO() - rc = runner.runner(["dump-dirnode", - "--basedir", "vdrive/test_one", - "--verbose", - self.bar_node.get_uri()], - stdout=out, stderr=err) - output = out.getvalue() - #print output - self.failUnlessEqual(rc, 0) - self.failUnless("dirnode uri: URI:DIR:myFURL" in output) - self.failUnless("write_enabler" in output) - self.failIf("write_enabler: None" in output) - self.failUnless("key baz\n" in output) - self.failUnless(" write: URI:DIR:myFURL:" in output) - self.failUnless(" read: URI:DIR-RO:myFURL:" in output) - self.failUnless("key file4\n" in output) - self.failUnless("H_key " in output) - - out,err = StringIO(), StringIO() - rc = runner.runner(["dump-dirnode", - "--basedir", "vdrive/test_one", - # non-verbose - "--uri", self.bar_node.get_uri()], - stdout=out, stderr=err) - output = out.getvalue() - #print output - self.failUnlessEqual(rc, 0) - self.failUnless("dirnode uri: URI:DIR:myFURL" in output) - self.failUnless("write_enabler" in output) - self.failIf("write_enabler: None" in output) - self.failUnless("key baz\n" in output) - self.failUnless(" write: URI:DIR:myFURL:" in output) - self.failUnless(" read: URI:DIR-RO:myFURL:" in output) - self.failUnless("key file4\n" in output) - self.failIf("H_key " in output) - - out,err = StringIO(), StringIO() - rc = runner.runner(["dump-dirnode", - "--basedir", "vdrive/test_one", - "--verbose", - self.bar_node_readonly.get_uri()], - stdout=out, stderr=err) - output = out.getvalue() - #print output - self.failUnlessEqual(rc, 0) - self.failUnless("dirnode uri: URI:DIR-RO:myFURL" in output) - self.failUnless("write_enabler: None" in output) - self.failUnless("key baz\n" in output) - self.failIf(" write: URI:DIR:myFURL:" in output) - self.failUnless(" read: URI:DIR-RO:myFURL:" in output) - self.failUnless("key file4\n" in output) - - def shouldFail(self, res, expected_failure, which, substring=None): - if isinstance(res, failure.Failure): - res.trap(expected_failure) - if substring: - self.failUnless(substring in str(res), - "substring '%s' not in '%s'" - % (substring, str(res))) - else: - self.fail("%s was supposed to raise %s, not get '%s'" % - (which, expected_failure, res)) - - def failUnlessKeysMatch(self, res, expected_keys): - self.failUnlessEqual(sorted(res.keys()), - sorted(expected_keys)) - return res - -def flip_bit(data, offset): - if offset < 0: - offset = len(data) + offset - return data[:offset] + chr(ord(data[offset]) ^ 0x01) + data[offset+1:] - -class Encryption(unittest.TestCase): - def test_loopback(self): - key = "k" * 16 - data = "This is some plaintext data." - crypttext = vdrive.encrypt(key, data) - plaintext = vdrive.decrypt(key, crypttext) - self.failUnlessEqual(data, plaintext) - - def test_hmac(self): - key = "j" * 16 - data = "This is some more plaintext data." - crypttext = vdrive.encrypt(key, data) - # flip a bit in the IV - self.failUnlessRaises(vdrive.IntegrityCheckError, - vdrive.decrypt, - key, flip_bit(crypttext, 0)) - # flip a bit in the crypttext - self.failUnlessRaises(vdrive.IntegrityCheckError, - vdrive.decrypt, - key, flip_bit(crypttext, 16)) - # flip a bit in the HMAC - self.failUnlessRaises(vdrive.IntegrityCheckError, - vdrive.decrypt, - key, flip_bit(crypttext, -1)) - plaintext = vdrive.decrypt(key, crypttext) - self.failUnlessEqual(data, plaintext) - diff --git a/src/allmydata/vdrive.py b/src/allmydata/vdrive.py deleted file mode 100644 index c1876598..00000000 --- a/src/allmydata/vdrive.py +++ /dev/null @@ -1,275 +0,0 @@ - -"""This is the client-side facility to manipulate virtual drives.""" - -import os.path -from zope.interface import implements -from twisted.internet import defer -from allmydata import uri -from allmydata.Crypto.Cipher import AES -from allmydata.util import hashutil, idlib -from allmydata.interfaces import IDirectoryNode, IFileNode - -class NotMutableError(Exception): - pass - - -def create_directory_node(client, diruri): - assert uri.is_dirnode_uri(diruri) - if uri.is_mutable_dirnode_uri(diruri): - dirnode_class = MutableDirectoryNode - else: - dirnode_class = ImmutableDirectoryNode - (furl, key) = uri.unpack_dirnode_uri(diruri) - d = client.tub.getReference(furl) - def _got(rref): - dirnode = dirnode_class(diruri, client, rref, key) - return dirnode - d.addCallback(_got) - return d - -IV_LENGTH = 14 -def encrypt(key, data): - IV = os.urandom(IV_LENGTH) - counterstart = IV + "\x00"*(16-IV_LENGTH) - assert len(counterstart) == 16, len(counterstart) - cryptor = AES.new(key=key, mode=AES.MODE_CTR, counterstart=counterstart) - crypttext = cryptor.encrypt(data) - mac = hashutil.hmac(key, IV + crypttext) - assert len(mac) == 32 - return IV + crypttext + mac - -class IntegrityCheckError(Exception): - pass - -def decrypt(key, data): - assert len(data) >= (32+IV_LENGTH), len(data) - IV, crypttext, mac = data[:IV_LENGTH], data[IV_LENGTH:-32], data[-32:] - if mac != hashutil.hmac(key, IV+crypttext): - raise IntegrityCheckError("HMAC does not match, crypttext is corrupted") - counterstart = IV + "\x00"*(16-IV_LENGTH) - assert len(counterstart) == 16, len(counterstart) - cryptor = AES.new(key=key, mode=AES.MODE_CTR, counterstart=counterstart) - plaintext = cryptor.decrypt(crypttext) - return plaintext - - -class ImmutableDirectoryNode: - implements(IDirectoryNode) - - def __init__(self, myuri, client, rref, readkey): - self._uri = myuri - self._client = client - self._tub = client.tub - self._rref = rref - self._readkey = readkey - self._writekey = None - self._write_enabler = None - self._index = hashutil.dir_index_hash(self._readkey) - self._mutable = False - - def dump(self): - return ["URI: %s" % self._uri, - "rk: %s" % idlib.b2a(self._readkey), - "index: %s" % idlib.b2a(self._index), - ] - - def is_mutable(self): - return self._mutable - - def get_uri(self): - return self._uri - - def get_immutable_uri(self): - # return the dirnode URI for a read-only form of this directory - if self._mutable: - return uri.make_immutable_dirnode_uri(self._uri) - else: - return self._uri - - def __hash__(self): - return hash((self.__class__, self._uri)) - def __cmp__(self, them): - if cmp(type(self), type(them)): - return cmp(type(self), type(them)) - if cmp(self.__class__, them.__class__): - return cmp(self.__class__, them.__class__) - return cmp(self._uri, them._uri) - - def _encrypt(self, key, data): - return encrypt(key, data) - - def _decrypt(self, key, data): - return decrypt(key, data) - - def _decrypt_child(self, E_write, E_read): - if E_write and self._writekey: - # we prefer read-write children when we can get them - return self._decrypt(self._writekey, E_write) - else: - return self._decrypt(self._readkey, E_read) - - def list(self): - d = self._rref.callRemote("list", self._index) - entries = {} - def _got(res): - dl = [] - for (E_name, E_write, E_read) in res: - name = self._decrypt(self._readkey, E_name) - child_uri = self._decrypt_child(E_write, E_read) - d2 = self._create_node(child_uri) - def _created(node, name): - entries[name] = node - d2.addCallback(_created, name) - dl.append(d2) - return defer.DeferredList(dl) - d.addCallback(_got) - d.addCallback(lambda res: entries) - return d - - def _hash_name(self, name): - return hashutil.dir_name_hash(self._readkey, name) - - def get(self, name): - H_name = self._hash_name(name) - d = self._rref.callRemote("get", self._index, H_name) - def _check_index_error(f): - f.trap(IndexError) - raise IndexError("get(index=%s): unable to find child named '%s'" - % (idlib.b2a(self._index), name)) - d.addErrback(_check_index_error) - d.addCallback(lambda (E_write, E_read): - self._decrypt_child(E_write, E_read)) - d.addCallback(self._create_node) - return d - - def _set(self, name, write_child, read_child): - if not self._mutable: - return defer.fail(NotMutableError()) - H_name = self._hash_name(name) - E_name = self._encrypt(self._readkey, name) - E_write = "" - if self._writekey and write_child: - E_write = self._encrypt(self._writekey, write_child) - E_read = self._encrypt(self._readkey, read_child) - d = self._rref.callRemote("set", self._index, self._write_enabler, - H_name, E_name, E_write, E_read) - return d - - def set_uri(self, name, child_uri): - write, read = self._split_uri(child_uri) - return self._set(name, write, read) - - def set_node(self, name, child): - d = self.set_uri(name, child.get_uri()) - d.addCallback(lambda res: child) - return d - - def delete(self, name): - if not self._mutable: - return defer.fail(NotMutableError()) - H_name = self._hash_name(name) - d = self._rref.callRemote("delete", self._index, self._write_enabler, - H_name) - return d - - def _create_node(self, child_uri): - if uri.is_dirnode_uri(child_uri): - return create_directory_node(self._client, child_uri) - else: - return defer.succeed(FileNode(child_uri, self._client)) - - def _split_uri(self, child_uri): - if uri.is_dirnode_uri(child_uri): - if uri.is_mutable_dirnode_uri(child_uri): - write = child_uri - read = uri.make_immutable_dirnode_uri(child_uri) - else: - write = None - read = child_uri - return (write, read) - return (None, child_uri) # file - - def create_empty_directory(self, name): - if not self._mutable: - return defer.fail(NotMutableError()) - child_writekey = hashutil.random_key() - my_furl, parent_writekey = uri.unpack_dirnode_uri(self._uri) - child_uri = uri.pack_dirnode_uri(my_furl, child_writekey) - child = MutableDirectoryNode(child_uri, self._client, self._rref, - child_writekey) - d = self._rref.callRemote("create_directory", - child._index, child._write_enabler) - d.addCallback(lambda index: self.set_node(name, child)) - return d - - def add_file(self, name, uploadable): - if not self._mutable: - return defer.fail(NotMutableError()) - uploader = self._client.getServiceNamed("uploader") - d = uploader.upload(uploadable) - d.addCallback(lambda uri: self.set_node(name, - FileNode(uri, self._client))) - return d - - def move_child_to(self, current_child_name, - new_parent, new_child_name=None): - if not (self._mutable and new_parent.is_mutable()): - return defer.fail(NotMutableError()) - if new_child_name is None: - new_child_name = current_child_name - d = self.get(current_child_name) - d.addCallback(lambda child: new_parent.set_node(new_child_name, child)) - d.addCallback(lambda child: self.delete(current_child_name)) - return d - -class MutableDirectoryNode(ImmutableDirectoryNode): - implements(IDirectoryNode) - - def __init__(self, myuri, client, rref, writekey): - readkey = hashutil.dir_read_key_hash(writekey) - ImmutableDirectoryNode.__init__(self, myuri, client, rref, readkey) - self._writekey = writekey - self._write_enabler = hashutil.dir_write_enabler_hash(writekey) - self._mutable = True - -def create_directory(client, furl): - write_key = hashutil.random_key() - (wk, we, rk, index) = \ - hashutil.generate_dirnode_keys_from_writekey(write_key) - myuri = uri.pack_dirnode_uri(furl, wk) - d = client.tub.getReference(furl) - def _got_vdrive_server(vdrive_server): - node = MutableDirectoryNode(myuri, client, vdrive_server, wk) - d2 = vdrive_server.callRemote("create_directory", index, we) - d2.addCallback(lambda res: node) - return d2 - d.addCallback(_got_vdrive_server) - return d - -class FileNode: - implements(IFileNode) - - def __init__(self, uri, client): - self.uri = uri - self._client = client - - def get_uri(self): - return self.uri - - def __hash__(self): - return hash((self.__class__, self.uri)) - def __cmp__(self, them): - if cmp(type(self), type(them)): - return cmp(type(self), type(them)) - if cmp(self.__class__, them.__class__): - return cmp(self.__class__, them.__class__) - return cmp(self.uri, them.uri) - - def download(self, target): - downloader = self._client.getServiceNamed("downloader") - return downloader.download(self.uri, target) - - def download_to_data(self): - downloader = self._client.getServiceNamed("downloader") - return downloader.download_to_data(self.uri) - diff --git a/src/allmydata/webish.py b/src/allmydata/webish.py index b27b7adb..a49fcc18 100644 --- a/src/allmydata/webish.py +++ b/src/allmydata/webish.py @@ -7,7 +7,7 @@ from nevow.static import File as nevow_File # TODO: merge with static.File? from allmydata.util import idlib from allmydata.uri import unpack_uri from allmydata.interfaces import IDownloadTarget, IDirectoryNode, IFileNode -from allmydata.vdrive import FileNode +from allmydata.dirnode import FileNode from allmydata import upload from zope.interface import implements, Interface import urllib