From ec77a227be3fb5ecb8343d4a6f88057dd038c70a Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Thu, 14 Jun 2007 20:14:34 -0700 Subject: [PATCH] revamp vdrive: nodes with furls. tests still fail. --- src/allmydata/client.py | 32 +++++-- src/allmydata/filetable.py | 115 ++++++++++++++----------- src/allmydata/interfaces.py | 72 +++++++++++++--- src/allmydata/introducer_and_vdrive.py | 7 +- src/allmydata/test/test_filetable.py | 11 +-- src/allmydata/test/test_system.py | 33 +++++-- src/allmydata/test/test_vdrive.py | 1 + src/allmydata/vdrive.py | 24 ++++++ src/allmydata/webish.py | 42 +++++---- 9 files changed, 232 insertions(+), 105 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 09acfefc..fa2cc684 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -13,7 +13,7 @@ from allmydata.Crypto.Util.number import bytes_to_long from allmydata.storageserver import StorageServer from allmydata.upload import Uploader from allmydata.download import Downloader -from allmydata.vdrive import VDrive +#from allmydata.vdrive import VDrive from allmydata.webish import WebishServer from allmydata.control import ControlServer from allmydata.introducer import IntroducerClient @@ -60,7 +60,7 @@ class Client(node.Node, Referenceable): f = open(GLOBAL_VDRIVE_FURL_FILE, "r") self.global_vdrive_furl = f.read().strip() f.close() - self.add_service(VDrive()) + #self.add_service(VDrive()) hotline_file = os.path.join(self.basedir, self.SUICIDE_PREVENTION_HOTLINE_FILE) @@ -110,23 +110,37 @@ class Client(node.Node, Referenceable): f.close() os.chmod("control.furl", 0600) - def _got_vdrive(self, vdrive_root): + def _got_vdrive(self, vdrive_server): + # vdrive_server implements RIVirtualDriveServer + self.log("connected to vdrive server") + d = vdrive_server.callRemote("get_public_root") + d.addCallback(self._got_vdrive_root, vdrive_server) + + def _got_vdrive_root(self, vdrive_root, vdrive_server): # vdrive_root implements RIMutableDirectoryNode - self.log("connected to vdrive") + self.log("got vdrive root") self._connected_to_vdrive = True - self.getServiceNamed("vdrive").set_root(vdrive_root) - if "webish" in self.namedServices: - self.getServiceNamed("webish").set_root_dirnode(vdrive_root) + self._vdrive_server = vdrive_server + self._vdrive_root = vdrive_root def _disconnected(): self._connected_to_vdrive = False vdrive_root.notifyOnDisconnect(_disconnected) + #vdrive = self.getServiceNamed("vdrive") + #vdrive.set_server(vdrive_server) + #vdrive.set_root(vdrive_root) + + if "webish" in self.namedServices: + webish = self.getServiceNamed("webish") + webish.set_vdrive(self.tub, vdrive_server, vdrive_root) + def remote_get_versions(self): return str(allmydata.__version__), str(self.OLDEST_SUPPORTED_VERSION) def remote_get_service(self, name): - # TODO: 'vdrive' should not be public in the medium term - return self.getServiceNamed(name) + if name in ("storageserver",): + return self.getServiceNamed(name) + raise RuntimeError("I am unwilling to give you service %s" % name) def get_remote_service(self, nodeid, servicename): if nodeid not in self.introducer_client.connections: diff --git a/src/allmydata/filetable.py b/src/allmydata/filetable.py index 601cc4b7..294f6a0d 100644 --- a/src/allmydata/filetable.py +++ b/src/allmydata/filetable.py @@ -2,9 +2,8 @@ import os from zope.interface import implements from foolscap import Referenceable -from allmydata.interfaces import RIMutableDirectoryNode +from allmydata.interfaces import RIVirtualDriveServer, RIMutableDirectoryNode, FileNode, DirectoryNode from allmydata.util import bencode, idlib -from allmydata.util.assertutil import _assert from twisted.application import service from twisted.python import log @@ -23,7 +22,7 @@ class MutableDirectoryNode(Referenceable): I am associated with a file on disk, using a randomly-generated (and hopefully unique) name. This file contains a serialized dictionary which maps child names to 'child specifications'. These specifications are - tuples, either of ('file', URI), or ('subdir', nodename). + tuples, either of ('file', URI), or ('subdir', FURL). """ implements(RIMutableDirectoryNode) @@ -41,38 +40,29 @@ class MutableDirectoryNode(Referenceable): self._name = self.create_filename() self._write_to_file({}) # start out empty - def make_subnode(self, name=None): - return self.__class__(self._basedir, name) - def _read_from_file(self): - f = open(os.path.join(self._basedir, self._name), "rb") - data = f.read() - f.close() - children_specifications = bencode.bdecode(data) - children = {} - for k,v in children_specifications.items(): - nodetype = v[0] - if nodetype == "file": - (uri, ) = v[1:] - child = uri - elif nodetype == "subdir": - (nodename, ) = v[1:] - child = self.make_subnode(nodename) + data = open(os.path.join(self._basedir, self._name), "rb").read() + children = bencode.bdecode(data) + child_nodes = {} + for k,v in children.iteritems(): + if v[0] == "file": + child_nodes[k] = FileNode(v[1]) + elif v[0] == "subdir": + child_nodes[k] = DirectoryNode(v[1]) else: - _assert("Unknown nodetype in node specification %s" % (v,)) - children[k] = child - return children + raise RuntimeError("unknown child spec '%s'" % (v[0],)) + return child_nodes def _write_to_file(self, children): - children_specifications = {} - for k,v in children.items(): - if isinstance(v, MutableDirectoryNode): - child = ("subdir", v._name) + child_nodes = {} + for k,v in children.iteritems(): + if isinstance(v, FileNode): + child_nodes[k] = ("file", v.uri) + elif isinstance(v, DirectoryNode): + child_nodes[k] = ("subdir", v.furl) else: - assert isinstance(v, str) - child = ("file", v) # URI - children_specifications[k] = child - data = bencode.bencode(children_specifications) + raise RuntimeError("unknown child node '%s'" % (v,)) + data = bencode.bencode(child_nodes) f = open(os.path.join(self._basedir, self._name), "wb") f.write(data) f.close() @@ -103,46 +93,71 @@ class MutableDirectoryNode(Referenceable): return children[name] remote_get = get - def add_directory(self, name): + def add(self, name, child): self.validate_name(name) children = self._read_from_file() if name in children: - raise BadDirectoryError("the directory already existed") - children[name] = child = self.make_subnode() + raise BadNameError("the child already existed") + children[name] = child self._write_to_file(children) return child - remote_add_directory = add_directory - - def add_file(self, name, uri): - self.validate_name(name) - children = self._read_from_file() - children[name] = uri - self._write_to_file(children) - remote_add_file = add_file + remote_add = add def remove(self, name): self.validate_name(name) children = self._read_from_file() if name not in children: raise BadFileError("cannot remove non-existent child") - dead_child = children[name] + child = children[name] del children[name] self._write_to_file(children) - #return dead_child + return child remote_remove = remove -class GlobalVirtualDrive(service.MultiService): +class NoPublicRootError(Exception): + pass + +class VirtualDriveServer(service.MultiService, Referenceable): + implements(RIVirtualDriveServer) name = "filetable" VDRIVEDIR = "vdrive" - def __init__(self, basedir="."): + def __init__(self, basedir=".", offer_public_root=True): service.MultiService.__init__(self) vdrive_dir = os.path.join(basedir, self.VDRIVEDIR) if not os.path.exists(vdrive_dir): os.mkdir(vdrive_dir) - self._root = MutableDirectoryNode(vdrive_dir, "root") - - def get_root(self): - return self._root - + self._vdrive_dir = vdrive_dir + self._root = None + if offer_public_root: + self._root = MutableDirectoryNode(vdrive_dir, "root") + + def startService(self): + service.MultiService.startService(self) + # _register_all_dirnodes exists to avoid hacking our Tub to + # automatically translate inbound your-reference names + # (Tub.getReferenceForName) into MutableDirectoryNode instances by + # looking in our basedir for them. Without that hack, we have to + # register all nodes at startup to make sure they'll be available to + # all callers. In addition, we must register any new ones that we + # create later on. + tub = self.parent.tub + self._register_all_dirnodes(tub) + + def _register_all_dirnodes(self, tub): + for name in os.listdir(self._vdrive_dir): + node = MutableDirectoryNode(self._vdrive_dir, name) + ignored_furl = tub.registerReference(node, name) + + def get_public_root(self): + if self._root: + return self._root + raise NoPublicRootError + remote_get_public_root = get_public_root + + def create_directory(self): + node = MutableDirectoryNode(self._vdrive_dir) + furl = self.parent.tub.registerReference(node, node._name) + return furl + remote_create_directory = create_directory diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index e0686a0d..bcd0f581 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -2,7 +2,7 @@ from zope.interface import Interface from foolscap.schema import StringConstraint, ListOf, TupleOf, SetOf, DictOf, \ ChoiceOf -from foolscap import RemoteInterface, Referenceable +from foolscap import RemoteInterface, Referenceable, Copyable, RemoteCopy HASH_SIZE=32 @@ -122,26 +122,78 @@ class RIStorageServer(RemoteInterface): # hm, we need a solution for forward references in schemas from foolscap.schema import Any RIMutableDirectoryNode_ = Any() # TODO: how can we avoid this? + +class DirectoryNode(Copyable, RemoteCopy): + """I have either a .furl attribute or a .get(tub) method.""" + typeToCopy = "allmydata.com/tahoe/interfaces/DirectoryNode/v1" + copytype = typeToCopy + def __init__(self, furl=None): + # RemoteCopy subclasses are always called without arguments + self.furl = furl + def getStateToCopy(self): + return {"furl": self.furl } + def setCopyableState(self, state): + self.furl = state['furl'] + def __hash__(self): + return hash((self.__class__, self.furl)) + def __cmp__(self, them): + if cmp(type(self), type(them)): + return cmp(type(self), type(them)) + if cmp(self.__class__, them.__class__): + return cmp(self.__class__, them.__class__) + return cmp(self.furl, them.furl) + +class FileNode(Copyable, RemoteCopy): + """I have a .uri attribute.""" + typeToCopy = "allmydata.com/tahoe/interfaces/FileNode/v1" + copytype = typeToCopy + def __init__(self, uri=None): + # RemoteCopy subclasses are always called without arguments + self.uri = uri + def getStateToCopy(self): + return {"uri": self.uri } + def setCopyableState(self, state): + self.uri = state['uri'] + def __hash__(self): + return hash((self.__class__, self.uri)) + def __cmp__(self, them): + if cmp(type(self), type(them)): + return cmp(type(self), type(them)) + if cmp(self.__class__, them.__class__): + return cmp(self.__class__, them.__class__) + return cmp(self.uri, them.uri) + +FileNode_ = Any() # TODO: foolscap needs constraints on copyables +DirectoryNode_ = Any() # TODO: same +AnyNode_ = ChoiceOf(FileNode_, DirectoryNode_) + class RIMutableDirectoryNode(RemoteInterface): def list(): return ListOf( TupleOf(str, # name, relative to directory - ChoiceOf(RIMutableDirectoryNode_, URI)), + AnyNode_, + ), maxLength=100, ) def get(name=str): - return ChoiceOf(RIMutableDirectoryNode_, URI) + return AnyNode_ - def add_directory(name=str): - return RIMutableDirectoryNode_ - - def add_file(name=str, uri=URI): - return None + def add(name=str, what=AnyNode_): + return AnyNode_ def remove(name=str): - return None + return AnyNode_ + + +class RIVirtualDriveServer(RemoteInterface): + def get_public_root(): + """If this vdrive server does not offer a public root, this will + raise an exception.""" + return DirectoryNode_ + + def create_directory(): + return DirectoryNode_ - # need more to move directories class ICodecEncoder(Interface): diff --git a/src/allmydata/introducer_and_vdrive.py b/src/allmydata/introducer_and_vdrive.py index 506331a8..5209facd 100644 --- a/src/allmydata/introducer_and_vdrive.py +++ b/src/allmydata/introducer_and_vdrive.py @@ -1,7 +1,7 @@ import os.path from allmydata import node -from allmydata.filetable import GlobalVirtualDrive +from allmydata.filetable import VirtualDriveServer from allmydata.introducer import Introducer @@ -21,9 +21,8 @@ class IntroducerAndVdrive(node.Node): f.write(self.urls["introducer"] + "\n") f.close() - gvd = self.add_service(GlobalVirtualDrive(self.basedir)) - self.urls["vdrive"] = self.tub.registerReference(gvd.get_root(), - "vdrive") + vds = self.add_service(VirtualDriveServer(self.basedir)) + self.urls["vdrive"] = self.tub.registerReference(vds, "vdrive") self.log(" vdrive is at %s" % self.urls["vdrive"]) f = open(os.path.join(self.basedir, "vdrive.furl"), "w") f.write(self.urls["vdrive"] + "\n") diff --git a/src/allmydata/test/test_filetable.py b/src/allmydata/test/test_filetable.py index 56a59040..ab19baaf 100644 --- a/src/allmydata/test/test_filetable.py +++ b/src/allmydata/test/test_filetable.py @@ -3,6 +3,7 @@ import os from twisted.trial import unittest from allmydata.filetable import (MutableDirectoryNode, BadDirectoryError, BadFileError, BadNameError) +from allmydata.interfaces import FileNode, DirectoryNode class FileTable(unittest.TestCase): @@ -10,10 +11,10 @@ class FileTable(unittest.TestCase): os.mkdir("filetable") root = MutableDirectoryNode(os.path.abspath("filetable"), "root") self.failUnlessEqual(root.list(), []) - root.add_file("one", "vid-one") - root.add_file("two", "vid-two") - self.failUnlessEqual(root.list(), [("one", "vid-one"), - ("two", "vid-two")]) + root.add("one", FileNode("vid-one")) + root.add("two", FileNode("vid-two")) + self.failUnlessEqual(root.list(), [("one", FileNode("vid-one")), + ("two", FileNode("vid-two"))]) root.remove("two") self.failUnlessEqual(root.list(), [("one", "vid-one")]) self.failUnlessRaises(BadFileError, root.remove, "two") @@ -26,7 +27,7 @@ class FileTable(unittest.TestCase): self.failUnlessRaises(BadNameError, root.get, ".") # dumb # now play with directories - subdir1 = root.add_directory("subdir1") + subdir1 = root.add("subdir1", DirectoryNode("subdir1.furl")) self.failUnless(isinstance(subdir1, MutableDirectoryNode)) subdir1a = root.get("subdir1") self.failUnless(isinstance(subdir1a, MutableDirectoryNode)) diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index 95355410..47e85f3a 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -3,7 +3,7 @@ import os from twisted.trial import unittest from twisted.internet import defer, reactor from twisted.application import service -from allmydata import client, uri, download +from allmydata import client, uri, download, vdrive from allmydata.introducer_and_vdrive import IntroducerAndVdrive from allmydata.util import idlib, fileutil, testutil from foolscap.eventual import flushEventualQueue @@ -234,17 +234,32 @@ class SystemTest(testutil.SignalMixin, unittest.TestCase): d = self.set_up_nodes() def _do_publish(res): log.msg("PUBLISHING") - v0 = self.clients[0].getServiceNamed("vdrive") - d1 = v0.make_directory("/", "subdir1") - d1.addCallback(lambda subdir1: - v0.put_file_by_data(subdir1, "mydata567", DATA)) + c0 = self.clients[0] + d1 = vdrive.mkdir(c0._vdrive_server, c0._vdrive_root, + "subdir1") + d1.addCallback(lambda subdir1_node: + c0.tub.getReference(subdir1_node.furl)) + def _put_file(subdir1): + uploader = c0.getServiceNamed("uploader") + d2 = uploader.upload_data(DATA) + def _stash_uri(uri): + self.uri = uri + return uri + d2.addCallback(_stash_uri) + d2.addCallback(lambda uri: vdrive.add_file(subdir1, "mydata567", uri)) + return d2 + d1.addCallback(_put_file) return d1 d.addCallback(_do_publish) - def _publish_done(uri): - self.uri = uri + def _publish_done(filenode): log.msg("publish finished") - v1 = self.clients[1].getServiceNamed("vdrive") - d1 = v1.get_file_to_data("/subdir1/mydata567") + + c1 = self.clients[1] + d1 = c1._vdrive_root.callRemote("get", "subdir1") + d1.addCallback(lambda subdir1_dirnode: + c1.tub.getReference(subdir1_dirnode.furl)) + d1.addCallback(lambda subdir1: subdir1.callRemote("get", "mydata567")) + d1.addCallback(lambda filenode: c1.getServiceNamed("downloader").download_to_data(filenode.uri)) return d1 d.addCallback(_publish_done) def _get_done(data): diff --git a/src/allmydata/test/test_vdrive.py b/src/allmydata/test/test_vdrive.py index 3610a66b..536f1c0c 100644 --- a/src/allmydata/test/test_vdrive.py +++ b/src/allmydata/test/test_vdrive.py @@ -66,3 +66,4 @@ class Traverse(unittest.TestCase): self.failUnlessEqual(sorted(files), ["2.a", "2.b", "d2.1"])) return d +del Traverse diff --git a/src/allmydata/vdrive.py b/src/allmydata/vdrive.py index 5b9146d0..9f98f785 100644 --- a/src/allmydata/vdrive.py +++ b/src/allmydata/vdrive.py @@ -5,10 +5,13 @@ from twisted.application import service from twisted.internet import defer from twisted.python import log from allmydata import upload, download +from allmydata.interfaces import FileNode, DirectoryNode class VDrive(service.MultiService): name = "vdrive" + def set_server(self, vdrive_server): + self.gvd_server = vdrive_server def set_root(self, root): self.gvd_root = root @@ -177,3 +180,24 @@ class VDrive(service.MultiService): def get_file_to_filehandle(self, from_where, filehandle): return self.get_file(from_where, download.FileHandle(filehandle)) + +# utility stuff +def add_file(parent_node, child_name, uri): + child_node = FileNode(uri) + d = parent_node.callRemote("add", child_name, child_node) + return d + +def mkdir(vdrive_server, parent_node, child_name): + d = vdrive_server.callRemote("create_directory") + d.addCallback(lambda newdir_furl: + parent_node.callRemote("add", child_name, DirectoryNode(newdir_furl))) + return d + +def add_shared_directory_furl(parent_node, child_name, furl): + child_node = DirectoryNode(furl) + d = parent_node.callRemote("add", child_name, child_node) + return d + +def create_anonymous_directory(vdrive_server): + d = vdrive_server.callRemote("create_directory") + return d diff --git a/src/allmydata/webish.py b/src/allmydata/webish.py index 043c645e..cedf32a7 100644 --- a/src/allmydata/webish.py +++ b/src/allmydata/webish.py @@ -5,8 +5,8 @@ from twisted.python import util, log from nevow import inevow, rend, loaders, appserver, url, tags as T from allmydata.util import idlib from allmydata.uri import unpack_uri -from allmydata.interfaces import IDownloadTarget#, IDownloader -from allmydata import upload +from allmydata.interfaces import IDownloadTarget, FileNode, DirectoryNode +from allmydata import upload, vdrive from zope.interface import implements, Interface import urllib from formless import annotate, webform @@ -21,8 +21,6 @@ def get_downloader_service(ctx): return IClient(ctx).getServiceNamed("downloader") def get_uploader_service(ctx): return IClient(ctx).getServiceNamed("uploader") -def get_vdrive_service(ctx): - return IClient(ctx).getServiceNamed("vdrive") class Welcome(rend.Page): addSlash = True @@ -99,7 +97,9 @@ class Directory(rend.Page): addSlash = True docFactory = getxmlfile("directory.xhtml") - def __init__(self, dirnode, dirname): + def __init__(self, tub, vdrive_server, dirnode, dirname): + self._tub = tub + self._vdrive_server = vdrive_server self._dirnode = dirnode self._dirname = dirname @@ -112,10 +112,16 @@ class Directory(rend.Page): dirname = self._dirname + "/" + name d = self._dirnode.callRemote("get", name) def _got_child(res): - if isinstance(res, str): + if isinstance(res, FileNode): dl = get_downloader_service(ctx) return Downloader(dl, name, res) - return Directory(res, dirname) + elif isinstance(res, DirectoryNode): + d2 = self._tub.getReference(res.furl) + d2.addCallback(lambda dirnode: + Directory(self._tub, self._vdrive_server, dirnode, dirname)) + return d2 + else: + raise RuntimeError("what is this %s" % res) d.addCallback(_got_child) return d @@ -215,16 +221,16 @@ class Directory(rend.Page): uploader = get_uploader_service(ctx) d = uploader.upload(upload.FileHandle(contents.file)) name = contents.filename - if privateupload: - d.addCallback(lambda vid: self.uploadprivate(name, vid)) - else: - d.addCallback(lambda vid: - self._dirnode.callRemote("add_file", name, vid)) + def _uploaded(uri): + if privateupload: + return self.uploadprivate(name, uri) + return vdrive.add_file(self._dirnode, name, uri) + d.addCallback(_uploaded) def _done(res): log.msg("webish upload complete") return res d.addCallback(_done) - return d + return d # TODO: huh? return url.here.add("results", "upload of '%s' complete!" % contents.filename) @@ -238,7 +244,7 @@ class Directory(rend.Page): def mkdir(self, name): """mkdir2""" log.msg("making new webish directory") - d = self._dirnode.callRemote("add_directory", name) + d = vdrive.mkdir(self._vdrive_server, self._dirnode, name) def _done(res): log.msg("webish mkdir complete") return res @@ -248,8 +254,8 @@ class Directory(rend.Page): def child__delete(self, ctx): # perform the delete, then redirect back to the directory page args = inevow.IRequest(ctx).args - vdrive = get_vdrive_service(ctx) - d = vdrive.remove(self._dirnode, args["name"][0]) + name = args["name"][0] + d = self._dirnode.callRemote("remove", name) def _deleted(res): return url.here.up() d.addCallback(_deleted) @@ -360,8 +366,8 @@ class WebishServer(service.MultiService): # apparently 'ISite' does not exist #self.site._client = self.parent - def set_root_dirnode(self, dirnode): - self.root.putChild("vdrive", Directory(dirnode, "/")) + def set_vdrive(self, tub, vdrive_server, dirnode): + self.root.putChild("vdrive", Directory(tub, vdrive_server, dirnode, "/")) # I tried doing it this way and for some reason it didn't seem to work #print "REMEMBERING", self.site, dl, IDownloader #self.site.remember(dl, IDownloader) -- 2.45.2