From 82c38d370add5c346c6770e132f755158bf97dd1 Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Fri, 15 Jun 2007 00:37:32 -0700 Subject: [PATCH] make new vdrive work, implement convenience wrapper, passes all tests --- src/allmydata/client.py | 19 ++-- src/allmydata/filetable.py | 14 +-- src/allmydata/interfaces.py | 94 +++++++++++--------- src/allmydata/test/test_filetable.py | 4 +- src/allmydata/test/test_system.py | 30 +++---- src/allmydata/vdrive.py | 126 +++++++++++++++++++++++---- src/allmydata/webish.py | 51 +++++------ 7 files changed, 212 insertions(+), 126 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index fa2cc684..a51cfcdc 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -13,7 +13,7 @@ from allmydata.Crypto.Util.number import bytes_to_long from allmydata.storageserver import StorageServer from allmydata.upload import Uploader from allmydata.download import Downloader -#from allmydata.vdrive import VDrive +from allmydata.vdrive import DirectoryNode from allmydata.webish import WebishServer from allmydata.control import ControlServer from allmydata.introducer import IntroducerClient @@ -113,18 +113,15 @@ class Client(node.Node, Referenceable): def _got_vdrive(self, vdrive_server): # vdrive_server implements RIVirtualDriveServer self.log("connected to vdrive server") - d = vdrive_server.callRemote("get_public_root") - d.addCallback(self._got_vdrive_root, vdrive_server) + d = vdrive_server.callRemote("get_public_root_furl") + d.addCallback(self._got_vdrive_root_furl, vdrive_server) - def _got_vdrive_root(self, vdrive_root, vdrive_server): - # vdrive_root implements RIMutableDirectoryNode + def _got_vdrive_root_furl(self, vdrive_root_furl, vdrive_server): + root = DirectoryNode(vdrive_root_furl, self) self.log("got vdrive root") - self._connected_to_vdrive = True self._vdrive_server = vdrive_server - self._vdrive_root = vdrive_root - def _disconnected(): - self._connected_to_vdrive = False - vdrive_root.notifyOnDisconnect(_disconnected) + self._vdrive_root = root + self._connected_to_vdrive = True #vdrive = self.getServiceNamed("vdrive") #vdrive.set_server(vdrive_server) @@ -132,7 +129,7 @@ class Client(node.Node, Referenceable): if "webish" in self.namedServices: webish = self.getServiceNamed("webish") - webish.set_vdrive(self.tub, vdrive_server, vdrive_root) + webish.set_vdrive_root(root) def remote_get_versions(self): return str(allmydata.__version__), str(self.OLDEST_SUPPORTED_VERSION) diff --git a/src/allmydata/filetable.py b/src/allmydata/filetable.py index 294f6a0d..c948ded9 100644 --- a/src/allmydata/filetable.py +++ b/src/allmydata/filetable.py @@ -2,7 +2,8 @@ import os from zope.interface import implements from foolscap import Referenceable -from allmydata.interfaces import RIVirtualDriveServer, RIMutableDirectoryNode, FileNode, DirectoryNode +from allmydata.interfaces import RIVirtualDriveServer, RIMutableDirectoryNode +from allmydata.vdrive import FileNode, DirectoryNode from allmydata.util import bencode, idlib from twisted.application import service from twisted.python import log @@ -61,7 +62,7 @@ class MutableDirectoryNode(Referenceable): elif isinstance(v, DirectoryNode): child_nodes[k] = ("subdir", v.furl) else: - raise RuntimeError("unknown child node '%s'" % (v,)) + raise RuntimeError("unknown child[%s] node '%s'" % (k,v)) data = bencode.bencode(child_nodes) f = open(os.path.join(self._basedir, self._name), "wb") f.write(data) @@ -143,6 +144,7 @@ class VirtualDriveServer(service.MultiService, Referenceable): # all callers. In addition, we must register any new ones that we # create later on. tub = self.parent.tub + self._root_furl = tub.registerReference(self._root, "root") self._register_all_dirnodes(tub) def _register_all_dirnodes(self, tub): @@ -150,14 +152,14 @@ class VirtualDriveServer(service.MultiService, Referenceable): node = MutableDirectoryNode(self._vdrive_dir, name) ignored_furl = tub.registerReference(node, name) - def get_public_root(self): + def get_public_root_furl(self): if self._root: - return self._root + return self._root_furl raise NoPublicRootError - remote_get_public_root = get_public_root + remote_get_public_root_furl = get_public_root_furl def create_directory(self): node = MutableDirectoryNode(self._vdrive_dir) furl = self.parent.tub.registerReference(node, node._name) - return furl + return DirectoryNode(furl) remote_create_directory = create_directory diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index bcd0f581..b7a88e82 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -2,7 +2,7 @@ from zope.interface import Interface from foolscap.schema import StringConstraint, ListOf, TupleOf, SetOf, DictOf, \ ChoiceOf -from foolscap import RemoteInterface, Referenceable, Copyable, RemoteCopy +from foolscap import RemoteInterface, Referenceable HASH_SIZE=32 @@ -123,46 +123,6 @@ class RIStorageServer(RemoteInterface): from foolscap.schema import Any RIMutableDirectoryNode_ = Any() # TODO: how can we avoid this? -class DirectoryNode(Copyable, RemoteCopy): - """I have either a .furl attribute or a .get(tub) method.""" - typeToCopy = "allmydata.com/tahoe/interfaces/DirectoryNode/v1" - copytype = typeToCopy - def __init__(self, furl=None): - # RemoteCopy subclasses are always called without arguments - self.furl = furl - def getStateToCopy(self): - return {"furl": self.furl } - def setCopyableState(self, state): - self.furl = state['furl'] - def __hash__(self): - return hash((self.__class__, self.furl)) - def __cmp__(self, them): - if cmp(type(self), type(them)): - return cmp(type(self), type(them)) - if cmp(self.__class__, them.__class__): - return cmp(self.__class__, them.__class__) - return cmp(self.furl, them.furl) - -class FileNode(Copyable, RemoteCopy): - """I have a .uri attribute.""" - typeToCopy = "allmydata.com/tahoe/interfaces/FileNode/v1" - copytype = typeToCopy - def __init__(self, uri=None): - # RemoteCopy subclasses are always called without arguments - self.uri = uri - def getStateToCopy(self): - return {"uri": self.uri } - def setCopyableState(self, state): - self.uri = state['uri'] - def __hash__(self): - return hash((self.__class__, self.uri)) - def __cmp__(self, them): - if cmp(type(self), type(them)): - return cmp(type(self), type(them)) - if cmp(self.__class__, them.__class__): - return cmp(self.__class__, them.__class__) - return cmp(self.uri, them.uri) - FileNode_ = Any() # TODO: foolscap needs constraints on copyables DirectoryNode_ = Any() # TODO: same AnyNode_ = ChoiceOf(FileNode_, DirectoryNode_) @@ -186,14 +146,62 @@ class RIMutableDirectoryNode(RemoteInterface): class RIVirtualDriveServer(RemoteInterface): - def get_public_root(): + def get_public_root_furl(): """If this vdrive server does not offer a public root, this will raise an exception.""" - return DirectoryNode_ + return FURL def create_directory(): return DirectoryNode_ +class IFileNode(Interface): + def download(target): + """Download the file's contents to a given IDownloadTarget""" + def download_to_data(): + pass + +class IDirectoryNode(Interface): + def list(): + """I return a Deferred that fires with a""" + pass + + def get(name): + """I return a Deferred that fires with a specific named child.""" + pass + + def add(name, child): + """I add a child at the specific name. I return a Deferred that fires + when the operation finishes.""" + + def add_file(name, uploadable): + """I upload a file (using the given IUploadable), then attach the + resulting FileNode to the directory at the given name.""" + + def remove(name): + """I remove the child at the specific name. I return a Deferred that + fires when the operation finishes.""" + + def create_empty_directory(name): + """I create and attach an empty directory at the given name. I return + a Deferred that fires when the operation finishes.""" + + def attach_shared_directory(name, furl): + """I attach a directory that was shared (possibly by someone else) + with IDirectoryNode.get_furl to this parent at the given name. I + return a Deferred that fires when the operation finishes.""" + + def get_shared_directory_furl(): + """I return a FURL that can be used to attach this directory to + somewhere else. The FURL is just a string, so it can be passed + through email or other out-of-band protocol. Use it by passing it in + to attach_shared_directory(). I return a Deferred that fires when the + operation finishes.""" + + def move_child_to(current_child_name, new_parent, new_child_name=None): + """I take one of my children and move them to a new parent. The child + is referenced by name. On the new parent, the child will live under + 'new_child_name', which defaults to 'current_child_name'. I return a + Deferred that fires when the operation finishes.""" class ICodecEncoder(Interface): diff --git a/src/allmydata/test/test_filetable.py b/src/allmydata/test/test_filetable.py index 8b912383..12a26ac5 100644 --- a/src/allmydata/test/test_filetable.py +++ b/src/allmydata/test/test_filetable.py @@ -2,8 +2,8 @@ import os from twisted.trial import unittest from allmydata.filetable import (MutableDirectoryNode, - BadDirectoryError, BadFileError, BadNameError) -from allmydata.interfaces import FileNode, DirectoryNode + BadFileError, BadNameError) +from allmydata.vdrive import FileNode, DirectoryNode class FileTable(unittest.TestCase): diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index 47e85f3a..85a96009 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -3,7 +3,7 @@ import os from twisted.trial import unittest from twisted.internet import defer, reactor from twisted.application import service -from allmydata import client, uri, download, vdrive +from allmydata import client, uri, download, upload from allmydata.introducer_and_vdrive import IntroducerAndVdrive from allmydata.util import idlib, fileutil, testutil from foolscap.eventual import flushEventualQueue @@ -234,32 +234,24 @@ class SystemTest(testutil.SignalMixin, unittest.TestCase): d = self.set_up_nodes() def _do_publish(res): log.msg("PUBLISHING") + ut = upload.Data(DATA) c0 = self.clients[0] - d1 = vdrive.mkdir(c0._vdrive_server, c0._vdrive_root, - "subdir1") + d1 = c0._vdrive_root.create_empty_directory("subdir1") d1.addCallback(lambda subdir1_node: - c0.tub.getReference(subdir1_node.furl)) - def _put_file(subdir1): - uploader = c0.getServiceNamed("uploader") - d2 = uploader.upload_data(DATA) - def _stash_uri(uri): - self.uri = uri - return uri - d2.addCallback(_stash_uri) - d2.addCallback(lambda uri: vdrive.add_file(subdir1, "mydata567", uri)) - return d2 - d1.addCallback(_put_file) + subdir1_node.add_file("mydata567", ut)) + def _stash_uri(filenode): + self.uri = filenode.uri + return filenode + d1.addCallback(_stash_uri) return d1 d.addCallback(_do_publish) def _publish_done(filenode): log.msg("publish finished") c1 = self.clients[1] - d1 = c1._vdrive_root.callRemote("get", "subdir1") - d1.addCallback(lambda subdir1_dirnode: - c1.tub.getReference(subdir1_dirnode.furl)) - d1.addCallback(lambda subdir1: subdir1.callRemote("get", "mydata567")) - d1.addCallback(lambda filenode: c1.getServiceNamed("downloader").download_to_data(filenode.uri)) + d1 = c1._vdrive_root.get("subdir1") + d1.addCallback(lambda subdir1: subdir1.get("mydata567")) + d1.addCallback(lambda filenode: filenode.download_to_data()) return d1 d.addCallback(_publish_done) def _get_done(data): diff --git a/src/allmydata/vdrive.py b/src/allmydata/vdrive.py index 9f98f785..7e8672d0 100644 --- a/src/allmydata/vdrive.py +++ b/src/allmydata/vdrive.py @@ -5,7 +5,7 @@ from twisted.application import service from twisted.internet import defer from twisted.python import log from allmydata import upload, download -from allmydata.interfaces import FileNode, DirectoryNode +from foolscap import Copyable, RemoteCopy class VDrive(service.MultiService): name = "vdrive" @@ -181,23 +181,113 @@ class VDrive(service.MultiService): return self.get_file(from_where, download.FileHandle(filehandle)) -# utility stuff -def add_file(parent_node, child_name, uri): - child_node = FileNode(uri) - d = parent_node.callRemote("add", child_name, child_node) - return d +class DirectoryNode(Copyable, RemoteCopy): + """I have either a .furl attribute or a .get(tub) method.""" + typeToCopy = "allmydata.com/tahoe/interfaces/DirectoryNode/v1" + copytype = typeToCopy + def __init__(self, furl=None, client=None): + # RemoteCopy subclasses are always called without arguments + self.furl = furl + self._set_client(client) + def _set_client(self, client): + self._client = client + return self + def getStateToCopy(self): + return {"furl": self.furl } + def setCopyableState(self, state): + self.furl = state['furl'] + def __hash__(self): + return hash((self.__class__, self.furl)) + def __cmp__(self, them): + if cmp(type(self), type(them)): + return cmp(type(self), type(them)) + if cmp(self.__class__, them.__class__): + return cmp(self.__class__, them.__class__) + return cmp(self.furl, them.furl) -def mkdir(vdrive_server, parent_node, child_name): - d = vdrive_server.callRemote("create_directory") - d.addCallback(lambda newdir_furl: - parent_node.callRemote("add", child_name, DirectoryNode(newdir_furl))) - return d + def list(self): + d = self._client.tub.getReference(self.furl) + d.addCallback(lambda node: node.callRemote("list")) + d.addCallback(lambda children: + [(name,child._set_client(self._client)) + for name,child in children]) + return d + + def get(self, name): + d = self._client.tub.getReference(self.furl) + d.addCallback(lambda node: node.callRemote("get", name)) + d.addCallback(lambda child: child._set_client(self._client)) + return d + + def add(self, name, child): + d = self._client.tub.getReference(self.furl) + d.addCallback(lambda node: node.callRemote("add", name, child)) + d.addCallback(lambda newnode: newnode._set_client(self._client)) + return d + + def add_file(self, name, uploadable): + uploader = self._client.getServiceNamed("uploader") + d = uploader.upload(uploadable) + d.addCallback(lambda uri: self.add(name, FileNode(uri, self._client))) + return d + + def remove(self, name): + d = self._client.tub.getReference(self.furl) + d.addCallback(lambda node: node.callRemote("remove", name)) + d.addCallback(lambda newnode: newnode._set_client(self._client)) + return d + + def create_empty_directory(self, name): + vdrive_server = self._client._vdrive_server + d = vdrive_server.callRemote("create_directory") + d.addCallback(lambda node: self.add(name, node)) + return d + + def attach_shared_directory(self, name, furl): + d = self.add(name, DirectoryNode(furl)) + return d + + def get_shared_directory_furl(self): + return defer.succeed(self.furl) + + def move_child_to(self, current_child_name, + new_parent, new_child_name=None): + if new_child_name is None: + new_child_name = current_child_name + d = self.get(current_child_name) + d.addCallback(lambda child: new_parent.add(new_child_name, child)) + d.addCallback(lambda child: self.remove(current_child_name)) + return d + +class FileNode(Copyable, RemoteCopy): + """I have a .uri attribute.""" + typeToCopy = "allmydata.com/tahoe/interfaces/FileNode/v1" + copytype = typeToCopy + def __init__(self, uri=None, client=None): + # RemoteCopy subclasses are always called without arguments + self.uri = uri + self._set_client(client) + def _set_client(self, client): + self._client = client + return self + def getStateToCopy(self): + return {"uri": self.uri } + def setCopyableState(self, state): + self.uri = state['uri'] + def __hash__(self): + return hash((self.__class__, self.uri)) + def __cmp__(self, them): + if cmp(type(self), type(them)): + return cmp(type(self), type(them)) + if cmp(self.__class__, them.__class__): + return cmp(self.__class__, them.__class__) + return cmp(self.uri, them.uri) + + def download(self, target): + downloader = self._client.getServiceNamed("downloader") + return downloader.download(self.uri, target) -def add_shared_directory_furl(parent_node, child_name, furl): - child_node = DirectoryNode(furl) - d = parent_node.callRemote("add", child_name, child_node) - return d + def download_to_data(self): + downloader = self._client.getServiceNamed("downloader") + return downloader.download_to_data(self.uri) -def create_anonymous_directory(vdrive_server): - d = vdrive_server.callRemote("create_directory") - return d diff --git a/src/allmydata/webish.py b/src/allmydata/webish.py index cedf32a7..b2728df7 100644 --- a/src/allmydata/webish.py +++ b/src/allmydata/webish.py @@ -5,8 +5,9 @@ from twisted.python import util, log from nevow import inevow, rend, loaders, appserver, url, tags as T from allmydata.util import idlib from allmydata.uri import unpack_uri -from allmydata.interfaces import IDownloadTarget, FileNode, DirectoryNode -from allmydata import upload, vdrive +from allmydata.interfaces import IDownloadTarget +from allmydata.vdrive import FileNode, DirectoryNode +from allmydata import upload from zope.interface import implements, Interface import urllib from formless import annotate, webform @@ -97,9 +98,7 @@ class Directory(rend.Page): addSlash = True docFactory = getxmlfile("directory.xhtml") - def __init__(self, tub, vdrive_server, dirnode, dirname): - self._tub = tub - self._vdrive_server = vdrive_server + def __init__(self, dirnode, dirname): self._dirnode = dirnode self._dirname = dirname @@ -110,16 +109,13 @@ class Directory(rend.Page): dirname = "/" + name else: dirname = self._dirname + "/" + name - d = self._dirnode.callRemote("get", name) + d = self._dirnode.get(name) def _got_child(res): if isinstance(res, FileNode): dl = get_downloader_service(ctx) return Downloader(dl, name, res) elif isinstance(res, DirectoryNode): - d2 = self._tub.getReference(res.furl) - d2.addCallback(lambda dirnode: - Directory(self._tub, self._vdrive_server, dirnode, dirname)) - return d2 + return Directory(res, dirname) else: raise RuntimeError("what is this %s" % res) d.addCallback(_got_child) @@ -132,18 +128,18 @@ class Directory(rend.Page): return "Directory of '%s':" % self._dirname def data_children(self, ctx, data): - d = self._dirnode.callRemote("list") + d = self._dirnode.list() return d def render_row(self, ctx, data): name, target = data - if isinstance(target, str): + if isinstance(target, FileNode): # file dlurl = urllib.quote(name) ctx.fillSlots("filename", T.a(href=dlurl)[html.escape(name)]) ctx.fillSlots("type", "FILE") - uri = target + uri = target.uri dl_uri_url = url.root.child("download_uri").child(uri) # add a filename= query argument to give it a Content-Type dl_uri_url = dl_uri_url.add("filename", name) @@ -156,13 +152,13 @@ class Directory(rend.Page): # to be invoked, which deletes the file and then redirects the # browser back to this directory del_url = url.here.child("_delete") - #del_url = del_url.add("uri", target) + #del_url = del_url.add("uri", target.uri) del_url = del_url.add("name", name) delete = T.form(action=del_url, method="post")[ T.input(type='submit', value='del', name="del"), ] ctx.fillSlots("delete", delete) - else: + elif isinstance(target, DirectoryNode): # directory subdir_url = urllib.quote(name) ctx.fillSlots("filename", @@ -171,6 +167,8 @@ class Directory(rend.Page): ctx.fillSlots("size", "-") ctx.fillSlots("uri", "-") ctx.fillSlots("delete", "-") + else: + raise RuntimeError("unknown thing %s" % (target,)) return ctx.tag def render_forms(self, ctx, data): @@ -224,7 +222,8 @@ class Directory(rend.Page): def _uploaded(uri): if privateupload: return self.uploadprivate(name, uri) - return vdrive.add_file(self._dirnode, name, uri) + else: + return self._dirnode.add(name, FileNode(uri)) d.addCallback(_uploaded) def _done(res): log.msg("webish upload complete") @@ -244,7 +243,7 @@ class Directory(rend.Page): def mkdir(self, name): """mkdir2""" log.msg("making new webish directory") - d = vdrive.mkdir(self._vdrive_server, self._dirnode, name) + d = self._dirnode.create_empty_directory(name) def _done(res): log.msg("webish mkdir complete") return res @@ -255,7 +254,7 @@ class Directory(rend.Page): # perform the delete, then redirect back to the directory page args = inevow.IRequest(ctx).args name = args["name"][0] - d = self._dirnode.callRemote("remove", name) + d = self._dirnode.remove(name) def _deleted(res): return url.here.up() d.addCallback(_deleted) @@ -291,10 +290,11 @@ class TypedFile(static.File): self.defaultType) class Downloader(resource.Resource): - def __init__(self, downloader, name, uri): + def __init__(self, downloader, name, filenode): self._downloader = downloader self._name = name - self._uri = uri + assert isinstance(filenode, FileNode) + self._filenode = filenode def render(self, ctx): req = inevow.IRequest(ctx) @@ -307,10 +307,7 @@ class Downloader(resource.Resource): if encoding: req.setHeader('content-encoding', encoding) - t = WebDownloadTarget(req) - #dl = IDownloader(ctx) - dl = self._downloader - dl.download(self._uri, t) + self._filenode.download(WebDownloadTarget(req)) return server.NOT_DONE_YET @@ -331,7 +328,7 @@ class Root(rend.Page): uri = req.args["uri"][0] else: return rend.NotFound - child = Downloader(dl, filename, uri) + child = Downloader(dl, filename, FileNode(uri, IClient(ctx))) return child, () return rend.Page.locateChild(self, ctx, segments) @@ -366,8 +363,8 @@ class WebishServer(service.MultiService): # apparently 'ISite' does not exist #self.site._client = self.parent - def set_vdrive(self, tub, vdrive_server, dirnode): - self.root.putChild("vdrive", Directory(tub, vdrive_server, dirnode, "/")) + def set_vdrive_root(self, root): + self.root.putChild("vdrive", Directory(root, "/")) # I tried doing it this way and for some reason it didn't seem to work #print "REMEMBERING", self.site, dl, IDownloader #self.site.remember(dl, IDownloader) -- 2.45.2