From: Brian Warner Date: Sat, 20 Jan 2007 11:52:53 +0000 (-0700) Subject: filetree: refactor INode serialization, start on tests X-Git-Tag: tahoe_v0.1.0-0-UNSTABLE~347 X-Git-Url: https://git.rkrishnan.org/components/com_hotproperty/flags/statistics?a=commitdiff_plain;h=c808d5a5efba0a7a6a9f7c951dbd39a5f5e90515;p=tahoe-lafs%2Ftahoe-lafs.git filetree: refactor INode serialization, start on tests --- diff --git a/src/allmydata/filetree/basenode.py b/src/allmydata/filetree/basenode.py index 664ff714..3287485b 100644 --- a/src/allmydata/filetree/basenode.py +++ b/src/allmydata/filetree/basenode.py @@ -12,7 +12,6 @@ class BaseDataNode(object): raise NotImplementedError # must be provided by subclass def serialize_node(self): return "%s:%s" % (self.prefix, self.get_base_data()) - def populate_node(self, data, node_maker): - assert data.startswith(self.prefix + ":") - self.set_base_data(data[len(self.prefix)+1:]) + def populate_node(self, body, node_maker): + self.set_base_data(body) diff --git a/src/allmydata/filetree/directory.py b/src/allmydata/filetree/directory.py index 2a7b023b..116a1f98 100644 --- a/src/allmydata/filetree/directory.py +++ b/src/allmydata/filetree/directory.py @@ -124,6 +124,7 @@ class _DirectorySubTree(object): def new(self): + # create a new, empty directory self.root = SubTreeNode(self) self.mutable = True # sure, why not @@ -135,7 +136,7 @@ class _DirectorySubTree(object): # to populate_from_data() raise NotImplementedError - def populate_from_data(self, data, node_maker): + def _populate_from_data(self, data, node_maker): self.root = SubTreeNode(self) self.root.populate_node(bencode.bdecode(data), node_maker) return self @@ -176,6 +177,35 @@ class _DirectorySubTree(object): break return (found_path, node, remaining_path) +class LocalFileSubTreeNode(BaseDataNode): + prefix = "LocalFileDirectory" + + def new(self, filename): + self.filename = filename + + def get_base_data(self): + return self.filename + def set_base_data(self, data): + self.filename = data + +class LocalFileSubTree(_DirectorySubTree): + def new(self, filename): + self.filename = filename + _DirectorySubTree.new(self) + + def populate_from_node(self, node, parent_is_mutable, node_maker, downloader): + self.mutable = True # probably + self.filename = node.filename + f = open(self.filename, "rb") + data = f.read() + f.close() + return defer.succeed(self._populate_from_data(node_maker)) + + def update(self, prepath, work_queue): + f = open(self.filename, "wb") + self.serialize_to_file(f) + f.close() + class CHKDirectorySubTreeNode(BaseDataNode): implements(ICHKDirectoryNode) prefix = "CHKDirectory" @@ -199,7 +229,7 @@ class CHKDirectorySubTree(_DirectorySubTree): assert ICHKDirectoryNode(node) self.mutable = parent_is_mutable d = downloader.download(node.get_uri(), download.Data()) - d.addCallback(self.populate_from_data, node_maker) + d.addCallback(self._populate_from_data, node_maker) return d def update(self, prepath, work_queue): @@ -228,10 +258,8 @@ class SSKDirectorySubTreeNode(object): def serialize_node(self): data = (self.read_cap, self.write_cap) return "%s:%s" % (self.prefix, bencode.bencode(data)) - def populate_node(self, data, node_maker): - assert data.startswith(self.prefix + ":") - capdata = data[len(self.prefix)+1:] - self.read_cap, self.write_cap = bencode.bdecode(capdata) + def populate_node(self, body, node_maker): + self.read_cap, self.write_cap = bencode.bdecode(body) def get_read_capability(self): return self.read_cap @@ -252,7 +280,7 @@ class SSKDirectorySubTree(_DirectorySubTree): self.write_capability = node.get_write_capability() self.mutable = bool(self.write_capability) d = downloader.download_ssk(self.read_capability, download.Data()) - d.addCallback(self.populate_from_data, node_maker) + d.addCallback(self._populate_from_data, node_maker) return d def set_version(self, version): diff --git a/src/allmydata/filetree/interfaces.py b/src/allmydata/filetree/interfaces.py index 972c81a3..ae7960a2 100644 --- a/src/allmydata/filetree/interfaces.py +++ b/src/allmydata/filetree/interfaces.py @@ -4,16 +4,21 @@ from zope.interface import Interface class INode(Interface): """This is some sort of retrievable node. All objects which implement other I*Node interfaces also implement this one.""" + + # the INode-implementing class must have an attribute named .prefix which + # contains a string. + def serialize_node(): """Return a data structure which contains enough information to build this node again in the future (by calling vdrive.make_node_from_serialized(). For IDirectoryNodes, this will be - a list. For all other nodes this will be a string.""" - def populate_node(data, node_maker): - """vdrive.make_node_from_serialized() will first use the prefix - inside 'data' to decide what kind of Node to create. It will then - call this function to populate the new Node from the data returned by - serialize_node.""" + a list. For all other nodes this will be a string of the form + 'prefix:body', where 'prefix' must be the same as the class attribute + .prefix .""" + def populate_node(body, node_maker): + """vdrive.make_node_from_serialized() will first use the prefix from + the .prefix attribute to decide what kind of Node to create. It will + then call this function with the body to populate the new Node.""" class IFileNode(Interface): """This is a file which can be retrieved.""" diff --git a/src/allmydata/filetree/opener.py b/src/allmydata/filetree/opener.py index ce6dd39e..16ef09b8 100644 --- a/src/allmydata/filetree/opener.py +++ b/src/allmydata/filetree/opener.py @@ -14,6 +14,9 @@ all_openable_subtree_types = [ redirect.QueenOrLocalFileRedirection, ] +# the Opener can turn an INode (which describes a subtree, like a directory +# or a redirection) into the fully-populated subtree. + class Opener(object): implements(interfaces.IOpener) def __init__(self, queen, downloader): diff --git a/src/allmydata/filetree/redirect.py b/src/allmydata/filetree/redirect.py index 3ceb86ac..86dd744b 100644 --- a/src/allmydata/filetree/redirect.py +++ b/src/allmydata/filetree/redirect.py @@ -37,6 +37,10 @@ class _BaseRedirection(object): class LocalFileRedirection(_BaseRedirection): stype = "LocalFileRedirection" + def new(self, handle, child_node): + self.filename = handle + _BaseRedirection.new(self, child_node) + def populate_from_node(self, node, parent_is_mutable, node_maker, downloader): # return a Deferred that fires (with self) when this node is ready # for use diff --git a/src/allmydata/filetree/vdrive.py b/src/allmydata/filetree/vdrive.py index 3bf5df75..233dd20a 100644 --- a/src/allmydata/filetree/vdrive.py +++ b/src/allmydata/filetree/vdrive.py @@ -28,9 +28,29 @@ class VirtualDrive(object): self.workqueue = workqueue workqueue.set_vdrive(self) # TODO: queen? + self.queen = None self.opener = opener.Opener(self.queen, downloader) self.root_node = root_node + # these are called when loading and creating nodes + def make_node_from_serialized(self, serialized): + # this turns a string into an INode, which contains information about + # the file or directory (like a URI), but does not contain the actual + # contents. An IOpener can be used later to retrieve the contents + # (which means downloading the file if this is an IFileNode, or + # perhaps creating a new subtree from the contents) + + # maybe include parent_is_mutable? + assert isinstance(serialized, str) + prefix, body = serialized.split(":", 2) + + for node_class in all_node_types: + if prefix == node_class.prefix: + node = node_class() + node.populate_node(body, self.make_node_from_serialized) + return node + raise RuntimeError("unable to handle subtree type '%s'" % prefix) + # these methods are used to walk through our subtrees def _get_root(self): @@ -127,25 +147,6 @@ class VirtualDrive(object): d.addCallback(_got_closest) return d - # these are called when loading and creating nodes - def make_node_from_serialized(self, serialized): - # this turns a string into an INode, which contains information about - # the file or directory (like a URI), but does not contain the actual - # contents. An IOpener can be used later to retrieve the contents - # (which means downloading the file if this is an IFileNode, or - # perhaps creating a new subtree from the contents) - - # maybe include parent_is_mutable? - assert isinstance(serialized, str) - colon = serialized.index(":") - prefix = serialized[:colon] - for node_class in all_node_types: - if prefix == node_class.prefix: - node = node_class() - node.populate_node(serialized, self.make_node_from_serialized) - return node - raise RuntimeError("unable to handle subtree type '%s'" % prefix) - # these are called by the workqueue def add(self, path, new_node): diff --git a/src/allmydata/test/test_filetree_new.py b/src/allmydata/test/test_filetree_new.py index c86adb0a..0891c774 100644 --- a/src/allmydata/test/test_filetree_new.py +++ b/src/allmydata/test/test_filetree_new.py @@ -1,17 +1,17 @@ -""" from zope.interface import implements from twisted.trial import unittest from twisted.internet import defer from allmydata.filetree.interfaces import IOpener, IDirectoryNode -from allmydata.filetree.directory import (ImmutableDirectorySubTree, +from allmydata.filetree.directory import (#ImmutableDirectorySubTree, SubTreeNode, CHKDirectorySubTree) -from allmydata.filetree.specification import (CHKFileSpecification, - CHKDirectorySpecification) +#from allmydata.filetree.specification import (CHKFileSpecification, +# CHKDirectorySpecification) from allmydata import workqueue from cStringIO import StringIO +""" class FakeOpener(object): implements(IOpener) def __init__(self, objects={}): @@ -311,3 +311,28 @@ del MultipleSubTrees class Redirect(unittest.TestCase): pass """ + +from allmydata.filetree import directory, redirect, vdrive + +class Load(unittest.TestCase): + + def testCreate(self): + # create some stuff, see if we can import everything + wq = workqueue.WorkQueue("test_filetree_new/Load/1.workqueue") + dl = None + + # create an empty directory (stored locally) as our root + root = directory.LocalFileSubTree() + root.new("dirtree.save") + + # and a node to point to it + root_node = directory.LocalFileSubTreeNode() + root_node.new("dirtree.save") + + v = vdrive.VirtualDrive(wq, dl, root_node) + + def start(): + root_node = redirect.LocalFileRedirectionNode() + root_node.new("handle", dirtree) + root = redirect.LocalFileRedirection() + # wow, bootstrapping is hard