From: Brian Warner Date: Sat, 20 Jan 2007 11:25:20 +0000 (-0700) Subject: more filetree hacking, still too early to test X-Git-Tag: tahoe_v0.1.0-0-UNSTABLE~348 X-Git-Url: https://git.rkrishnan.org/simplejson/__init__.py.html?a=commitdiff_plain;h=e843abe54294b4d0b3c8dd46876456668b16de05;p=tahoe-lafs%2Ftahoe-lafs.git more filetree hacking, still too early to test --- diff --git a/src/allmydata/filetree/basenode.py b/src/allmydata/filetree/basenode.py index 5c73bbc4..664ff714 100644 --- a/src/allmydata/filetree/basenode.py +++ b/src/allmydata/filetree/basenode.py @@ -2,15 +2,17 @@ from zope.interface import implements from allmydata.filetree.interfaces import INode -class BaseURINode(object): +class BaseDataNode(object): implements(INode) prefix = None # must be set by subclass - def is_directory(self): - return False + def get_base_data(self): + raise NotImplementedError # must be provided by subclass + def set_base_data(self, data): + raise NotImplementedError # must be provided by subclass def serialize_node(self): - return "%s:%s" % (self.prefix, self.uri) + return "%s:%s" % (self.prefix, self.get_base_data()) def populate_node(self, data, node_maker): assert data.startswith(self.prefix + ":") - self.uri = data[len(self.prefix)+1:] + self.set_base_data(data[len(self.prefix)+1:]) diff --git a/src/allmydata/filetree/directory.py b/src/allmydata/filetree/directory.py index fe1533f5..2a7b023b 100644 --- a/src/allmydata/filetree/directory.py +++ b/src/allmydata/filetree/directory.py @@ -5,7 +5,7 @@ from allmydata.filetree.interfaces import ( ICHKDirectoryNode, ISSKDirectoryNode, NoSuchChildError, ) -from allmydata.filetree.basenode import BaseURINode +from allmydata.filetree.basenode import BaseDataNode from allmydata import download from allmydata.util import bencode @@ -44,9 +44,6 @@ class SubTreeNode: # # then self.child_specifications["foo.jpg"] = ("CHKFILE","fooURI") # self.child_specifications = {} - def is_directory(self): - return True - def list(self): return sorted(self.children.keys()) @@ -179,10 +176,15 @@ class _DirectorySubTree(object): break return (found_path, node, remaining_path) -class CHKDirectorySubTreeNode(BaseURINode): +class CHKDirectorySubTreeNode(BaseDataNode): implements(ICHKDirectoryNode) prefix = "CHKDirectory" + def get_base_data(self): + return self.uri + def set_base_data(self, data): + self.uri = data + def get_uri(self): return self.uri @@ -190,9 +192,6 @@ class CHKDirectorySubTreeNode(BaseURINode): class CHKDirectorySubTree(_DirectorySubTree): # maybe mutable, maybe not - def mutation_affects_parent(self): - return True - def set_uri(self, uri): self.old_uri = uri @@ -209,6 +208,7 @@ class CHKDirectorySubTree(_DirectorySubTree): self.serialize_to_file(f) f.close() boxname = work_queue.create_boxname() + # mutation affects our parent work_queue.add_upload_chk(filename, boxname) work_queue.add_delete_tempfile(filename) work_queue.add_retain_uri_from_box(boxname) @@ -225,8 +225,6 @@ class SSKDirectorySubTreeNode(object): implements(INode, ISSKDirectoryNode) prefix = "SSKDirectory" - def is_directory(self): - return False def serialize_node(self): data = (self.read_cap, self.write_cap) return "%s:%s" % (self.prefix, bencode.bencode(data)) @@ -248,9 +246,6 @@ class SSKDirectorySubTree(_DirectorySubTree): self.version = 0 # TODO: populate - def mutation_affects_parent(self): - return False - def populate_from_node(self, node, parent_is_mutable, node_maker, downloader): node = ISSKDirectoryNode(node) self.read_capability = node.get_read_capability() @@ -268,6 +263,7 @@ class SSKDirectorySubTree(_DirectorySubTree): f, filename = work_queue.create_tempfile(".sskdir") self.serialize_to_file(f) f.close() + # mutation does not affect our parent work_queue.add_upload_ssk(filename, self.write_capability, self.version) self.version = self.version + 1 diff --git a/src/allmydata/filetree/file.py b/src/allmydata/filetree/file.py index 3eb440f2..5745e799 100644 --- a/src/allmydata/filetree/file.py +++ b/src/allmydata/filetree/file.py @@ -1,13 +1,18 @@ from zope.interface import implements from allmydata.filetree.interfaces import INode, IFileNode -from allmydata.filetree.basenode import BaseURINode +from allmydata.filetree.basenode import BaseDataNode from allmydata.util import bencode -class CHKFileNode(BaseURINode): +class CHKFileNode(BaseDataNode): implements(IFileNode) prefix = "CHKFile" + def get_base_data(self): + return self.uri + def set_base_data(self, data): + self.uri = data + def get_uri(self): return self.uri @@ -15,8 +20,6 @@ class SSKFileNode(object): implements(INode, IFileNode) prefix = "SSKFile" - def is_directory(self): - return False def serialize_node(self): data = (self.read_cap, self.write_cap) return "%s:%s" % (self.prefix, bencode.bencode(data)) diff --git a/src/allmydata/filetree/interfaces.py b/src/allmydata/filetree/interfaces.py index f209e3c7..972c81a3 100644 --- a/src/allmydata/filetree/interfaces.py +++ b/src/allmydata/filetree/interfaces.py @@ -4,17 +4,16 @@ from zope.interface import Interface class INode(Interface): """This is some sort of retrievable node. All objects which implement other I*Node interfaces also implement this one.""" - def is_directory(): - """Return True if this node is an internal directory node.""" def serialize_node(): """Return a data structure which contains enough information to build - this node again in the future (by calling vdrive.make_node(). For - IDirectoryNodes, this will be a list. For all other nodes this will - be a string.""" + this node again in the future (by calling + vdrive.make_node_from_serialized(). For IDirectoryNodes, this will be + a list. For all other nodes this will be a string.""" def populate_node(data, node_maker): - """vdrive.make_node() will first use the prefix inside 'data' to - decide what kind of Node to create. It will then call this function - to populate the new Node from the data returned by serialize_node.""" + """vdrive.make_node_from_serialized() will first use the prefix + inside 'data' to decide what kind of Node to create. It will then + call this function to populate the new Node from the data returned by + serialize_node.""" class IFileNode(Interface): """This is a file which can be retrieved.""" @@ -83,12 +82,6 @@ class ISubTree(Interface): calls). """ - def populate_from_data(data): - """Used internally by populate_from_node. This is called with a - sequence of bytes that describes the contents of the subtree, - probably a bencoded tuple or s-expression. Returns self. - """ - def is_mutable(): """This returns True if we have the ability to modify this subtree. If this returns True, this reference may be adapted to @@ -194,7 +187,7 @@ class IVirtualDrive(Interface): # internal methods - def make_node(serialized): + def make_node_from_serialized(serialized): """Given a string produced by original_node.serialize_node(), produce an equivalent node. """ diff --git a/src/allmydata/filetree/redirect.py b/src/allmydata/filetree/redirect.py index 1c4fe245..3ceb86ac 100644 --- a/src/allmydata/filetree/redirect.py +++ b/src/allmydata/filetree/redirect.py @@ -1,98 +1,183 @@ +from cStringIO import StringIO +from zope.interface import implements +from twisted.internet import defer + +from allmydata.filetree.interfaces import ISubTree +from allmydata.filetree.basenode import BaseDataNode from allmydata.util import bencode -class LocalFileRedirection(object): +class LocalFileRedirectionNode(BaseDataNode): + prefix = "LocalFileRedirection" + + def new(self, handle): + self.handle = handle + + def get_base_data(self): + return self.handle + def set_base_data(self, data): + self.handle = data + +class _BaseRedirection(object): + implements(ISubTree) + + def new(self, child_node): + self.child_node = child_node + + def get_node_for_path(self, path): + return ([], self.child_node, path) + + def serialize_subtree_to_file(self, f): + return self.child_node.serialize_node() + + def _populate_from_data(self, data, node_maker): + self.child_node = node_maker(data) + return self + +class LocalFileRedirection(_BaseRedirection): stype = "LocalFileRedirection" - def populate_from_specification(self, spec, parent_is_mutable, downloader): + def populate_from_node(self, node, parent_is_mutable, node_maker, downloader): # return a Deferred that fires (with self) when this node is ready # for use - (stype, filename) = spec - assert stype == self.stype - #filename = spec.get_filename() - # there is a local file which contains a bencoded serialized - # subtree specification. - - # TODO: will this enable outsiders to cause us to read from - # arbitrary files? Think about this. - f = open(filename, "rb") + assert isinstance(node, LocalFileRedirectionNode) + self.filename = node.handle + # there is a local file which contains a bencoded serialized subtree + # specification. + + # TODO: will this enable outsiders to cause us to read from arbitrary + # files? Think about this. It is probably a good idea to restrict the + # filename to be a single component, and to always put them in a + # well-known directory that contains nothing else, and maybe make + # them unguessable. + f = open(self.filename, "rb") data = f.read() f.close() # note: we don't cache the contents of the file. TODO: consider # doing this based upon mtime. It is important that we be able to # notice if the file has been changed. + return defer.succeed(self._populate_from_data(data, node_maker)) - return self.populate_from_data(data) + def is_mutable(self): + return True - def populate_from_data(self, data): - # data is a subtree specification for our one child - self.child_spec = bencode.bdecode(data) - return self + def update(self, prepath, workqueue): + f = open(self.filename, "wb") + self.serialize_subtree_to_file(f) + f.close() + + +class QueenRedirectionNode(LocalFileRedirectionNode): + prefix = "QueenRedirection" -class QueenRedirection(object): - stype = "QueenRedirection" +class QueenRedirection(_BaseRedirection): + style = "QueenRedirection" - def populate_from_specification(self, spec, parent_is_mutable, downloader): - # this specifies a handle for which the Queen maintains a - # serialized subtree specification. - (stype, handle) = spec + def new(self, handle): + self.handle = handle + + def populate_from_node(self, node, parent_is_mutable, node_maker, downloader): + # this specifies a handle for which the Queen maintains a serialized + # subtree specification. + assert isinstance(node, QueenRedirectionNode) + self.handle = node.handle # TODO: queen? - d = self._queen.callRemote("lookup_handle", handle) - d.addCallback(self.populate_from_data) + d = self._queen.callRemote("lookup_handle", self.handle) + d.addCallback(self._populate_from_data, node_maker) return d - def populate_from_data(self, data): - self.child_spec = bencode.bdecode(data) - return self + def is_mutable(self): + return True # TODO: maybe, maybe not + + def update(self, prepath, workqueue): + f = StringIO() + self.serialize_subtree_to_file(f) + d = self._queen.callRemote("set_handle", self.handle, f.getvalue()) + return d + +class QueenOrLocalFileRedirectionNode(LocalFileRedirectionNode): + prefix = "QueenOrLocalFileRedirection" -class QueenOrLocalFileRedirection(object): +class QueenOrLocalFileRedirection(_BaseRedirection): stype = "QueenOrLocalFileRedirection" - def populate_from_specification(self, spec, parent_is_mutable, downloader): + def new(self, handle, child_node): + self.handle = handle + self.version = 0 + self.child_node = child_node + # TODO + + def populate_from_node(self, node, parent_is_mutable, node_maker, downloader): # there is a local file which contains a bencoded serialized # subtree specification. The queen also has a copy. Whomever has # the higher version number wins. - (stype, filename, handle) = spec + assert isinstance(node, QueenOrLocalFileRedirectionNode) + self.filename = self.handle = node.handle - f = open(filename, "rb") + f = open(self.filename, "rb") #local_version, local_data = bencode.bdecode(f.read()) local_version_and_data = f.read() f.close() # TODO: queen? # TODO: pubsub so we can cache the queen's results - d = self._queen.callRemote("lookup_handle", handle) - d.addCallback(self._choose_winner, local_version_and_data) + d = self._queen.callRemote("lookup_handle", self.handle) + d.addCallback(self._choose_winner, local_version_and_data, node_maker) return d - def _choose_winner(self, queen_version_and_data, local_version_and_data): + def _choose_winner(self, queen_version_and_data, local_version_and_data, node_maker): queen_version, queen_data = bencode.bdecode(queen_version_and_data) local_version, local_data = bencode.bdecode(local_version_and_data) if queen_version > local_version: data = queen_data + self.version = queen_version else: data = local_data - return self.populate_from_data(data) - - def populate_from_data(self, data): + self.version = local_version # NOTE: two layers of bencoding here, TODO - self.child_spec = bencode.bdecode(data) - return self + return self._populate_from_data(data, node_maker) + + def is_mutable(self): + return True + + def update(self, prepath, workqueue): + self.version += 1 + f = StringIO() + self.serialize_subtree_to_file(f) + version_and_data = bencode.bencode((self.version, f.getvalue())) + f = open(self.filename, "wb") + f.write(version_and_data) + f.close() + d = self._queen.callRemote("set_handle", self.handle, version_and_data) + return d + +class HTTPRedirectionNode(BaseDataNode): + prefix = "HTTPRedirection" -class HTTPRedirection(object): + def new(self, url): + self.url = url + + def get_base_data(self): + return self.url + def set_base_data(self, data): + self.url = data + +class HTTPRedirection(_BaseRedirection): stype = "HTTPRedirection" - def populate_from_specification(self, spec, parent_is_mutable, downloader): + def populate_from_node(self, node, parent_is_mutable, node_maker, downloader): # this specifies a URL at which there is a bencoded serialized # subtree specification. - (stype, url) = spec + assert isinstance(node, HTTPRedirectionNode) from twisted.web import client - d = client.getPage(url) - d.addCallback(self.populate_from_data) + d = client.getPage(node.url) + d.addCallback(self._populate_from_data, node_maker) return d - def populate_from_data(self, data): - self.child_spec = bencode.bdecode(data) - return self + def is_mutable(self): + return False + + def update(self, prepath, workqueue): + raise RuntimeError("HTTPRedirection is not mutable") diff --git a/src/allmydata/filetree/vdrive.py b/src/allmydata/filetree/vdrive.py index 581b9f69..3bf5df75 100644 --- a/src/allmydata/filetree/vdrive.py +++ b/src/allmydata/filetree/vdrive.py @@ -8,6 +8,9 @@ from allmydata.filetree.interfaces import ( ) from allmydata.upload import IUploadable +# this list is used by VirtualDrive.make_node_from_serialized() to convert +# node specification strings (found inside the serialized form of subtrees) +# into Nodes (which live in the in-RAM form of subtrees). all_node_types = [ directory.CHKDirectorySubTreeNode, directory.SSKDirectorySubTreeNode, @@ -57,7 +60,7 @@ class VirtualDrive(object): def _get_closest_node_2(self, res, parent_is_mutable): (found_path, node, remaining_path) = res - if node.is_directory(): + if IDirectoryNode.providedBy(node): # traversal done return (node, remaining_path) # otherwise, we must open and recurse into a new subtree @@ -125,7 +128,7 @@ class VirtualDrive(object): return d # these are called when loading and creating nodes - def make_node(self, serialized): + def make_node_from_serialized(self, serialized): # this turns a string into an INode, which contains information about # the file or directory (like a URI), but does not contain the actual # contents. An IOpener can be used later to retrieve the contents @@ -139,7 +142,7 @@ class VirtualDrive(object): for node_class in all_node_types: if prefix == node_class.prefix: node = node_class() - node.populate_node(serialized, self.make_node) + node.populate_node(serialized, self.make_node_from_serialized) return node raise RuntimeError("unable to handle subtree type '%s'" % prefix)