from zope.interface import implements
-from allmydata.filetree.interfaces import (INode,
- IDirectoryNode,
- ISubTree,
- ICHKDirectoryNode, ISSKDirectoryNode,
- NoSuchChildError,
- )
+from allmydata.filetree.interfaces import (
+ INode, IDirectoryNode, ISubTree,
+ ICHKDirectoryNode, ISSKDirectoryNode,
+ NoSuchChildError,
+ )
+from allmydata.filetree.basenode import BaseURINode
from allmydata import download
from allmydata.util import bencode
# each time the vdrive changes, update the local drive to match, and
# vice versa.
-
-def to_node(spec):
- # TODO
- pass
-def to_spec(node):
- # TODO
- pass
-
+# from the itertools 'recipes' page
+from itertools import izip, tee
+def pairwise(iterable):
+ "s -> (s0,s1), (s1,s2), (s2, s3), ..."
+ a, b = tee(iterable)
+ try:
+ b.next()
+ except StopIteration:
+ pass
+ return izip(a, b)
class SubTreeNode:
implements(INode, IDirectoryNode)
def __init__(self, tree):
self.enclosing_tree = tree
- # subdirectory_node_children maps child name to another SubTreeNode
- # instance. This is only for internal directory nodes. All other
- # nodes are listed in child_specifications instead.
- self.subdirectory_node_children = {}
- # child_specifications maps child name to a specification tuple which
- # describes how to obtain the actual child. For example, if "foo.jpg"
- # in this node represents a CHK-encoded FILE with a uri of "fooURI",
- # then self.child_specifications["foo.jpg"] = ("CHKFILE","fooURI")
- self.child_specifications = {}
+ self.children = {}
+# # subdirectory_node_children maps child name to another SubTreeNode
+# # instance. This is only for internal directory nodes. All other
+# # nodes are listed in child_specifications instead.
+# self.subdirectory_node_children = {}
+# # child_specifications maps child name to a specification tuple which
+# # describes how to obtain the actual child. For example, if "foo.jpg"
+# # in this node represents a CHK-encoded FILE with a uri of "fooURI",
+# # then self.child_specifications["foo.jpg"] = ("CHKFILE","fooURI")
+# self.child_specifications = {}
def is_directory(self):
return True
def list(self):
- return sorted(self.subdirectory_node_children.keys() +
- self.child_specifications.keys())
+ return sorted(self.children.keys())
def get(self, childname):
- if childname in self.subdirectory_node_children:
- return self.subdirectory_node_children[childname]
- elif childname in self.child_specifications:
- return to_node(self.child_specifications[childname])
+ if childname in self.children:
+ return self.children[childname]
else:
raise NoSuchChildError("no child named '%s'" % (childname,))
def delete(self, childname):
assert self.enclosing_tree.is_mutable()
- if childname in self.subdirectory_node_children:
- del self.subdirectory_node_children[childname]
- elif childname in self.child_specifications:
- del self.child_specifications[childname]
+ if childname in self.children:
+ del self.children[childname]
else:
raise NoSuchChildError("no child named '%s'" % (childname,))
def add_subdir(self, childname):
- assert childname not in self.subdirectory_node_children
- assert childname not in self.child_specifications
+ assert childname not in self.children
newnode = SubTreeNode(self.enclosing_tree)
- self.subdirectory_node_children[childname] = newnode
+ self.children[childname] = newnode
return newnode
def add(self, childname, node):
- assert childname not in self.subdirectory_node_children
- assert childname not in self.child_specifications
- spec = to_spec(node)
- self.child_specifications[childname] = spec
+ assert childname not in self.children
+ assert INode(node)
+ self.children[childname] = node
return self
- def serialize_to_sexprs(self):
+ def serialize_node(self):
# note: this is a one-pass recursive serialization that will result
# in the whole file table being held in memory. This is only
# appropriate for directories with fewer than, say, 10k nodes. If we
# generator instead, and write the serialized data directly to a
# tempfile.
#
- # ["DIRECTORY", name1, child1, name2, child2..]
+ # [name1, child1, name2, child2..]
+ #
+ # child1 is either a list for subdirs, or a string for non-subdirs
- data = ["DIRECTORY"]
- for name in sorted(self.node_children.keys()):
+ data = []
+ for name in sorted(self.children.keys()):
data.append(name)
- data.append(self.node_children[name].serialize())
- for name in sorted(self.child_specifications.keys()):
- data.append(name)
- data.append(self.child_specifications[name].serialize())
+ data.append(self.children[name].serialize_node())
return data
- def populate_from_sexprs(self, data):
- assert data[0] == "DIRECTORY"
- assert len(data) % 2 == 1
- for i in range(1, len(data), 2):
- name = data[i]
- child_data = data[i+1]
- assert isinstance(child_data, (list, tuple))
- child_type = child_data[0]
- if child_type == "DIRECTORY":
+ def populate_node(self, data, node_maker):
+ assert len(data) % 2 == 0
+ for (name, child_data) in pairwise(data):
+ if isinstance(child_data, (list, tuple)):
child = SubTreeNode(self.enclosing_tree)
- child.populate_from_sexprs(child_data)
- self.node_children[name] = child
+ child.populate_node(child_data)
else:
- self.child_specifications[name] = child_data
+ assert isinstance(child_data, str)
+ child = node_maker(child_data)
+ self.children[name] = child
self.root = SubTreeNode(self)
self.mutable = True # sure, why not
- def populate_from_specification(self, spec, parent_is_mutable, downloader):
- return self.populate_from_node(to_node(spec),
- parent_is_mutable, downloader)
+ def populate_from_node(self, node, parent_is_mutable, node_maker, downloader):
+ # self.populate_from_node must be defined by the subclass (CHK or
+ # SSK), since it controls how the spec is interpreted. It will
+ # probably use the contents of the node to figure out what to
+ # download from the mesh, then pass this downloaded serialized data
+ # to populate_from_data()
+ raise NotImplementedError
- def populate_from_data(self, data):
- self.root = SubTreeNode()
- self.root.populate_from_sexprs(bencode.bdecode(data))
+ def populate_from_data(self, data, node_maker):
+ self.root = SubTreeNode(self)
+ self.root.populate_node(bencode.bdecode(data), node_maker)
return self
- def serialize(self):
- """Return a series of nested lists which describe my structure
- in a form that can be bencoded."""
- return self.root.serialize_to_sexprs()
-
- def serialize_to_file(self, f):
- f.write(bencode.bencode(self.serialize()))
+ def serialize_subtree_to_file(self, f):
+ sexprs = self.root.serialize_node()
+ bencode.bwrite(sexprs, f)
def is_mutable(self):
return self.mutable
node = self.root
while remaining_path:
name = remaining_path[0]
- if name in node.node_children:
- node = node.node_children[name]
- assert isinstance(node, SubTreeNode)
+ try:
+ childnode = node.get(name)
+ except NoSuchChildError:
+ # The node *would* be in this subtree if it existed, but it
+ # doesn't. Leave found_path and remaining_path alone, and
+ # node points at the last parent node that was on the path.
+ break
+ if IDirectoryNode.providedBy(childnode):
+ # recurse
+ node = childnode
found_path.append(name)
remaining_path.pop(0)
continue
- if name in node.child_specifications:
+ else:
# the path takes us out of this subtree and into another
- next_subtree_spec = node.child_specifications[name]
- node = to_node(next_subtree_spec)
+ node = childnode # next subtree node
found_path.append(name)
remaining_path.pop(0)
break
- # The node *would* be in this subtree if it existed, but it
- # doesn't. Leave found_path and remaining_path alone, and node
- # points at the last parent node that was on the path.
- break
return (found_path, node, remaining_path)
+class CHKDirectorySubTreeNode(BaseURINode):
+ implements(ICHKDirectoryNode)
+ prefix = "CHKDirectory"
+
+ def get_uri(self):
+ return self.uri
+
+
class CHKDirectorySubTree(_DirectorySubTree):
# maybe mutable, maybe not
def set_uri(self, uri):
self.old_uri = uri
- def populate_from_node(self, node, parent_is_mutable, downloader):
- node = ICHKDirectoryNode(node)
+ def populate_from_node(self, node, parent_is_mutable, node_maker, downloader):
+ assert ICHKDirectoryNode(node)
self.mutable = parent_is_mutable
d = downloader.download(node.get_uri(), download.Data())
- d.addCallback(self.populate_from_data)
+ d.addCallback(self.populate_from_data, node_maker)
return d
def update(self, prepath, work_queue):
# this needs investigation.
return boxname
+
+class SSKDirectorySubTreeNode(object):
+ implements(INode, ISSKDirectoryNode)
+ prefix = "SSKDirectory"
+
+ def is_directory(self):
+ return False
+ def serialize_node(self):
+ data = (self.read_cap, self.write_cap)
+ return "%s:%s" % (self.prefix, bencode.bencode(data))
+ def populate_node(self, data, node_maker):
+ assert data.startswith(self.prefix + ":")
+ capdata = data[len(self.prefix)+1:]
+ self.read_cap, self.write_cap = bencode.bdecode(capdata)
+
+ def get_read_capability(self):
+ return self.read_cap
+ def get_write_capability(self):
+ return self.write_cap
+
+
class SSKDirectorySubTree(_DirectorySubTree):
def new(self):
def mutation_affects_parent(self):
return False
- def populate_from_node(self, node, parent_is_mutable, downloader):
+ def populate_from_node(self, node, parent_is_mutable, node_maker, downloader):
node = ISSKDirectoryNode(node)
self.read_capability = node.get_read_capability()
self.write_capability = node.get_write_capability()
self.mutable = bool(self.write_capability)
d = downloader.download_ssk(self.read_capability, download.Data())
- d.addCallback(self.populate_from_data)
+ d.addCallback(self.populate_from_data, node_maker)
return d
def set_version(self, version):
other I*Node interfaces also implement this one."""
def is_directory():
"""Return True if this node is an internal directory node."""
+ def serialize_node():
+ """Return a data structure which contains enough information to build
+ this node again in the future (by calling vdrive.make_node(). For
+ IDirectoryNodes, this will be a list. For all other nodes this will
+ be a string."""
+ def populate_node(data, node_maker):
+ """vdrive.make_node() will first use the prefix inside 'data' to
+ decide what kind of Node to create. It will then call this function
+ to populate the new Node from the data returned by serialize_node."""
class IFileNode(Interface):
"""This is a file which can be retrieved."""
+ # TODO: not sure which of these to provide.. should URIs contain "CHK" or
+ # "SSK" in them? Or should that be a detail of IDownloader?
def get_uri():
"""Return the URI of the target file. This URI can be passed
to an IDownloader to retrieve the data."""
+ def download(downloader, target):
+ """Download the file to the given target (using the provided
+ downloader). Return a deferred that fires (with 'target') when the
+ download is complete."""
class IDirectoryNode(Interface):
"""This is a directory which can be listed."""
a DirectoryNode, or it might be a FileNode.
"""
- def populate_from_specification(spec, parent_is_mutable, downloader):
- """Given a specification tuple, arrange to populate this subtree by
- pulling data from some source (possibly the mesh, or the queen, or an
- HTTP server, or the local filesystem). Return a Deferred that will
- fire (with self) when this subtree is ready for use (specifically
- when it is ready for get() and add() calls).
+ def populate_from_node(node, parent_is_mutable, node_maker, downloader):
+ """Subtrees are created by opener.open() being called with an INode
+ which describes both the kind of subtree to be created and a way to
+ obtain its contents. open() uses the node to create a new instance of
+ the appropriate subtree type, then calls this populate_from_node()
+ method.
+
+ Each subtree's populate_from_node() method is expected to use the
+ downloader to obtain a file with the subtree's serialized contents
+ (probably by pulling data from some source, like the mesh, the queen,
+ an HTTP server, or somewhere on the local filesystem), then
+ unserialize them and populate the subtree's state.
+
+ Return a Deferred that will fire (with self) when this subtree is
+ ready for use (specifically when it is ready for get() and add()
+ calls).
"""
- def populate_from_node(node, parent_is_mutable, downloader):
- """Like populate_from_specification."""
-
def populate_from_data(data):
- """Used internally by populate_from_specification. This is called
- with a sequence of bytes that describes the contents of the subtree,
+ """Used internally by populate_from_node. This is called with a
+ sequence of bytes that describes the contents of the subtree,
probably a bencoded tuple or s-expression. Returns self.
"""
- def unserialize(serialized_data):
- """Populate all nodes from serialized_data, previously created by
- calling my serialize() method. 'serialized_data' is a series of
- nested lists (s-expressions), probably recorded in bencoded form."""
-
def is_mutable():
"""This returns True if we have the ability to modify this subtree.
If this returns True, this reference may be adapted to
"""
+ def serialize_subtree_to_file(f):
+ """Create a string which describes my structure and write it to the
+ given filehandle (using only .write()). This string should be
+ suitable for uploading to the mesh or storing in a local file."""
+
def update(prepath, workqueue):
"""Perform and schedule whatever work is necessary to record this
subtree to persistent storage and update the parent at 'prepath'
some subtree is updated which does not require notifying the parent.
"""
- def serialize():
- """Return a series of nested lists which describe my structure
- in a form that can be bencoded."""
-
#class IMutableSubTree(Interface):
# def mutation_affects_parent():
class IVirtualDrive(Interface):
+ def __init__(workqueue, downloader, root_node):
+ pass
+
+ # internal methods
+
+ def make_node(serialized):
+ """Given a string produced by original_node.serialize_node(), produce
+ an equivalent node.
+ """
+
# commands to manipulate files
def list(path):
# TODO
class ICHKDirectoryNode(Interface):
- pass
+ def get_uri():
+ pass
class ISSKDirectoryNode(Interface):
- pass
+ def get_read_capability():
+ pass
+ def get_write_capability():
+ pass
from zope.interface import implements
-from allmydata.filetree import opener
-from allmydata.filetree.interfaces import (IVirtualDrive, ISubTree, IFileNode,
- IDirectoryNode, NoSuchDirectoryError,
- NoSuchChildError, PathAlreadyExistsError,
- PathDoesNotExistError,
- )
+from allmydata.filetree import opener, directory, redirect
+from allmydata.filetree.interfaces import (
+ IVirtualDrive, INode, ISubTree, IFileNode, IDirectoryNode,
+ NoSuchDirectoryError, NoSuchChildError, PathAlreadyExistsError,
+ PathDoesNotExistError,
+ )
from allmydata.upload import IUploadable
+all_node_types = [
+ directory.CHKDirectorySubTreeNode,
+ directory.SSKDirectorySubTreeNode,
+ redirect.LocalFileRedirectionNode,
+ redirect.QueenRedirectionNode,
+ redirect.HTTPRedirectionNode,
+ redirect.QueenOrLocalFileRedirectionNode,
+]
+
class VirtualDrive(object):
implements(IVirtualDrive)
- def __init__(self, workqueue, downloader, root_specification):
+ def __init__(self, workqueue, downloader, root_node):
+ assert INode(root_node)
self.workqueue = workqueue
workqueue.set_vdrive(self)
# TODO: queen?
self.opener = opener.Opener(self.queen, downloader)
- self.root_specification = root_specification
+ self.root_node = root_node
# these methods are used to walk through our subtrees
def _get_root(self):
- return self.opener.open(self.root_specification, False)
+ return self.opener.open(self.root_node, False)
def _get_node(self, path):
d = self._get_closest_node(path)
d.addCallback(_got_closest)
return d
+ # these are called when loading and creating nodes
+ def make_node(self, serialized):
+ # this turns a string into an INode, which contains information about
+ # the file or directory (like a URI), but does not contain the actual
+ # contents. An IOpener can be used later to retrieve the contents
+ # (which means downloading the file if this is an IFileNode, or
+ # perhaps creating a new subtree from the contents)
+
+ # maybe include parent_is_mutable?
+ assert isinstance(serialized, str)
+ colon = serialized.index(":")
+ prefix = serialized[:colon]
+ for node_class in all_node_types:
+ if prefix == node_class.prefix:
+ node = node_class()
+ node.populate_node(serialized, self.make_node)
+ return node
+ raise RuntimeError("unable to handle subtree type '%s'" % prefix)
+
# these are called by the workqueue
def add(self, path, new_node):
projects / tahoe-lafs / tahoe-lafs.git / commitdiff