From: Brian Warner Date: Tue, 26 Jun 2007 03:34:19 +0000 (-0700) Subject: remove old filetree code X-Git-Tag: allmydata-tahoe-0.4.0~31 X-Git-Url: https://git.rkrishnan.org/listings/...?a=commitdiff_plain;h=4a0682874b41543769a60e88498d389d368795f8;p=tahoe-lafs%2Ftahoe-lafs.git remove old filetree code --- diff --git a/setup.py b/setup.py index 5036b8e4..726dd47b 100644 --- a/setup.py +++ b/setup.py @@ -79,7 +79,7 @@ majority of the nodes are no longer available.""", url='http://allmydata.org/', license='GNU GPL', packages=["allmydata", "allmydata.test", "allmydata.util", - "allmydata.filetree", "allmydata.scripts",], + "allmydata.scripts",], package_dir={ "allmydata": "src/allmydata",}, scripts = ["bin/allmydata-tahoe"], package_data={ 'allmydata': ['web/*.xhtml', 'web/*.css'] }, diff --git a/src/allmydata/filetree/__init__.py b/src/allmydata/filetree/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/src/allmydata/filetree/basenode.py b/src/allmydata/filetree/basenode.py deleted file mode 100644 index 44ce1d76..00000000 --- a/src/allmydata/filetree/basenode.py +++ /dev/null @@ -1,21 +0,0 @@ - -from zope.interface import implements -from allmydata.filetree.interfaces import INode - -class BaseDataNode(object): - implements(INode) - prefix = None # must be set by subclass - - def new(self, data): - self.set_base_data(data) - return self - - def get_base_data(self): - raise NotImplementedError # must be provided by subclass - def set_base_data(self, data): - raise NotImplementedError # must be provided by subclass - def serialize_node(self): - return "%s:%s" % (self.prefix, self.get_base_data()) - def populate_node(self, body, node_maker): - self.set_base_data(body) - diff --git a/src/allmydata/filetree/directory.py b/src/allmydata/filetree/directory.py deleted file mode 100644 index 32908f92..00000000 --- a/src/allmydata/filetree/directory.py +++ /dev/null @@ -1,439 +0,0 @@ - -from zope.interface import implements -from twisted.internet import defer -from cStringIO import StringIO -from allmydata.filetree.interfaces import ( - INode, INodeMaker, IDirectoryNode, ISubTree, - ICHKDirectoryNode, ISSKDirectoryNode, - NoSuchChildError, - ) -from allmydata.filetree.basenode import BaseDataNode -from allmydata import download -from allmydata.util import bencode - -# interesting feature ideas: -# pubsub for MutableDirectoryNode: get rapid notification of changes -# caused by someone else -# -# bind a local physical directory to the MutableDirectoryNode contents: -# each time the vdrive changes, update the local drive to match, and -# vice versa. - -from itertools import islice, izip -def in_pairs(iterable): - "s -> (s0,s1), (s2,s3), (s4,s5), ..." - a = islice(iterable, 0, None, 2) - b = islice(iterable, 1, None, 2) - return izip(a, b) - - -class SubTreeNode: - implements(INode, IDirectoryNode) - - def __init__(self, tree): - self.enclosing_tree = tree - self.children = {} -# # subdirectory_node_children maps child name to another SubTreeNode -# # instance. This is only for internal directory nodes. All other -# # nodes are listed in child_specifications instead. -# self.subdirectory_node_children = {} -# # child_specifications maps child name to a specification tuple which -# # describes how to obtain the actual child. For example, if "foo.jpg" -# # in this node represents a CHK-encoded FILE with a uri of "fooURI", -# # then self.child_specifications["foo.jpg"] = ("CHKFILE","fooURI") -# self.child_specifications = {} - - def list(self): - return self.children - - def get(self, childname): - if childname in self.children: - return self.children[childname] - else: - raise NoSuchChildError("no child named '%s'" % (childname,)) - - def get_subtree(self): - return self.enclosing_tree - - def delete(self, childname): - assert self.enclosing_tree.is_mutable() - if childname in self.children: - del self.children[childname] - else: - raise NoSuchChildError("no child named '%s'" % (childname,)) - - def add_subdir(self, childname): - assert childname not in self.children - newnode = SubTreeNode(self.enclosing_tree) - self.children[childname] = newnode - return newnode - - def add(self, childname, node): - assert childname not in self.children - assert INode(node) - self.children[childname] = node - return self - - def serialize_node(self): - # note: this is a one-pass recursive serialization that will result - # in the whole file table being held in memory. This is only - # appropriate for directories with fewer than, say, 10k nodes. If we - # support larger directories, we should turn this into some kind of - # generator instead, and write the serialized data directly to a - # tempfile. - # - # [name1, child1, name2, child2..] - # - # child1 is either a list for subdirs, or a string for non-subdirs - - data = [] - for name in sorted(self.children.keys()): - data.append(name) - data.append(self.children[name].serialize_node()) - return data - - def populate_dirnode(self, data, node_maker): - assert INodeMaker(node_maker) - assert len(data) % 2 == 0 - for (name, child_data) in in_pairs(data): - if isinstance(child_data, (list, tuple)): - child = SubTreeNode(self.enclosing_tree) - child.populate_dirnode(child_data, node_maker) - else: - assert isinstance(child_data, str) - child = node_maker.make_node_from_serialized(child_data) - self.children[name] = child - - def is_leaf_subtree(self): - return False - - -class _DirectorySubTree(object): - """I represent a set of connected directories that all share the same - access control: any given person can read or write anything in this tree - as a group, and it is not possible to give access to some pieces of this - tree and not to others. Read-only access to individual files can be - granted independently, of course, but through an unnamed URI, not as a - subdirectory. - - Each internal directory is represented by a separate Node. - - This is an abstract base class. Individual subclasses will implement - various forms of serialization, persistence, and mutability. - - """ - implements(ISubTree) - - - def new(self): - # create a new, empty directory - self.root = SubTreeNode(self) - self.mutable = True # sure, why not - return self - - def populate_from_node(self, node, parent_is_mutable, node_maker, downloader): - # self.populate_from_node must be defined by the subclass (CHK or - # SSK), since it controls how the spec is interpreted. It will - # probably use the contents of the node to figure out what to - # download from the grid, then pass this downloaded serialized data - # to populate_from_data() - raise NotImplementedError - - def _populate_from_data(self, data, node_maker): - self.root = SubTreeNode(self) - self.root.populate_dirnode(bencode.bdecode(data), node_maker) - return self - - def serialize_subtree_to_file(self, f): - sexprs = self.root.serialize_node() - bencode.bwrite(sexprs, f) - - def is_mutable(self): - return self.mutable - - def get_node_for_path(self, path): - # this is restricted to traversing our own subtree. Returns - # (found_path, node, remaining_path) - found_path = [] - remaining_path = path[:] - node = self.root - while remaining_path: - name = remaining_path[0] - try: - childnode = node.get(name) - except NoSuchChildError: - # The node *would* be in this subtree if it existed, but it - # doesn't. Leave found_path and remaining_path alone, and - # node points at the last parent node that was on the path. - break - if IDirectoryNode.providedBy(childnode): - # recurse - node = childnode - found_path.append(name) - remaining_path.pop(0) - continue - else: - # the path takes us out of this subtree and into another - node = childnode # next subtree node - found_path.append(name) - remaining_path.pop(0) - break - return (found_path, node, remaining_path) - - def put_node_at_path(self, path, new_node): - assert len(path) > 0 - child_name = path[-1] - - # first step: get (or create) the parent directory - node = self.root - for subdir_name in path[:-1]: - # TODO: peeking at private attributes is naughty, but using - # node.get() and catching NoSuchChildError would be slightly - # ugly. Reconsider the IDirectoryNode.get() API. - childnode = node.children.get(subdir_name) - if childnode: - assert IDirectoryNode.providedBy(childnode) - else: - # we have to create new directories until the parent exists - childnode = node.add_subdir(subdir_name) - node = childnode - - # 'node' is now pointing at the parent directory - if child_name in node.children: - # oops, there's already something there. We can replace it. - # TODO: How do we free the subtree that was just orphaned? - node.delete(child_name) - - # now we can finally add the new node - node.add(child_name, new_node) - - def delete_node_at_path(self, path): - assert len(path) > 0 - child_name = path[-1] - - # first step: get the parent directory - node = self.root - for subdir_name in path[:-1]: - subdir_node = node.get(subdir_name) # may raise NoSuchChildError - node = subdir_node - - # 'node' is now pointing at the parent directory. Let's make sure the - # path they want to delete actually exists. We don't really care what - # the child *is*, just that it exists. - node.get(child_name) # may raise NoSuchChildError - - # now delete it - # TODO: How do we free the subtree that was just orphaned? - node.delete(child_name) - - -class LocalFileSubTreeNode(BaseDataNode): - prefix = "LocalFileDirectory" - - def new(self, filename): - self.filename = filename - return self - - def get_base_data(self): - return self.filename - def set_base_data(self, data): - self.filename = data - - def is_leaf_subtree(self): - return False - -class LocalFileSubTree(_DirectorySubTree): - node_class = LocalFileSubTreeNode - - def new(self, filename): - self.filename = filename - return _DirectorySubTree.new(self) - - def populate_from_node(self, node, parent_is_mutable, node_maker, downloader): - self.mutable = True # probably - self.filename = node.filename - f = open(self.filename, "rb") - data = f.read() - f.close() - d = defer.succeed(data) - d.addCallback(self._populate_from_data, node_maker) - return d - - def mutation_modifies_parent(self): - return False - - def create_node_now(self): - return LocalFileSubTreeNode().new(self.filename) - - def _update(self): - f = open(self.filename, "wb") - self.serialize_subtree_to_file(f) - f.close() - - def update_now(self, uploader): - self._update() - return self.create_node_now() - - def update(self, work_queue): - # TODO: this may suffer from the same execute-too-early problem as - # redirect.LocalFileRedirection - self._update() - return None - - -class CHKDirectorySubTreeNode(BaseDataNode): - implements(ICHKDirectoryNode) - prefix = "CHKDirectory" - - def get_base_data(self): - return self.uri - def set_base_data(self, data): - self.uri = data - - def get_uri(self): - return self.uri - - def is_leaf_subtree(self): - return False - - -class CHKDirectorySubTree(_DirectorySubTree): - # maybe mutable, maybe not - node_class = CHKDirectorySubTreeNode - - def new(self): - self.uri = None - return _DirectorySubTree.new(self) - - def set_uri(self, uri): - self.uri = uri - - def populate_from_node(self, node, parent_is_mutable, node_maker, downloader): - assert ICHKDirectoryNode(node) - self.mutable = parent_is_mutable - d = downloader.download(node.get_uri(), download.Data()) - d.addCallback(self._populate_from_data, node_maker) - def _populated(res): - self.uri = node.get_uri() - return self - d.addCallback(_populated) - return d - - def mutation_modifies_parent(self): - return True - - def create_node_now(self): - return CHKDirectorySubTreeNode().new(self.uri) - - def update_now(self, uploader): - f = StringIO() - self.serialize_subtree_to_file(f) - data = f.getvalue() - d = uploader.upload_data(data) - def _uploaded(uri): - self.uri = uri - return self.create_node_now() - d.addCallback(_uploaded) - return d - - def update(self, workqueue): - # this is the CHK form - old_uri = self.uri - f, filename = workqueue.create_tempfile(".chkdir") - self.serialize_subtree_to_file(f) - f.close() - boxname = workqueue.create_boxname() - workqueue.add_upload_chk(filename, boxname) - workqueue.add_delete_tempfile(filename) - workqueue.add_retain_uri_from_box(boxname) - workqueue.add_delete_box(boxname) - if old_uri: - workqueue.add_unlink_uri(old_uri) - # TODO: think about how self.old_uri will get updated. I *think* that - # this whole instance will get replaced, so it ought to be ok. But - # this needs investigation. - - # mutation affects our parent, so we return a boxname for them - return boxname - - -class SSKDirectorySubTreeNode(object): - implements(INode, ISSKDirectoryNode) - prefix = "SSKDirectory" - - def serialize_node(self): - data = (self.read_cap, self.write_cap) - return "%s:%s" % (self.prefix, bencode.bencode(data)) - def populate_node(self, body, node_maker): - self.read_cap, self.write_cap = bencode.bdecode(body) - - def get_read_capability(self): - return self.read_cap - def get_write_capability(self): - return self.write_cap - def set_read_capability(self, read_cap): - self.read_cap = read_cap - def set_write_capability(self, write_cap): - self.write_cap = write_cap - - def is_leaf_subtree(self): - return False - - -class SSKDirectorySubTree(_DirectorySubTree): - node_class = SSKDirectorySubTreeNode - - def new(self): - _DirectorySubTree.new(self) - self.version = 0 - # TODO: populate - return self - - def populate_from_node(self, node, parent_is_mutable, node_maker, downloader): - node = ISSKDirectoryNode(node) - self.read_capability = node.get_read_capability() - self.write_capability = node.get_write_capability() - self.mutable = bool(self.write_capability) - d = downloader.download_ssk(self.read_capability, download.Data()) - d.addCallback(self._populate_from_data, node_maker) - return d - - def set_version(self, version): - self.version = version - - def mutation_modifies_parent(self): - return False - - def create_node_now(self): - node = SSKDirectorySubTreeNode() - node.set_read_capability(self.read_capability) - node.set_write_capability(self.write_capability) - return node - - def update_now(self, uploader): - if not self.write_capability: - raise RuntimeError("This SSKDirectorySubTree is not mutable") - - f = StringIO() - self.serialize_subtree_to_file(f) - data = f.getvalue() - - self.version += 1 - d = uploader.upload_ssk_data(self.write_capability, self.version, data) - d.addCallback(lambda ignored: self.create_node_now()) - return d - - def update(self, workqueue): - # this is the SSK form - f, filename = workqueue.create_tempfile(".sskdir") - self.serialize_subtree_to_file(f) - f.close() - - oldversion = self.version - self.version = self.version + 1 - - workqueue.add_upload_ssk(self.write_capability, oldversion, filename) - workqueue.add_delete_tempfile(filename) - workqueue.add_retain_ssk(self.read_capability) - # mutation does not affect our parent - return None diff --git a/src/allmydata/filetree/file.py b/src/allmydata/filetree/file.py deleted file mode 100644 index 25c9e05b..00000000 --- a/src/allmydata/filetree/file.py +++ /dev/null @@ -1,51 +0,0 @@ - -from zope.interface import implements -from allmydata.filetree.interfaces import INode, IFileNode -from allmydata.filetree.basenode import BaseDataNode -from allmydata.util import bencode - -class CHKFileNode(BaseDataNode): - implements(IFileNode) - prefix = "CHKFile" - - def new(self, uri): - self.uri = uri - return self - - def put_node_at_path(self, path, node): - raise RuntimeError - - def get_base_data(self): - return self.uri - def set_base_data(self, data): - self.uri = data - - def get_uri(self): - return self.uri - - def is_leaf_subtree(self): - return True - -class SSKFileNode(object): - implements(INode, IFileNode) - prefix = "SSKFile" - - def put_node_at_path(self, path, node): - raise RuntimeError - - def serialize_node(self): - data = (self.read_cap, self.write_cap) - return "%s:%s" % (self.prefix, bencode.bencode(data)) - def populate_node(self, data, node_maker): - assert data.startswith(self.prefix + ":") - capdata = data[len(self.prefix)+1:] - self.read_cap, self.write_cap = bencode.bdecode(capdata) - - def get_read_capability(self): - return self.read_cap - def get_write_capability(self): - return self.write_cap - - def is_leaf_subtree(self): - return True - diff --git a/src/allmydata/filetree/interfaces.py b/src/allmydata/filetree/interfaces.py deleted file mode 100644 index 04d18e80..00000000 --- a/src/allmydata/filetree/interfaces.py +++ /dev/null @@ -1,338 +0,0 @@ - -from zope.interface import Interface - -class INode(Interface): - """This is some sort of retrievable node. All objects which implement - other I*Node interfaces also implement this one.""" - - # the INode-implementing class must have an attribute named .prefix which - # contains a string. - - def serialize_node(): - """Return a data structure which contains enough information to build - this node again in the future (by calling - INodeMaker.make_node_from_serialized(). For IDirectoryNodes, this - will be a list. For all other nodes this will be a string of the form - 'prefix:body', where 'prefix' must be the same as the class attribute - .prefix .""" - def populate_node(body, node_maker): - """INodeMaker.make_node_from_serialized() will first use the prefix - from the .prefix attribute to decide what kind of Node to create. - They will then call this populate_node() method with the body to - populate the new Node. 'node_maker' provides INodeMaker, which - provides that same make_node_from_serialized function to create any - internal child nodes that might be necessary.""" - - def is_leaf_subtree(): - """Return True if this node does not refer to a traversable - subtree. When searching for the node that describes a path, the - search will stop at the first leaf node found. IFileNodes should - return True here. - """ -# TODO: there is a slightly confusing mixture of IDirectoryNodes and all -# other kinds of nodes. It is convenient to mix them because that way list() -# can point at nodes of all sorts, but IDirectoryNodes are very different -# than the rest (because they to not represent distinct subtrees). There -# might be a better way to factor this. - -# TODO: 'node' is a problematic term here. It refers to nodes in the graph of -# connected subtrees. It also refers to nodes in the graph of directories and -# links within a single subtree. And the interface named INode is -# unfortunately a homonym with "inode", the data structure we previously used -# to represent information about an uploaded file which was too large to keep -# locally (the list of blockids), which meant the inode itself was uploaded. -# We no longer use inodes, but using a word that sounds just like it may -# cause confusion. - -class IFileNode(Interface): - """This is a file which can be retrieved.""" - # TODO: not sure which of these to provide.. should URIs contain "CHK" or - # "SSK" in them? Or should that be a detail of IDownloader? - def get_uri(): - """Return the URI of the target file. This URI can be passed - to an IDownloader to retrieve the data.""" - def download(downloader, target): - """Download the file to the given target (using the provided - downloader). Return a deferred that fires (with 'target') when the - download is complete.""" - -class IDirectoryNode(Interface): - """This is a directory which can be listed.""" - # these calls do not modify the subtree - def list(): - """Return a dictionary mapping each childname to a node. These nodes - implement various I*Node interfaces depending upon what they can do.""" - def get(childname): - """Return a child node. Raises NoSuchChildError if there is no - child of that name.""" - def get_subtree(): - """Return the ISubTree which contains this node.""" - - # the following calls modify the subtree. After calling them, you must - # tell the enclosing subtree to serialize and upload itself. They can - # only be called if this directory node is associated with a mutable - # subtree. - def delete(childname): - """Delete any child referenced by this name.""" - def add_subdir(childname): - """Create a new directory node, and return it.""" - def add(childname, node): - """Add a new node to this path. Returns self.""" - -class ISubTree(Interface): - """A subtree is a collection of Nodes: files, directories, other trees. - - A subtree represents a set of connected directories and files that all - share the same access control: any given person can read or write - anything in this tree as a group, and it is not possible to give access - to some pieces of this tree and not to others. Read-only access to - individual files can be granted independently, of course, but through an - unnamed URI, not as a subdirectory. - - Each internal directory is represented by a separate Node. This might be - a DirectoryNode, or it might be a FileNode. - """ - - # All ISubTree-providing instances must have a class-level attribute - # named .node_class which references the matching INode-providing class. - # This is used by the ISubTreeMaker to turn nodes into subtrees. - - def populate_from_node(node, parent_is_mutable, node_maker, downloader): - """Subtrees are created by ISubTreeMaker.open() being called with an - INode which describes both the kind of subtree to be created and a - way to obtain its contents. open() uses the node to create a new - instance of the appropriate subtree type, then calls this - populate_from_node() method. - - Each subtree's populate_from_node() method is expected to use the - downloader to obtain a file with the subtree's serialized contents - (probably by pulling data from some source, like the grid, the vdrive - server, an HTTP server, or somewhere on the local filesystem), then - unserialize them and populate the subtree's state. - - Return a Deferred that will fire (with self) when this subtree is - ready for use (specifically when it is ready for get() and add() - calls). - """ - - def is_mutable(): - """This returns True if we have the ability to modify this subtree. - If this returns True, this reference may be adapted to - IMutableSubTree to actually exercise these mutation rights. - """ - - def mutation_modifies_parent(): - """This returns True if any modification to this subtree will result - in it getting a new identity, and thus requiring its parent be - notified. This is True for CHKDirectorySubTree, but False for - SSKDirectorySubTree and all redirections. - """ - - def get_node_for_path(path): - """Ask this subtree to follow the path through its internal nodes. - - Returns a tuple of (found_path, node, remaining_path). This method - operations synchronously, and does not return a Deferred. - - (found_path=path, found_node, []) - If the path terminates within this subtree, found_path=path and - remaining_path=[], and the node will be an internal IDirectoryNode. - - (found_path, last_node, remaining_path) - If the path does not terminate within this subtree but neither does - it exit this subtree, the last internal IDirectoryNode that *was* on - the path will be returned in 'node'. The path components that led to - this node will be in found_path, and the remaining components will be - in remaining_path. If you want to create the target node, loop over - remaining_path as follows:: - - while remaining_path: - node = node.add_subdir(remaining_path.pop(0)) - - (found_path, exit_node, remaining_path) - If the path leaves this subtree, 'node' will be a different kind of - INode (probably one that points at a child directory of some sort), - found_path will be the components that led to this point, and - remaining_path will be the remaining components. If you still wish to - locate the target, use 'node' to open a new subtree, then provide - 'remaining_path' to the new subtree's get_node_for_path() method. - - """ - - def put_node_at_path(path, node): - """Add the given node to this subtree, at 'path'. - - This may create internal directory subnodes as necessary. This must - run synchronously, and returns None. - """ - - def delete_node_at_path(path): - """Delete the node at the the given path. - - This must run synchronously, and returns None. - """ - - def serialize_subtree_to_file(f): - """Create a string which describes my structure and write it to the - given filehandle (using only .write()). This string should be - suitable for uploading to the grid or storing in a local file.""" - - def update_now(uploader): - """Perform whatever work is necessary to record this subtree to - persistent storage. - - This returns an INode, or a Deferred that fires (with an INode) when - the subtree has been persisted. - - For directory subtrees, this will cause the subtree to serialize - itself to a file, then upload this file to the grid, then create an - INode-providing instance which describes where the file wound up. For - redirections, this will cause the subtree to modify the redirection's - persistent storage, then return the (unmodified) INode that describes - the redirection. - - This form does not use the workqueue. If the node is shut down before - the Deferred fires, a redirection or SSK subtree might be left in its - previous state, or it might have been updated. - """ - - def update(workqueue): - """Perform and schedule whatever work is necessary to record this - subtree to persistent storage. - - Returns a boxname or None, synchronously. This function does not - return a Deferred. - - If the parent subtree needs to be modified with the new identity of - this subtree (i.e. for CHKDirectorySubTree instances), this will - return a boxname in which the serialized INode will be placed once - the added workqueue steps have completed. The caller should add - 'addpath' steps to the workqueue using this boxname (which will - eventually cause recursion on other subtrees, until some subtree is - updated which does not require notifying the parent). update() will - add steps to delete the box at the end of the workqueue. - - If the parent subtree does not need to be modified (i.e. for - SSKDirectorySubTree instances, or redirections), this will return - None. - - This is like update_now(), but uses the workqueue to insure - consistency in the face of node shutdowns. Once our intentions have - been recorded in the workqueue, if the node is shut down before the - upload steps have completed, the update will eventually complete the - next time the node is started. - """ - - def create_node_now(): - # TODO: this is no longer just for testing.. vdrive.addpath needs it - """FOR TESTING ONLY. Immediately create and return an INode which - describes the current state of this subtree. This does not perform - any upload or persistence work, and thus depends upon any internal - state having been previously set correctly. In general this will - return the correct value for subtrees which have just been created - (and not yet mutated). It will also return the correct value for - subtrees which do not change their identity when they are mutated - (SSKDirectorySubTrees and redirections). - """ - -class INodeMaker(Interface): - def make_node_from_serialized(serialized): - """Turn a string into an INode, which contains information about the - file or directory (like a URI), but does not contain the actual - contents. An ISubTreeMaker can be used later to retrieve the contents - (which means downloading the file if this is an IFileNode, or perhaps - creating a new subtree from the contents).""" - -class ISubTreeMaker(Interface): - def make_subtree_from_node(node, parent_is_mutable): - """Turn an INode into an ISubTree. - - I accept an INode-providing specification of a subtree, and return a - Deferred that fires with an ISubTree-providing instance. I will - perform network IO and download the serialized data that the INode - references, if necessary, or ask the vdrive server (or other provider) - for a pointer, or read it from local disk. - """ - - -class IVirtualDrive(Interface): - - def __init__(workqueue, downloader, root_node): - pass - - # commands to manipulate files - - def list(path): - """List the contents of the directory at the given path. - - 'path' is a list of strings (empty to refer to the root directory) - and must refer to a DIRECTORY node. This method returns a Deferred - that fires with a dictionary that maps strings to filetypes. The - strings are useful as path name components. The filetypes are - Interfaces: either IDirectoryNode if path+[childname] can be used in - a 'list' method, or IFileNode if path+[childname] can be used in a - 'download' method. - - The Deferred will errback (with NoSuchDirectoryError) if the path - does not point to an actual directory. - """ - - def download(path, target): - """Download the file at the given path to 'target'. - - 'path' must refer to a FILE. 'target' must implement IDownloadTarget. - This returns a Deferred that fires (with 'target') when the download - is complete. - """ - - def upload_data(path, data): - """Upload a string to the given path. The path must not already exist. - - path[:-1] must refer to a writable DIRECTORY node. - - This uses the workqueue, and returns None. - """ - - def upload(path, filename): - """Upload a file from disk to the given path. - - This uses the workqueue, and returns None. - """ - - def delete(path): - """Delete the file or directory at the given path. - - Returns a Deferred that fires (with self) when the delete is - complete. - """ - - def add_node(path, node): - """Add a node to the given path. Use the workqueue. - """ - - # commands to manipulate subtrees - - # ... detach subtree, merge subtree, etc - - -# TODO - -class ICHKDirectoryNode(Interface): - def get_uri(): - pass -class ISSKDirectoryNode(Interface): - def get_read_capability(): - pass - def get_write_capability(): - pass - - - -class NoSuchChildError(Exception): - pass -class NoSuchDirectoryError(Exception): - pass -class PathAlreadyExistsError(Exception): - pass -class PathDoesNotExistError(Exception): - pass diff --git a/src/allmydata/filetree/nodemaker.py b/src/allmydata/filetree/nodemaker.py deleted file mode 100644 index 05a1364a..00000000 --- a/src/allmydata/filetree/nodemaker.py +++ /dev/null @@ -1,41 +0,0 @@ - -from zope.interface import implements -from allmydata.filetree import directory, file, redirect -from allmydata.filetree.interfaces import INodeMaker - -# this list is used by NodeMaker to convert node specification strings (found -# inside the serialized form of subtrees) into Nodes (which live in the -# in-RAM form of subtrees). -all_node_types = [ - directory.LocalFileSubTreeNode, - directory.CHKDirectorySubTreeNode, - directory.SSKDirectorySubTreeNode, - file.CHKFileNode, - file.SSKFileNode, - redirect.LocalFileRedirectionNode, - redirect.VdriveRedirectionNode, - redirect.HTTPRedirectionNode, - redirect.VdriveOrLocalFileRedirectionNode, -] - -class NodeMaker(object): - implements(INodeMaker) - - def make_node_from_serialized(self, serialized): - # this turns a string into an INode, which contains information about - # the file or directory (like a URI), but does not contain the actual - # contents. An ISubTreeMaker can be used later to retrieve the - # contents (which means downloading the file if this is an IFileNode, - # or perhaps creating a new subtree from the contents) - - # maybe include parent_is_mutable? - assert isinstance(serialized, str) - prefix, body = serialized.split(":", 2) - - for node_class in all_node_types: - if prefix == node_class.prefix: - node = node_class() - node.populate_node(body, self) - return node - raise RuntimeError("unable to handle node type '%s'" % prefix) - diff --git a/src/allmydata/filetree/redirect.py b/src/allmydata/filetree/redirect.py deleted file mode 100644 index 90bbcc2b..00000000 --- a/src/allmydata/filetree/redirect.py +++ /dev/null @@ -1,273 +0,0 @@ - -from cStringIO import StringIO -from zope.interface import implements -from twisted.internet import defer - -from allmydata.filetree.interfaces import ISubTree, INodeMaker -from allmydata.filetree.basenode import BaseDataNode -from allmydata.util import bencode - -class LocalFileRedirectionNode(BaseDataNode): - prefix = "LocalFileRedirection" - - def new(self, handle): - self.handle = handle - return self - - def get_base_data(self): - return self.handle - def set_base_data(self, data): - self.handle = data - - def is_leaf_subtree(self): - return False - -class _BaseRedirection(object): - implements(ISubTree) - - def new(self, child_node): - self.child_node = child_node - return self - - def mutation_modifies_parent(self): - return False - - def get_node_for_path(self, path): - return ([], self.child_node, path) - - def put_node_at_path(self, path, node): - assert path == [] - self.child_node = node - - def serialize_subtree_to_file(self, f): - f.write(self.child_node.serialize_node()) - - def _populate_from_data(self, data, node_maker): - assert INodeMaker(node_maker) - self.child_node = node_maker.make_node_from_serialized(data) - return self - - -class LocalFileRedirection(_BaseRedirection): - node_class = LocalFileRedirectionNode - - def new(self, handle, child_node): - self.filename = handle - return _BaseRedirection.new(self, child_node) - - def populate_from_node(self, node, parent_is_mutable, node_maker, downloader): - # return a Deferred that fires (with self) when this node is ready - # for use - - assert isinstance(node, LocalFileRedirectionNode) - self.filename = node.handle - # there is a local file which contains a bencoded serialized subtree - # specification. - - # TODO: will this enable outsiders to cause us to read from arbitrary - # files? Think about this. It is probably a good idea to restrict the - # filename to be a single component, and to always put them in a - # well-known directory that contains nothing else, and maybe make - # them unguessable. - f = open(self.filename, "rb") - data = f.read() - f.close() - # note: we don't cache the contents of the file. TODO: consider - # doing this based upon mtime. It is important that we be able to - # notice if the file has been changed. - d = defer.succeed(data) - d.addCallback(self._populate_from_data, node_maker) - return d - - def is_mutable(self): - return True - - def create_node_now(self): - return LocalFileRedirectionNode().new(self.filename) - - def _update(self): - f = open(self.filename, "wb") - self.serialize_subtree_to_file(f) - f.close() - - def update_now(self, uploader): - self._update() - return self.create_node_now() - - def update(self, workqueue): - # TODO: this happens too early, before earlier items in the workqueue - # have been executed. This might not be a problem, if our update() - # method isn't actually called until everything earlier has been - # executed anyways. Need to ponder this. - self._update() - return None - -class VdriveRedirectionNode(LocalFileRedirectionNode): - prefix = "VdriveRedirection" - -class VdriveRedirection(_BaseRedirection): - node_class = VdriveRedirectionNode - - def new(self, handle): - self.handle = handle - return self - - def populate_from_node(self, node, parent_is_mutable, node_maker, downloader): - # this specifies a handle for which the Vdrive maintains a serialized - # subtree specification. - assert isinstance(node, VdriveRedirectionNode) - self.handle = node.handle - - # TODO: vdrive? - d = self._vdrive.callRemote("lookup_handle", self.handle) - d.addCallback(self._populate_from_data, node_maker) - return d - - def is_mutable(self): - return True # TODO: maybe, maybe not - - def create_node_now(self): - return VdriveRedirectionNode().new(self.handle) - - def update_now(self, uploader): - f = StringIO() - self.serialize_subtree_to_file(f) - d = self._vdrive.callRemote("set_handle", self.handle, f.getvalue()) - def _done(res): - return self.create_node_now() - d.addCallback(_done) - return d - - def update(self, workqueue): - f, filename = workqueue.create_tempfile(".tovdrive") - self.serialize_subtree_to_file(f) - f.close() - workqueue.add_vdrive_update_handle(self.handle, filename) - workqueue.add_delete_tempfile(filename) - return None - -class VdriveOrLocalFileRedirectionNode(LocalFileRedirectionNode): - prefix = "VdriveOrLocalFileRedirection" - -class VdriveOrLocalFileRedirection(_BaseRedirection): - node_class = VdriveOrLocalFileRedirectionNode - - def new(self, handle, child_node): - self.handle = handle - self.version = 0 - self.child_node = child_node - # TODO - return self - - def populate_from_node(self, node, parent_is_mutable, node_maker, downloader): - # there is a local file which contains a bencoded serialized - # subtree specification. The vdrive also has a copy. Whomever has - # the higher version number wins. - assert isinstance(node, VdriveOrLocalFileRedirectionNode) - self.filename = self.handle = node.handle - - f = open(self.filename, "rb") - #local_version, local_data = bencode.bdecode(f.read()) - local_version_and_data = f.read() - f.close() - - # TODO: vdrive? - # TODO: pubsub so we can cache the vdrive's results - d = self._vdrive.callRemote("lookup_handle", self.handle) - d.addCallback(self._choose_winner, local_version_and_data) - d.addCallback(self._populate_from_data, node_maker) - return d - - def _choose_winner(self, vdrive_version_and_data, local_version_and_data): - vdrive_version, vdrive_data = bencode.bdecode(vdrive_version_and_data) - local_version, local_data = bencode.bdecode(local_version_and_data) - if vdrive_version > local_version: - data = vdrive_data - self.version = vdrive_version - else: - data = local_data - self.version = local_version - # NOTE: two layers of bencoding here, TODO - return data - - def is_mutable(self): - return True - - def create_node_now(self): - return VdriveOrLocalFileRedirectionNode().new(self.handle) - - def _update(self): - self.version += 1 - f = StringIO() - self.serialize_subtree_to_file(f) - version_and_data = bencode.bencode((self.version, f.getvalue())) - return version_and_data - - def update_now(self, uploader): - version_and_data = self._update() - f = open(self.filename, "wb") - f.write(version_and_data) - f.close() - - d = self._vdrive.callRemote("set_handle", self.handle, version_and_data) - def _done(res): - return self.create_node_now() - d.addCallback(_done) - return d - - def update(self, workqueue): - version_and_data = self._update() - # TODO: this may have the same problem as LocalFileRedirection.update - f = open(self.filename, "wb") - f.write(version_and_data) - f.close() - - f, filename = workqueue.create_tempfile(".tovdrive") - self.serialize_subtree_to_file(f) - f.close() - workqueue.add_vdrive_update_handle(self.handle, filename) - workqueue.add_delete_tempfile(filename) - return None - -class HTTPRedirectionNode(BaseDataNode): - prefix = "HTTPRedirection" - - def new(self, url): - self.url = url - return self - - def get_base_data(self): - return self.url - def set_base_data(self, data): - self.url = data - - def is_leaf_subtree(self): - return False - -class HTTPRedirection(_BaseRedirection): - node_class = HTTPRedirectionNode - - def new(self, url): - self.url = url - - def populate_from_node(self, node, parent_is_mutable, node_maker, downloader): - # this specifies a URL at which there is a bencoded serialized - # subtree specification. - self.url = node.url - assert isinstance(node, HTTPRedirectionNode) - from twisted.web import client - d = client.getPage(self.url) - d.addCallback(self._populate_from_data, node_maker) - return d - - def is_mutable(self): - return False - - def create_node_now(self): - return HTTPRedirectionNode().new(self.url) - - def update_now(self, uploader): - raise RuntimeError("HTTPRedirection is not mutable") - - def update(self, workqueue): - raise RuntimeError("HTTPRedirection is not mutable") diff --git a/src/allmydata/filetree/vdrive.py b/src/allmydata/filetree/vdrive.py deleted file mode 100644 index 883a1750..00000000 --- a/src/allmydata/filetree/vdrive.py +++ /dev/null @@ -1,352 +0,0 @@ - -import os.path -from zope.interface import implements -from twisted.internet import defer -from allmydata.filetree import directory, redirect -from allmydata.filetree.interfaces import ( - IVirtualDrive, ISubTreeMaker, - INodeMaker, INode, ISubTree, IFileNode, IDirectoryNode, - NoSuchDirectoryError, NoSuchChildError, PathAlreadyExistsError, - PathDoesNotExistError, - ) -from allmydata.interfaces import IDownloader, IUploader, IWorkQueue - -from allmydata.filetree.nodemaker import NodeMaker - -all_openable_subtree_types = [ - directory.LocalFileSubTree, - directory.CHKDirectorySubTree, - directory.SSKDirectorySubTree, - redirect.LocalFileRedirection, - redirect.VdriveRedirection, - redirect.VdriveOrLocalFileRedirection, - redirect.HTTPRedirection, - ] - -class SubTreeMaker(object): - implements(ISubTreeMaker) - - def __init__(self, downloader): - # this is created with everything it might need to download and - # create subtrees. That means a Downloader and in the future (?) a - # reference to the vdrive. - assert IDownloader(downloader) - self._downloader = downloader - self._node_maker = NodeMaker() - self._cache = {} - - def _create(self, node, parent_is_mutable): - assert INode(node) - assert INodeMaker(self._node_maker) - for subtree_class in all_openable_subtree_types: - if isinstance(node, subtree_class.node_class): - subtree = subtree_class() - d = subtree.populate_from_node(node, - parent_is_mutable, - self._node_maker, - self._downloader) - return d - raise RuntimeError("unable to handle subtree specification '%s'" - % (node,)) - - def make_subtree_from_node(self, node, parent_is_mutable): - assert INode(node) - assert not isinstance(node, IDirectoryNode) - - # is it in cache? To check this we need to use the node's serialized - # form, since nodes are instances and don't compare by value - node_s = node.serialize_node() - if node_s in self._cache: - return defer.succeed(self._cache[node_s]) - - d = defer.maybeDeferred(self._create, node, parent_is_mutable) - d.addCallback(self._add_to_cache, node_s) - return d - - def _add_to_cache(self, subtree, node_s): - self._cache[node_s] = subtree - # TODO: remove things from the cache eventually - return subtree - - - -class VirtualDrive(object): - implements(IVirtualDrive) - debug = False - - def __init__(self, workqueue, downloader, uploader, root_node): - assert IWorkQueue(workqueue) - assert IDownloader(downloader) - assert IUploader(uploader) - assert INode(root_node) - self.workqueue = workqueue - workqueue.set_vdrive(self) - workqueue.set_uploader(uploader) - self._downloader = downloader - self._uploader = uploader - self.root_node = root_node - self.subtree_maker = SubTreeMaker(downloader) - - # these methods are used to walk through our subtrees - - def _get_root(self): - return self.subtree_maker.make_subtree_from_node(self.root_node, False) - - def _get_node(self, path): - d = self._get_closest_node(path) - def _got_node((node, remaining_path)): - if remaining_path: - return None - return node - d.addCallback(_got_node) - return d - - def _get_closest_node(self, path): - """Find the closest directory node parent for the desired path. - Return a Deferred that fires with (node, remaining_path). - """ - d = self._get_root() - d.addCallback(self._get_closest_node_1, path) - return d - - def _get_closest_node_1(self, subtree, path): - (found_path, node, remaining_path) = subtree.get_node_for_path(path) - parent_is_mutable = subtree.is_mutable() - if IDirectoryNode.providedBy(node) or node.is_leaf_subtree(): - # traversal done - return (node, remaining_path) - # otherwise, we must open and recurse into a new subtree - d = self.subtree_maker.make_subtree_from_node(node, parent_is_mutable) - def _opened(next_subtree): - next_subtree = ISubTree(next_subtree) - return self._get_closest_node_1(next_subtree, remaining_path) - d.addCallback(_opened) - return d - - def _get_directory(self, path): - """Return a Deferred that fires with the IDirectoryNode at the given - path, or raise NoSuchDirectoryError if there is no such node. This - will traverse subtrees as necessary.""" - d = self._get_node(path) - def _got_directory(node): - if not node: - raise NoSuchDirectoryError - assert IDirectoryNode(node) - return node - d.addCallback(_got_directory) - return d - - def _get_file(self, path): - """Return a Deferred that files with an IFileNode at the given path, - or raises a NoSuchDirectoryError or NoSuchChildError, or some other - error if the path refers to something other than a file.""" - d = self._get_node(path) - def _got_node(node): - if not node: - raise NoSuchChildError - return IFileNode(node) - d.addCallback(_got_node) - return d - - def _get_file_uri(self, path): - d = self._get_file(path) - d.addCallback(lambda filenode: filenode.get_uri()) - return d - - def _child_should_not_exist(self, path): - d = self._get_node(path) - def _got_node(node): - if node is not None: - raise PathAlreadyExistsError - d.addCallback(_got_node) - return d - - def _child_should_exist(self, path): - d = self._get_node(path) - def _got_node(node): - if node is None: - raise PathDoesNotExistError - d.addCallback(_got_node) - return d - - def _get_closest_node_and_prepath(self, path): - d = self._get_closest_node(path) - def _got_closest((node, remaining_path)): - prepath_len = len(path) - len(remaining_path) - prepath = path[:prepath_len] - assert path[prepath_len:] == remaining_path, "um, path=%s, prepath=%s, prepath_len=%d, remaining_path=%s" % (path, prepath, prepath_len, remaining_path) - return (prepath, node, remaining_path) - d.addCallback(_got_closest) - return d - - def get_subtrees_for_path(self, path): - # compute a list of [(subtree1, subpath1), ...], which represents - # which parts of 'path' traverse which subtrees. This can be used to - # present the virtual drive to the user in a form that includes - # redirection nodes (which do not consume path segments), or to - # figure out which subtrees need to be updated when the identity of a - # lower subtree (i.e. CHK) is changed. - - # TODO: it might be useful to add some items to the return value. - # Like if there is a node already present at that path, to return it. - d = self._get_root() - results = [] - d.addCallback(self._get_subtrees_for_path_1, results, path) - return d - - def _get_subtrees_for_path_1(self, subtree, results, path): - (found_path, node, remaining_path) = subtree.get_node_for_path(path) - if IDirectoryNode.providedBy(node): - # traversal done. We are looking at the final subtree, and the - # entire path (found_path + remaining_path) will live in here. - r = (subtree, (found_path + remaining_path)) - results.append(r) - return results - if node.is_leaf_subtree(): - # for this assert to fail, we found a File or something where we - # were expecting to find another subdirectory. - assert len(remaining_path) == 0 - results.append((subtree, found_path)) - return results - # otherwise we must open and recurse into a new subtree - results.append((subtree, found_path)) - parent_is_mutable = subtree.is_mutable() - d = self.subtree_maker.make_subtree_from_node(node, parent_is_mutable) - def _opened(next_subtree): - next_subtree = ISubTree(next_subtree) - return self._get_subtrees_for_path_1(next_subtree, results, - remaining_path) - d.addCallback(_opened) - return d - - - # these are called by the workqueue - - def addpath_with_node(self, path, new_node): - new_node_boxname = self.workqueue.create_boxname(new_node) - self.workqueue.add_delete_box(new_node_boxname) - return self.addpath(path, new_node_boxname) - - def addpath(self, path, new_node_boxname): - # this adds a block of steps to the workqueue which, when complete, - # will result in the new_node existing in the virtual drive at - # 'path'. - - # First we figure out which subtrees are involved - d = self.get_subtrees_for_path(path) - - # then we walk through them from the bottom, arranging to modify them - # as necessary - def _got_subtrees(subtrees, new_node_boxname): - for (subtree, subpath) in reversed(subtrees): - if self.debug: - print "SUBTREE", subtree, subpath - assert subtree.is_mutable() - must_update = subtree.mutation_modifies_parent() - subtree_node = subtree.create_node_now() - new_subtree_boxname = None - if must_update: - new_subtree_boxname = self.workqueue.create_boxname() - self.workqueue.add_delete_box(new_subtree_boxname) - self.workqueue.add_modify_subtree(subtree_node, subpath, - new_node_boxname, - new_subtree_boxname) - # the box filled by the modify_subtree will be propagated - # upwards - new_node_boxname = new_subtree_boxname - else: - # the buck stops here - self.workqueue.add_modify_subtree(subtree_node, subpath, - new_node_boxname) - return - d.addCallback(_got_subtrees, new_node_boxname) - return d - - def deletepath(self, path): - if self.debug: - print "DELETEPATH(%s)" % (path,) - return self.addpath(path, None) - - def modify_subtree(self, subtree_node, localpath, new_node, - new_subtree_boxname=None): - # TODO: I'm lying here, we don't know who the parent is, so we can't - # really say whether they're mutable or not. But we're pretty sure - # that the new subtree is supposed to be mutable, because we asserted - # that earlier (although I suppose perhaps someone could change a - # VdriveRedirection or an SSK file while we're offline in the middle - # of our workqueue..). Tell the new subtree that their parent is - # mutable so we can be sure it will believe that it itself is - # mutable. - parent_is_mutable = True - d = self.subtree_maker.make_subtree_from_node(subtree_node, - parent_is_mutable) - def _got_subtree(subtree): - assert subtree.is_mutable() - if new_node: - subtree.put_node_at_path(localpath, new_node) - else: - subtree.delete_node_at_path(localpath) - return subtree.update_now(self._uploader) - d.addCallback(_got_subtree) - if new_subtree_boxname: - d.addCallback(lambda new_subtree_node: - self.workqueue.write_to_box(new_subtree_boxname, - new_subtree_node)) - return d - - - # these are user-visible - - def list(self, path): - assert isinstance(path, list) - d = self._get_directory(path) - d.addCallback(lambda node: node.list()) - return d - - def download(self, path, target): - # TODO: does this mean download it right now? or schedule it in the - # workqueue for eventual download? should we add download steps to - # the workqueue? - assert isinstance(path, list) - d = self._get_file_uri(path) - d.addCallback(lambda uri: self._downloader.download(uri, target)) - return d - - def download_as_data(self, path): - # TODO: this is kind of goofy.. think of a better download API that - # is appropriate for this class - from allmydata import download - target = download.Data() - return self.download(path, target) - - def upload_data(self, path, data): - assert isinstance(path, list) - f, tempfilename = self.workqueue.create_tempfile() - f.write(data) - f.close() - boxname = self.workqueue.create_boxname() - self.workqueue.add_upload_chk(tempfilename, boxname) - self.workqueue.add_addpath(boxname, path) - self.workqueue.add_delete_box(boxname) - self.workqueue.add_delete_tempfile(tempfilename) - - def upload(self, path, filename): - assert isinstance(path, list) - filename = os.path.abspath(filename) - boxname = self.workqueue.create_boxname() - self.workqueue.add_upload_chk(filename, boxname) - self.workqueue.add_addpath(boxname, path) - self.workqueue.add_delete_box(boxname) - - def delete(self, path): - assert isinstance(path, list) - self.workqueue.add_deletepath(path) - - def add_node(self, path, node): - assert isinstance(path, list) - assert INode(node) - assert not IDirectoryNode.providedBy(node) - boxname = self.workqueue.create_boxname(node) - self.workqueue.add_addpath(boxname, path) - self.workqueue.add_delete_box(boxname) - diff --git a/src/allmydata/test/test_filetree_new.py b/src/allmydata/test/test_filetree_new.py deleted file mode 100644 index 9925fb47..00000000 --- a/src/allmydata/test/test_filetree_new.py +++ /dev/null @@ -1,667 +0,0 @@ - -from zope.interface import implements -from twisted.trial import unittest -from twisted.internet import defer -from allmydata.interfaces import IDownloader, IUploader -#from allmydata.filetree.directory import (ImmutableDirectorySubTree, -# SubTreeNode, -# CHKDirectorySubTree) -#from allmydata.filetree.specification import (CHKFileSpecification, -# CHKDirectorySpecification) -from allmydata import workqueue -from cStringIO import StringIO - -class FakeGrid(object): - implements(IDownloader, IUploader) - -""" -class FakeOpener(object): - implements(IOpener) - def __init__(self, objects={}): - self.objects = objects - def open(self, subtree_specification, parent_is_mutable): - #print "open", subtree_specification, subtree_specification.serialize(), parent_is_mutable - return defer.succeed(self.objects[subtree_specification.serialize()]) - - -class FakeWorkQueue(object): - implements(workqueue.IWorkQueue) - def __init__(self): - self.first_commands = [] - self.last_commands = [] - self.tempfile_number = 0 - self.boxname_number = 0 - def dump_commands(self): - return self.first_commands + self.last_commands - def clear_commands(self): - self.first_commands = [] - self.last_commands = [] - - def create_tempfile(self, suffix=""): - self.tempfile_number += 1 - self.first_commands.append("create_tempfile-%d" % self.tempfile_number) - return (StringIO(), "dummy_filename-%d" % self.tempfile_number) - def create_boxname(self): - self.boxname_number += 1 - self.first_commands.append("create_boxname-%d" % self.boxname_number) - return "dummy_boxname-%d" % self.boxname_number - def add_upload_chk(self, source_filename, stash_uri_in_boxname): - self.first_commands.append(("upload_chk", source_filename, - stash_uri_in_boxname)) - def add_upload_ssk(self, source_filename, write_capability, - previous_version): - self.first_commands.append(("upload_ssk", source_filename, - write_capability, previous_version)) - def add_retain_ssk(self, read_capability): - self.last_commands.append(("retain_ssk", read_capability)) - def add_unlink_ssk(self, write_capability): - self.last_commands.append(("unlink_ssk", write_capability)) - def add_retain_uri_from_box(self, boxname): - self.last_commands.append(("retain_uri_from_box", boxname)) - def add_addpath(self, boxname, path): - self.first_commands.append(("addpath", boxname, path)) - def add_unlink_uri(self, uri): - self.last_commands.append(("unlink_uri", uri)) - def add_delete_tempfile(self, filename): - self.first_commands.append(("delete_tempfile", filename)) - def add_delete_box(self, boxname): - self.last_commands.append(("delete_box", boxname)) - - - -class OneSubTree(unittest.TestCase): - def test_create_empty_immutable(self): - st = ImmutableDirectorySubTree() - st.new() - self.failIf(st.is_mutable()) - d = st.get([], FakeOpener()) - def _got_root(root): - self.failUnless(IDirectoryNode.providedBy(root)) - self.failUnlessEqual(root.list(), []) - d.addCallback(_got_root) - return d - - def test_immutable_1(self): - st = ImmutableDirectorySubTree() - st.new() - # now populate it (by modifying the internal data structures) with - # some internal directories - one = SubTreeNode(st) - two = SubTreeNode(st) - three = SubTreeNode(st) - st.root.node_children["one"] = one - st.root.node_children["two"] = two - two.node_children["three"] = three - - # now examine it - self.failIf(st.is_mutable()) - o = FakeOpener() - d = st.get([], o) - def _got_root(root): - self.failUnless(IDirectoryNode.providedBy(root)) - self.failUnlessEqual(root.list(), ["one", "two"]) - d.addCallback(_got_root) - d.addCallback(lambda res: st.get(["one"], o)) - def _got_one(_one): - self.failUnlessIdentical(one, _one) - self.failUnless(IDirectoryNode.providedBy(_one)) - self.failUnlessEqual(_one.list(), []) - d.addCallback(_got_one) - d.addCallback(lambda res: st.get(["two"], o)) - def _got_two(_two): - self.failUnlessIdentical(two, _two) - self.failUnless(IDirectoryNode.providedBy(_two)) - self.failUnlessEqual(_two.list(), ["three"]) - d.addCallback(_got_two) - d.addCallback(lambda res: st.get(["two", "three"], o)) - def _got_three(_three): - self.failUnlessIdentical(three, _three) - self.failUnless(IDirectoryNode.providedBy(_three)) - self.failUnlessEqual(_three.list(), []) - d.addCallback(_got_three) - d.addCallback(lambda res: st.get(["missing"], o)) - d.addCallback(self.failUnlessEqual, None) - return d - - def test_mutable_1(self): - o = FakeOpener() - wq = FakeWorkQueue() - st = MutableCHKDirectorySubTree() - st.new() - st.set_uri(None) - self.failUnless(st.is_mutable()) - d = st.get([], o) - def _got_root(root): - self.failUnless(IDirectoryNode.providedBy(root)) - self.failUnlessEqual(root.list(), []) - d.addCallback(_got_root) - file_three = CHKFileSpecification() - file_three.set_uri("file_three_uri") - d.addCallback(lambda res: st.add(["one", "two", "three"], file_three, - o, wq)) - d.addCallback(lambda res: st.get(["one"], o)) - def _got_one(one): - self.failUnless(IDirectoryNode.providedBy(one)) - self.failUnlessEqual(one.list(), ["two"]) - d.addCallback(_got_one) - d.addCallback(lambda res: st.get(["one", "two"], o)) - def _got_two(two): - self.failUnless(IDirectoryNode.providedBy(two)) - self.failUnlessEqual(two.list(), ["three"]) - self.failUnlessIdentical(two.child_specifications["three"], - file_three) - d.addCallback(_got_two) - return d - - def test_addpath(self): - o = FakeOpener() - wq = FakeWorkQueue() - st = MutableCHKDirectorySubTree() - st.new() - st.set_uri(None) - file_three = CHKFileSpecification() - file_three.set_uri("file_three_uri") - d = st.add(["one", "two", "three"], file_three, o, wq) - def _done(res): - expected = [ - "create_tempfile-1", - "create_boxname-1", - ('upload_chk', 'dummy_filename-1', 'dummy_boxname-1'), - ('delete_tempfile', 'dummy_filename-1'), - ('addpath', 'dummy_boxname-1', []), - ('retain_uri_from_box', 'dummy_boxname-1'), - ('delete_box', 'dummy_boxname-1'), - ('unlink_uri', None), - ] - self.failUnlessEqual(wq.dump_commands(), expected) - #print - #for c in wq.dump_commands(): - # print c - d.addCallback(_done) - return d - - def test_serialize(self): - st = ImmutableDirectorySubTree() - st.new() - one = SubTreeNode(st) - two = SubTreeNode(st) - three = SubTreeNode(st) - st.root.node_children["one"] = one - st.root.node_children["two"] = two - two.node_children["three"] = three - file_four = CHKFileSpecification() - file_four.set_uri("file_four_uri") - two.child_specifications["four"] = file_four - data = st.serialize() - st_new = ImmutableDirectorySubTree() - st_new.unserialize(data) - - st_four = ImmutableDirectorySubTree() - st_four.new() - st_four.root.node_children["five"] = SubTreeNode(st_four) - - o = FakeOpener({("CHK-File", "file_four_uri"): st_four}) - d = st.get([], o) - def _got_root(root): - self.failUnless(IDirectoryNode.providedBy(root)) - self.failUnlessEqual(root.list(), ["one", "two"]) - d.addCallback(_got_root) - d.addCallback(lambda res: st.get(["two"], o)) - def _got_two(_two): - self.failUnless(IDirectoryNode.providedBy(_two)) - self.failUnlessEqual(_two.list(), ["four", "three"]) - d.addCallback(_got_two) - - d.addCallback(lambda res: st.get(["two", "four"], o)) - def _got_four(_four): - self.failUnless(IDirectoryNode.providedBy(_four)) - self.failUnlessEqual(_four.list(), ["five"]) - d.addCallback(_got_four) - -class MultipleSubTrees(unittest.TestCase): - - def test_open(self): - st = ImmutableDirectorySubTree() - st.new() - # populate it with some internal directories and child links and see - # if we can follow them - one = SubTreeNode(st) - two = SubTreeNode(st) - three = SubTreeNode(st) - st.root.node_children["one"] = one - st.root.node_children["two"] = two - two.node_children["three"] = three - - def test_addpath(self): - wq = FakeWorkQueue() - st1 = MutableCHKDirectorySubTree() - st1.new() - st1.set_uri(None) - one = SubTreeNode(st1) - two = SubTreeNode(st1) - st1.root.node_children["one"] = one - one.node_children["two"] = two - three = CHKDirectorySpecification() - three.set_uri("dir_three_uri") - two.child_specifications["three"] = three - - st2 = MutableCHKDirectorySubTree() - st2.new() - st2.set_uri(None) - four = SubTreeNode(st2) - five = SubTreeNode(st2) - st2.root.node_children["four"] = four - four.node_children["five"] = five - - file_six = CHKFileSpecification() - file_six.set_uri("file_six_uri") - - o = FakeOpener({("CHK-Directory", "dir_three_uri"): st2}) - - d = defer.succeed(None) - d.addCallback(lambda res: - st1.get(["one", "two", "three", "four", "five"], o)) - def _got_five(res): - self.failUnless(IDirectoryNode.providedBy(res)) - self.failUnlessIdentical(res, five) - d.addCallback(_got_five) - - d.addCallback(lambda res: - st1.add(["one", "two", "six"], - file_six, o, wq)) - def _done(res): - expected = [ - "create_tempfile-1", - "create_boxname-1", - ('upload_chk', 'dummy_filename-1', 'dummy_boxname-1'), - ('delete_tempfile', 'dummy_filename-1'), - # one/two/six only modifies the top-most CHKDirectory, so - # the addpath that gets scheduled is targeted at the root - ('addpath', 'dummy_boxname-1', []), - ('retain_uri_from_box', 'dummy_boxname-1'), - ('delete_box', 'dummy_boxname-1'), - ('unlink_uri', None), - ] - self.failUnlessEqual(wq.dump_commands(), expected) - wq.clear_commands() - d.addCallback(_done) - - d.addCallback(lambda res: - st1.add(["one", "two", "three", "four", "six"], - file_six, o, wq)) - def _done2(res): - expected = [ - "create_tempfile-2", - "create_boxname-2", - ('upload_chk', 'dummy_filename-2', 'dummy_boxname-2'), - ('delete_tempfile', 'dummy_filename-2'), - # one/two/three/four/six modifies the lower CHKDirectory, so - # we schedule an addpath of the link that points from the - # upper CHKDirectory to the lower one (at one/two/three). - ('addpath', 'dummy_boxname-2', ["one", "two", "three"]), - ('retain_uri_from_box', 'dummy_boxname-2'), - ('delete_box', 'dummy_boxname-2'), - ('unlink_uri', None), - ] - self.failUnlessEqual(wq.dump_commands(), expected) - d.addCallback(_done2) - - - return d - -del OneSubTree -del MultipleSubTrees - -class Redirect(unittest.TestCase): - pass -""" - -import os.path -from twisted.python.failure import Failure -from allmydata.filetree import directory, redirect, vdrive -from allmydata.filetree.interfaces import (ISubTree, INode, IDirectoryNode, - IFileNode, NoSuchDirectoryError, - NoSuchChildError) -from allmydata.filetree.file import CHKFileNode -from allmydata import upload -from allmydata.interfaces import IDownloader -from allmydata.util import bencode - -class Utils(unittest.TestCase): - def test_in_pairs(self): - l = range(8) - pairs = list(directory.in_pairs(l)) - self.failUnlessEqual(pairs, [(0,1), (2,3), (4,5), (6,7)]) - -class FakeGrid(object): - implements(IDownloader, IUploader) - debug = False - - def __init__(self): - self.files = {} - - def upload(self, uploadable): - uri = "stub-uri-%d" % len(self.files) - if self.debug: - print "FakeGrid.upload -> %s" % uri - assert upload.IUploadable.providedBy(uploadable) - f = uploadable.get_filehandle() - data = f.read() - uploadable.close_filehandle(f) - self.files[uri] = data - return defer.succeed(uri) - - def upload_filename(self, filename): - if self.debug: - print "FakeGrid.upload_filename(%s)" % filename - return self.upload(upload.FileName(filename)) - - def upload_data(self, data): - if self.debug: - print "FakeGrid.upload_data(%s)" % data - return self.upload(upload.Data(data)) - - def download(self, uri, target): - if self.debug: - print "FakeGrid.download(%s)" % uri - target.open() - target.write(self.files[uri]) - target.close() - return defer.maybeDeferred(target.finish) - - -class VDrive(unittest.TestCase): - - def makeVirtualDrive(self, basedir, root_node=None, grid=None): - wq = workqueue.WorkQueue(os.path.join("test_filetree", - "VDrive", - basedir, "1.workqueue")) - if grid: - assert IUploader.providedBy(grid) - assert IDownloader.providedBy(grid) - dl = ul = grid - else: - dl = ul = FakeGrid() - if not root_node: - root_node = directory.LocalFileSubTreeNode() - root_node.new("rootdirtree.save") - v = vdrive.VirtualDrive(wq, dl, ul, root_node) - return v - - def makeLocalTree(self, basename): - # create a LocalFileRedirection pointing at a LocalFileSubTree. - # Returns a VirtualDrive instance. - topdir = directory.LocalFileSubTree().new("%s-dirtree.save" % basename) - topdir.update_now(None) - root = redirect.LocalFileRedirection().new("%s-root" % basename, - topdir.create_node_now()) - root.update_now(None) - v = self.makeVirtualDrive("%s-vdrive" % basename, - root.create_node_now()) - return v - - def makeCHKTree(self, basename): - # create a LocalFileRedirection pointing at a CHKDirectorySubTree. - # Returns a VirtualDrive instance. - grid = FakeGrid() - topdir = directory.CHKDirectorySubTree().new() - d = topdir.update_now(grid) - def _updated(topnode): - root = redirect.LocalFileRedirection() - root.new("%s-root" % basename, topnode) - return root.update_now(grid) - d.addCallback(_updated) - d.addCallback(lambda rootnode: - self.makeVirtualDrive("%s-vdrive" % basename, - rootnode, grid)) - return d - - def failUnlessListsAreEqual(self, list1, list2): - self.failUnlessEqual(sorted(list1), sorted(list2)) - - def failUnlessContentsAreEqual(self, c1, c2): - c1a = dict([(k,v.serialize_node()) for k,v in c1.items()]) - c2a = dict([(k,v.serialize_node()) for k,v in c2.items()]) - self.failUnlessEqual(c1a, c2a) - - def testDirectory(self): - stm = vdrive.SubTreeMaker(FakeGrid()) - - # create an empty directory (stored locally) - subtree = directory.LocalFileSubTree() - subtree.new("dirtree.save") - self.failUnless(ISubTree.providedBy(subtree)) - - # get the root IDirectoryNode (which is still empty) and examine it - (found_path, root, remaining_path) = subtree.get_node_for_path([]) - self.failUnlessEqual(found_path, []) - self.failUnlessEqual(remaining_path, []) - self.failUnless(INode.providedBy(root)) - self.failUnless(IDirectoryNode.providedBy(root)) - self.failUnlessListsAreEqual(root.list().keys(), []) - self.failUnlessIdentical(root.get_subtree(), subtree) - - # now add some children to it - subdir1 = root.add_subdir("subdir1") - file1 = CHKFileNode() - file1.new("uri1") - root.add("foo.txt", file1) - self.failUnlessListsAreEqual(root.list().keys(), - ["foo.txt", "subdir1"]) - self.failUnlessIdentical(root.get("foo.txt"), file1) - subdir1a = root.get("subdir1") - self.failUnlessIdentical(subdir1, subdir1a) - del subdir1a - self.failUnless(IDirectoryNode.providedBy(subdir1)) - self.failUnlessListsAreEqual(subdir1.list().keys(), []) - self.failUnlessIdentical(subdir1.get_subtree(), subtree) - - subdir2 = subdir1.add_subdir("subdir2") - subdir3 = subdir2.add_subdir("subdir3") - subdir4 = subdir2.add_subdir("subdir4") - - subdir2.delete("subdir4") - self.failUnlessListsAreEqual(subdir2.list().keys(), ["subdir3"]) - - del root, subdir1, subdir2, subdir3, subdir4 - # leaving file1 for later use - - # now serialize it and examine the results - f = StringIO() - subtree.serialize_subtree_to_file(f) - data = f.getvalue() - #print data - unpacked = bencode.bdecode(data) - #print unpacked - del f, data, unpacked - - node = subtree.create_node_now() - self.failUnless(isinstance(node, directory.LocalFileSubTreeNode)) - node_s = node.serialize_node() - self.failUnless(isinstance(node_s, str)) - self.failUnless(node_s.startswith("LocalFileDirectory:")) - self.failUnless("dirtree.save" in node_s) - del node, node_s - - d = defer.maybeDeferred(subtree.update_now, None) - def _updated(node): - # now reconstruct it - return stm.make_subtree_from_node(node, False) - d.addCallback(_updated) - - def _opened(new_subtree): - res = new_subtree.get_node_for_path([]) - (found_path, root, remaining_path) = res - self.failUnlessEqual(found_path, []) - self.failUnlessEqual(remaining_path, []) - self.failUnless(INode.providedBy(root)) - self.failUnless(IDirectoryNode.providedBy(root)) - self.failUnlessListsAreEqual(root.list().keys(), - ["foo.txt", "subdir1"]) - file1a = root.get("foo.txt") - self.failUnless(INode(file1a)) - self.failUnless(isinstance(file1a, CHKFileNode)) - self.failUnless(IFileNode(file1a)) - self.failUnlessEqual(file1a.get_uri(), "uri1") - subdir1 = root.get("subdir1") - subdir2 = subdir1.get("subdir2") - self.failUnlessListsAreEqual(subdir2.list().keys(), ["subdir3"]) - subdir2.delete("subdir3") - self.failUnlessListsAreEqual(subdir2.list().keys(), []) - d.addCallback(_opened) - return d - - def shouldFail(self, res, expected_failure, which): - if isinstance(res, Failure): - res.trap(expected_failure) - else: - self.fail("%s was supposed to raise %s, not get '%s'" % - (which, expected_failure, res)) - - def testVdrive(self): - v = self.makeLocalTree("vdrive") - - d = v.list([]) - def _listed(contents): - self.failUnlessEqual(contents, {}) - d.addCallback(_listed) - - child1 = CHKFileNode().new("uri1") - d.addCallback(lambda res: v.add_node(["a"], child1)) - d.addCallback(lambda res: v.workqueue.flush()) - d.addCallback(lambda res: v.list([])) - def _listed2(contents): - self.failUnlessListsAreEqual(contents.keys(), ["a"]) - self.failUnlessContentsAreEqual(contents, {"a": child1}) - d.addCallback(_listed2) - child2 = CHKFileNode().new("uri2") - child3 = CHKFileNode().new("uri3") - d.addCallback(lambda res: v.add_node(["b","c"], child2)) - d.addCallback(lambda res: v.add_node(["b","d"], child3)) - d.addCallback(lambda res: v.workqueue.flush()) - d.addCallback(lambda res: v.list([])) - def _listed3(contents): - self.failUnlessListsAreEqual(contents.keys(), ["a","b"]) - d.addCallback(_listed3) - d.addCallback(lambda res: v.list(["b"])) - def _listed4(contents): - self.failUnlessListsAreEqual(contents.keys(), ["c","d"]) - self.failUnlessContentsAreEqual(contents, - {"c": child2, "d": child3}) - d.addCallback(_listed4) - - d.addCallback(lambda res: v._get_file_uri(["b","c"])) - d.addCallback(self.failUnlessEqual, "uri2") - - d.addCallback(lambda res: v.list(["bogus"])) - d.addBoth(self.shouldFail, NoSuchDirectoryError, "list(bogus)") - - d.addCallback(lambda res: v._get_file_uri(["b", "bogus"])) - d.addBoth(self.shouldFail, NoSuchChildError, "_get_file_uri(b/bogus)") - - return d - - def testUpload(self): - v = self.makeLocalTree("upload") - filename = "upload1" - DATA = "here is some data\n" - f = open(filename, "wb") - f.write(DATA) - f.close() - - rc = v.upload(["a","b","upload1"], filename) - self.failUnlessIdentical(rc, None) - - d = v.workqueue.flush() - - d.addCallback(lambda res: v.list([])) - d.addCallback(lambda contents: - self.failUnlessListsAreEqual(contents.keys(), ["a"])) - d.addCallback(lambda res: v.list(["a"])) - d.addCallback(lambda contents: - self.failUnlessListsAreEqual(contents.keys(), ["b"])) - d.addCallback(lambda res: v.list(["a","b"])) - d.addCallback(lambda contents: - self.failUnlessListsAreEqual(contents.keys(), - ["upload1"])) - d.addCallback(lambda res: v.download_as_data(["a","b","upload1"])) - d.addCallback(self.failUnlessEqual, DATA) - - return d - - def testCHKDirUpload(self): - DATA = "here is some data\n" - filename = "upload1" - f = open(filename, "wb") - f.write(DATA) - f.close() - - d = defer.maybeDeferred(self.makeCHKTree, "chk-upload") - def _made(v): - self.v = v - - rc = v.upload(["a","b","upload1"], filename) - self.failUnlessIdentical(rc, None) - - return v.workqueue.flush() - d.addCallback(_made) - - d.addCallback(lambda res: self.v.list([])) - d.addCallback(lambda contents: - self.failUnlessListsAreEqual(contents.keys(), ["a"])) - d.addCallback(lambda res: self.v.list(["a"])) - d.addCallback(lambda contents: - self.failUnlessListsAreEqual(contents.keys(), ["b"])) - d.addCallback(lambda res: self.v.list(["a","b"])) - d.addCallback(lambda contents: - self.failUnlessListsAreEqual(contents.keys(), - ["upload1"])) - d.addCallback(lambda res: self.v.download_as_data(["a","b","upload1"])) - d.addCallback(self.failUnlessEqual, DATA) - - return d - - def testCHKDirDelete(self): - DATA = "here is some data\n" - filename = "upload1" - f = open(filename, "wb") - f.write(DATA) - f.close() - - d = defer.maybeDeferred(self.makeCHKTree, "chk-delete") - def _made(v): - self.v = v - d.addCallback(_made) - - d.addCallback(lambda r: - self.v.upload(["a","b","upload1"], filename)) - d.addCallback(lambda r: - self.v.upload_data(["a","b","upload2"], DATA)) - d.addCallback(lambda r: - self.v.upload(["a","c","upload3"], filename)) - d.addCallback(lambda r: - self.v.workqueue.flush()) - - d.addCallback(lambda r: self.v.list([])) - d.addCallback(lambda contents: - self.failUnlessListsAreEqual(contents.keys(), ["a"])) - d.addCallback(lambda r: self.v.list(["a"])) - d.addCallback(lambda contents: - self.failUnlessListsAreEqual(contents.keys(), ["b","c"])) - d.addCallback(lambda r: self.v.list(["a","b"])) - d.addCallback(lambda contents: - self.failUnlessListsAreEqual(contents.keys(), - ["upload1", "upload2"])) - #d.addCallback(lambda r: self.v.download_as_data(["a","b","upload1"])) - #d.addCallback(self.failUnlessEqual, DATA) - - # now delete it - d.addCallback(lambda r: self.v.delete(["a","b","upload2"])) - d.addCallback(lambda r: self.v.workqueue.flush()) - d.addCallback(lambda r: self.v.list(["a","b"])) - d.addCallback(lambda contents: - self.failUnlessListsAreEqual(contents.keys(), - ["upload1"])) - - - return d diff --git a/src/allmydata/test/test_vdrive.py b/src/allmydata/test/test_vdrive.py index 472bffcd..4ae201a9 100644 --- a/src/allmydata/test/test_vdrive.py +++ b/src/allmydata/test/test_vdrive.py @@ -255,61 +255,3 @@ class Test(unittest.TestCase): sorted(expected_keys)) return res - -""" -class Traverse(unittest.TestCase): - def make_tree(self, basedir): - os.makedirs(basedir) - root = LocalDirNode(basedir) - self.d1 = d1 = root.add_directory("d1") - self.d2 = d2 = root.add_directory("d2") - root.add_file("a", "a") - root.add_file("b", "b") - d1.add_file("1.a", "1.a") - d1.add_file("1.b", "1.b") - d2.add_file("2.a", "2.a") - d2.add_file("2.b", "2.b") - return root - - def test_one(self): - basedir = "test_vdrive/one" - root = self.make_tree(basedir) - v = vdrive.VDrive() - v.set_root(root) - - d = v.get_dir("") - d.addCallback(lambda dir: self.failUnlessEqual(dir, root)) - d.addCallback(lambda res: v.get_dir("/d1")) - def _check(dir): - self.failUnless(isinstance(dir, LocalDirNode)) - self.failUnlessEqual(dir._basedir, self.d1._basedir) - d.addCallback(_check) - - - d.addCallback(lambda res: v.listdir("")) - d.addCallback(lambda files: - self.failUnlessEqual(sorted(files), - ["a", "b", "d1", "d2"])) - d.addCallback(lambda res: v.listdir("/")) - d.addCallback(lambda files: - self.failUnlessEqual(sorted(files), - ["a", "b", "d1", "d2"])) - d.addCallback(lambda res: v.listdir("d1")) - d.addCallback(lambda files: - self.failUnlessEqual(sorted(files), - ["1.a", "1.b"])) - - d.addCallback(lambda res: v.make_directory("", "d3")) - d.addCallback(lambda res: v.listdir("")) - d.addCallback(lambda files: - self.failUnlessEqual(sorted(files), - ["a", "b", "d1", "d2", "d3"])) - - d.addCallback(lambda res: v.make_directory("d2", "d2.1")) - d.addCallback(lambda res: v.listdir("/d2")) - d.addCallback(lambda files: - self.failUnlessEqual(sorted(files), - ["2.a", "2.b", "d2.1"])) - return d -del Traverse -""" diff --git a/src/allmydata/test/test_workqueue.py b/src/allmydata/test/test_workqueue.py deleted file mode 100644 index 6f205948..00000000 --- a/src/allmydata/test/test_workqueue.py +++ /dev/null @@ -1,166 +0,0 @@ - -import os -from twisted.trial import unittest -from twisted.internet import defer -from allmydata import workqueue -from allmydata.util import idlib -from allmydata.filetree.file import CHKFileNode - -class FakeWorkQueue(workqueue.WorkQueue): - - def __init__(self, basedir): - workqueue.WorkQueue.__init__(self, basedir) - self.dispatched_steps = [] - - def dispatch_step(self, steptype, lines): - self.dispatched_steps.append((steptype, lines)) - return defer.succeed(None) - -class Reuse(unittest.TestCase): - def wq(self, testname): - return FakeWorkQueue("test_workqueue/Reuse/%s/workqueue" % testname) - - def testOne(self): - wq = self.wq("testOne") - # steps must be retained from one session to the next - wq.add_upload_chk("source_filename", "box1") - wq.add_unlink_uri("someuri") - # files in the tmpdir are not: these are either in the process of - # being added or in the process of being removed. - tmpfile = os.path.join(wq.tmpdir, "foo") - f = open(tmpfile, "wb") - f.write("foo") - f.close() - # files created with create_tempfile *are* retained, however - f, filename = wq.create_tempfile() - filename = os.path.join(wq.filesdir, filename) - f.write("bar") - f.close() - - del wq - wq2 = self.wq("testOne") - steps = wq2.get_all_steps() - self.failUnlessEqual(steps[0], ("upload_chk", - ["source_filename", "box1"])) - self.failUnlessEqual(steps[1], ("unlink_uri", ["someuri"])) - self.failIf(os.path.exists(tmpfile)) - self.failUnless(os.path.exists(filename)) - - -class Items(unittest.TestCase): - def wq(self, testname): - return FakeWorkQueue("test_workqueue/Items/%s/workqueue" % testname) - - def testTempfile(self): - wq = self.wq("testTempfile") - (f, filename) = wq.create_tempfile(".chkdir") - self.failUnless(filename.endswith(".chkdir")) - data = "this is some random data: %s\n" % idlib.b2a(os.urandom(15)) - f.write(data) - f.close() - f2 = wq.open_tempfile(filename) - data2 = f2.read() - f2.close() - self.failUnlessEqual(data, data2) - - def testBox(self): - wq = self.wq("testBox") - boxname = wq.create_boxname() - wq.write_to_box(boxname, CHKFileNode().new("uri goes here")) - out = wq.read_from_box(boxname) - self.failUnless(isinstance(out, CHKFileNode)) - self.failUnlessEqual(out.get_uri(), "uri goes here") - - def testCHK(self): - wq = self.wq("testCHK") - wq.add_upload_chk("source_filename", "box1") - wq.add_retain_uri_from_box("box1") - wq.add_addpath("box1", ["home", "warner", "foo.txt"]) - wq.add_delete_box("box1") - wq.add_unlink_uri("olduri") - - self.failUnlessEqual(wq.count_pending_steps(), 5) - stepname, steptype, lines = wq.get_next_step() - self.failUnlessEqual(steptype, "upload_chk") - steps = wq.get_all_steps() - self.failUnlessEqual(steps[0], ("upload_chk", - ["source_filename", "box1"])) - self.failUnlessEqual(steps[1], ("retain_uri_from_box", - ["box1"])) - self.failUnlessEqual(steps[2], ("addpath", - ["box1", "home", "warner", "foo.txt"])) - self.failUnlessEqual(steps[3], ("delete_box", - ["box1"])) - self.failUnlessEqual(steps[4], ("unlink_uri", - ["olduri"])) - - def testCHK2(self): - wq = self.wq("testCHK2") - wq.add_upload_chk("source_filename", "box1") - wq.add_retain_uri_from_box("box1") - wq.add_addpath("box1", ["home", "warner", "foo.txt"]) - wq.add_delete_box("box1") - wq.add_unlink_uri("olduri") - - # then this batch happens a bit later - (f, tmpfilename) = wq.create_tempfile(".chkdir") - f.write("some data") - f.close() - wq.add_upload_chk(os.path.join(wq.filesdir, tmpfilename), "box2") - wq.add_delete_tempfile(tmpfilename) - wq.add_retain_uri_from_box("box2") - wq.add_delete_box("box2") - wq.add_unlink_uri("oldchk") - - self.failUnlessEqual(wq.count_pending_steps(), 10) - steps = wq.get_all_steps() - - self.failUnlessEqual(steps[0], ("upload_chk", - ["source_filename", "box1"])) - self.failUnlessEqual(steps[1], ("retain_uri_from_box", - ["box1"])) - self.failUnlessEqual(steps[2], ("addpath", - ["box1", "home", "warner", "foo.txt"])) - self.failUnlessEqual(steps[3], - ("upload_chk", - [os.path.join(wq.filesdir, tmpfilename), - "box2"])) - self.failUnlessEqual(steps[4], - ("retain_uri_from_box", ["box2"])) - self.failUnlessEqual(steps[5], ("delete_box", - ["box1"])) - self.failUnlessEqual(steps[6], ("unlink_uri", - ["olduri"])) - self.failUnlessEqual(steps[7], - ("delete_tempfile", [tmpfilename])) - self.failUnlessEqual(steps[8], ("delete_box", ["box2"])) - self.failUnlessEqual(steps[9], ("unlink_uri", ["oldchk"])) - - def testRun(self): - wq = self.wq("testRun") - wq.add_upload_chk("source_filename", "box1") - wq.add_retain_uri_from_box("box1") - wq.add_addpath("box1", ["home", "warner", "foo.txt"]) - wq.add_delete_box("box1") - wq.add_unlink_uri("olduri") - - # this tempfile should be deleted after the last step completes - (f, tmpfilename) = wq.create_tempfile(".dummy") - tmpfilename = os.path.join(wq.filesdir, tmpfilename) - f.write("stuff") - f.close() - self.failUnless(os.path.exists(tmpfilename)) - # likewise this unreferenced box should get deleted - boxname = wq.create_boxname() - wq.write_to_box(boxname, CHKFileNode().new("uri here")) - boxfile = os.path.join(wq.boxesdir, boxname) - self.failUnless(os.path.exists(boxfile)) - - d = wq.flush() - def _check(res): - self.failUnlessEqual(len(wq.dispatched_steps), 5) - self.failUnlessEqual(wq.dispatched_steps[0][0], "upload_chk") - self.failIf(os.path.exists(tmpfilename)) - self.failIf(os.path.exists(boxfile)) - d.addCallback(_check) - return d diff --git a/src/allmydata/vdrive.py b/src/allmydata/vdrive.py index bd11576c..8a30e932 100644 --- a/src/allmydata/vdrive.py +++ b/src/allmydata/vdrive.py @@ -3,10 +3,8 @@ import os.path from zope.interface import implements -from twisted.application import service from twisted.internet import defer -from twisted.python import log -from allmydata import upload, download, uri +from allmydata import uri from allmydata.Crypto.Cipher import AES from allmydata.util import hashutil, idlib from allmydata.interfaces import IDirectoryNode, IFileNode @@ -14,179 +12,6 @@ from allmydata.interfaces import IDirectoryNode, IFileNode class NotMutableError(Exception): pass -class VDrive(service.MultiService): - name = "vdrive" - - def set_server(self, vdrive_server): - self.gvd_server = vdrive_server - def set_root(self, root): - self.gvd_root = root - - def dirpath(self, dir_or_path): - if isinstance(dir_or_path, str): - return self.get_dir(dir_or_path) - return defer.succeed(dir_or_path) - - def get_dir(self, path): - """Return a Deferred that fires with a RemoteReference to a - MutableDirectoryNode at the given /-delimited path.""" - d = defer.succeed(self.gvd_root) - if path.startswith("/"): - path = path[1:] - if path == "": - return d - for piece in path.split("/"): - d.addCallback(lambda parent: parent.callRemote("list")) - def _find(table, subdir): - for name,target in table: - if name == subdir: - return target - else: - raise KeyError("no such directory '%s' in '%s'" % - (subdir, [t[0] for t in table])) - d.addCallback(_find, piece) - def _check(subdir): - assert not isinstance(subdir, str), "Hey, %s shouldn't be a string" % subdir - return subdir - d.addCallback(_check) - return d - - def get_uri_from_parent(self, parent, filename): - assert not isinstance(parent, str), "'%s' isn't a directory node" % (parent,) - d = parent.callRemote("list") - def _find(table): - for name,target in table: - if name == filename: - assert isinstance(target, str), "Hey, %s isn't a file" % filename - return target - else: - raise KeyError("no such file '%s' in '%s'" % - (filename, [t[0] for t in table])) - d.addCallback(_find) - return d - - def get_root(self): - return self.gvd_root - - def listdir(self, dir_or_path): - d = self.dirpath(dir_or_path) - d.addCallback(lambda parent: parent.callRemote("list")) - def _list(table): - return [t[0] for t in table] - d.addCallback(_list) - return d - - def put_file(self, dir_or_path, name, uploadable): - """Upload an IUploadable and add it to the virtual drive (as an entry - called 'name', in 'dir_or_path') 'dir_or_path' must either be a - string like 'root/subdir1/subdir2', or a directory node (either the - root directory node returned by get_root(), or a subdirectory - returned by list() ). - - The uploadable can be an instance of allmydata.upload.Data, - FileHandle, or FileName. - - I return a deferred that will fire when the operation is complete. - """ - - log.msg("putting file to '%s'" % name) - ul = self.parent.getServiceNamed("uploader") - d = self.dirpath(dir_or_path) - def _got_dir(dirnode): - d1 = ul.upload(uploadable) - def _add(uri): - d2 = dirnode.callRemote("add_file", name, uri) - d2.addCallback(lambda res: uri) - return d2 - d1.addCallback(_add) - return d1 - d.addCallback(_got_dir) - def _done(res): - log.msg("finished putting file to '%s'" % name) - return res - d.addCallback(_done) - return d - - def put_file_by_filename(self, dir_or_path, name, filename): - return self.put_file(dir_or_path, name, upload.FileName(filename)) - def put_file_by_data(self, dir_or_path, name, data): - return self.put_file(dir_or_path, name, upload.Data(data)) - def put_file_by_filehandle(self, dir_or_path, name, filehandle): - return self.put_file(dir_or_path, name, upload.FileHandle(filehandle)) - - def make_directory(self, dir_or_path, name): - d = self.dirpath(dir_or_path) - d.addCallback(lambda parent: parent.callRemote("add_directory", name)) - return d - - def remove(self, parent, name): - assert not isinstance(parent, str) - log.msg("vdrive removing %s" % name) - # first find the uri - d = self.get_uri_from_parent(parent, name) - def _got_uri(vid): - # TODO: delete the file's shares using this - pass - d.addCallback(_got_uri) - def _delete_from_parent(res): - return parent.callRemote("remove", name) - d.addCallback(_delete_from_parent) - def _done(res): - log.msg("vdrive done removing %s" % name) - d.addCallback(_done) - return d - - - def get_file(self, dir_and_name_or_path, download_target): - """Retrieve a file from the virtual drive and put it somewhere. - - The file to be retrieved may either be specified as a (dir, name) - tuple or as a full /-delimited pathname. In the former case, 'dir' - can be either a DirectoryNode or a pathname. - - The download target must be an IDownloadTarget instance like - allmydata.download.Data, .FileName, or .FileHandle . - """ - - log.msg("getting file from %s" % (dir_and_name_or_path,)) - dl = self.parent.getServiceNamed("downloader") - - if isinstance(dir_and_name_or_path, tuple): - dir_or_path, name = dir_and_name_or_path - d = self.dirpath(dir_or_path) - def _got_dir(dirnode): - return self.get_uri_from_parent(dirnode, name) - d.addCallback(_got_dir) - else: - rslash = dir_and_name_or_path.rfind("/") - if rslash == -1: - # we're looking for a file in the root directory - dir = self.gvd_root - name = dir_and_name_or_path - d = self.get_uri_from_parent(dir, name) - else: - dirpath = dir_and_name_or_path[:rslash] - name = dir_and_name_or_path[rslash+1:] - d = self.dirpath(dirpath) - d.addCallback(lambda dir: - self.get_uri_from_parent(dir, name)) - - def _got_uri(uri): - return dl.download(uri, download_target) - d.addCallback(_got_uri) - def _done(res): - log.msg("finished getting file") - return res - d.addCallback(_done) - return d - - def get_file_to_filename(self, from_where, filename): - return self.get_file(from_where, download.FileName(filename)) - def get_file_to_data(self, from_where): - return self.get_file(from_where, download.Data()) - def get_file_to_filehandle(self, from_where, filehandle): - return self.get_file(from_where, download.FileHandle(filehandle)) - def create_directory_node(client, diruri): assert uri.is_dirnode_uri(diruri)