filetree: make delete() work
authorBrian Warner <warner@allmydata.com>
Wed, 24 Jan 2007 22:10:53 +0000 (15:10 -0700)
committerBrian Warner <warner@allmydata.com>
Wed, 24 Jan 2007 22:10:53 +0000 (15:10 -0700)
src/allmydata/filetree/directory.py
src/allmydata/filetree/interfaces.py
src/allmydata/filetree/vdrive.py
src/allmydata/interfaces.py
src/allmydata/test/test_filetree_new.py
src/allmydata/workqueue.py

index 81287718555eb8a8373d2874f170a37c34bf2475..02a9397b5e3e2de9066d5de802471cd096e3887e 100644 (file)
@@ -207,6 +207,25 @@ class _DirectorySubTree(object):
         # now we can finally add the new node
         node.add(child_name, new_node)
 
+    def delete_node_at_path(self, path):
+        assert len(path) > 0
+        child_name = path[-1]
+
+        # first step: get the parent directory
+        node = self.root
+        for subdir_name in path[:-1]:
+            subdir_node = node.get(subdir_name) # may raise NoSuchChildError
+            node = subdir_node
+
+        # 'node' is now pointing at the parent directory. Let's make sure the
+        # path they want to delete actually exists. We don't really care what
+        # the child *is*, just that it exists.
+        node.get(child_name) # may raise NoSuchChildError
+
+        # now delete it
+        # TODO: How do we free the subtree that was just orphaned?
+        node.delete(child_name)
+
 
 class LocalFileSubTreeNode(BaseDataNode):
     prefix = "LocalFileDirectory"
index 76465bb1a31d0e736965cf51e747f42c21788f31..700c2046f98abbedae7176479df286fdea7dcf88 100644 (file)
@@ -166,6 +166,12 @@ class ISubTree(Interface):
         run synchronously, and returns None.
         """
 
+    def delete_node_at_path(path):
+        """Delete the node at the the given path.
+
+        This must run synchronously, and returns None.
+        """
+
     def serialize_subtree_to_file(f):
         """Create a string which describes my structure and write it to the
         given filehandle (using only .write()). This string should be
@@ -279,16 +285,18 @@ class IVirtualDrive(Interface):
         is complete.
         """
 
-    def upload_now(path, uploadable):
-        """Upload a file to the given path. The path must not already exist.
+    def upload_data(path, data):
+        """Upload a string to the given path. The path must not already exist.
 
-        path[:-1] must refer to a writable DIRECTORY node. 'uploadable' must
-        implement IUploadable. This returns a Deferred that fires (with
-        'uploadable') when the upload is complete. Do not use the workqueue.
+        path[:-1] must refer to a writable DIRECTORY node.
+
+        This uses the workqueue, and returns None.
         """
 
-    def upload_later(path, filename):
-        """Upload a file from disk to the given path. Use the workqueue.
+    def upload(path, filename):
+        """Upload a file from disk to the given path.
+
+        This uses the workqueue, and returns None.
         """
 
     def delete(path):
index a8494e158b21ca694c37cb8ee0116804c8a2d376..438b4ac5f5ff07917b18e102685301be8242b6f6 100644 (file)
@@ -74,6 +74,7 @@ class SubTreeMaker(object):
 
 class VirtualDrive(object):
     implements(IVirtualDrive)
+    debug = False
 
     def __init__(self, workqueue, downloader, uploader, root_node):
         assert IWorkQueue(workqueue)
@@ -242,6 +243,8 @@ class VirtualDrive(object):
         # as necessary
         def _got_subtrees(subtrees, new_node_boxname):
             for (subtree, subpath) in reversed(subtrees):
+                if self.debug:
+                    print "SUBTREE", subtree, subpath
                 assert subtree.is_mutable()
                 must_update = subtree.mutation_modifies_parent()
                 subtree_node = subtree.create_node_now()
@@ -263,6 +266,11 @@ class VirtualDrive(object):
         d.addCallback(_got_subtrees, new_node_boxname)
         return d
 
+    def deletepath(self, path):
+        if self.debug:
+            print "DELETEPATH(%s)" % (path,)
+        return self.addpath(path, None)
+
     def modify_subtree(self, subtree_node, localpath, new_node,
                        new_subtree_boxname=None):
         # TODO: I'm lying here, we don't know who the parent is, so we can't
@@ -278,7 +286,10 @@ class VirtualDrive(object):
                                                       parent_is_mutable)
         def _got_subtree(subtree):
             assert subtree.is_mutable()
-            subtree.put_node_at_path(localpath, new_node)
+            if new_node:
+                subtree.put_node_at_path(localpath, new_node)
+            else:
+                subtree.delete_node_at_path(localpath)
             return subtree.update_now(self._uploader)
         d.addCallback(_got_subtree)
         if new_subtree_boxname:
@@ -312,26 +323,18 @@ class VirtualDrive(object):
         target = download.Data()
         return self.download(path, target)
 
-    def upload_now(self, path, uploadable):
+    def upload_data(self, path, data):
         assert isinstance(path, list)
-        # note: the first few steps of this do not use the workqueue, but I
-        # think things should remain consistent anyways. If the node is shut
-        # down before the file has finished uploading, then we forget all
-        # abou the file.
-        uploadable = IUploadable(uploadable)
-        d = self._child_should_not_exist(path)
-        # then we upload the file
-        d.addCallback(lambda ignored: self._uploader.upload(uploadable))
-        def _uploaded(uri):
-            assert isinstance(uri, str)
-            new_node = file.CHKFileNode().new(uri)
-            boxname = self.workqueue.create_boxname(new_node)
-            self.workqueue.add_addpath(boxname, path)
-            self.workqueue.add_delete_box(boxname)
-        d.addCallback(_uploaded)
-        return d
+        f, tempfilename = self.workqueue.create_tempfile()
+        f.write(data)
+        f.close()
+        boxname = self.workqueue.create_boxname()
+        self.workqueue.add_upload_chk(tempfilename, boxname)
+        self.workqueue.add_addpath(boxname, path)
+        self.workqueue.add_delete_box(boxname)
+        self.workqueue.add_delete_tempfile(tempfilename)
 
-    def upload_later(self, path, filename):
+    def upload(self, path, filename):
         assert isinstance(path, list)
         filename = os.path.abspath(filename)
         boxname = self.workqueue.create_boxname()
@@ -341,21 +344,7 @@ class VirtualDrive(object):
 
     def delete(self, path):
         assert isinstance(path, list)
-        parent_path = path[:-1]
-        orphan_path = path[-1]
-        d = self._get_closest_node_and_prepath(parent_path)
-        def _got_parent((prepath, node, remaining_path)):
-            assert not remaining_path
-            node.delete(orphan_path)
-            # now serialize and upload
-            subtree = node.get_subtree()
-            boxname = subtree.update(self.workqueue)
-            if boxname:
-                self.workqueue.add_addpath(boxname, prepath)
-                self.workqueue.add_delete_box(boxname)
-            return self
-        d.addCallback(_got_parent)
-        return d
+        self.workqueue.add_deletepath(path)
 
     def add_node(self, path, node):
         assert isinstance(path, list)
index d1cd48ea32e2f0d60fd0df1df15f400e7edf1d82..024d7ff533e9aa262f2b21b370fe14da72d9eba0 100644 (file)
@@ -224,6 +224,15 @@ class IUploader(Interface):
     def upload_ssk(write_capability, new_version, uploadable):
         pass # TODO
 
+    def upload_data(data):
+        """Like upload(), but accepts a string."""
+
+    def upload_filename(filename):
+        """Like upload(), but accepts an absolute pathname."""
+
+    def upload_filehandle(filehane):
+        """Like upload(), but accepts an open filehandle."""
+
 
 class IWorkQueue(Interface):
     """Each filetable root is associated a work queue, which is persisted on
@@ -315,6 +324,13 @@ class IWorkQueue(Interface):
         steps to be added to the workqueue.
         """
 
+    def add_deletepath(path):
+        """When executed, finds the subtree that contains the node at 'path'
+        and modifies it (and any necessary parent subtrees) to delete that
+        path. This will probably cause one or more 'add_modify_subtree' or
+        'add_modify_redirection' steps to be added to the workqueue.
+        """
+
     def add_modify_subtree(subtree_node, localpath, new_node_boxname,
                            new_subtree_boxname=None):
         """When executed, this step retrieves the subtree specified by
@@ -322,7 +338,8 @@ class IWorkQueue(Interface):
         then modifies it such that a subtree-relative 'localpath' points to
         the new node. It then serializes the subtree in its new form, and
         optionally puts a node that describes the new subtree in
-        'new_node_boxname'.
+        'new_node_boxname'. If 'new_node_boxname' is None, this deletes the
+        given path.
 
         The idea is that 'subtree_node' will refer a CHKDirectorySubTree, and
         'new_node_boxname' will contain the CHKFileNode that points to a
index 7c6d05eed328a4e1db6a59f59d899d1cec7b14c5..4d478112d71588af0f0f01a0e6425005b67efd92 100644 (file)
@@ -323,10 +323,11 @@ from allmydata.filetree.interfaces import (ISubTree, INode, IDirectoryNode,
                                            IFileNode, NoSuchDirectoryError,
                                            NoSuchChildError)
 from allmydata.filetree.file import CHKFileNode
+from allmydata import upload
 from allmydata.interfaces import IDownloader
 from allmydata.util import bencode
 
-class InPairs(unittest.TestCase):
+class Utils(unittest.TestCase):
     def test_in_pairs(self):
         l = range(8)
         pairs = list(directory.in_pairs(l))
@@ -338,19 +339,28 @@ class FakeMesh(object):
 
     def __init__(self):
         self.files = {}
-    def upload_filename(self, filename):
+
+    def upload(self, uploadable):
         uri = "stub-uri-%d" % len(self.files)
         if self.debug:
-            print "FakeMesh.upload_filename(%s) -> %s" % (filename, uri)
-        data = open(filename,"r").read()
+            print "FakeMesh.upload -> %s" % uri
+        assert upload.IUploadable.providedBy(uploadable)
+        f = uploadable.get_filehandle()
+        data = f.read()
+        uploadable.close_filehandle(f)
         self.files[uri] = data
         return defer.succeed(uri)
+
+    def upload_filename(self, filename):
+        if self.debug:
+            print "FakeMesh.upload_filename(%s)" % filename
+        return self.upload(upload.FileName(filename))
+
     def upload_data(self, data):
-        uri = "stub-uri-%d" % len(self.files)
         if self.debug:
-            print "FakeMesh.upload_data(%s) -> %s" % (data, uri)
-        self.files[uri] = data
-        return defer.succeed(uri)
+            print "FakeMesh.upload_data(%s)" % data
+        return self.upload(upload.Data(data))
+
     def download(self, uri, target):
         if self.debug:
             print "FakeMesh.download(%s)" % uri
@@ -363,7 +373,9 @@ class FakeMesh(object):
 class VDrive(unittest.TestCase):
 
     def makeVirtualDrive(self, basedir, root_node=None, mesh=None):
-        wq = workqueue.WorkQueue(os.path.join(basedir, "1.workqueue"))
+        wq = workqueue.WorkQueue(os.path.join("test_filetree",
+                                              "VDrive",
+                                              basedir, "1.workqueue"))
         if mesh:
             assert IUploader.providedBy(mesh)
             assert IDownloader.providedBy(mesh)
@@ -557,7 +569,7 @@ class VDrive(unittest.TestCase):
         f.write(DATA)
         f.close()
 
-        rc = v.upload_later(["a","b","upload1"], filename)
+        rc = v.upload(["a","b","upload1"], filename)
         self.failUnlessIdentical(rc, None)
 
         d = v.workqueue.flush()
@@ -579,16 +591,16 @@ class VDrive(unittest.TestCase):
 
     def testCHKDirUpload(self):
         DATA = "here is some data\n"
-        d = defer.maybeDeferred(self.makeCHKTree, "upload")
+        filename = "upload1"
+        f = open(filename, "w")
+        f.write(DATA)
+        f.close()
+
+        d = defer.maybeDeferred(self.makeCHKTree, "chk-upload")
         def _made(v):
             self.v = v
 
-            filename = "upload1"
-            f = open(filename, "w")
-            f.write(DATA)
-            f.close()
-
-            rc = v.upload_later(["a","b","upload1"], filename)
+            rc = v.upload(["a","b","upload1"], filename)
             self.failUnlessIdentical(rc, None)
 
             return v.workqueue.flush()
@@ -609,3 +621,47 @@ class VDrive(unittest.TestCase):
 
         return d
 
+    def testCHKDirDelete(self):
+        DATA = "here is some data\n"
+        filename = "upload1"
+        f = open(filename, "w")
+        f.write(DATA)
+        f.close()
+
+        d = defer.maybeDeferred(self.makeCHKTree, "chk-delete")
+        def _made(v):
+            self.v = v
+        d.addCallback(_made)
+
+        d.addCallback(lambda r:
+                      self.v.upload(["a","b","upload1"], filename))
+        d.addCallback(lambda r:
+                      self.v.upload_data(["a","b","upload2"], DATA))
+        d.addCallback(lambda r:
+                      self.v.upload(["a","c","upload3"], filename))
+        d.addCallback(lambda r:
+                      self.v.workqueue.flush())
+
+        d.addCallback(lambda r: self.v.list([]))
+        d.addCallback(lambda contents:
+                      self.failUnlessListsAreEqual(contents.keys(), ["a"]))
+        d.addCallback(lambda r: self.v.list(["a"]))
+        d.addCallback(lambda contents:
+                      self.failUnlessListsAreEqual(contents.keys(), ["b","c"]))
+        d.addCallback(lambda r: self.v.list(["a","b"]))
+        d.addCallback(lambda contents:
+                      self.failUnlessListsAreEqual(contents.keys(),
+                                                   ["upload1", "upload2"]))
+        #d.addCallback(lambda r: self.v.download_as_data(["a","b","upload1"]))
+        #d.addCallback(self.failUnlessEqual, DATA)
+
+        # now delete it
+        d.addCallback(lambda r: self.v.delete(["a","b","upload2"]))
+        d.addCallback(lambda r: self.v.workqueue.flush())
+        d.addCallback(lambda r: self.v.list(["a","b"]))
+        d.addCallback(lambda contents:
+                      self.failUnlessListsAreEqual(contents.keys(),
+                                                   ["upload1"]))
+
+
+        return d
index 3b037c59e8f900e6effc6355b6ac4c66cc235d63..1ad47a9eb80bc3d31b4d0664947153daa238c2f2 100644 (file)
@@ -177,6 +177,12 @@ class WorkQueue(object):
         lines.extend(path)
         self._create_step_first(lines)
 
+    def add_deletepath(self, path):
+        assert isinstance(path, (list, tuple))
+        lines = ["deletepath"]
+        lines.extend(path)
+        self._create_step_first(lines)
+
     def add_modify_subtree(self, subtree_node, localpath, new_node_boxname,
                            new_subtree_boxname=None):
         assert isinstance(localpath, (list, tuple))
@@ -184,6 +190,8 @@ class WorkQueue(object):
         self.add_delete_box(box1)
         # TODO: it would probably be easier if steps were represented in
         # directories, with a separate file for each argument
+        if new_node_boxname is None:
+            new_node_boxname = ""
         if new_subtree_boxname is None:
             new_subtree_boxname = ""
         lines = ["modify_subtree",
@@ -323,6 +331,13 @@ class WorkQueue(object):
             print "STEP_ADDPATH(%s -> %s)" % (boxname, "/".join(path))
         path = list(path)
         return self.vdrive.addpath(path, boxname)
+
+    def step_deletepath(self, *path):
+        if self.debug:
+            print "STEP_DELETEPATH(%s)" % "/".join(path)
+        path = list(path)
+        return self.vdrive.deletepath(path)
+
     def step_modify_subtree(self, subtree_node_boxname, new_node_boxname,
                             new_subtree_boxname, *localpath):
         # the weird order of arguments is a consequence of the fact that
@@ -330,7 +345,9 @@ class WorkQueue(object):
         if not new_subtree_boxname:
             new_subtree_boxname = None
         subtree_node = self.read_from_box(subtree_node_boxname)
-        new_node = self.read_from_box(new_node_boxname)
+        new_node = None
+        if new_node_boxname:
+            new_node = self.read_from_box(new_node_boxname)
         localpath = list(localpath)
         return self.vdrive.modify_subtree(subtree_node, localpath,
                                           new_node, new_subtree_boxname)