From: Daira Hopwood <daira@jacaranda.org>
Date: Mon, 19 Oct 2015 14:14:28 +0000 (+0100)
Subject: WIP Merge branch '1710.actually-unlink-files.0' of https://github.com/meejah/tahoe... 
X-Git-Url: https://git.rkrishnan.org/%5B/frontends/(%5B%5E?a=commitdiff_plain;h=cfb23549bdf0b957a6143a600171845d50df7579;p=tahoe-lafs%2Ftahoe-lafs.git

WIP Merge branch '1710.actually-unlink-files.0' of https://github.com/meejah/tahoe-lafs into 2521.magic-db-schema.1

# Conflicts:
#	src/allmydata/frontends/magic_folder.py
#	src/allmydata/test/test_magic_folder.py
---

diff --git a/Makefile b/Makefile
index 6df6bd6a..abda237a 100644
--- a/Makefile
+++ b/Makefile
@@ -83,6 +83,11 @@ _tmpfstest: make-version
 	sudo umount '$(TMPDIR)'
 	rmdir '$(TMPDIR)'
 
+.PHONY: smoketest
+smoketest:
+	-python ./src/allmydata/test/check_magicfolder_smoke.py kill
+	-rm -rf smoke_magicfolder/
+	python ./src/allmydata/test/check_magicfolder_smoke.py
 
 # code coverage: install the "coverage" package from PyPI, do "make test-coverage" to
 # do a unit test run with coverage-gathering enabled, then use "make coverage-output" to
diff --git a/src/allmydata/frontends/magic_folder.py b/src/allmydata/frontends/magic_folder.py
index a53cd375..16adb032 100644
--- a/src/allmydata/frontends/magic_folder.py
+++ b/src/allmydata/frontends/magic_folder.py
@@ -618,18 +618,28 @@ class Downloader(QueueMixin, WriteFileMixin):
         self._log("_process(%r)" % (item,))
         if now is None:
             now = time.time()
+
         (relpath_u, file_node, metadata) = item
         fp = self._get_filepath(relpath_u)
         abspath_u = unicode_from_filepath(fp)
+        deleted = metadata.get('deleted', False)
+        is_directory = relpath_u.endswith(u"/")
 
         d = defer.succeed(None)
-        if relpath_u.endswith(u"/"):
-            self._log("mkdir(%r)" % (abspath_u,))
-            d.addCallback(lambda ign: fileutil.make_dirs(abspath_u))
-            d.addCallback(lambda ign: abspath_u)
+        if is_directory:
+            if deleted:
+                self._log("rmdir(%r)" % (abspath_u,))
+                # what to do here, if anything?
+            else:
+                self._log("mkdir(%r)" % (abspath_u,))
+                d.addCallback(lambda ign: fileutil.make_dirs(abspath_u))
+                d.addCallback(lambda ign: abspath_u)
         else:
-            d.addCallback(lambda ign: file_node.download_best_version())
-            d.addCallback(lambda contents: self._write_downloaded_file(abspath_u, contents, is_conflict=False))
+            if deleted:
+                d.addCallback(lambda ign: self._unlink_deleted_file(abspath_u))
+            else:
+                d.addCallback(lambda ign: file_node.download_best_version())
+                d.addCallback(lambda contents: self._write_downloaded_file(abspath_u, contents, is_conflict=False))
 
         def do_update_db(written_abspath_u):
             filecap = file_node.get_uri()
@@ -637,7 +647,7 @@ class Downloader(QueueMixin, WriteFileMixin):
             last_downloaded_uri = filecap
             last_downloaded_timestamp = now
             written_pathinfo = get_pathinfo(written_abspath_u)
-            if not written_pathinfo.exists:
+            if not deleted and not written_pathinfo.exists:
                 raise Exception("downloaded object %s disappeared" % quote_local_unicode_path(written_abspath_u))
 
             self._db.did_upload_version(relpath_u, metadata['version'], last_uploaded_uri,
@@ -653,3 +663,44 @@ class Downloader(QueueMixin, WriteFileMixin):
             return res
         d.addBoth(remove_from_pending)
         return d
+
+    FUDGE_SECONDS = 10.0
+
+    def _unlink_deleted_file(self, abspath_u):
+        fileutil.remove(abspath_u)
+        return abspath_u
+
+    @classmethod
+    def _write_downloaded_file(cls, abspath_u, file_contents, is_conflict=False, now=None):
+        # 1. Write a temporary file, say .foo.tmp.
+        # 2. is_conflict determines whether this is an overwrite or a conflict.
+        # 3. Set the mtime of the replacement file to be T seconds before the
+        #    current local time.
+        # 4. Perform a file replacement with backup filename foo.backup,
+        #    replaced file foo, and replacement file .foo.tmp. If any step of
+        #    this operation fails, reclassify as a conflict and stop.
+        #
+        # Returns the path of the destination file.
+
+        precondition_abspath(abspath_u)
+        replacement_path_u = abspath_u + u".tmp"  # FIXME more unique
+        backup_path_u = abspath_u + u".backup"
+        if now is None:
+            now = time.time()
+
+        fileutil.write(replacement_path_u, file_contents)
+        os.utime(replacement_path_u, (now, now - cls.FUDGE_SECONDS))
+        if is_conflict:
+            return cls._rename_conflicted_file(abspath_u, replacement_path_u)
+        else:
+            try:
+                fileutil.replace_file(abspath_u, replacement_path_u, backup_path_u)
+                return abspath_u
+            except fileutil.ConflictError:
+                return cls._rename_conflicted_file(abspath_u, replacement_path_u)
+
+    @classmethod
+    def _rename_conflicted_file(self, abspath_u, replacement_path_u):
+        conflict_path_u = abspath_u + u".conflict"
+        fileutil.rename_no_overwrite(replacement_path_u, conflict_path_u)
+        return conflict_path_u
diff --git a/src/allmydata/test/check_magicfolder_smoke.py b/src/allmydata/test/check_magicfolder_smoke.py
new file mode 100644
index 00000000..501eb881
--- /dev/null
+++ b/src/allmydata/test/check_magicfolder_smoke.py
@@ -0,0 +1,295 @@
+#!/usr/bin/env python
+
+# this is a smoke-test using "./bin/tahoe" to:
+#
+# 1. create an introducer
+# 2. create 5 storage nodes
+# 3. create 2 client nodes (client0 = alice, client1 = bob)
+# 4. Alice creates a magic-folder ("magik:")
+# 5. Alice invites Bob
+# 6. Bob joins
+#
+# After that, some basic tests are performed; see the "if True:"
+# blocks to turn some on or off. Could benefit from some cleanups
+# etc. but this seems useful out of the gate for quick testing.
+#
+# TO RUN:
+# from top-level of your checkout (we use "./bin/tahoe"):
+# python src/allmydata/test/check_magicfolder_smoke.py
+#
+# This will create "./smoke_magicfolder" (which is disposable) and
+# contains all the Tahoe basedirs for the introducer, storage nodes,
+# clients, and the clients' magic-folders. NOTE that if these
+# directories already exist they will NOT be re-created. So kill the
+# grid and then "rm -rf smoke_magicfolder" if you want to re-run the
+# tests cleanly.
+#
+# Run the script with a single arg, "kill" to run "tahoe stop" on all
+# the nodes.
+#
+# This will have "tahoe start" -ed all the nodes, so you can continue
+# to play around after the script exits.
+
+from __future__ import print_function
+
+import sys
+import time
+import subprocess
+from os.path import join, abspath, curdir, exists
+from os import mkdir, listdir, unlink
+
+tahoe_base = abspath(curdir)
+data_base = join(tahoe_base, 'smoke_magicfolder')
+tahoe_bin = join(tahoe_base, 'bin', 'tahoe')
+
+if not exists(data_base):
+    print("Creating", data_base)
+    mkdir(data_base)
+
+if not exists(tahoe_bin):
+    raise RuntimeError("Can't find 'tahoe' binary at '{}'".format(tahoe_bin))
+
+if 'kill' in sys.argv:
+    print("Killing the grid")
+    for d in listdir(data_base):
+        print("killing", d)
+        subprocess.call(
+            [
+                tahoe_bin, 'stop', join(data_base, d),
+            ]
+        )
+    sys.exit(0)
+
+if not exists(join(data_base, 'introducer')):
+    subprocess.check_call(
+        [
+            tahoe_bin, 'create-introducer', join(data_base, 'introducer'),
+        ]
+    )
+with open(join(data_base, 'introducer', 'tahoe.cfg'), 'w') as f:
+    f.write('''
+[node]
+nickname = introducer0
+web.port = 4560
+''')
+subprocess.check_call(
+    [
+        tahoe_bin, 'start', join(data_base, 'introducer'),
+    ]
+)
+
+furl_fname = join(data_base, 'introducer', 'private', 'introducer.furl')
+while not exists(furl_fname):
+    time.sleep(1)
+furl = open(furl_fname, 'r').read()
+print("FURL", furl)
+
+for x in range(5):
+    data_dir = join(data_base, 'node%d' % x)
+    if not exists(data_dir):
+        subprocess.check_call(
+            [
+                tahoe_bin, 'create-node', data_dir,
+            ]
+        )
+        with open(join(data_dir, 'tahoe.cfg'), 'w') as f:
+            f.write('''
+[node]
+nickname = node{node_id}
+web.port =
+web.static = public_html
+tub.location = localhost:{tub_port}
+
+[client]
+# Which services should this client connect to?
+introducer.furl = {furl}
+shares.needed = 2
+shares.happy = 3
+shares.total = 4
+'''.format(node_id=x, furl=furl, tub_port=(9900 + x)))
+    subprocess.check_call(
+        [
+            tahoe_bin, 'start', data_dir,
+        ]
+    )
+
+
+
+# alice and bob clients
+do_invites = False
+for x in range(2):
+    data_dir = join(data_base, 'client%d' % x)
+    magic_dir = join(data_base, 'client%d-magic' % x)
+    mkdir(magic_dir)
+    if not exists(data_dir):
+        do_invites = True
+        subprocess.check_call(
+            [
+                tahoe_bin, 'create-node', data_dir,
+            ]
+        )
+        with open(join(data_dir, 'tahoe.cfg'), 'w') as f:
+            f.write('''
+[node]
+nickname = client{node_id}
+web.port = tcp:998{node_id}:interface=localhost
+web.static = public_html
+
+[client]
+# Which services should this client connect to?
+introducer.furl = {furl}
+shares.needed = 2
+shares.happy = 3
+shares.total = 4
+'''.format(node_id=x, furl=furl, magic_dir=magic_dir))
+    subprocess.check_call(
+        [
+            tahoe_bin, 'start', data_dir,
+        ]
+    )
+
+# okay, now we have alice + bob (client0, client1)
+# now we have alice create a magic-folder, and invite bob to it
+
+if do_invites:
+    data_dir = join(data_base, 'client0')
+    # alice/client0 creates her folder, invites bob
+    print("Alice creates a magic-folder")
+    subprocess.check_call(
+        [
+            tahoe_bin, 'magic-folder', 'create', '--basedir', data_dir, 'magik:', 'alice',
+            join(data_base, 'client0-magic'),
+        ]
+    )
+    print("Alice invites Bob")
+    invite = subprocess.check_output(
+        [
+            tahoe_bin, 'magic-folder', 'invite', '--basedir', data_dir, 'magik:', 'bob',
+        ]
+    )
+    print("  invite:", invite)
+
+    # now we let "bob"/client1 join
+    print("Bob joins Alice's magic folder")
+    data_dir = join(data_base, 'client1')
+    subprocess.check_call(
+        [
+            tahoe_bin, 'magic-folder', 'join', '--basedir', data_dir, invite,
+            join(data_base, 'client1-magic'),
+        ]
+    )
+    print("Bob has joined.")
+
+    print("Restarting alice + bob clients")
+    subprocess.check_call(
+        [
+            tahoe_bin, 'restart', '--basedir', join(data_base, 'client0'),
+        ]
+    )
+    subprocess.check_call(
+        [
+            tahoe_bin, 'restart', '--basedir', join(data_base, 'client1'),
+        ]
+    )
+
+if True:
+    for x in range(2):
+        with open(join(data_base, 'client{}'.format(x), 'private', 'magic_folder_dircap'), 'r') as f:
+            print("dircap{}: {}".format(x, f.read().strip()))
+
+if True:
+    # alice writes a file; bob should get it
+    alice_foo = join(data_base, 'client0-magic', 'first_file')
+    bob_foo = join(data_base, 'client1-magic', 'first_file')
+    with open(alice_foo, 'w') as f:
+        f.write("line one\n")
+
+    print("Waiting for:", bob_foo)
+    while True:
+        if exists(bob_foo):
+            print("  found", bob_foo)
+            with open(bob_foo, 'r') as f:
+                if f.read() == "line one\n":
+                    break
+                print("  file contents still mismatched")
+        time.sleep(1)
+
+if True:
+    # bob writes a file; alice should get it
+    alice_bar = join(data_base, 'client0-magic', 'second_file')
+    bob_bar = join(data_base, 'client1-magic', 'second_file')
+    with open(bob_bar, 'w') as f:
+        f.write("line one\n")
+
+    print("Waiting for:", alice_bar)
+    while True:
+        if exists(bob_bar):
+            print("  found", bob_bar)
+            with open(bob_bar, 'r') as f:
+                if f.read() == "line one\n":
+                    break
+                print("  file contents still mismatched")
+        time.sleep(1)
+
+if True:
+    # bob deletes alice's "first_file"; alice should also delete it
+    alice_foo = join(data_base, 'client0-magic', 'first_file')
+    bob_foo = join(data_base, 'client1-magic', 'first_file')
+    unlink(alice_foo)
+
+    print("Waiting for '%s' to disappear" % (bob_foo,))
+    while True:
+        if not exists(bob_foo):
+            print("  disappeared", bob_foo)
+            break
+        time.sleep(1)
+
+if True:
+    # re-write 'second_file'
+    alice_foo = join(data_base, 'client0-magic', 'second_file')
+    bob_foo = join(data_base, 'client1-magic', 'second_file')
+    gold_content = "line one\nsecond line\n"
+
+    with open(bob_foo, 'w') as f:
+        f.write(gold_content)
+
+    print("Waiting for:", alice_foo)
+    while True:
+        if exists(alice_foo):
+            print("  found", alice_foo)
+            with open(alice_foo, 'r') as f:
+                content = f.read()
+                if content == gold_content:
+                    break
+                print("  file contents still mismatched:\n")
+        time.sleep(1)
+
+if True:
+    # restore 'first_file' but with different contents
+    alice_foo = join(data_base, 'client0-magic', 'first_file')
+    bob_foo = join(data_base, 'client1-magic', 'first_file')
+    gold_content = "see it again for the first time\n"
+
+    with open(bob_foo, 'w') as f:
+        f.write(gold_content)
+
+    print("Waiting for:", alice_foo)
+    while True:
+        if exists(alice_foo):
+            print("  found", alice_foo)
+            with open(alice_foo, 'r') as f:
+                content = f.read()
+                if content == gold_content:
+                    break
+                print("  file contents still mismatched:\n")
+        time.sleep(1)
+
+
+
+# XXX test .backup (delete a file)
+
+# port david's clock.advance stuff
+# fix clock.advance()
+# subdirectory
+# file deletes
+# conflicts
diff --git a/src/allmydata/test/test_magic_folder.py b/src/allmydata/test/test_magic_folder.py
index 39a71ed1..fe5255c5 100644
--- a/src/allmydata/test/test_magic_folder.py
+++ b/src/allmydata/test/test_magic_folder.py
@@ -241,6 +241,43 @@ class MagicFolderTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, ReallyEqual
         d.addBoth(self.cleanup)
         return d
 
+    @defer.inlineCallbacks
+    def X_test_delete(self):
+        self.set_up_grid()
+        self.local_dir = os.path.join(self.basedir, u"local_dir")
+        self.mkdir_nonascii(self.local_dir)
+
+        yield self.create_invite_join_magic_folder(u"Alice\u0101", self.local_dir)
+        yield self._restart_client(None)
+
+        try:
+            # create a file
+            uploaded_d = self.magicfolder.uploader.set_hook('processed')
+            downloaded_d = self.magicfolder.downloader.set_hook('processed')
+            path = os.path.join(self.local_dir, u'foo')
+            with open(path, 'w') as f:
+                f.write('foo\n')
+            self.notify(to_filepath(path), self.inotify.IN_CLOSE_WRITE)
+            yield uploaded_d
+            #yield downloaded_d
+            self.assertTrue(os.path.exists(path))
+
+            # the real test part: delete the file
+            uploaded_d2 = self.magicfolder.uploader.set_hook('processed')
+            os.unlink(path)
+            self.notify(to_filepath(path), self.inotify.IN_DELETE)
+            yield uploaded_d2
+            yield downloaded_d
+            self.assertFalse(os.path.exists(path))
+
+            # ensure we still have a DB entry, and that the version is 1
+            node, metadata = yield self.magicfolder.downloader._get_collective_latest_file(u'foo')
+            self.assertTrue(node is not None, "Failed to find '{}' in DMD".format(path))
+            self.failUnlessEqual(metadata['version'], 1)
+
+        finally:
+            yield self.cleanup(None)
+
     def test_magic_folder(self):
         self.set_up_grid()
         self.local_dir = os.path.join(self.basedir, self.unicode_or_fallback(u"loc\u0101l_dir", u"local_dir"))
@@ -336,6 +373,10 @@ class MagicFolderTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, ReallyEqual
         #print "_check_version_in_local_db: %r has version %s" % (relpath_u, version)
         self.failUnlessEqual(version, expected_version)
 
+    def _check_file_gone(self, magicfolder, relpath_u):
+        path = os.path.join(magicfolder.uploader._local_path_u, relpath_u)
+        self.assertTrue(not os.path.exists(path))
+
     def test_alice_bob(self):
         alice_clock = task.Clock()
         bob_clock = task.Clock()
@@ -402,6 +443,7 @@ class MagicFolderTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, ReallyEqual
 
         d.addCallback(lambda ign: self._check_version_in_local_db(self.bob_magicfolder, u"file1", 1))
         d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file1", 1))
+        d.addCallback(lambda ign: self._check_file_gone(self.bob_magicfolder, u"file1"))
         d.addCallback(_check_downloader_count, 'objects_failed', 0)
         d.addCallback(_check_downloader_count, 'objects_downloaded', 2)