From: Daira Hopwood Date: Mon, 19 Oct 2015 14:14:28 +0000 (+0100) Subject: WIP Merge branch '1710.actually-unlink-files.0' of https://github.com/meejah/tahoe... X-Git-Url: https://git.rkrishnan.org/reliability?a=commitdiff_plain;h=cfb23549bdf0b957a6143a600171845d50df7579;p=tahoe-lafs%2Ftahoe-lafs.git WIP Merge branch '1710.actually-unlink-files.0' of https://github.com/meejah/tahoe-lafs into 2521.magic-db-schema.1 # Conflicts: # src/allmydata/frontends/magic_folder.py # src/allmydata/test/test_magic_folder.py --- diff --git a/Makefile b/Makefile index 6df6bd6a..abda237a 100644 --- a/Makefile +++ b/Makefile @@ -83,6 +83,11 @@ _tmpfstest: make-version sudo umount '$(TMPDIR)' rmdir '$(TMPDIR)' +.PHONY: smoketest +smoketest: + -python ./src/allmydata/test/check_magicfolder_smoke.py kill + -rm -rf smoke_magicfolder/ + python ./src/allmydata/test/check_magicfolder_smoke.py # code coverage: install the "coverage" package from PyPI, do "make test-coverage" to # do a unit test run with coverage-gathering enabled, then use "make coverage-output" to diff --git a/src/allmydata/frontends/magic_folder.py b/src/allmydata/frontends/magic_folder.py index a53cd375..16adb032 100644 --- a/src/allmydata/frontends/magic_folder.py +++ b/src/allmydata/frontends/magic_folder.py @@ -618,18 +618,28 @@ class Downloader(QueueMixin, WriteFileMixin): self._log("_process(%r)" % (item,)) if now is None: now = time.time() + (relpath_u, file_node, metadata) = item fp = self._get_filepath(relpath_u) abspath_u = unicode_from_filepath(fp) + deleted = metadata.get('deleted', False) + is_directory = relpath_u.endswith(u"/") d = defer.succeed(None) - if relpath_u.endswith(u"/"): - self._log("mkdir(%r)" % (abspath_u,)) - d.addCallback(lambda ign: fileutil.make_dirs(abspath_u)) - d.addCallback(lambda ign: abspath_u) + if is_directory: + if deleted: + self._log("rmdir(%r)" % (abspath_u,)) + # what to do here, if anything? + else: + self._log("mkdir(%r)" % (abspath_u,)) + d.addCallback(lambda ign: fileutil.make_dirs(abspath_u)) + d.addCallback(lambda ign: abspath_u) else: - d.addCallback(lambda ign: file_node.download_best_version()) - d.addCallback(lambda contents: self._write_downloaded_file(abspath_u, contents, is_conflict=False)) + if deleted: + d.addCallback(lambda ign: self._unlink_deleted_file(abspath_u)) + else: + d.addCallback(lambda ign: file_node.download_best_version()) + d.addCallback(lambda contents: self._write_downloaded_file(abspath_u, contents, is_conflict=False)) def do_update_db(written_abspath_u): filecap = file_node.get_uri() @@ -637,7 +647,7 @@ class Downloader(QueueMixin, WriteFileMixin): last_downloaded_uri = filecap last_downloaded_timestamp = now written_pathinfo = get_pathinfo(written_abspath_u) - if not written_pathinfo.exists: + if not deleted and not written_pathinfo.exists: raise Exception("downloaded object %s disappeared" % quote_local_unicode_path(written_abspath_u)) self._db.did_upload_version(relpath_u, metadata['version'], last_uploaded_uri, @@ -653,3 +663,44 @@ class Downloader(QueueMixin, WriteFileMixin): return res d.addBoth(remove_from_pending) return d + + FUDGE_SECONDS = 10.0 + + def _unlink_deleted_file(self, abspath_u): + fileutil.remove(abspath_u) + return abspath_u + + @classmethod + def _write_downloaded_file(cls, abspath_u, file_contents, is_conflict=False, now=None): + # 1. Write a temporary file, say .foo.tmp. + # 2. is_conflict determines whether this is an overwrite or a conflict. + # 3. Set the mtime of the replacement file to be T seconds before the + # current local time. + # 4. Perform a file replacement with backup filename foo.backup, + # replaced file foo, and replacement file .foo.tmp. If any step of + # this operation fails, reclassify as a conflict and stop. + # + # Returns the path of the destination file. + + precondition_abspath(abspath_u) + replacement_path_u = abspath_u + u".tmp" # FIXME more unique + backup_path_u = abspath_u + u".backup" + if now is None: + now = time.time() + + fileutil.write(replacement_path_u, file_contents) + os.utime(replacement_path_u, (now, now - cls.FUDGE_SECONDS)) + if is_conflict: + return cls._rename_conflicted_file(abspath_u, replacement_path_u) + else: + try: + fileutil.replace_file(abspath_u, replacement_path_u, backup_path_u) + return abspath_u + except fileutil.ConflictError: + return cls._rename_conflicted_file(abspath_u, replacement_path_u) + + @classmethod + def _rename_conflicted_file(self, abspath_u, replacement_path_u): + conflict_path_u = abspath_u + u".conflict" + fileutil.rename_no_overwrite(replacement_path_u, conflict_path_u) + return conflict_path_u diff --git a/src/allmydata/test/check_magicfolder_smoke.py b/src/allmydata/test/check_magicfolder_smoke.py new file mode 100644 index 00000000..501eb881 --- /dev/null +++ b/src/allmydata/test/check_magicfolder_smoke.py @@ -0,0 +1,295 @@ +#!/usr/bin/env python + +# this is a smoke-test using "./bin/tahoe" to: +# +# 1. create an introducer +# 2. create 5 storage nodes +# 3. create 2 client nodes (client0 = alice, client1 = bob) +# 4. Alice creates a magic-folder ("magik:") +# 5. Alice invites Bob +# 6. Bob joins +# +# After that, some basic tests are performed; see the "if True:" +# blocks to turn some on or off. Could benefit from some cleanups +# etc. but this seems useful out of the gate for quick testing. +# +# TO RUN: +# from top-level of your checkout (we use "./bin/tahoe"): +# python src/allmydata/test/check_magicfolder_smoke.py +# +# This will create "./smoke_magicfolder" (which is disposable) and +# contains all the Tahoe basedirs for the introducer, storage nodes, +# clients, and the clients' magic-folders. NOTE that if these +# directories already exist they will NOT be re-created. So kill the +# grid and then "rm -rf smoke_magicfolder" if you want to re-run the +# tests cleanly. +# +# Run the script with a single arg, "kill" to run "tahoe stop" on all +# the nodes. +# +# This will have "tahoe start" -ed all the nodes, so you can continue +# to play around after the script exits. + +from __future__ import print_function + +import sys +import time +import subprocess +from os.path import join, abspath, curdir, exists +from os import mkdir, listdir, unlink + +tahoe_base = abspath(curdir) +data_base = join(tahoe_base, 'smoke_magicfolder') +tahoe_bin = join(tahoe_base, 'bin', 'tahoe') + +if not exists(data_base): + print("Creating", data_base) + mkdir(data_base) + +if not exists(tahoe_bin): + raise RuntimeError("Can't find 'tahoe' binary at '{}'".format(tahoe_bin)) + +if 'kill' in sys.argv: + print("Killing the grid") + for d in listdir(data_base): + print("killing", d) + subprocess.call( + [ + tahoe_bin, 'stop', join(data_base, d), + ] + ) + sys.exit(0) + +if not exists(join(data_base, 'introducer')): + subprocess.check_call( + [ + tahoe_bin, 'create-introducer', join(data_base, 'introducer'), + ] + ) +with open(join(data_base, 'introducer', 'tahoe.cfg'), 'w') as f: + f.write(''' +[node] +nickname = introducer0 +web.port = 4560 +''') +subprocess.check_call( + [ + tahoe_bin, 'start', join(data_base, 'introducer'), + ] +) + +furl_fname = join(data_base, 'introducer', 'private', 'introducer.furl') +while not exists(furl_fname): + time.sleep(1) +furl = open(furl_fname, 'r').read() +print("FURL", furl) + +for x in range(5): + data_dir = join(data_base, 'node%d' % x) + if not exists(data_dir): + subprocess.check_call( + [ + tahoe_bin, 'create-node', data_dir, + ] + ) + with open(join(data_dir, 'tahoe.cfg'), 'w') as f: + f.write(''' +[node] +nickname = node{node_id} +web.port = +web.static = public_html +tub.location = localhost:{tub_port} + +[client] +# Which services should this client connect to? +introducer.furl = {furl} +shares.needed = 2 +shares.happy = 3 +shares.total = 4 +'''.format(node_id=x, furl=furl, tub_port=(9900 + x))) + subprocess.check_call( + [ + tahoe_bin, 'start', data_dir, + ] + ) + + + +# alice and bob clients +do_invites = False +for x in range(2): + data_dir = join(data_base, 'client%d' % x) + magic_dir = join(data_base, 'client%d-magic' % x) + mkdir(magic_dir) + if not exists(data_dir): + do_invites = True + subprocess.check_call( + [ + tahoe_bin, 'create-node', data_dir, + ] + ) + with open(join(data_dir, 'tahoe.cfg'), 'w') as f: + f.write(''' +[node] +nickname = client{node_id} +web.port = tcp:998{node_id}:interface=localhost +web.static = public_html + +[client] +# Which services should this client connect to? +introducer.furl = {furl} +shares.needed = 2 +shares.happy = 3 +shares.total = 4 +'''.format(node_id=x, furl=furl, magic_dir=magic_dir)) + subprocess.check_call( + [ + tahoe_bin, 'start', data_dir, + ] + ) + +# okay, now we have alice + bob (client0, client1) +# now we have alice create a magic-folder, and invite bob to it + +if do_invites: + data_dir = join(data_base, 'client0') + # alice/client0 creates her folder, invites bob + print("Alice creates a magic-folder") + subprocess.check_call( + [ + tahoe_bin, 'magic-folder', 'create', '--basedir', data_dir, 'magik:', 'alice', + join(data_base, 'client0-magic'), + ] + ) + print("Alice invites Bob") + invite = subprocess.check_output( + [ + tahoe_bin, 'magic-folder', 'invite', '--basedir', data_dir, 'magik:', 'bob', + ] + ) + print(" invite:", invite) + + # now we let "bob"/client1 join + print("Bob joins Alice's magic folder") + data_dir = join(data_base, 'client1') + subprocess.check_call( + [ + tahoe_bin, 'magic-folder', 'join', '--basedir', data_dir, invite, + join(data_base, 'client1-magic'), + ] + ) + print("Bob has joined.") + + print("Restarting alice + bob clients") + subprocess.check_call( + [ + tahoe_bin, 'restart', '--basedir', join(data_base, 'client0'), + ] + ) + subprocess.check_call( + [ + tahoe_bin, 'restart', '--basedir', join(data_base, 'client1'), + ] + ) + +if True: + for x in range(2): + with open(join(data_base, 'client{}'.format(x), 'private', 'magic_folder_dircap'), 'r') as f: + print("dircap{}: {}".format(x, f.read().strip())) + +if True: + # alice writes a file; bob should get it + alice_foo = join(data_base, 'client0-magic', 'first_file') + bob_foo = join(data_base, 'client1-magic', 'first_file') + with open(alice_foo, 'w') as f: + f.write("line one\n") + + print("Waiting for:", bob_foo) + while True: + if exists(bob_foo): + print(" found", bob_foo) + with open(bob_foo, 'r') as f: + if f.read() == "line one\n": + break + print(" file contents still mismatched") + time.sleep(1) + +if True: + # bob writes a file; alice should get it + alice_bar = join(data_base, 'client0-magic', 'second_file') + bob_bar = join(data_base, 'client1-magic', 'second_file') + with open(bob_bar, 'w') as f: + f.write("line one\n") + + print("Waiting for:", alice_bar) + while True: + if exists(bob_bar): + print(" found", bob_bar) + with open(bob_bar, 'r') as f: + if f.read() == "line one\n": + break + print(" file contents still mismatched") + time.sleep(1) + +if True: + # bob deletes alice's "first_file"; alice should also delete it + alice_foo = join(data_base, 'client0-magic', 'first_file') + bob_foo = join(data_base, 'client1-magic', 'first_file') + unlink(alice_foo) + + print("Waiting for '%s' to disappear" % (bob_foo,)) + while True: + if not exists(bob_foo): + print(" disappeared", bob_foo) + break + time.sleep(1) + +if True: + # re-write 'second_file' + alice_foo = join(data_base, 'client0-magic', 'second_file') + bob_foo = join(data_base, 'client1-magic', 'second_file') + gold_content = "line one\nsecond line\n" + + with open(bob_foo, 'w') as f: + f.write(gold_content) + + print("Waiting for:", alice_foo) + while True: + if exists(alice_foo): + print(" found", alice_foo) + with open(alice_foo, 'r') as f: + content = f.read() + if content == gold_content: + break + print(" file contents still mismatched:\n") + time.sleep(1) + +if True: + # restore 'first_file' but with different contents + alice_foo = join(data_base, 'client0-magic', 'first_file') + bob_foo = join(data_base, 'client1-magic', 'first_file') + gold_content = "see it again for the first time\n" + + with open(bob_foo, 'w') as f: + f.write(gold_content) + + print("Waiting for:", alice_foo) + while True: + if exists(alice_foo): + print(" found", alice_foo) + with open(alice_foo, 'r') as f: + content = f.read() + if content == gold_content: + break + print(" file contents still mismatched:\n") + time.sleep(1) + + + +# XXX test .backup (delete a file) + +# port david's clock.advance stuff +# fix clock.advance() +# subdirectory +# file deletes +# conflicts diff --git a/src/allmydata/test/test_magic_folder.py b/src/allmydata/test/test_magic_folder.py index 39a71ed1..fe5255c5 100644 --- a/src/allmydata/test/test_magic_folder.py +++ b/src/allmydata/test/test_magic_folder.py @@ -241,6 +241,43 @@ class MagicFolderTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, ReallyEqual d.addBoth(self.cleanup) return d + @defer.inlineCallbacks + def X_test_delete(self): + self.set_up_grid() + self.local_dir = os.path.join(self.basedir, u"local_dir") + self.mkdir_nonascii(self.local_dir) + + yield self.create_invite_join_magic_folder(u"Alice\u0101", self.local_dir) + yield self._restart_client(None) + + try: + # create a file + uploaded_d = self.magicfolder.uploader.set_hook('processed') + downloaded_d = self.magicfolder.downloader.set_hook('processed') + path = os.path.join(self.local_dir, u'foo') + with open(path, 'w') as f: + f.write('foo\n') + self.notify(to_filepath(path), self.inotify.IN_CLOSE_WRITE) + yield uploaded_d + #yield downloaded_d + self.assertTrue(os.path.exists(path)) + + # the real test part: delete the file + uploaded_d2 = self.magicfolder.uploader.set_hook('processed') + os.unlink(path) + self.notify(to_filepath(path), self.inotify.IN_DELETE) + yield uploaded_d2 + yield downloaded_d + self.assertFalse(os.path.exists(path)) + + # ensure we still have a DB entry, and that the version is 1 + node, metadata = yield self.magicfolder.downloader._get_collective_latest_file(u'foo') + self.assertTrue(node is not None, "Failed to find '{}' in DMD".format(path)) + self.failUnlessEqual(metadata['version'], 1) + + finally: + yield self.cleanup(None) + def test_magic_folder(self): self.set_up_grid() self.local_dir = os.path.join(self.basedir, self.unicode_or_fallback(u"loc\u0101l_dir", u"local_dir")) @@ -336,6 +373,10 @@ class MagicFolderTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, ReallyEqual #print "_check_version_in_local_db: %r has version %s" % (relpath_u, version) self.failUnlessEqual(version, expected_version) + def _check_file_gone(self, magicfolder, relpath_u): + path = os.path.join(magicfolder.uploader._local_path_u, relpath_u) + self.assertTrue(not os.path.exists(path)) + def test_alice_bob(self): alice_clock = task.Clock() bob_clock = task.Clock() @@ -402,6 +443,7 @@ class MagicFolderTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, ReallyEqual d.addCallback(lambda ign: self._check_version_in_local_db(self.bob_magicfolder, u"file1", 1)) d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file1", 1)) + d.addCallback(lambda ign: self._check_file_gone(self.bob_magicfolder, u"file1")) d.addCallback(_check_downloader_count, 'objects_failed', 0) d.addCallback(_check_downloader_count, 'objects_downloaded', 2)