self._log("_process(%r)" % (item,))
if now is None:
now = time.time()
+
(relpath_u, file_node, metadata) = item
fp = self._get_filepath(relpath_u)
abspath_u = unicode_from_filepath(fp)
+ deleted = metadata.get('deleted', False)
+ is_directory = relpath_u.endswith(u"/")
d = defer.succeed(None)
- if relpath_u.endswith(u"/"):
- self._log("mkdir(%r)" % (abspath_u,))
- d.addCallback(lambda ign: fileutil.make_dirs(abspath_u))
- d.addCallback(lambda ign: abspath_u)
+ if is_directory:
+ if deleted:
+ self._log("rmdir(%r)" % (abspath_u,))
+ # what to do here, if anything?
+ else:
+ self._log("mkdir(%r)" % (abspath_u,))
+ d.addCallback(lambda ign: fileutil.make_dirs(abspath_u))
+ d.addCallback(lambda ign: abspath_u)
else:
- d.addCallback(lambda ign: file_node.download_best_version())
- d.addCallback(lambda contents: self._write_downloaded_file(abspath_u, contents, is_conflict=False))
+ if deleted:
+ d.addCallback(lambda ign: self._unlink_deleted_file(abspath_u))
+ else:
+ d.addCallback(lambda ign: file_node.download_best_version())
+ d.addCallback(lambda contents: self._write_downloaded_file(abspath_u, contents, is_conflict=False))
def do_update_db(written_abspath_u):
filecap = file_node.get_uri()
last_downloaded_uri = filecap
last_downloaded_timestamp = now
written_pathinfo = get_pathinfo(written_abspath_u)
- if not written_pathinfo.exists:
+ if not deleted and not written_pathinfo.exists:
raise Exception("downloaded object %s disappeared" % quote_local_unicode_path(written_abspath_u))
self._db.did_upload_version(relpath_u, metadata['version'], last_uploaded_uri,
return res
d.addBoth(remove_from_pending)
return d
+
+ FUDGE_SECONDS = 10.0
+
+ def _unlink_deleted_file(self, abspath_u):
+ fileutil.remove(abspath_u)
+ return abspath_u
+
+ @classmethod
+ def _write_downloaded_file(cls, abspath_u, file_contents, is_conflict=False, now=None):
+ # 1. Write a temporary file, say .foo.tmp.
+ # 2. is_conflict determines whether this is an overwrite or a conflict.
+ # 3. Set the mtime of the replacement file to be T seconds before the
+ # current local time.
+ # 4. Perform a file replacement with backup filename foo.backup,
+ # replaced file foo, and replacement file .foo.tmp. If any step of
+ # this operation fails, reclassify as a conflict and stop.
+ #
+ # Returns the path of the destination file.
+
+ precondition_abspath(abspath_u)
+ replacement_path_u = abspath_u + u".tmp" # FIXME more unique
+ backup_path_u = abspath_u + u".backup"
+ if now is None:
+ now = time.time()
+
+ fileutil.write(replacement_path_u, file_contents)
+ os.utime(replacement_path_u, (now, now - cls.FUDGE_SECONDS))
+ if is_conflict:
+ return cls._rename_conflicted_file(abspath_u, replacement_path_u)
+ else:
+ try:
+ fileutil.replace_file(abspath_u, replacement_path_u, backup_path_u)
+ return abspath_u
+ except fileutil.ConflictError:
+ return cls._rename_conflicted_file(abspath_u, replacement_path_u)
+
+ @classmethod
+ def _rename_conflicted_file(self, abspath_u, replacement_path_u):
+ conflict_path_u = abspath_u + u".conflict"
+ fileutil.rename_no_overwrite(replacement_path_u, conflict_path_u)
+ return conflict_path_u
--- /dev/null
+#!/usr/bin/env python
+
+# this is a smoke-test using "./bin/tahoe" to:
+#
+# 1. create an introducer
+# 2. create 5 storage nodes
+# 3. create 2 client nodes (client0 = alice, client1 = bob)
+# 4. Alice creates a magic-folder ("magik:")
+# 5. Alice invites Bob
+# 6. Bob joins
+#
+# After that, some basic tests are performed; see the "if True:"
+# blocks to turn some on or off. Could benefit from some cleanups
+# etc. but this seems useful out of the gate for quick testing.
+#
+# TO RUN:
+# from top-level of your checkout (we use "./bin/tahoe"):
+# python src/allmydata/test/check_magicfolder_smoke.py
+#
+# This will create "./smoke_magicfolder" (which is disposable) and
+# contains all the Tahoe basedirs for the introducer, storage nodes,
+# clients, and the clients' magic-folders. NOTE that if these
+# directories already exist they will NOT be re-created. So kill the
+# grid and then "rm -rf smoke_magicfolder" if you want to re-run the
+# tests cleanly.
+#
+# Run the script with a single arg, "kill" to run "tahoe stop" on all
+# the nodes.
+#
+# This will have "tahoe start" -ed all the nodes, so you can continue
+# to play around after the script exits.
+
+from __future__ import print_function
+
+import sys
+import time
+import subprocess
+from os.path import join, abspath, curdir, exists
+from os import mkdir, listdir, unlink
+
+tahoe_base = abspath(curdir)
+data_base = join(tahoe_base, 'smoke_magicfolder')
+tahoe_bin = join(tahoe_base, 'bin', 'tahoe')
+
+if not exists(data_base):
+ print("Creating", data_base)
+ mkdir(data_base)
+
+if not exists(tahoe_bin):
+ raise RuntimeError("Can't find 'tahoe' binary at '{}'".format(tahoe_bin))
+
+if 'kill' in sys.argv:
+ print("Killing the grid")
+ for d in listdir(data_base):
+ print("killing", d)
+ subprocess.call(
+ [
+ tahoe_bin, 'stop', join(data_base, d),
+ ]
+ )
+ sys.exit(0)
+
+if not exists(join(data_base, 'introducer')):
+ subprocess.check_call(
+ [
+ tahoe_bin, 'create-introducer', join(data_base, 'introducer'),
+ ]
+ )
+with open(join(data_base, 'introducer', 'tahoe.cfg'), 'w') as f:
+ f.write('''
+[node]
+nickname = introducer0
+web.port = 4560
+''')
+subprocess.check_call(
+ [
+ tahoe_bin, 'start', join(data_base, 'introducer'),
+ ]
+)
+
+furl_fname = join(data_base, 'introducer', 'private', 'introducer.furl')
+while not exists(furl_fname):
+ time.sleep(1)
+furl = open(furl_fname, 'r').read()
+print("FURL", furl)
+
+for x in range(5):
+ data_dir = join(data_base, 'node%d' % x)
+ if not exists(data_dir):
+ subprocess.check_call(
+ [
+ tahoe_bin, 'create-node', data_dir,
+ ]
+ )
+ with open(join(data_dir, 'tahoe.cfg'), 'w') as f:
+ f.write('''
+[node]
+nickname = node{node_id}
+web.port =
+web.static = public_html
+tub.location = localhost:{tub_port}
+
+[client]
+# Which services should this client connect to?
+introducer.furl = {furl}
+shares.needed = 2
+shares.happy = 3
+shares.total = 4
+'''.format(node_id=x, furl=furl, tub_port=(9900 + x)))
+ subprocess.check_call(
+ [
+ tahoe_bin, 'start', data_dir,
+ ]
+ )
+
+
+
+# alice and bob clients
+do_invites = False
+for x in range(2):
+ data_dir = join(data_base, 'client%d' % x)
+ magic_dir = join(data_base, 'client%d-magic' % x)
+ mkdir(magic_dir)
+ if not exists(data_dir):
+ do_invites = True
+ subprocess.check_call(
+ [
+ tahoe_bin, 'create-node', data_dir,
+ ]
+ )
+ with open(join(data_dir, 'tahoe.cfg'), 'w') as f:
+ f.write('''
+[node]
+nickname = client{node_id}
+web.port = tcp:998{node_id}:interface=localhost
+web.static = public_html
+
+[client]
+# Which services should this client connect to?
+introducer.furl = {furl}
+shares.needed = 2
+shares.happy = 3
+shares.total = 4
+'''.format(node_id=x, furl=furl, magic_dir=magic_dir))
+ subprocess.check_call(
+ [
+ tahoe_bin, 'start', data_dir,
+ ]
+ )
+
+# okay, now we have alice + bob (client0, client1)
+# now we have alice create a magic-folder, and invite bob to it
+
+if do_invites:
+ data_dir = join(data_base, 'client0')
+ # alice/client0 creates her folder, invites bob
+ print("Alice creates a magic-folder")
+ subprocess.check_call(
+ [
+ tahoe_bin, 'magic-folder', 'create', '--basedir', data_dir, 'magik:', 'alice',
+ join(data_base, 'client0-magic'),
+ ]
+ )
+ print("Alice invites Bob")
+ invite = subprocess.check_output(
+ [
+ tahoe_bin, 'magic-folder', 'invite', '--basedir', data_dir, 'magik:', 'bob',
+ ]
+ )
+ print(" invite:", invite)
+
+ # now we let "bob"/client1 join
+ print("Bob joins Alice's magic folder")
+ data_dir = join(data_base, 'client1')
+ subprocess.check_call(
+ [
+ tahoe_bin, 'magic-folder', 'join', '--basedir', data_dir, invite,
+ join(data_base, 'client1-magic'),
+ ]
+ )
+ print("Bob has joined.")
+
+ print("Restarting alice + bob clients")
+ subprocess.check_call(
+ [
+ tahoe_bin, 'restart', '--basedir', join(data_base, 'client0'),
+ ]
+ )
+ subprocess.check_call(
+ [
+ tahoe_bin, 'restart', '--basedir', join(data_base, 'client1'),
+ ]
+ )
+
+if True:
+ for x in range(2):
+ with open(join(data_base, 'client{}'.format(x), 'private', 'magic_folder_dircap'), 'r') as f:
+ print("dircap{}: {}".format(x, f.read().strip()))
+
+if True:
+ # alice writes a file; bob should get it
+ alice_foo = join(data_base, 'client0-magic', 'first_file')
+ bob_foo = join(data_base, 'client1-magic', 'first_file')
+ with open(alice_foo, 'w') as f:
+ f.write("line one\n")
+
+ print("Waiting for:", bob_foo)
+ while True:
+ if exists(bob_foo):
+ print(" found", bob_foo)
+ with open(bob_foo, 'r') as f:
+ if f.read() == "line one\n":
+ break
+ print(" file contents still mismatched")
+ time.sleep(1)
+
+if True:
+ # bob writes a file; alice should get it
+ alice_bar = join(data_base, 'client0-magic', 'second_file')
+ bob_bar = join(data_base, 'client1-magic', 'second_file')
+ with open(bob_bar, 'w') as f:
+ f.write("line one\n")
+
+ print("Waiting for:", alice_bar)
+ while True:
+ if exists(bob_bar):
+ print(" found", bob_bar)
+ with open(bob_bar, 'r') as f:
+ if f.read() == "line one\n":
+ break
+ print(" file contents still mismatched")
+ time.sleep(1)
+
+if True:
+ # bob deletes alice's "first_file"; alice should also delete it
+ alice_foo = join(data_base, 'client0-magic', 'first_file')
+ bob_foo = join(data_base, 'client1-magic', 'first_file')
+ unlink(alice_foo)
+
+ print("Waiting for '%s' to disappear" % (bob_foo,))
+ while True:
+ if not exists(bob_foo):
+ print(" disappeared", bob_foo)
+ break
+ time.sleep(1)
+
+if True:
+ # re-write 'second_file'
+ alice_foo = join(data_base, 'client0-magic', 'second_file')
+ bob_foo = join(data_base, 'client1-magic', 'second_file')
+ gold_content = "line one\nsecond line\n"
+
+ with open(bob_foo, 'w') as f:
+ f.write(gold_content)
+
+ print("Waiting for:", alice_foo)
+ while True:
+ if exists(alice_foo):
+ print(" found", alice_foo)
+ with open(alice_foo, 'r') as f:
+ content = f.read()
+ if content == gold_content:
+ break
+ print(" file contents still mismatched:\n")
+ time.sleep(1)
+
+if True:
+ # restore 'first_file' but with different contents
+ alice_foo = join(data_base, 'client0-magic', 'first_file')
+ bob_foo = join(data_base, 'client1-magic', 'first_file')
+ gold_content = "see it again for the first time\n"
+
+ with open(bob_foo, 'w') as f:
+ f.write(gold_content)
+
+ print("Waiting for:", alice_foo)
+ while True:
+ if exists(alice_foo):
+ print(" found", alice_foo)
+ with open(alice_foo, 'r') as f:
+ content = f.read()
+ if content == gold_content:
+ break
+ print(" file contents still mismatched:\n")
+ time.sleep(1)
+
+
+
+# XXX test .backup (delete a file)
+
+# port david's clock.advance stuff
+# fix clock.advance()
+# subdirectory
+# file deletes
+# conflicts
d.addBoth(self.cleanup)
return d
+ @defer.inlineCallbacks
+ def X_test_delete(self):
+ self.set_up_grid()
+ self.local_dir = os.path.join(self.basedir, u"local_dir")
+ self.mkdir_nonascii(self.local_dir)
+
+ yield self.create_invite_join_magic_folder(u"Alice\u0101", self.local_dir)
+ yield self._restart_client(None)
+
+ try:
+ # create a file
+ uploaded_d = self.magicfolder.uploader.set_hook('processed')
+ downloaded_d = self.magicfolder.downloader.set_hook('processed')
+ path = os.path.join(self.local_dir, u'foo')
+ with open(path, 'w') as f:
+ f.write('foo\n')
+ self.notify(to_filepath(path), self.inotify.IN_CLOSE_WRITE)
+ yield uploaded_d
+ #yield downloaded_d
+ self.assertTrue(os.path.exists(path))
+
+ # the real test part: delete the file
+ uploaded_d2 = self.magicfolder.uploader.set_hook('processed')
+ os.unlink(path)
+ self.notify(to_filepath(path), self.inotify.IN_DELETE)
+ yield uploaded_d2
+ yield downloaded_d
+ self.assertFalse(os.path.exists(path))
+
+ # ensure we still have a DB entry, and that the version is 1
+ node, metadata = yield self.magicfolder.downloader._get_collective_latest_file(u'foo')
+ self.assertTrue(node is not None, "Failed to find '{}' in DMD".format(path))
+ self.failUnlessEqual(metadata['version'], 1)
+
+ finally:
+ yield self.cleanup(None)
+
def test_magic_folder(self):
self.set_up_grid()
self.local_dir = os.path.join(self.basedir, self.unicode_or_fallback(u"loc\u0101l_dir", u"local_dir"))
#print "_check_version_in_local_db: %r has version %s" % (relpath_u, version)
self.failUnlessEqual(version, expected_version)
+ def _check_file_gone(self, magicfolder, relpath_u):
+ path = os.path.join(magicfolder.uploader._local_path_u, relpath_u)
+ self.assertTrue(not os.path.exists(path))
+
def test_alice_bob(self):
alice_clock = task.Clock()
bob_clock = task.Clock()
d.addCallback(lambda ign: self._check_version_in_local_db(self.bob_magicfolder, u"file1", 1))
d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file1", 1))
+ d.addCallback(lambda ign: self._check_file_gone(self.bob_magicfolder, u"file1"))
d.addCallback(_check_downloader_count, 'objects_failed', 0)
d.addCallback(_check_downloader_count, 'objects_downloaded', 2)
projects / tahoe-lafs / tahoe-lafs.git / commitdiff