From e68b09b081603b3f86bee4eaff67299125786d26 Mon Sep 17 00:00:00 2001 From: Daira Hopwood Date: Tue, 21 Jul 2015 00:42:15 +0100 Subject: [PATCH] Rename drop-upload to Magic Folder. fixes ticket:2405 Signed-off-by: Daira Hopwood --- .../{drop-upload.rst => magic-folder.rst} | 37 ++--- src/allmydata/client.py | 26 ++-- .../{drop_upload.py => magic_folder.py} | 44 +++--- src/allmydata/test/test_client.py | 52 +++---- ...st_drop_upload.py => test_magic_folder.py} | 138 +++++++++--------- 5 files changed, 150 insertions(+), 147 deletions(-) rename docs/frontends/{drop-upload.rst => magic-folder.rst} (84%) rename src/allmydata/frontends/{drop_upload.py => magic_folder.py} (85%) rename src/allmydata/test/{test_drop_upload.py => test_magic_folder.py} (76%) diff --git a/docs/frontends/drop-upload.rst b/docs/frontends/magic-folder.rst similarity index 84% rename from docs/frontends/drop-upload.rst rename to docs/frontends/magic-folder.rst index b6fcd92c..7835c990 100644 --- a/docs/frontends/drop-upload.rst +++ b/docs/frontends/magic-folder.rst @@ -1,8 +1,8 @@ .. -*- coding: utf-8-with-signature -*- -=============================== -Tahoe-LAFS Drop-Upload Frontend -=============================== +================================ +Tahoe-LAFS Magic Folder Frontend +================================ 1. `Introduction`_ 2. `Configuration`_ @@ -12,7 +12,7 @@ Tahoe-LAFS Drop-Upload Frontend Introduction ============ -The drop-upload frontend allows an upload to a Tahoe-LAFS grid to be triggered +The Magic Folder frontend allows an upload to a Tahoe-LAFS grid to be triggered automatically whenever a file is created or changed in a specific local directory. It currently works on Linux and Windows. @@ -30,18 +30,18 @@ suggestions to improve its usability, functionality, and reliability. Configuration ============= -The drop-upload frontend runs as part of a gateway node. To set it up, you +The Magic Folder frontend runs as part of a gateway node. To set it up, you need to choose the local directory to monitor for file changes, and a mutable directory on the grid to which files will be uploaded. -These settings are configured in the ``[drop_upload]`` section of the +These settings are configured in the ``[magic_folder]`` section of the gateway's ``tahoe.cfg`` file. -``[drop_upload]`` +``[magic_folder]`` ``enabled = (boolean, optional)`` - If this is ``True``, drop-upload will be enabled. The default value is + If this is ``True``, Magic Folder will be enabled. The default value is ``False``. ``local.directory = (UTF-8 path)`` @@ -51,10 +51,11 @@ gateway's ``tahoe.cfg`` file. in UTF-8 regardless of the system's filesystem encoding. Relative paths will be interpreted starting from the node's base directory. -In addition, the file ``private/drop_upload_dircap`` must contain a -writecap pointing to an existing mutable directory to be used as the target -of uploads. It will start with ``URI:DIR2:``, and cannot include an alias -or path. +In addition: + * the file ``private/magic_folder_dircap`` must contain a writecap pointing + to an existing mutable directory to be used as the target of uploads. + It will start with ``URI:DIR2:``, and cannot include an alias or path. + * the file ``private/collective_dircap`` must contain a readcap After setting the above fields and starting or restarting the gateway, you can confirm that the feature is working by copying a file into the @@ -91,11 +92,11 @@ The only way to determine whether uploads have failed is to look at the 'Operational Statistics' page linked from the Welcome page. This only shows a count of failures, not the names of files. Uploads are never retried. -The drop-upload frontend performs its uploads sequentially (i.e. it waits +The Magic Folder frontend performs its uploads sequentially (i.e. it waits until each upload is finished before starting the next), even when there would be enough memory and bandwidth to efficiently perform them in parallel. -A drop-upload can occur in parallel with an upload by a different frontend, -though. (`#1459`_) +A Magic Folder upload can occur in parallel with an upload by a different +frontend, though. (`#1459`_) On Linux, if there are a large number of near-simultaneous file creation or change events (greater than the number specified in the file @@ -126,8 +127,8 @@ up-to-date. (`#1440`_) Files deleted from the local directory will not be unlinked from the upload directory. (`#1710`_) -The ``private/drop_upload_dircap`` file cannot use an alias or path to -specify the upload directory. (`#1711`_) +The ``private/magic_folder_dircap`` and ``private/collective_dircap`` files +cannot use an alias or path to specify the upload directory. (`#1711`_) Files are always uploaded as immutable. If there is an existing mutable file of the same name in the upload directory, it will be unlinked and replaced @@ -146,7 +147,7 @@ The expected encoding is that printed by On Windows, local directories with non-ASCII names are not currently working. (`#2219`_) -On Windows, when a node has drop-upload enabled, it is unresponsive to Ctrl-C +On Windows, when a node has Magic Folder enabled, it is unresponsive to Ctrl-C (it can only be killed using Task Manager or similar). (`#2218`_) .. _`#1105`: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/1105 diff --git a/src/allmydata/client.py b/src/allmydata/client.py index b0d598ba..86ae9ac2 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -151,7 +151,7 @@ class Client(node.Node, pollmixin.PollMixin): # ControlServer and Helper are attached after Tub startup self.init_ftp_server() self.init_sftp_server() - self.init_drop_uploader() + self.init_magic_folder() # If the node sees an exit_trigger file, it will poll every second to see # whether the file still exists, and what its mtime is. If the file does not @@ -492,33 +492,33 @@ class Client(node.Node, pollmixin.PollMixin): sftp_portstr, pubkey_file, privkey_file) s.setServiceParent(self) - def init_drop_uploader(self): + def init_magic_folder(self): if self.get_config("drop_upload", "enabled", False, boolean=True): - if self.get_config("drop_upload", "upload.dircap", None): - raise OldConfigOptionError("The [drop_upload]upload.dircap option is no longer supported; please " - "put the cap in a 'private/drop_upload_dircap' file, and delete this option.") + raise OldConfigOptionError("The [drop_upload] section must be renamed to [magic_folder].\n" + "See docs/frontends/magic-folder.rst for more information.") - upload_dircap = self.get_or_create_private_config("drop_upload_dircap") - local_dir_config = self.get_config("drop_upload", "local.directory").decode("utf-8") + if self.get_config("magic_folder", "enabled", False, boolean=True): + upload_dircap = self.get_or_create_private_config("magic_folder_dircap") + local_dir_config = self.get_config("magic_folder", "local.directory").decode("utf-8") local_dir = abspath_expanduser_unicode(local_dir_config, base=self.basedir) try: - from allmydata.frontends import drop_upload + from allmydata.frontends import magic_folder dbfile = os.path.join(self.basedir, "private", "magicfolderdb.sqlite") dbfile = abspath_expanduser_unicode(dbfile) - parent_dircap_path = os.path.join(self.basedir, "private", "magic_folder_parent_dircap") - parent_dircap_path = abspath_expanduser_unicode(parent_dircap_path) - parent_dircap = fileutil.read(parent_dircap_path).strip() + collective_dircap_path = os.path.join(self.basedir, "private", "collective_dircap") + collective_dircap_path = abspath_expanduser_unicode(collective_dircap_path) + collective_dircap = fileutil.read(collective_dircap_path).strip() - s = drop_upload.DropUploader(self, upload_dircap, parent_dircap, local_dir, dbfile) + s = magic_folder.MagicFolder(self, upload_dircap, collective_dircap, local_dir, dbfile) s.setServiceParent(self) s.startService() # start processing the upload queue when we've connected to enough servers self.upload_ready_d.addCallback(s.upload_ready) except Exception, e: - self.log("couldn't start drop-uploader: %r", args=(e,)) + self.log("couldn't start Magic Folder: %r", args=(e,)) def _check_exit_trigger(self, exit_trigger_file): if os.path.exists(exit_trigger_file): diff --git a/src/allmydata/frontends/drop_upload.py b/src/allmydata/frontends/magic_folder.py similarity index 85% rename from src/allmydata/frontends/drop_upload.py rename to src/allmydata/frontends/magic_folder.py index 0be45f12..79dff178 100644 --- a/src/allmydata/frontends/drop_upload.py +++ b/src/allmydata/frontends/magic_folder.py @@ -38,10 +38,10 @@ def get_inotify_module(): raise -class DropUploader(service.MultiService): - name = 'drop-upload' +class MagicFolder(service.MultiService): + name = 'magic-folder' - def __init__(self, client, upload_dircap, parent_dircap, local_dir, dbfile, inotify=None, + def __init__(self, client, upload_dircap, collective_dircap, local_dir, dbfile, inotify=None, pending_delay=1.0): precondition_abspath(local_dir) @@ -61,20 +61,20 @@ class DropUploader(service.MultiService): self._inotify = inotify or get_inotify_module() if not self._local_path.exists(): - raise AssertionError("The '[drop_upload] local.directory' parameter was %s " + raise AssertionError("The '[magic_folder] local.directory' parameter was %s " "but there is no directory at that location." % quote_local_unicode_path(local_dir)) if not self._local_path.isdir(): - raise AssertionError("The '[drop_upload] local.directory' parameter was %s " + raise AssertionError("The '[magic_folder] local.directory' parameter was %s " "but the thing at that location is not a directory." % quote_local_unicode_path(local_dir)) # TODO: allow a path rather than a cap URI. - self._parent = self._client.create_node_from_uri(upload_dircap) - if not IDirectoryNode.providedBy(self._parent): - raise AssertionError("The URI in 'private/drop_upload_dircap' does not refer to a directory.") - if self._parent.is_unknown() or self._parent.is_readonly(): - raise AssertionError("The URI in 'private/drop_upload_dircap' is not a writecap to a directory.") + self._upload_dirnode = self._client.create_node_from_uri(upload_dircap) + if not IDirectoryNode.providedBy(self._upload_dirnode): + raise AssertionError("The URI in 'private/magic_folder_dircap' does not refer to a directory.") + if self._upload_dirnode.is_unknown() or self._upload_dirnode.is_readonly(): + raise AssertionError("The URI in 'private/magic_folder_dircap' is not a writecap to a directory.") self._processed_callback = lambda ign: None self._ignore_count = 0 @@ -150,7 +150,7 @@ class DropUploader(service.MultiService): self._scan(self._local_dir) - self._stats_provider.count('drop_upload.dirs_monitored', 1) + self._stats_provider.count('magic_folder.dirs_monitored', 1) return d def upload_ready(self): @@ -163,7 +163,7 @@ class DropUploader(service.MultiService): def _append_to_deque(self, path): self._upload_deque.append(path) self._pending.add(path) - self._stats_provider.count('drop_upload.objects_queued', 1) + self._stats_provider.count('magic_folder.objects_queued', 1) if self.is_upload_ready: reactor.callLater(0, self._turn_deque) @@ -188,16 +188,16 @@ class DropUploader(service.MultiService): def _add_file(name): u = FileName(path, self._convergence) - return self._parent.add_file(name, u, overwrite=True) + return self._upload_dirnode.add_file(name, u, overwrite=True) def _add_dir(name): self._notifier.watch(to_filepath(path), mask=self.mask, callbacks=[self._notify], recursive=True) u = Data("", self._convergence) name += "@_" - d2 = self._parent.add_file(name, u, overwrite=True) + d2 = self._upload_dirnode.add_file(name, u, overwrite=True) def _succeeded(ign): self._log("created subdirectory %r" % (path,)) - self._stats_provider.count('drop_upload.directories_created', 1) + self._stats_provider.count('magic_folder.directories_created', 1) def _failed(f): self._log("failed to create subdirectory %r" % (path,)) return f @@ -213,7 +213,7 @@ class DropUploader(service.MultiService): if not os.path.exists(path): self._log("drop-upload: notified object %r disappeared " "(this is normal for temporary objects)" % (path,)) - self._stats_provider.count('drop_upload.objects_disappeared', 1) + self._stats_provider.count('magic_folder.objects_disappeared', 1) return None elif os.path.islink(path): raise Exception("symlink not being processed") @@ -229,7 +229,7 @@ class DropUploader(service.MultiService): ctime = s[stat.ST_CTIME] mtime = s[stat.ST_MTIME] self._db.did_upload_file(filecap, path, mtime, ctime, size) - self._stats_provider.count('drop_upload.files_uploaded', 1) + self._stats_provider.count('magic_folder.files_uploaded', 1) d2.addCallback(add_db_entry) return d2 else: @@ -238,12 +238,12 @@ class DropUploader(service.MultiService): d.addCallback(_maybe_upload) def _succeeded(res): - self._stats_provider.count('drop_upload.objects_queued', -1) - self._stats_provider.count('drop_upload.objects_succeeded', 1) + self._stats_provider.count('magic_folder.objects_queued', -1) + self._stats_provider.count('magic_folder.objects_succeeded', 1) return res def _failed(f): - self._stats_provider.count('drop_upload.objects_queued', -1) - self._stats_provider.count('drop_upload.objects_failed', 1) + self._stats_provider.count('magic_folder.objects_queued', -1) + self._stats_provider.count('magic_folder.objects_failed', 1) self._log("%r while processing %r" % (f, path)) return f d.addCallbacks(_succeeded, _failed) @@ -267,7 +267,7 @@ class DropUploader(service.MultiService): def finish(self, for_tests=False): self._notifier.stopReading() - self._stats_provider.count('drop_upload.dirs_monitored', -1) + self._stats_provider.count('magic_folder.dirs_monitored', -1) if for_tests and hasattr(self._notifier, 'wait_until_stopped'): return self._notifier.wait_until_stopped() else: diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py index 0547d7a0..74ed4f74 100644 --- a/src/allmydata/test/test_client.py +++ b/src/allmydata/test/test_client.py @@ -4,7 +4,7 @@ from twisted.trial import unittest from twisted.application import service import allmydata -import allmydata.frontends.drop_upload +import allmydata.frontends.magic_folder import allmydata.util.log from allmydata.node import Node, OldConfigError, OldConfigOptionError, MissingConfigEntry, UnescapedHashError @@ -302,20 +302,21 @@ class Basic(testutil.ReallyEqualMixin, testutil.NonASCIIPathMixin, unittest.Test _check("helper.furl = None", None) _check("helper.furl = pb://blah\n", "pb://blah") - def test_create_drop_uploader(self): - class MockDropUploader(service.MultiService): - name = 'drop-upload' + def test_create_magic_folder(self): + class MockMagicFolder(service.MultiService): + name = 'magic-folder' - def __init__(self, client, upload_dircap, parent_dircap, local_dir, dbfile, inotify=None, + def __init__(self, client, upload_dircap, collective_dircap, local_dir, dbfile, inotify=None, pending_delay=1.0): service.MultiService.__init__(self) self.client = client self.upload_dircap = upload_dircap + self.collective_dircap = collective_dircap self.local_dir = local_dir self.dbfile = dbfile self.inotify = inotify - self.patch(allmydata.frontends.drop_upload, 'DropUploader', MockDropUploader) + self.patch(allmydata.frontends.magic_folder, 'MagicFolder', MockMagicFolder) upload_dircap = "URI:DIR2:blah" local_dir_u = self.unicode_or_fallback(u"loc\u0101l_dir", u"local_dir") @@ -323,10 +324,10 @@ class Basic(testutil.ReallyEqualMixin, testutil.NonASCIIPathMixin, unittest.Test config = (BASECONFIG + "[storage]\n" + "enabled = false\n" + - "[drop_upload]\n" + + "[magic_folder]\n" + "enabled = true\n") - basedir1 = "test_client.Basic.test_create_drop_uploader1" + basedir1 = "test_client.Basic.test_create_magic_folder1" os.mkdir(basedir1) fileutil.write(os.path.join(basedir1, "tahoe.cfg"), @@ -334,48 +335,49 @@ class Basic(testutil.ReallyEqualMixin, testutil.NonASCIIPathMixin, unittest.Test self.failUnlessRaises(MissingConfigEntry, client.Client, basedir1) fileutil.write(os.path.join(basedir1, "tahoe.cfg"), config) - fileutil.write(os.path.join(basedir1, "private", "drop_upload_dircap"), "URI:DIR2:blah") - fileutil.write(os.path.join(basedir1, "private", "magic_folder_parent_dircap"), "URI:DIR2:meow") + fileutil.write(os.path.join(basedir1, "private", "magic_folder_dircap"), "URI:DIR2:blah") + fileutil.write(os.path.join(basedir1, "private", "collective_dircap"), "URI:DIR2:meow") self.failUnlessRaises(MissingConfigEntry, client.Client, basedir1) fileutil.write(os.path.join(basedir1, "tahoe.cfg"), - config + "upload.dircap = " + upload_dircap + "\n") + config.replace("[magic_folder]\n", "[drop_upload]\n")) self.failUnlessRaises(OldConfigOptionError, client.Client, basedir1) fileutil.write(os.path.join(basedir1, "tahoe.cfg"), config + "local.directory = " + local_dir_utf8 + "\n") c1 = client.Client(basedir1) - uploader = c1.getServiceNamed('drop-upload') - self.failUnless(isinstance(uploader, MockDropUploader), uploader) - self.failUnlessReallyEqual(uploader.client, c1) - self.failUnlessReallyEqual(uploader.upload_dircap, upload_dircap) - self.failUnlessReallyEqual(os.path.basename(uploader.local_dir), local_dir_u) - self.failUnless(uploader.inotify is None, uploader.inotify) - self.failUnless(uploader.running) + magicfolder = c1.getServiceNamed('magic-folder') + self.failUnless(isinstance(magicfolder, MockMagicFolder), magicfolder) + self.failUnlessReallyEqual(magicfolder.client, c1) + self.failUnlessReallyEqual(magicfolder.upload_dircap, upload_dircap) + self.failUnlessReallyEqual(os.path.basename(magicfolder.local_dir), local_dir_u) + self.failUnless(magicfolder.inotify is None, magicfolder.inotify) + self.failUnless(magicfolder.running) class Boom(Exception): pass - def BoomDropUploader(client, upload_dircap, local_dir_utf8, inotify=None): + def BoomMagicFolder(self, client, upload_dircap, collective_dircap, local_dir, dbfile, + inotify=None, pending_delay=1.0): raise Boom() logged_messages = [] def mock_log(*args, **kwargs): logged_messages.append("%r %r" % (args, kwargs)) self.patch(allmydata.util.log, 'msg', mock_log) - self.patch(allmydata.frontends.drop_upload, 'DropUploader', BoomDropUploader) + self.patch(allmydata.frontends.magic_folder, 'MagicFolder', BoomMagicFolder) - basedir2 = "test_client.Basic.test_create_drop_uploader2" + basedir2 = "test_client.Basic.test_create_magic_folder2" os.mkdir(basedir2) os.mkdir(os.path.join(basedir2, "private")) fileutil.write(os.path.join(basedir2, "tahoe.cfg"), BASECONFIG + - "[drop_upload]\n" + + "[magic_folder]\n" + "enabled = true\n" + "local.directory = " + local_dir_utf8 + "\n") - fileutil.write(os.path.join(basedir2, "private", "drop_upload_dircap"), "URI:DIR2:blah") - fileutil.write(os.path.join(basedir2, "private", "magic_folder_parent_dircap"), "URI:DIR2:meow") + fileutil.write(os.path.join(basedir2, "private", "magic_folder_dircap"), "URI:DIR2:blah") + fileutil.write(os.path.join(basedir2, "private", "collective_dircap"), "URI:DIR2:meow") c2 = client.Client(basedir2) - self.failUnlessRaises(KeyError, c2.getServiceNamed, 'drop-upload') + self.failUnlessRaises(KeyError, c2.getServiceNamed, 'magic-folder') self.failUnless([True for arg in logged_messages if "Boom" in arg], logged_messages) diff --git a/src/allmydata/test/test_drop_upload.py b/src/allmydata/test/test_magic_folder.py similarity index 76% rename from src/allmydata/test/test_drop_upload.py rename to src/allmydata/test/test_magic_folder.py index 6be6d33c..80e2c8e9 100644 --- a/src/allmydata/test/test_drop_upload.py +++ b/src/allmydata/test/test_magic_folder.py @@ -13,13 +13,13 @@ from allmydata.test.no_network import GridTestMixin from allmydata.test.common_util import ReallyEqualMixin, NonASCIIPathMixin from allmydata.test.common import ShouldFailMixin -from allmydata.frontends import drop_upload -from allmydata.frontends.drop_upload import DropUploader +from allmydata.frontends import magic_folder +from allmydata.frontends.magic_folder import MagicFolder from allmydata import backupdb from allmydata.util.fileutil import abspath_expanduser_unicode -class DropUploadTestMixin(GridTestMixin, ShouldFailMixin, ReallyEqualMixin, NonASCIIPathMixin): +class MagicFolderTestMixin(GridTestMixin, ShouldFailMixin, ReallyEqualMixin, NonASCIIPathMixin): """ These tests will be run both with a mock notifier, and (on platforms that support it) with the real INotify. @@ -29,7 +29,7 @@ class DropUploadTestMixin(GridTestMixin, ShouldFailMixin, ReallyEqualMixin, NonA GridTestMixin.setUp(self) temp = self.mktemp() self.basedir = abspath_expanduser_unicode(temp.decode(get_filesystem_encoding())) - self.uploader = None + self.magicfolder = None self.dir_node = None def _get_count(self, name): @@ -50,20 +50,20 @@ class DropUploadTestMixin(GridTestMixin, ShouldFailMixin, ReallyEqualMixin, NonA self.failUnless(IDirectoryNode.providedBy(n)) self.upload_dirnode = n self.upload_dircap = n.get_uri() - self.parent_dircap = "abc123" + self.collective_dircap = "abc123" - def _create_uploader(self, ign): + def _create_magicfolder(self, ign): dbfile = abspath_expanduser_unicode(u"magicfolderdb.sqlite", base=self.basedir) - self.uploader = DropUploader(self.client, self.upload_dircap, self.parent_dircap, self.local_dir, - dbfile, inotify=self.inotify, pending_delay=0.2) - self.uploader.setServiceParent(self.client) - self.uploader.upload_ready() + self.magicfolder = MagicFolder(self.client, self.upload_dircap, self.collective_dircap, self.local_dir, + dbfile, inotify=self.inotify, pending_delay=0.2) + self.magicfolder.setServiceParent(self.client) + self.magicfolder.upload_ready() # Prevent unclean reactor errors. def _cleanup(self, res): d = defer.succeed(None) - if self.uploader is not None: - d.addCallback(lambda ign: self.uploader.finish(for_tests=True)) + if self.magicfolder is not None: + d.addCallback(lambda ign: self.magicfolder.finish(for_tests=True)) d.addCallback(lambda ign: res) return d @@ -100,7 +100,7 @@ class DropUploadTestMixin(GridTestMixin, ShouldFailMixin, ReallyEqualMixin, NonA r = db.check_file(path) self.failUnless(r.was_uploaded()) - def test_uploader_start_service(self): + def test_magicfolder_start_service(self): self.set_up_grid() self.local_dir = abspath_expanduser_unicode(self.unicode_or_fallback(u"l\u00F8cal_dir", u"local_dir"), @@ -112,10 +112,10 @@ class DropUploadTestMixin(GridTestMixin, ShouldFailMixin, ReallyEqualMixin, NonA d = self.client.create_dirnode() d.addCallback(self._made_upload_dir) - d.addCallback(self._create_uploader) - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('drop_upload.dirs_monitored'), 1)) + d.addCallback(self._create_magicfolder) + d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('magic_folder.dirs_monitored'), 1)) d.addBoth(self._cleanup) - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('drop_upload.dirs_monitored'), 0)) + d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('magic_folder.dirs_monitored'), 0)) return d def test_move_tree(self): @@ -139,46 +139,46 @@ class DropUploadTestMixin(GridTestMixin, ShouldFailMixin, ReallyEqualMixin, NonA d = self.client.create_dirnode() d.addCallback(self._made_upload_dir) - d.addCallback(self._create_uploader) + d.addCallback(self._create_magicfolder) def _check_move_empty_tree(res): self.mkdir_nonascii(empty_tree_dir) d2 = defer.Deferred() - self.uploader.set_processed_callback(d2.callback, ignore_count=0) + self.magicfolder.set_processed_callback(d2.callback, ignore_count=0) os.rename(empty_tree_dir, new_empty_tree_dir) self.notify(to_filepath(new_empty_tree_dir), self.inotify.IN_MOVED_TO) return d2 d.addCallback(_check_move_empty_tree) - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('drop_upload.objects_succeeded'), 1)) - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('drop_upload.files_uploaded'), 0)) - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('drop_upload.objects_queued'), 0)) - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('drop_upload.directories_created'), 1)) + d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('magic_folder.objects_succeeded'), 1)) + d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('magic_folder.files_uploaded'), 0)) + d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('magic_folder.objects_queued'), 0)) + d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('magic_folder.directories_created'), 1)) def _check_move_small_tree(res): self.mkdir_nonascii(small_tree_dir) fileutil.write(abspath_expanduser_unicode(u"what", base=small_tree_dir), "say when") d2 = defer.Deferred() - self.uploader.set_processed_callback(d2.callback, ignore_count=1) + self.magicfolder.set_processed_callback(d2.callback, ignore_count=1) os.rename(small_tree_dir, new_small_tree_dir) self.notify(to_filepath(new_small_tree_dir), self.inotify.IN_MOVED_TO) return d2 d.addCallback(_check_move_small_tree) - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('drop_upload.objects_succeeded'), 3)) - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('drop_upload.files_uploaded'), 1)) - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('drop_upload.objects_queued'), 0)) - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('drop_upload.directories_created'), 2)) + d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('magic_folder.objects_succeeded'), 3)) + d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('magic_folder.files_uploaded'), 1)) + d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('magic_folder.objects_queued'), 0)) + d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('magic_folder.directories_created'), 2)) def _check_moved_tree_is_watched(res): d2 = defer.Deferred() - self.uploader.set_processed_callback(d2.callback, ignore_count=0) + self.magicfolder.set_processed_callback(d2.callback, ignore_count=0) fileutil.write(abspath_expanduser_unicode(u"another", base=new_small_tree_dir), "file") self.notify(to_filepath(abspath_expanduser_unicode(u"another", base=new_small_tree_dir)), self.inotify.IN_CLOSE_WRITE) return d2 d.addCallback(_check_moved_tree_is_watched) - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('drop_upload.objects_succeeded'), 4)) - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('drop_upload.files_uploaded'), 2)) - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('drop_upload.objects_queued'), 0)) - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('drop_upload.directories_created'), 2)) + d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('magic_folder.objects_succeeded'), 4)) + d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('magic_folder.files_uploaded'), 2)) + d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('magic_folder.objects_queued'), 0)) + d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('magic_folder.directories_created'), 2)) # Files that are moved out of the upload directory should no longer be watched. def _move_dir_away(ign): @@ -192,10 +192,10 @@ class DropUploadTestMixin(GridTestMixin, ShouldFailMixin, ReallyEqualMixin, NonA return d.addCallback(create_file) d.addCallback(lambda ign: time.sleep(1)) - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('drop_upload.objects_succeeded'), 4)) - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('drop_upload.files_uploaded'), 2)) - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('drop_upload.objects_queued'), 0)) - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('drop_upload.directories_created'), 2)) + d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('magic_folder.objects_succeeded'), 4)) + d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('magic_folder.files_uploaded'), 2)) + d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('magic_folder.objects_queued'), 0)) + d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('magic_folder.directories_created'), 2)) d.addBoth(self._cleanup) return d @@ -203,9 +203,9 @@ class DropUploadTestMixin(GridTestMixin, ShouldFailMixin, ReallyEqualMixin, NonA def test_persistence(self): """ Perform an upload of a given file and then stop the client. - Start a new client and uploader... and verify that the file is NOT uploaded + Start a new client and magic-folder service... and verify that the file is NOT uploaded a second time. This test is meant to test the database persistence along with - the startup and shutdown code paths of the uploader. + the startup and shutdown code paths of the magic-folder service. """ self.set_up_grid() self.local_dir = abspath_expanduser_unicode(u"test_persistence", base=self.basedir) @@ -215,18 +215,18 @@ class DropUploadTestMixin(GridTestMixin, ShouldFailMixin, ReallyEqualMixin, NonA self.stats_provider = self.client.stats_provider d = self.client.create_dirnode() d.addCallback(self._made_upload_dir) - d.addCallback(self._create_uploader) + d.addCallback(self._create_magicfolder) def create_file(val): d2 = defer.Deferred() - self.uploader.set_processed_callback(d2.callback) + self.magicfolder.set_processed_callback(d2.callback) test_file = abspath_expanduser_unicode(u"what", base=self.local_dir) fileutil.write(test_file, "meow") self.notify(to_filepath(test_file), self.inotify.IN_CLOSE_WRITE) return d2 d.addCallback(create_file) - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('drop_upload.objects_succeeded'), 1)) - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('drop_upload.objects_queued'), 0)) + d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('magic_folder.objects_succeeded'), 1)) + d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('magic_folder.objects_queued'), 0)) d.addCallback(self._cleanup) def _restart(ign): @@ -234,14 +234,14 @@ class DropUploadTestMixin(GridTestMixin, ShouldFailMixin, ReallyEqualMixin, NonA self.client = self.g.clients[0] self.stats_provider = self.client.stats_provider d.addCallback(_restart) - d.addCallback(self._create_uploader) + d.addCallback(self._create_magicfolder) d.addCallback(lambda ign: time.sleep(3)) - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('drop_upload.objects_succeeded'), 0)) - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('drop_upload.objects_queued'), 0)) + d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('magic_folder.objects_succeeded'), 0)) + d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('magic_folder.objects_queued'), 0)) d.addBoth(self._cleanup) return d - def test_drop_upload(self): + def test_magic_folder(self): self.set_up_grid() self.local_dir = os.path.join(self.basedir, self.unicode_or_fallback(u"loc\u0101l_dir", u"local_dir")) self.mkdir_nonascii(self.local_dir) @@ -252,7 +252,7 @@ class DropUploadTestMixin(GridTestMixin, ShouldFailMixin, ReallyEqualMixin, NonA d = self.client.create_dirnode() d.addCallback(self._made_upload_dir) - d.addCallback(self._create_uploader) + d.addCallback(self._create_magicfolder) # Write something short enough for a LIT file. d.addCallback(lambda ign: self._check_file(u"short", "test")) @@ -271,20 +271,20 @@ class DropUploadTestMixin(GridTestMixin, ShouldFailMixin, ReallyEqualMixin, NonA d.addCallback(lambda ign: self._check_file(name_u, "test"*100)) # TODO: test that causes an upload failure. - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('drop_upload.files_failed'), 0)) + d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('magic_folder.files_failed'), 0)) d.addBoth(self._cleanup) return d def _check_file(self, name_u, data, temporary=False): - previously_uploaded = self._get_count('drop_upload.objects_succeeded') - previously_disappeared = self._get_count('drop_upload.objects_disappeared') + previously_uploaded = self._get_count('magic_folder.objects_succeeded') + previously_disappeared = self._get_count('magic_folder.objects_disappeared') d = defer.Deferred() # Note: this relies on the fact that we only get one IN_CLOSE_WRITE notification per file # (otherwise we would get a defer.AlreadyCalledError). Should we be relying on that? - self.uploader.set_processed_callback(d.callback) + self.magicfolder.set_processed_callback(d.callback) path_u = abspath_expanduser_unicode(name_u, base=self.local_dir) path = to_filepath(path_u) @@ -307,28 +307,28 @@ class DropUploadTestMixin(GridTestMixin, ShouldFailMixin, ReallyEqualMixin, NonA if temporary: d.addCallback(lambda ign: self.shouldFail(NoSuchChildError, 'temp file not uploaded', None, self.upload_dirnode.get, name_u)) - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('drop_upload.objects_disappeared'), + d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('magic_folder.objects_disappeared'), previously_disappeared + 1)) else: d.addCallback(lambda ign: self.upload_dirnode.get(name_u)) d.addCallback(download_to_data) d.addCallback(lambda actual_data: self.failUnlessReallyEqual(actual_data, data)) - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('drop_upload.objects_succeeded'), + d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('magic_folder.objects_succeeded'), previously_uploaded + 1)) - d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('drop_upload.objects_queued'), 0)) + d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('magic_folder.objects_queued'), 0)) return d -class MockTest(DropUploadTestMixin, unittest.TestCase): +class MockTest(MagicFolderTestMixin, unittest.TestCase): """This can run on any platform, and even if twisted.internet.inotify can't be imported.""" def setUp(self): - DropUploadTestMixin.setUp(self) + MagicFolderTestMixin.setUp(self) self.inotify = fake_inotify def notify(self, path, mask): - self.uploader._notifier.event(path, mask) + self.magicfolder._notifier.event(path, mask) def test_errors(self): self.set_up_grid() @@ -348,37 +348,37 @@ class MockTest(DropUploadTestMixin, unittest.TestCase): readonly_dircap = n.get_readonly_uri() self.shouldFail(AssertionError, 'nonexistent local.directory', 'there is no directory', - DropUploader, client, upload_dircap, '', doesnotexist, magicfolderdb, inotify=fake_inotify) + MagicFolder, client, upload_dircap, '', doesnotexist, magicfolderdb, inotify=fake_inotify) self.shouldFail(AssertionError, 'non-directory local.directory', 'is not a directory', - DropUploader, client, upload_dircap, '', not_a_dir, magicfolderdb, inotify=fake_inotify) + MagicFolder, client, upload_dircap, '', not_a_dir, magicfolderdb, inotify=fake_inotify) self.shouldFail(AssertionError, 'bad upload.dircap', 'does not refer to a directory', - DropUploader, client, 'bad', '', errors_dir, magicfolderdb, inotify=fake_inotify) + MagicFolder, client, 'bad', '', errors_dir, magicfolderdb, inotify=fake_inotify) self.shouldFail(AssertionError, 'non-directory upload.dircap', 'does not refer to a directory', - DropUploader, client, 'URI:LIT:foo', '', errors_dir, magicfolderdb, inotify=fake_inotify) + MagicFolder, client, 'URI:LIT:foo', '', errors_dir, magicfolderdb, inotify=fake_inotify) self.shouldFail(AssertionError, 'readonly upload.dircap', 'is not a writecap to a directory', - DropUploader, client, readonly_dircap, '', errors_dir, magicfolderdb, inotify=fake_inotify) + MagicFolder, client, readonly_dircap, '', errors_dir, magicfolderdb, inotify=fake_inotify) def _not_implemented(): raise NotImplementedError("blah") - self.patch(drop_upload, 'get_inotify_module', _not_implemented) + self.patch(magic_folder, 'get_inotify_module', _not_implemented) self.shouldFail(NotImplementedError, 'unsupported', 'blah', - DropUploader, client, upload_dircap, '', errors_dir, magicfolderdb) + MagicFolder, client, upload_dircap, '', errors_dir, magicfolderdb) d.addCallback(_check_errors) return d -class RealTest(DropUploadTestMixin, unittest.TestCase): +class RealTest(MagicFolderTestMixin, unittest.TestCase): """This is skipped unless both Twisted and the platform support inotify.""" def setUp(self): - DropUploadTestMixin.setUp(self) - self.inotify = drop_upload.get_inotify_module() + MagicFolderTestMixin.setUp(self) + self.inotify = magic_folder.get_inotify_module() def notify(self, path, mask): # Writing to the filesystem causes the notification. pass try: - drop_upload.get_inotify_module() + magic_folder.get_inotify_module() except NotImplementedError: - RealTest.skip = "Drop-upload support can only be tested for-real on an OS that supports inotify or equivalent." + RealTest.skip = "Magic Folder support can only be tested for-real on an OS that supports inotify or equivalent." -- 2.45.2