src/allmydata/test/test_magic_folder.py

   1
   2 import os, sys
   3 import shutil
   4
   5 from twisted.trial import unittest
   6 from twisted.internet import defer, task
   7
   8 from allmydata.interfaces import IDirectoryNode
   9 from allmydata.util.assertutil import precondition
  10
  11 from allmydata.util import fake_inotify, fileutil
  12 from allmydata.util.deferredutil import DeferredListShouldSucceed
  13 from allmydata.util.encodingutil import get_filesystem_encoding, to_filepath
  14 from allmydata.util.consumer import download_to_data
  15 from allmydata.test.no_network import GridTestMixin
  16 from allmydata.test.common_util import ReallyEqualMixin, NonASCIIPathMixin
  17 from allmydata.test.common import ShouldFailMixin
  18 from .test_cli_magic_folder import MagicFolderCLITestMixin
  19
  20 from allmydata.frontends import magic_folder
  21 from allmydata.frontends.magic_folder import MagicFolder, Downloader, WriteFileMixin
  22 from allmydata import magicfolderdb, magicpath
  23 from allmydata.util.fileutil import get_pathinfo
  24 from allmydata.util.fileutil import abspath_expanduser_unicode
  25 from allmydata.immutable.upload import Data
  26
  27
  28 class MagicFolderDbTests(unittest.TestCase):
  29
  30     def setUp(self):
  31         self.temp = abspath_expanduser_unicode(unicode(self.mktemp()))
  32         os.mkdir(self.temp)
  33         dbfile = abspath_expanduser_unicode(u"testdb.sqlite", base=self.temp)
  34         self.db = magicfolderdb.get_magicfolderdb(dbfile, create_version=(magicfolderdb.SCHEMA_v1, 1))
  35         self.failUnless(self.db, "unable to create magicfolderdb from %r" % (dbfile,))
  36         self.failUnlessEqual(self.db.VERSION, 1)
  37
  38     def tearDown(self):
  39         shutil.rmtree(self.temp)
  40
  41     def test_create(self):
  42         self.db.did_upload_version(
  43             relpath_u=u'fake_path',
  44             version=0,
  45             last_uploaded_uri=None,
  46             last_downloaded_uri='URI:foo',
  47             last_downloaded_timestamp=1234.5,
  48             pathinfo=get_pathinfo(self.temp),  # a directory, but should be fine for test
  49         )
  50
  51         entry = self.db.get_db_entry(u'fake_path')
  52         self.assertTrue(entry is not None)
  53         self.assertEqual(entry.last_downloaded_uri, 'URI:foo')
  54
  55     def test_update(self):
  56         self.db.did_upload_version(
  57             relpath_u=u'fake_path',
  58             version=0,
  59             last_uploaded_uri=None,
  60             last_downloaded_uri='URI:foo',
  61             last_downloaded_timestamp=1234.5,
  62             pathinfo=get_pathinfo(self.temp),  # a directory, but should be fine for test
  63         )
  64         self.db.did_upload_version(
  65             relpath_u=u'fake_path',
  66             version=1,
  67             last_uploaded_uri=None,
  68             last_downloaded_uri='URI:bar',
  69             last_downloaded_timestamp=1234.5,
  70             pathinfo=get_pathinfo(self.temp),  # a directory, but should be fine for test
  71         )
  72
  73         entry = self.db.get_db_entry(u'fake_path')
  74         self.assertTrue(entry is not None)
  75         self.assertEqual(entry.last_downloaded_uri, 'URI:bar')
  76         self.assertEqual(entry.version, 1)
  77
  78     def test_same_content_different_path(self):
  79         content_uri = 'URI:CHK:27d2yruqwk6zb2w7hkbbfxxbue:ipmszjysmn4vdeaxz7rtxtv3gwv6vrqcg2ktrdmn4oxqqucltxxq:2:4:1052835840'
  80         self.db.did_upload_version(
  81             relpath_u=u'path0',
  82             version=0,
  83             last_uploaded_uri=None,
  84             last_downloaded_uri=content_uri,
  85             last_downloaded_timestamp=1234.5,
  86             pathinfo=get_pathinfo(self.temp),  # a directory, but should be fine for test
  87         )
  88         self.db.did_upload_version(
  89             relpath_u=u'path1',
  90             version=0,
  91             last_uploaded_uri=None,
  92             last_downloaded_uri=content_uri,
  93             last_downloaded_timestamp=1234.5,
  94             pathinfo=get_pathinfo(self.temp),  # a directory, but should be fine for test
  95         )
  96
  97         entry = self.db.get_db_entry(u'path0')
  98         self.assertTrue(entry is not None)
  99         self.assertEqual(entry.last_downloaded_uri, content_uri)
 100
 101         entry = self.db.get_db_entry(u'path1')
 102         self.assertTrue(entry is not None)
 103         self.assertEqual(entry.last_downloaded_uri, content_uri)
 104
 105
 106 class MagicFolderTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, ReallyEqualMixin, NonASCIIPathMixin):
 107     """
 108     These tests will be run both with a mock notifier, and (on platforms that support it)
 109     with the real INotify.
 110     """
 111
 112     def setUp(self):
 113         GridTestMixin.setUp(self)
 114         temp = self.mktemp()
 115         self.basedir = abspath_expanduser_unicode(temp.decode(get_filesystem_encoding()))
 116         self.magicfolder = None
 117         self.patch(Downloader, 'REMOTE_SCAN_INTERVAL', 0)
 118
 119     def _get_count(self, name, client=None):
 120         counters = (client or self.get_client()).stats_provider.get_stats()["counters"]
 121         return counters.get('magic_folder.%s' % (name,), 0)
 122
 123     def _createdb(self):
 124         dbfile = abspath_expanduser_unicode(u"magicfolderdb.sqlite", base=self.basedir)
 125         mdb = magicfolderdb.get_magicfolderdb(dbfile, create_version=(magicfolderdb.SCHEMA_v1, 1))
 126         self.failUnless(mdb, "unable to create magicfolderdb from %r" % (dbfile,))
 127         self.failUnlessEqual(mdb.VERSION, 1)
 128         return mdb
 129
 130     def _restart_client(self, ign):
 131         #print "_restart_client"
 132         d = self.restart_client()
 133         d.addCallback(self._wait_until_started)
 134         return d
 135
 136     def _wait_until_started(self, ign):
 137         #print "_wait_until_started"
 138         self.magicfolder = self.get_client().getServiceNamed('magic-folder')
 139         return self.magicfolder.ready()
 140
 141     def test_db_basic(self):
 142         fileutil.make_dirs(self.basedir)
 143         self._createdb()
 144
 145     def test_db_persistence(self):
 146         """Test that a file upload creates an entry in the database."""
 147
 148         fileutil.make_dirs(self.basedir)
 149         db = self._createdb()
 150
 151         relpath1 = u"myFile1"
 152         pathinfo = fileutil.PathInfo(isdir=False, isfile=True, islink=False,
 153                                      exists=True, size=1, mtime=123, ctime=456)
 154         db.did_upload_version(relpath1, 0, 'URI:LIT:1', 'URI:LIT:0', 0, pathinfo)
 155
 156         c = db.cursor
 157         c.execute("SELECT size, mtime, ctime"
 158                   " FROM local_files"
 159                   " WHERE path=?",
 160                   (relpath1,))
 161         row = c.fetchone()
 162         self.failUnlessEqual(row, (pathinfo.size, pathinfo.mtime, pathinfo.ctime))
 163
 164         # Second test uses magic_folder.is_new_file instead of SQL query directly
 165         # to confirm the previous upload entry in the db.
 166         relpath2 = u"myFile2"
 167         path2 = os.path.join(self.basedir, relpath2)
 168         fileutil.write(path2, "meow\n")
 169         pathinfo = fileutil.get_pathinfo(path2)
 170         db.did_upload_version(relpath2, 0, 'URI:LIT:2', 'URI:LIT:1', 0, pathinfo)
 171         db_entry = db.get_db_entry(relpath2)
 172         self.failUnlessFalse(magic_folder.is_new_file(pathinfo, db_entry))
 173
 174         different_pathinfo = fileutil.PathInfo(isdir=False, isfile=True, islink=False,
 175                                                exists=True, size=0, mtime=pathinfo.mtime, ctime=pathinfo.ctime)
 176         self.failUnlessTrue(magic_folder.is_new_file(different_pathinfo, db_entry))
 177
 178     def test_magicfolder_start_service(self):
 179         self.set_up_grid()
 180
 181         self.local_dir = abspath_expanduser_unicode(self.unicode_or_fallback(u"l\u00F8cal_dir", u"local_dir"),
 182                                                     base=self.basedir)
 183         self.mkdir_nonascii(self.local_dir)
 184
 185         d = defer.succeed(None)
 186         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.dirs_monitored'), 0))
 187
 188         d.addCallback(lambda ign: self.create_invite_join_magic_folder(u"Alice", self.local_dir))
 189         d.addCallback(self._restart_client)
 190
 191         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.dirs_monitored'), 1))
 192         d.addBoth(self.cleanup)
 193         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.dirs_monitored'), 0))
 194         return d
 195
 196     def test_scan_once_on_startup(self):
 197         self.set_up_grid()
 198         self.local_dir = abspath_expanduser_unicode(u"test_scan_once_on_startup", base=self.basedir)
 199         self.mkdir_nonascii(self.local_dir)
 200         self.collective_dircap = ""
 201
 202         alice_clock = task.Clock()
 203         bob_clock = task.Clock()
 204         d = self.setup_alice_and_bob(alice_clock, bob_clock)
 205
 206         def upload_stuff(ignore):
 207             uploadable = Data("", self.alice_magicfolder._client.convergence)
 208             return self.alice_magicfolder._client.upload(uploadable)
 209         d.addCallback(upload_stuff)
 210         def check_is_upload(ignore):
 211             alice_clock.advance(99)
 212             d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 0, magic=self.alice_magicfolder))
 213             d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0, magic=self.alice_magicfolder))
 214             d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
 215             d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 0, magic=self.alice_magicfolder))
 216             d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
 217             d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 0, magic=self.alice_magicfolder))
 218
 219         d.addCallback(check_is_upload)
 220         def _cleanup(ign, magicfolder, clock):
 221             if magicfolder is not None:
 222                 d2 = magicfolder.finish()
 223                 clock.advance(0)
 224                 return d2
 225         def cleanup_Alice_and_Bob(result):
 226             print "cleanup alice bob test\n"
 227             d = defer.succeed(None)
 228             d.addCallback(_cleanup, self.alice_magicfolder, alice_clock)
 229             d.addCallback(_cleanup, self.bob_magicfolder, bob_clock)
 230             d.addCallback(lambda ign: result)
 231             return d
 232
 233         d.addBoth(cleanup_Alice_and_Bob)
 234         return d
 235
 236     def test_move_tree(self):
 237         self.set_up_grid()
 238
 239         self.local_dir = abspath_expanduser_unicode(self.unicode_or_fallback(u"l\u00F8cal_dir", u"local_dir"),
 240                                                     base=self.basedir)
 241         self.mkdir_nonascii(self.local_dir)
 242
 243         empty_tree_name = self.unicode_or_fallback(u"empty_tr\u00EAe", u"empty_tree")
 244         empty_tree_dir = abspath_expanduser_unicode(empty_tree_name, base=self.basedir)
 245         new_empty_tree_dir = abspath_expanduser_unicode(empty_tree_name, base=self.local_dir)
 246
 247         small_tree_name = self.unicode_or_fallback(u"small_tr\u00EAe", u"empty_tree")
 248         small_tree_dir = abspath_expanduser_unicode(small_tree_name, base=self.basedir)
 249         new_small_tree_dir = abspath_expanduser_unicode(small_tree_name, base=self.local_dir)
 250
 251         d = self.create_invite_join_magic_folder(u"Alice", self.local_dir)
 252         d.addCallback(self._restart_client)
 253
 254         def _check_move_empty_tree(res):
 255             print "_check_move_empty_tree"
 256             uploaded_d = self.magicfolder.uploader.set_hook('processed')
 257             self.mkdir_nonascii(empty_tree_dir)
 258             os.rename(empty_tree_dir, new_empty_tree_dir)
 259             self.notify(to_filepath(new_empty_tree_dir), self.inotify.IN_MOVED_TO)
 260
 261             return uploaded_d
 262         d.addCallback(_check_move_empty_tree)
 263         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 264         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 1))
 265         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.files_uploaded'), 0))
 266         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 267         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.directories_created'), 1))
 268
 269         def _check_move_small_tree(res):
 270             print "_check_move_small_tree"
 271             uploaded_d = self.magicfolder.uploader.set_hook('processed', ignore_count=1)
 272             self.mkdir_nonascii(small_tree_dir)
 273             what_path = abspath_expanduser_unicode(u"what", base=small_tree_dir)
 274             fileutil.write(what_path, "say when")
 275             os.rename(small_tree_dir, new_small_tree_dir)
 276             self.notify(to_filepath(new_small_tree_dir), self.inotify.IN_MOVED_TO)
 277
 278             return uploaded_d
 279         d.addCallback(_check_move_small_tree)
 280         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 281         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 3))
 282         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.files_uploaded'), 1))
 283         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 284         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.directories_created'), 2))
 285
 286         def _check_moved_tree_is_watched(res):
 287             print "_check_moved_tree_is_watched"
 288             uploaded_d = self.magicfolder.uploader.set_hook('processed')
 289             another_path = abspath_expanduser_unicode(u"another", base=new_small_tree_dir)
 290             fileutil.write(another_path, "file")
 291             self.notify(to_filepath(another_path), self.inotify.IN_CLOSE_WRITE)
 292
 293             return uploaded_d
 294         d.addCallback(_check_moved_tree_is_watched)
 295         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 296         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 4))
 297         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.files_uploaded'), 2))
 298         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 299         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.directories_created'), 2))
 300
 301         # Files that are moved out of the upload directory should no longer be watched.
 302         #def _move_dir_away(ign):
 303         #    os.rename(new_empty_tree_dir, empty_tree_dir)
 304         #    # Wuh? Why don't we get this event for the real test?
 305         #    #self.notify(to_filepath(new_empty_tree_dir), self.inotify.IN_MOVED_FROM)
 306         #d.addCallback(_move_dir_away)
 307         #def create_file(val):
 308         #    test_file = abspath_expanduser_unicode(u"what", base=empty_tree_dir)
 309         #    fileutil.write(test_file, "meow")
 310         #    #self.notify(...)
 311         #    return
 312         #d.addCallback(create_file)
 313         #d.addCallback(lambda ign: time.sleep(1))  # XXX ICK
 314         #d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 315         #d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 4))
 316         #d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.files_uploaded'), 2))
 317         #d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 318         #d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.directories_created'), 2))
 319
 320         d.addBoth(self.cleanup)
 321         return d
 322
 323     def test_persistence(self):
 324         """
 325         Perform an upload of a given file and then stop the client.
 326         Start a new client and magic-folder service... and verify that the file is NOT uploaded
 327         a second time. This test is meant to test the database persistence along with
 328         the startup and shutdown code paths of the magic-folder service.
 329         """
 330         self.set_up_grid()
 331         self.local_dir = abspath_expanduser_unicode(u"test_persistence", base=self.basedir)
 332         self.mkdir_nonascii(self.local_dir)
 333         self.collective_dircap = ""
 334
 335         d = defer.succeed(None)
 336         d.addCallback(lambda ign: self.create_invite_join_magic_folder(u"Alice", self.local_dir))
 337         d.addCallback(self._restart_client)
 338
 339         def create_test_file(filename):
 340             d2 = self.magicfolder.uploader.set_hook('processed')
 341             test_file = abspath_expanduser_unicode(filename, base=self.local_dir)
 342             fileutil.write(test_file, "meow %s" % filename)
 343             self.notify(to_filepath(test_file), self.inotify.IN_CLOSE_WRITE)
 344             return d2
 345         d.addCallback(lambda ign: create_test_file(u"what1"))
 346         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 347         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 1))
 348         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 349         d.addCallback(self.cleanup)
 350
 351         d.addCallback(self._restart_client)
 352         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 353         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 1))
 354         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 355         d.addCallback(lambda ign: create_test_file(u"what2"))
 356         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 357         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 2))
 358         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 359         d.addBoth(self.cleanup)
 360         return d
 361
 362     @defer.inlineCallbacks
 363     def test_delete(self):
 364         self.set_up_grid()
 365         self.local_dir = os.path.join(self.basedir, u"local_dir")
 366         self.mkdir_nonascii(self.local_dir)
 367
 368         yield self.create_invite_join_magic_folder(u"Alice\u0101", self.local_dir)
 369         yield self._restart_client(None)
 370
 371         try:
 372             # create a file
 373             up_proc = self.magicfolder.uploader.set_hook('processed')
 374             # down_proc = self.magicfolder.downloader.set_hook('processed')
 375             path = os.path.join(self.local_dir, u'foo')
 376             fileutil.write(path, 'foo\n')
 377             self.notify(to_filepath(path), self.inotify.IN_CLOSE_WRITE)
 378             yield up_proc
 379             self.assertTrue(os.path.exists(path))
 380
 381             # the real test part: delete the file
 382             up_proc = self.magicfolder.uploader.set_hook('processed')
 383             os.unlink(path)
 384             self.notify(to_filepath(path), self.inotify.IN_DELETE)
 385             yield up_proc
 386             self.assertFalse(os.path.exists(path))
 387
 388             # ensure we still have a DB entry, and that the version is 1
 389             node, metadata = yield self.magicfolder.downloader._get_collective_latest_file(u'foo')
 390             self.assertTrue(node is not None, "Failed to find %r in DMD" % (path,))
 391             self.failUnlessEqual(metadata['version'], 1)
 392
 393         finally:
 394             yield self.cleanup(None)
 395
 396     @defer.inlineCallbacks
 397     def test_delete_and_restore(self):
 398         self.set_up_grid()
 399         self.local_dir = os.path.join(self.basedir, u"local_dir")
 400         self.mkdir_nonascii(self.local_dir)
 401
 402         yield self.create_invite_join_magic_folder(u"Alice\u0101", self.local_dir)
 403         yield self._restart_client(None)
 404
 405         try:
 406             # create a file
 407             up_proc = self.magicfolder.uploader.set_hook('processed')
 408             # down_proc = self.magicfolder.downloader.set_hook('processed')
 409             path = os.path.join(self.local_dir, u'foo')
 410             fileutil.write(path, 'foo\n')
 411             self.notify(to_filepath(path), self.inotify.IN_CLOSE_WRITE)
 412             yield up_proc
 413             self.assertTrue(os.path.exists(path))
 414
 415             # delete the file
 416             up_proc = self.magicfolder.uploader.set_hook('processed')
 417             os.unlink(path)
 418             self.notify(to_filepath(path), self.inotify.IN_DELETE)
 419             yield up_proc
 420             self.assertFalse(os.path.exists(path))
 421
 422             # ensure we still have a DB entry, and that the version is 1
 423             node, metadata = yield self.magicfolder.downloader._get_collective_latest_file(u'foo')
 424             self.assertTrue(node is not None, "Failed to find %r in DMD" % (path,))
 425             self.failUnlessEqual(metadata['version'], 1)
 426
 427             # restore the file, with different contents
 428             up_proc = self.magicfolder.uploader.set_hook('processed')
 429             path = os.path.join(self.local_dir, u'foo')
 430             fileutil.write(path, 'bar\n')
 431             self.notify(to_filepath(path), self.inotify.IN_CLOSE_WRITE)
 432             yield up_proc
 433
 434             # ensure we still have a DB entry, and that the version is 2
 435             node, metadata = yield self.magicfolder.downloader._get_collective_latest_file(u'foo')
 436             self.assertTrue(node is not None, "Failed to find %r in DMD" % (path,))
 437             self.failUnlessEqual(metadata['version'], 2)
 438
 439         finally:
 440             yield self.cleanup(None)
 441
 442     @defer.inlineCallbacks
 443     def test_alice_delete_bob_restore(self):
 444         alice_clock = task.Clock()
 445         bob_clock = task.Clock()
 446         yield self.setup_alice_and_bob(alice_clock, bob_clock)
 447         alice_dir = self.alice_magicfolder.uploader._local_path_u
 448         bob_dir = self.bob_magicfolder.uploader._local_path_u
 449         alice_fname = os.path.join(alice_dir, 'blam')
 450         bob_fname = os.path.join(bob_dir, 'blam')
 451
 452         try:
 453             # alice creates a file, bob downloads it
 454             alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
 455             bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
 456
 457             fileutil.write(alice_fname, 'contents0\n')
 458             self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 459
 460             alice_clock.advance(0)
 461             yield alice_proc  # alice uploads
 462
 463             bob_clock.advance(0)
 464             yield bob_proc    # bob downloads
 465
 466             # check the state
 467             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
 468             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 0)
 469             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
 470             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 0)
 471             yield self.failUnlessReallyEqual(
 472                 self._get_count('downloader.objects_failed', client=self.bob_magicfolder._client),
 473                 0
 474             )
 475             yield self.failUnlessReallyEqual(
 476                 self._get_count('downloader.objects_downloaded', client=self.bob_magicfolder._client),
 477                 1
 478             )
 479
 480             print("BOB DELETE")
 481             # now bob deletes it (bob should upload, alice download)
 482             bob_proc = self.bob_magicfolder.uploader.set_hook('processed')
 483             alice_proc = self.alice_magicfolder.downloader.set_hook('processed')
 484             os.unlink(bob_fname)
 485             self.notify(to_filepath(bob_fname), self.inotify.IN_DELETE, magic=self.bob_magicfolder)
 486
 487             bob_clock.advance(0)
 488             yield bob_proc
 489             alice_clock.advance(0)
 490             yield alice_proc
 491
 492             # check versions
 493             node, metadata = yield self.alice_magicfolder.downloader._get_collective_latest_file(u'blam')
 494             self.assertTrue(metadata['deleted'])
 495             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
 496             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 1)
 497             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
 498             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 1)
 499
 500             print("ALICE RESTORE")
 501             # now alice restores it (alice should upload, bob download)
 502             alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
 503             bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
 504             fileutil.write(alice_fname, 'new contents\n')
 505             self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 506
 507             alice_clock.advance(0)
 508             yield alice_proc
 509             bob_clock.advance(0)
 510             yield bob_proc
 511
 512             # check versions
 513             node, metadata = yield self.alice_magicfolder.downloader._get_collective_latest_file(u'blam')
 514             self.assertTrue('deleted' not in metadata or not metadata['deleted'])
 515             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 2)
 516             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 2)
 517             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 2)
 518             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 2)
 519
 520         finally:
 521             # cleanup
 522             d0 = self.alice_magicfolder.finish()
 523             alice_clock.advance(0)
 524             yield d0
 525
 526             d1 = self.bob_magicfolder.finish()
 527             bob_clock.advance(0)
 528             yield d1
 529
 530     @defer.inlineCallbacks
 531     def test_alice_sees_bobs_delete_with_error(self):
 532         # alice creates a file, bob deletes it -- and we also arrange
 533         # for Alice's file to have "gone missing" as well.
 534         alice_clock = task.Clock()
 535         bob_clock = task.Clock()
 536         yield self.setup_alice_and_bob(alice_clock, bob_clock)
 537         alice_dir = self.alice_magicfolder.uploader._local_path_u
 538         bob_dir = self.bob_magicfolder.uploader._local_path_u
 539         alice_fname = os.path.join(alice_dir, 'blam')
 540         bob_fname = os.path.join(bob_dir, 'blam')
 541
 542         try:
 543             # alice creates a file, bob downloads it
 544             alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
 545             bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
 546
 547             fileutil.write(alice_fname, 'contents0\n')
 548             self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 549
 550             alice_clock.advance(0)
 551             yield alice_proc  # alice uploads
 552
 553             bob_clock.advance(0)
 554             yield bob_proc    # bob downloads
 555
 556             # check the state
 557             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
 558             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 0)
 559             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
 560             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 0)
 561             yield self.failUnlessReallyEqual(
 562                 self._get_count('downloader.objects_failed', client=self.bob_magicfolder._client),
 563                 0
 564             )
 565             yield self.failUnlessReallyEqual(
 566                 self._get_count('downloader.objects_downloaded', client=self.bob_magicfolder._client),
 567                 1
 568             )
 569
 570             # now bob deletes it (bob should upload, alice download)
 571             bob_proc = self.bob_magicfolder.uploader.set_hook('processed')
 572             alice_proc = self.alice_magicfolder.downloader.set_hook('processed')
 573             os.unlink(bob_fname)
 574             self.notify(to_filepath(bob_fname), self.inotify.IN_DELETE, magic=self.bob_magicfolder)
 575             # just after notifying bob, we also delete alice's,
 576             # covering the 'except' flow in _rename_deleted_file()
 577             os.unlink(alice_fname)
 578
 579             bob_clock.advance(0)
 580             yield bob_proc
 581             alice_clock.advance(0)
 582             yield alice_proc
 583
 584             # check versions
 585             node, metadata = yield self.alice_magicfolder.downloader._get_collective_latest_file(u'blam')
 586             self.assertTrue(metadata['deleted'])
 587             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
 588             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 1)
 589             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
 590             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 1)
 591
 592         finally:
 593             # cleanup
 594             d0 = self.alice_magicfolder.finish()
 595             alice_clock.advance(0)
 596             yield d0
 597
 598             d1 = self.bob_magicfolder.finish()
 599             bob_clock.advance(0)
 600             yield d1
 601
 602     @defer.inlineCallbacks
 603     def test_alice_create_bob_update(self):
 604         alice_clock = task.Clock()
 605         bob_clock = task.Clock()
 606         yield self.setup_alice_and_bob(alice_clock, bob_clock)
 607         alice_dir = self.alice_magicfolder.uploader._local_path_u
 608         bob_dir = self.bob_magicfolder.uploader._local_path_u
 609         alice_fname = os.path.join(alice_dir, 'blam')
 610         bob_fname = os.path.join(bob_dir, 'blam')
 611
 612         try:
 613             # alice creates a file, bob downloads it
 614             alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
 615             bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
 616
 617             fileutil.write(alice_fname, 'contents0\n')
 618             self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 619
 620             alice_clock.advance(0)
 621             yield alice_proc  # alice uploads
 622
 623             bob_clock.advance(0)
 624             yield bob_proc    # bob downloads
 625
 626             # check the state
 627             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
 628             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 0)
 629             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
 630             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 0)
 631             yield self.failUnlessReallyEqual(
 632                 self._get_count('downloader.objects_failed', client=self.bob_magicfolder._client),
 633                 0
 634             )
 635             yield self.failUnlessReallyEqual(
 636                 self._get_count('downloader.objects_downloaded', client=self.bob_magicfolder._client),
 637                 1
 638             )
 639
 640             # now bob updates it (bob should upload, alice download)
 641             bob_proc = self.bob_magicfolder.uploader.set_hook('processed')
 642             alice_proc = self.alice_magicfolder.downloader.set_hook('processed')
 643             fileutil.write(bob_fname, 'bob wuz here\n')
 644             self.notify(to_filepath(bob_fname), self.inotify.IN_CLOSE_WRITE, magic=self.bob_magicfolder)
 645
 646             bob_clock.advance(0)
 647             yield bob_proc
 648             alice_clock.advance(0)
 649             yield alice_proc
 650
 651             # check the state
 652             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
 653             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 1)
 654             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
 655             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 1)
 656
 657         finally:
 658             # cleanup
 659             d0 = self.alice_magicfolder.finish()
 660             alice_clock.advance(0)
 661             yield d0
 662
 663             d1 = self.bob_magicfolder.finish()
 664             bob_clock.advance(0)
 665             yield d1
 666
 667     @defer.inlineCallbacks
 668     def test_alice_delete_and_restore(self):
 669         alice_clock = task.Clock()
 670         bob_clock = task.Clock()
 671         yield self.setup_alice_and_bob(alice_clock, bob_clock)
 672         alice_dir = self.alice_magicfolder.uploader._local_path_u
 673         bob_dir = self.bob_magicfolder.uploader._local_path_u
 674         alice_fname = os.path.join(alice_dir, 'blam')
 675         bob_fname = os.path.join(bob_dir, 'blam')
 676
 677         try:
 678             # alice creates a file, bob downloads it
 679             alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
 680             bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
 681
 682             fileutil.write(alice_fname, 'contents0\n')
 683             self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 684
 685             alice_clock.advance(0)
 686             yield alice_proc  # alice uploads
 687
 688             bob_clock.advance(0)
 689             yield bob_proc    # bob downloads
 690
 691             # check the state
 692             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
 693             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 0)
 694             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
 695             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 0)
 696             yield self.failUnlessReallyEqual(
 697                 self._get_count('downloader.objects_failed', client=self.bob_magicfolder._client),
 698                 0
 699             )
 700             yield self.failUnlessReallyEqual(
 701                 self._get_count('downloader.objects_downloaded', client=self.bob_magicfolder._client),
 702                 1
 703             )
 704             self.failUnless(os.path.exists(bob_fname))
 705
 706             # now alice deletes it (alice should upload, bob download)
 707             alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
 708             bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
 709             os.unlink(alice_fname)
 710             self.notify(to_filepath(alice_fname), self.inotify.IN_DELETE, magic=self.alice_magicfolder)
 711
 712             alice_clock.advance(0)
 713             yield alice_proc
 714             bob_clock.advance(0)
 715             yield bob_proc
 716
 717             # check the state
 718             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
 719             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 1)
 720             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
 721             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 1)
 722             self.failIf(os.path.exists(bob_fname))
 723
 724             # now alice restores the file (with new contents)
 725             alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
 726             bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
 727             fileutil.write(alice_fname, 'alice wuz here\n')
 728             self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 729
 730             alice_clock.advance(0)
 731             yield alice_proc
 732             bob_clock.advance(0)
 733             yield bob_proc
 734
 735             # check the state
 736             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 2)
 737             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 2)
 738             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 2)
 739             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 2)
 740             self.failUnless(os.path.exists(bob_fname))
 741
 742         finally:
 743             # cleanup
 744             d0 = self.alice_magicfolder.finish()
 745             alice_clock.advance(0)
 746             yield d0
 747
 748             d1 = self.bob_magicfolder.finish()
 749             bob_clock.advance(0)
 750             yield d1
 751
 752     def test_magic_folder(self):
 753         self.set_up_grid()
 754         self.local_dir = os.path.join(self.basedir, self.unicode_or_fallback(u"loc\u0101l_dir", u"local_dir"))
 755         self.mkdir_nonascii(self.local_dir)
 756
 757         d = self.create_invite_join_magic_folder(u"Alice\u0101", self.local_dir)
 758         d.addCallback(self._restart_client)
 759
 760         # Write something short enough for a LIT file.
 761         d.addCallback(lambda ign: self._check_file(u"short", "test"))
 762
 763         # Write to the same file again with different data.
 764         d.addCallback(lambda ign: self._check_file(u"short", "different"))
 765
 766         # Test that temporary files are not uploaded.
 767         d.addCallback(lambda ign: self._check_file(u"tempfile", "test", temporary=True))
 768
 769         # Test creation of a subdirectory.
 770         d.addCallback(lambda ign: self._check_mkdir(u"directory"))
 771
 772         # Write something longer, and also try to test a Unicode name if the fs can represent it.
 773         name_u = self.unicode_or_fallback(u"l\u00F8ng", u"long")
 774         d.addCallback(lambda ign: self._check_file(name_u, "test"*100))
 775
 776         # TODO: test that causes an upload failure.
 777         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 778
 779         d.addBoth(self.cleanup)
 780         return d
 781
 782     def _check_mkdir(self, name_u):
 783         return self._check_file(name_u + u"/", "", directory=True)
 784
 785     def _check_file(self, name_u, data, temporary=False, directory=False):
 786         precondition(not (temporary and directory), temporary=temporary, directory=directory)
 787
 788         print "%r._check_file(%r, %r, temporary=%r, directory=%r)" % (self, name_u, data, temporary, directory)
 789         previously_uploaded = self._get_count('uploader.objects_succeeded')
 790         previously_disappeared = self._get_count('uploader.objects_disappeared')
 791
 792         d = self.magicfolder.uploader.set_hook('processed')
 793
 794         path_u = abspath_expanduser_unicode(name_u, base=self.local_dir)
 795         path = to_filepath(path_u)
 796
 797         if directory:
 798             os.mkdir(path_u)
 799             event_mask = self.inotify.IN_CREATE | self.inotify.IN_ISDIR
 800         else:
 801             # We don't use FilePath.setContent() here because it creates a temporary file that
 802             # is renamed into place, which causes events that the test is not expecting.
 803             f = open(path_u, "wb")
 804             try:
 805                 if temporary and sys.platform != "win32":
 806                     os.unlink(path_u)
 807                 f.write(data)
 808             finally:
 809                 f.close()
 810             if temporary and sys.platform == "win32":
 811                 os.unlink(path_u)
 812                 self.notify(path, self.inotify.IN_DELETE, flush=False)
 813             event_mask = self.inotify.IN_CLOSE_WRITE
 814
 815         self.notify(path, event_mask)
 816         encoded_name_u = magicpath.path2magic(name_u)
 817
 818         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 819         if temporary:
 820             d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_disappeared'),
 821                                                                  previously_disappeared + 1))
 822         else:
 823             def _here(res, n):
 824                 print "here %r %r" % (n, res)
 825                 return res
 826             d.addBoth(_here, 1)
 827             d.addCallback(lambda ign: self.upload_dirnode.list())
 828             d.addBoth(_here, 1.5)
 829             d.addCallback(lambda ign: self.upload_dirnode.get(encoded_name_u))
 830             d.addBoth(_here, 2)
 831             d.addCallback(download_to_data)
 832             d.addBoth(_here, 3)
 833             d.addCallback(lambda actual_data: self.failUnlessReallyEqual(actual_data, data))
 834             d.addBoth(_here, 4)
 835             d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'),
 836                                                                  previously_uploaded + 1))
 837
 838         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 839         return d
 840
 841     def _check_version_in_dmd(self, magicfolder, relpath_u, expected_version):
 842         encoded_name_u = magicpath.path2magic(relpath_u)
 843         d = magicfolder.downloader._get_collective_latest_file(encoded_name_u)
 844         def check_latest(result):
 845             if result[0] is not None:
 846                 node, metadata = result
 847                 d.addCallback(lambda ign: self.failUnlessEqual(metadata['version'], expected_version))
 848         d.addCallback(check_latest)
 849         return d
 850
 851     def _check_version_in_local_db(self, magicfolder, relpath_u, expected_version):
 852         db_entry = magicfolder._db.get_db_entry(relpath_u)
 853         if db_entry is not None:
 854             #print "_check_version_in_local_db: %r has version %s" % (relpath_u, version)
 855             self.failUnlessEqual(db_entry.version, expected_version)
 856
 857     def _check_file_gone(self, magicfolder, relpath_u):
 858         path = os.path.join(magicfolder.uploader._local_path_u, relpath_u)
 859         self.assertTrue(not os.path.exists(path))
 860
 861     def _check_uploader_count(self, name, expected, magic=None):
 862         self.failUnlessReallyEqual(self._get_count('uploader.'+name, client=(magic or self.alice_magicfolder)._client),
 863                                    expected)
 864
 865     def _check_downloader_count(self, name, expected, magic=None):
 866         self.failUnlessReallyEqual(self._get_count('downloader.'+name, client=(magic or self.bob_magicfolder)._client),
 867                                    expected)
 868
 869     def test_alice_bob(self):
 870         alice_clock = task.Clock()
 871         bob_clock = task.Clock()
 872         d = self.setup_alice_and_bob(alice_clock, bob_clock)
 873
 874         def _wait_for_Alice(ign, downloaded_d):
 875             print "Now waiting for Alice to download\n"
 876             alice_clock.advance(0)
 877             return downloaded_d
 878
 879         def _wait_for_Bob(ign, downloaded_d):
 880             print "Now waiting for Bob to download\n"
 881             bob_clock.advance(0)
 882             return downloaded_d
 883
 884         def _wait_for(ign, something_to_do, alice=True):
 885             if alice:
 886                 downloaded_d = self.bob_magicfolder.downloader.set_hook('processed')
 887                 uploaded_d = self.alice_magicfolder.uploader.set_hook('processed')
 888             else:
 889                 downloaded_d = self.alice_magicfolder.downloader.set_hook('processed')
 890                 uploaded_d = self.bob_magicfolder.uploader.set_hook('processed')
 891             something_to_do()
 892             if alice:
 893                 print "Waiting for Alice to upload\n"
 894                 alice_clock.advance(0)
 895                 uploaded_d.addCallback(_wait_for_Bob, downloaded_d)
 896             else:
 897                 print "Waiting for Bob to upload\n"
 898                 bob_clock.advance(0)
 899                 uploaded_d.addCallback(_wait_for_Alice, downloaded_d)
 900             return uploaded_d
 901
 902         def Alice_to_write_a_file():
 903             print "Alice writes a file\n"
 904             self.file_path = abspath_expanduser_unicode(u"file1", base=self.alice_magicfolder.uploader._local_path_u)
 905             fileutil.write(self.file_path, "meow, meow meow. meow? meow meow! meow.")
 906             self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 907         d.addCallback(_wait_for, Alice_to_write_a_file)
 908
 909         d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file1", 0))
 910         d.addCallback(lambda ign: self._check_version_in_local_db(self.alice_magicfolder, u"file1", 0))
 911         d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0))
 912         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 1))
 913         d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 1))
 914         d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0))
 915         d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0))
 916         d.addCallback(lambda ign: self._check_uploader_count('objects_conflicted', 0))
 917         d.addCallback(lambda ign: self._check_uploader_count('objects_conflicted', 0, magic=self.bob_magicfolder))
 918
 919         d.addCallback(lambda ign: self._check_version_in_local_db(self.bob_magicfolder, u"file1", 0))
 920         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0))
 921         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 1))
 922         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 0, magic=self.bob_magicfolder))
 923
 924         def check_delete_file(ign):
 925             d_bob = self.bob_magicfolder.uploader.set_hook('processed')
 926             def Alice_to_delete_file():
 927                 print "Alice deletes the file!\n"
 928                 os.unlink(self.file_path)
 929                 self.notify(to_filepath(self.file_path), self.inotify.IN_DELETE, magic=self.alice_magicfolder)
 930
 931             d_alice = defer.succeed(None)
 932             d_alice.addCallback(_wait_for, Alice_to_delete_file)
 933
 934             p = abspath_expanduser_unicode(u"file1", base=self.bob_magicfolder.uploader._local_path_u)
 935             if sys.platform == "win32":
 936                 self.notify(to_filepath(p), self.inotify.IN_MOVED_FROM, magic=self.bob_magicfolder, flush=False)
 937                 self.notify(to_filepath(p + u'.backup'), self.inotify.IN_MOVED_TO, magic=self.bob_magicfolder)
 938             else:
 939                 self.notify(to_filepath(p + u'.backup'), self.inotify.IN_CREATE, magic=self.bob_magicfolder, flush=False)
 940                 self.notify(to_filepath(p), self.inotify.IN_DELETE, magic=self.bob_magicfolder)
 941
 942             d_alice.addCallback(lambda ign: bob_clock.advance(0))
 943             return DeferredListShouldSucceed([d_alice, d_bob])
 944         d.addCallback(check_delete_file)
 945
 946         d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file1", 1))
 947         d.addCallback(lambda ign: self._check_version_in_local_db(self.alice_magicfolder, u"file1", 1))
 948         d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0))
 949         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 2))
 950         d.addCallback(lambda ign: self._check_uploader_count('objects_not_uploaded', 1, magic=self.bob_magicfolder))
 951         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 1, magic=self.bob_magicfolder))
 952
 953         d.addCallback(lambda ign: self._check_version_in_local_db(self.bob_magicfolder, u"file1", 1))
 954         d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file1", 1))
 955         d.addCallback(lambda ign: self._check_file_gone(self.bob_magicfolder, u"file1"))
 956         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0))
 957         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 2))
 958
 959         def Alice_to_rewrite_file():
 960             print "Alice rewrites file\n"
 961             self.file_path = abspath_expanduser_unicode(u"file1", base=self.alice_magicfolder.uploader._local_path_u)
 962             fileutil.write(self.file_path, "Alice suddenly sees the white rabbit running into the forest.")
 963             self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 964         d.addCallback(_wait_for, Alice_to_rewrite_file)
 965
 966         d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file1", 2))
 967         d.addCallback(lambda ign: self._check_version_in_local_db(self.alice_magicfolder, u"file1", 2))
 968         d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0))
 969         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 3))
 970         d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 3))
 971         d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0))
 972         d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0))
 973         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
 974         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0))
 975
 976         d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file1", 2))
 977         d.addCallback(lambda ign: self._check_version_in_local_db(self.bob_magicfolder, u"file1", 2))
 978         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0))
 979         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 3))
 980
 981         path_u = u"/tmp/magic_folder_test"
 982         encoded_path_u = magicpath.path2magic(u"/tmp/magic_folder_test")
 983
 984         def Alice_tries_to_p0wn_Bob(ign):
 985             print "Alice tries to p0wn Bob\n"
 986             processed_d = self.bob_magicfolder.downloader.set_hook('processed')
 987
 988             # upload a file that would provoke the security bug from #2506
 989             uploadable = Data("", self.alice_magicfolder._client.convergence)
 990             alice_dmd = self.alice_magicfolder.uploader._upload_dirnode
 991
 992             d2 = alice_dmd.add_file(encoded_path_u, uploadable, metadata={"version": 0}, overwrite=True)
 993             d2.addCallback(lambda ign: self.failUnless(alice_dmd.has_child(encoded_path_u)))
 994             d2.addCallback(_wait_for_Bob, processed_d)
 995             return d2
 996         d.addCallback(Alice_tries_to_p0wn_Bob)
 997
 998         d.addCallback(lambda ign: self.failIf(os.path.exists(path_u)))
 999         d.addCallback(lambda ign: self._check_version_in_local_db(self.bob_magicfolder, encoded_path_u, None))
1000         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 3))
1001         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
1002         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0))
1003
1004         def Bob_to_rewrite_file():
1005             print "Bob rewrites file\n"
1006             self.file_path = abspath_expanduser_unicode(u"file1", base=self.bob_magicfolder.uploader._local_path_u)
1007             print "---- bob's file is %r" % (self.file_path,)
1008             fileutil.write(self.file_path, "No white rabbit to be found.")
1009             self.magicfolder = self.bob_magicfolder
1010             self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE)
1011         d.addCallback(lambda ign: _wait_for(None, Bob_to_rewrite_file, alice=False))
1012
1013         d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file1", 3))
1014         d.addCallback(lambda ign: self._check_version_in_local_db(self.bob_magicfolder, u"file1", 3))
1015         d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0, magic=self.bob_magicfolder))
1016         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 2, magic=self.bob_magicfolder))
1017         d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 1, magic=self.bob_magicfolder))
1018         d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0, magic=self.bob_magicfolder))
1019         d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0, magic=self.bob_magicfolder))
1020         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0))
1021
1022         d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file1", 3))
1023         d.addCallback(lambda ign: self._check_version_in_local_db(self.alice_magicfolder, u"file1", 3))
1024         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
1025         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 1, magic=self.alice_magicfolder))
1026         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
1027
1028         def Alice_conflicts_with_Bobs_last_downloaded_uri():
1029             print "Alice conflicts with Bob\n"
1030             downloaded_d = self.bob_magicfolder.downloader.set_hook('processed')
1031             uploadable = Data("do not follow the white rabbit", self.alice_magicfolder._client.convergence)
1032             alice_dmd = self.alice_magicfolder.uploader._upload_dirnode
1033             d2 = alice_dmd.add_file(u"file1", uploadable,
1034                                     metadata={"version": 5,
1035                                               "last_downloaded_uri" : "URI:LIT:" },
1036                                     overwrite=True)
1037             print "Waiting for Alice to upload\n"
1038             d2.addCallback(lambda ign: bob_clock.advance(6))
1039             d2.addCallback(lambda ign: downloaded_d)
1040             d2.addCallback(lambda ign: self.failUnless(alice_dmd.has_child(encoded_path_u)))
1041             return d2
1042
1043         d.addCallback(lambda ign: Alice_conflicts_with_Bobs_last_downloaded_uri())
1044         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 4))
1045         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 1))
1046         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 1, magic=self.alice_magicfolder))
1047         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
1048         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
1049
1050         # prepare to perform another conflict test
1051         def Alice_to_write_file2():
1052             print "Alice writes a file\n"
1053             self.file_path = abspath_expanduser_unicode(u"file2", base=self.alice_magicfolder.uploader._local_path_u)
1054             fileutil.write(self.file_path, "something")
1055             self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
1056         d.addCallback(_wait_for, Alice_to_write_file2)
1057         d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file2", 0))
1058         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
1059         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
1060
1061         def Bob_to_rewrite_file2():
1062             print "Bob rewrites file\n"
1063             self.file_path = abspath_expanduser_unicode(u"file2", base=self.bob_magicfolder.uploader._local_path_u)
1064             print "---- bob's file is %r" % (self.file_path,)
1065             fileutil.write(self.file_path, "roger roger. what vector?")
1066             self.magicfolder = self.bob_magicfolder
1067             self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE)
1068         d.addCallback(lambda ign: _wait_for(None, Bob_to_rewrite_file2, alice=False))
1069         d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file2", 1))
1070         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 5))
1071         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 1))
1072         d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0, magic=self.bob_magicfolder))
1073         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 3, magic=self.bob_magicfolder))
1074         d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 2, magic=self.bob_magicfolder))
1075         d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0, magic=self.bob_magicfolder))
1076         d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0, magic=self.bob_magicfolder))
1077         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
1078         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
1079         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 2, magic=self.alice_magicfolder))
1080
1081         # XXX here we advance the clock and then test again to make sure no values are monotonically increasing
1082         # with each queue turn ;-p
1083         alice_clock.advance(6)
1084         bob_clock.advance(6)
1085         d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file2", 1))
1086         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 5))
1087         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 1))
1088         d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0, magic=self.bob_magicfolder))
1089         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 3, magic=self.bob_magicfolder))
1090         d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 2, magic=self.bob_magicfolder))
1091         d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0, magic=self.bob_magicfolder))
1092         d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0, magic=self.bob_magicfolder))
1093         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
1094         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
1095         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 2, magic=self.alice_magicfolder))
1096
1097         def Alice_conflicts_with_Bobs_last_uploaded_uri():
1098             print "Alice conflicts with Bob\n"
1099             encoded_path_u = magicpath.path2magic(u"file2")
1100             downloaded_d = self.bob_magicfolder.downloader.set_hook('processed')
1101             uploadable = Data("rabbits with sharp fangs", self.alice_magicfolder._client.convergence)
1102             alice_dmd = self.alice_magicfolder.uploader._upload_dirnode
1103             d2 = alice_dmd.add_file(u"file2", uploadable,
1104                                     metadata={"version": 5,
1105                                               "last_uploaded_uri" : "URI:LIT:" },
1106                                     overwrite=True)
1107             print "Waiting for Alice to upload\n"
1108             d2.addCallback(lambda ign: bob_clock.advance(6))
1109             d2.addCallback(lambda ign: downloaded_d)
1110             d2.addCallback(lambda ign: self.failUnless(alice_dmd.has_child(encoded_path_u)))
1111             return d2
1112         d.addCallback(lambda ign: Alice_conflicts_with_Bobs_last_uploaded_uri())
1113         d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file2", 5))
1114         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 6))
1115         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 2))
1116         d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0, magic=self.bob_magicfolder))
1117         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 3, magic=self.bob_magicfolder))
1118         d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 2, magic=self.bob_magicfolder))
1119         d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0, magic=self.bob_magicfolder))
1120         d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0, magic=self.bob_magicfolder))
1121         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
1122         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
1123         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 2, magic=self.alice_magicfolder))
1124
1125         alice_clock.advance(6)
1126         bob_clock.advance(6)
1127         alice_clock.advance(6)
1128         bob_clock.advance(6)
1129
1130         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 2, magic=self.alice_magicfolder))
1131         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
1132         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 2))
1133         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 6))
1134
1135         # prepare to perform another conflict test
1136         def Alice_to_write_file3():
1137             print "Alice writes a file\n"
1138             self.file_path = abspath_expanduser_unicode(u"file3", base=self.alice_magicfolder.uploader._local_path_u)
1139             fileutil.write(self.file_path, "something")
1140             self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
1141         d.addCallback(_wait_for, Alice_to_write_file3)
1142         d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file3", 0))
1143         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
1144         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 7))
1145         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 2, magic=self.alice_magicfolder))
1146         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 2))
1147         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
1148
1149         def Bob_to_rewrite_file3():
1150             print "Bob rewrites file\n"
1151             self.file_path = abspath_expanduser_unicode(u"file3", base=self.bob_magicfolder.uploader._local_path_u)
1152             print "---- bob's file is %r" % (self.file_path,)
1153             fileutil.write(self.file_path, "roger roger")
1154             self.magicfolder = self.bob_magicfolder
1155             self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE)
1156         d.addCallback(lambda ign: _wait_for(None, Bob_to_rewrite_file3, alice=False))
1157         d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file3", 1))
1158         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 7))
1159         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 2))
1160         d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0, magic=self.bob_magicfolder))
1161         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 4, magic=self.bob_magicfolder))
1162         d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 3, magic=self.bob_magicfolder))
1163         d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0, magic=self.bob_magicfolder))
1164         d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0, magic=self.bob_magicfolder))
1165         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
1166         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
1167         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 3, magic=self.alice_magicfolder))
1168
1169
1170
1171         def _cleanup(ign, magicfolder, clock):
1172             if magicfolder is not None:
1173                 d2 = magicfolder.finish()
1174                 clock.advance(0)
1175                 return d2
1176
1177         def cleanup_Alice_and_Bob(result):
1178             print "cleanup alice bob test\n"
1179             d = defer.succeed(None)
1180             d.addCallback(_cleanup, self.alice_magicfolder, alice_clock)
1181             d.addCallback(_cleanup, self.bob_magicfolder, bob_clock)
1182             d.addCallback(lambda ign: result)
1183             return d
1184         d.addBoth(cleanup_Alice_and_Bob)
1185         return d
1186
1187
1188 class MockTest(MagicFolderTestMixin, unittest.TestCase):
1189     """This can run on any platform, and even if twisted.internet.inotify can't be imported."""
1190
1191     def setUp(self):
1192         MagicFolderTestMixin.setUp(self)
1193         self.inotify = fake_inotify
1194         self.patch(magic_folder, 'get_inotify_module', lambda: self.inotify)
1195
1196     def notify(self, path, mask, magic=None, flush=True):
1197         if magic is None:
1198             magic = self.magicfolder
1199         magic.uploader._notifier.event(path, mask)
1200         # no flush for the mock test.
1201
1202     def test_errors(self):
1203         self.set_up_grid()
1204
1205         errors_dir = abspath_expanduser_unicode(u"errors_dir", base=self.basedir)
1206         os.mkdir(errors_dir)
1207         not_a_dir = abspath_expanduser_unicode(u"NOT_A_DIR", base=self.basedir)
1208         fileutil.write(not_a_dir, "")
1209         magicfolderdb = abspath_expanduser_unicode(u"magicfolderdb", base=self.basedir)
1210         doesnotexist  = abspath_expanduser_unicode(u"doesnotexist", base=self.basedir)
1211
1212         client = self.g.clients[0]
1213         d = client.create_dirnode()
1214         def _check_errors(n):
1215             self.failUnless(IDirectoryNode.providedBy(n))
1216             upload_dircap = n.get_uri()
1217             readonly_dircap = n.get_readonly_uri()
1218
1219             self.shouldFail(AssertionError, 'nonexistent local.directory', 'there is no directory',
1220                             MagicFolder, client, upload_dircap, '', doesnotexist, magicfolderdb, 0077)
1221             self.shouldFail(AssertionError, 'non-directory local.directory', 'is not a directory',
1222                             MagicFolder, client, upload_dircap, '', not_a_dir, magicfolderdb, 0077)
1223             self.shouldFail(AssertionError, 'bad upload.dircap', 'does not refer to a directory',
1224                             MagicFolder, client, 'bad', '', errors_dir, magicfolderdb, 0077)
1225             self.shouldFail(AssertionError, 'non-directory upload.dircap', 'does not refer to a directory',
1226                             MagicFolder, client, 'URI:LIT:foo', '', errors_dir, magicfolderdb, 0077)
1227             self.shouldFail(AssertionError, 'readonly upload.dircap', 'is not a writecap to a directory',
1228                             MagicFolder, client, readonly_dircap, '', errors_dir, magicfolderdb, 0077)
1229             self.shouldFail(AssertionError, 'collective dircap', 'is not a readonly cap to a directory',
1230                             MagicFolder, client, upload_dircap, upload_dircap, errors_dir, magicfolderdb, 0077)
1231
1232             def _not_implemented():
1233                 raise NotImplementedError("blah")
1234             self.patch(magic_folder, 'get_inotify_module', _not_implemented)
1235             self.shouldFail(NotImplementedError, 'unsupported', 'blah',
1236                             MagicFolder, client, upload_dircap, '', errors_dir, magicfolderdb, 0077)
1237         d.addCallback(_check_errors)
1238         return d
1239
1240     def test_write_downloaded_file(self):
1241         workdir = u"cli/MagicFolder/write-downloaded-file"
1242         local_file = fileutil.abspath_expanduser_unicode(os.path.join(workdir, "foobar"))
1243
1244         class TestWriteFileMixin(WriteFileMixin):
1245             def _log(self, msg):
1246                 pass
1247
1248         writefile = TestWriteFileMixin()
1249         writefile._umask = 0077
1250
1251         # create a file with name "foobar" with content "foo"
1252         # write downloaded file content "bar" into "foobar" with is_conflict = False
1253         fileutil.make_dirs(workdir)
1254         fileutil.write(local_file, "foo")
1255
1256         # if is_conflict is False, then the .conflict file shouldn't exist.
1257         writefile._write_downloaded_file(local_file, "bar", False, None)
1258         conflicted_path = local_file + u".conflict"
1259         self.failIf(os.path.exists(conflicted_path))
1260
1261         # At this point, the backup file should exist with content "foo"
1262         backup_path = local_file + u".backup"
1263         self.failUnless(os.path.exists(backup_path))
1264         self.failUnlessEqual(fileutil.read(backup_path), "foo")
1265
1266         # .tmp file shouldn't exist
1267         self.failIf(os.path.exists(local_file + u".tmp"))
1268
1269         # .. and the original file should have the new content
1270         self.failUnlessEqual(fileutil.read(local_file), "bar")
1271
1272         # now a test for conflicted case
1273         writefile._write_downloaded_file(local_file, "bar", True, None)
1274         self.failUnless(os.path.exists(conflicted_path))
1275
1276         # .tmp file shouldn't exist
1277         self.failIf(os.path.exists(local_file + u".tmp"))
1278
1279     def test_periodic_full_scan(self):
1280         self.set_up_grid()
1281         self.local_dir = abspath_expanduser_unicode(u"test_periodic_full_scan",base=self.basedir)
1282         self.mkdir_nonascii(self.local_dir)
1283
1284         alice_clock = task.Clock()
1285         d = self.do_create_magic_folder(0)
1286         d.addCallback(lambda ign: self.do_invite(0, u"Alice\u00F8"))
1287         def get_invite_code(result):
1288             self.invite_code = result[1].strip()
1289         d.addCallback(get_invite_code)
1290         d.addCallback(lambda ign: self.do_join(0, self.local_dir, self.invite_code))
1291         def get_alice_caps(ign):
1292             self.alice_collective_dircap, self.alice_upload_dircap = self.get_caps_from_files(0)
1293         d.addCallback(get_alice_caps)
1294         d.addCallback(lambda ign: self.check_joined_config(0, self.alice_upload_dircap))
1295         d.addCallback(lambda ign: self.check_config(0, self.local_dir))
1296         def get_Alice_magicfolder(result):
1297             self.magicfolder = self.init_magicfolder(0, self.alice_upload_dircap,
1298                                                            self.alice_collective_dircap,
1299                                                            self.local_dir, alice_clock)
1300             return result
1301         d.addCallback(get_Alice_magicfolder)
1302         empty_tree_name = self.unicode_or_fallback(u"empty_tr\u00EAe", u"empty_tree")
1303         empty_tree_dir = abspath_expanduser_unicode(empty_tree_name, base=self.basedir)
1304         new_empty_tree_dir = abspath_expanduser_unicode(empty_tree_name, base=self.local_dir)
1305
1306         def _check_move_empty_tree(res):
1307             print "CHECK MOVE EMPTY TREE"
1308             uploaded_d = self.magicfolder.uploader.set_hook('processed')
1309             self.mkdir_nonascii(empty_tree_dir)
1310             os.rename(empty_tree_dir, new_empty_tree_dir)
1311             self.notify(to_filepath(new_empty_tree_dir), self.inotify.IN_MOVED_TO)
1312             return uploaded_d
1313         d.addCallback(_check_move_empty_tree)
1314         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
1315         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 1))
1316         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.files_uploaded'), 0))
1317         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
1318         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.directories_created'), 1))
1319
1320         def _create_file_without_event(res):
1321             print "CREATE FILE WITHOUT EMITTING EVENT"
1322             processed_d = self.magicfolder.uploader.set_hook('processed')
1323             what_path = abspath_expanduser_unicode(u"what", base=new_empty_tree_dir)
1324             fileutil.write(what_path, "say when")
1325             print "ADVANCE CLOCK"
1326             alice_clock.advance(self.magicfolder.uploader._periodic_full_scan_duration + 1)
1327             return processed_d
1328         d.addCallback(_create_file_without_event)
1329         def _advance_clock(res):
1330             print "_advance_clock"
1331             processed_d = self.magicfolder.uploader.set_hook('processed')
1332             alice_clock.advance(0)
1333             return processed_d
1334         d.addCallback(_advance_clock)
1335         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.files_uploaded'), 1))
1336         def cleanup(res):
1337             d2 = self.magicfolder.finish()
1338             alice_clock.advance(0)
1339             return d2
1340         d.addCallback(cleanup)
1341         return d
1342
1343 class RealTest(MagicFolderTestMixin, unittest.TestCase):
1344     """This is skipped unless both Twisted and the platform support inotify."""
1345
1346     def setUp(self):
1347         MagicFolderTestMixin.setUp(self)
1348         self.inotify = magic_folder.get_inotify_module()
1349
1350     def notify(self, path, mask, magic=None, flush=True):
1351         # Writing to the filesystem causes the notification.
1352         # However, flushing filesystem buffers may be necessary on Windows.
1353         if flush:
1354             fileutil.flush_volume(path.path)
1355
1356 try:
1357     magic_folder.get_inotify_module()
1358 except NotImplementedError:
1359     RealTest.skip = "Magic Folder support can only be tested for-real on an OS that supports inotify or equivalent."