src/allmydata/test/test_magic_folder.py

   1
   2 import os, sys
   3
   4 from twisted.trial import unittest
   5 from twisted.internet import defer, task
   6
   7 from allmydata.interfaces import IDirectoryNode
   8 from allmydata.util.assertutil import precondition
   9
  10 from allmydata.util import fake_inotify, fileutil
  11 from allmydata.util.deferredutil import DeferredListShouldSucceed
  12 from allmydata.util.encodingutil import get_filesystem_encoding, to_filepath
  13 from allmydata.util.consumer import download_to_data
  14 from allmydata.test.no_network import GridTestMixin
  15 from allmydata.test.common_util import ReallyEqualMixin, NonASCIIPathMixin
  16 from allmydata.test.common import ShouldFailMixin
  17 from .test_cli_magic_folder import MagicFolderCLITestMixin
  18
  19 from allmydata.frontends import magic_folder
  20 from allmydata.frontends.magic_folder import MagicFolder, Downloader, WriteFileMixin
  21 from allmydata import magicfolderdb, magicpath
  22 from allmydata.util.fileutil import abspath_expanduser_unicode
  23 from allmydata.immutable.upload import Data
  24
  25
  26 class MagicFolderTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, ReallyEqualMixin, NonASCIIPathMixin):
  27     """
  28     These tests will be run both with a mock notifier, and (on platforms that support it)
  29     with the real INotify.
  30     """
  31
  32     def setUp(self):
  33         GridTestMixin.setUp(self)
  34         temp = self.mktemp()
  35         self.basedir = abspath_expanduser_unicode(temp.decode(get_filesystem_encoding()))
  36         self.magicfolder = None
  37         self.patch(Downloader, 'REMOTE_SCAN_INTERVAL', 0)
  38
  39     def _get_count(self, name, client=None):
  40         counters = (client or self.get_client()).stats_provider.get_stats()["counters"]
  41         return counters.get('magic_folder.%s' % (name,), 0)
  42
  43     def _createdb(self):
  44         dbfile = abspath_expanduser_unicode(u"magicfolderdb.sqlite", base=self.basedir)
  45         mdb = magicfolderdb.get_magicfolderdb(dbfile, create_version=(magicfolderdb.SCHEMA_v1, 1))
  46         self.failUnless(mdb, "unable to create magicfolderdb from %r" % (dbfile,))
  47         self.failUnlessEqual(mdb.VERSION, 1)
  48         return mdb
  49
  50     def _restart_client(self, ign):
  51         #print "_restart_client"
  52         d = self.restart_client()
  53         d.addCallback(self._wait_until_started)
  54         return d
  55
  56     def _wait_until_started(self, ign):
  57         #print "_wait_until_started"
  58         self.magicfolder = self.get_client().getServiceNamed('magic-folder')
  59         return self.magicfolder.ready()
  60
  61     def test_db_basic(self):
  62         fileutil.make_dirs(self.basedir)
  63         self._createdb()
  64
  65     def test_db_persistence(self):
  66         """Test that a file upload creates an entry in the database."""
  67
  68         fileutil.make_dirs(self.basedir)
  69         db = self._createdb()
  70
  71         relpath1 = u"myFile1"
  72         pathinfo = fileutil.PathInfo(isdir=False, isfile=True, islink=False,
  73                                      exists=True, size=1, mtime=123, ctime=456)
  74         db.did_upload_version(relpath1, 0, 'URI:LIT:1', 'URI:LIT:0', 0, pathinfo)
  75
  76         c = db.cursor
  77         c.execute("SELECT size, mtime, ctime"
  78                   " FROM local_files"
  79                   " WHERE path=?",
  80                   (relpath1,))
  81         row = c.fetchone()
  82         self.failUnlessEqual(row, (pathinfo.size, pathinfo.mtime, pathinfo.ctime))
  83
  84         # Second test uses magic_folder.is_new_file instead of SQL query directly
  85         # to confirm the previous upload entry in the db.
  86         relpath2 = u"myFile2"
  87         path2 = os.path.join(self.basedir, relpath2)
  88         fileutil.write(path2, "meow\n")
  89         pathinfo = fileutil.get_pathinfo(path2)
  90         db.did_upload_version(relpath2, 0, 'URI:LIT:2', 'URI:LIT:1', 0, pathinfo)
  91         db_entry = db.get_db_entry(relpath2)
  92         self.failUnlessFalse(magic_folder.is_new_file(pathinfo, db_entry))
  93
  94         different_pathinfo = fileutil.PathInfo(isdir=False, isfile=True, islink=False,
  95                                                exists=True, size=0, mtime=pathinfo.mtime, ctime=pathinfo.ctime)
  96         self.failUnlessTrue(magic_folder.is_new_file(different_pathinfo, db_entry))
  97
  98     def test_magicfolder_start_service(self):
  99         self.set_up_grid()
 100
 101         self.local_dir = abspath_expanduser_unicode(self.unicode_or_fallback(u"l\u00F8cal_dir", u"local_dir"),
 102                                                     base=self.basedir)
 103         self.mkdir_nonascii(self.local_dir)
 104
 105         d = defer.succeed(None)
 106         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.dirs_monitored'), 0))
 107
 108         d.addCallback(lambda ign: self.create_invite_join_magic_folder(u"Alice", self.local_dir))
 109         d.addCallback(self._restart_client)
 110
 111         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.dirs_monitored'), 1))
 112         d.addBoth(self.cleanup)
 113         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.dirs_monitored'), 0))
 114         return d
 115
 116     def test_scan_once_on_startup(self):
 117         self.set_up_grid()
 118         self.local_dir = abspath_expanduser_unicode(u"test_scan_once_on_startup", base=self.basedir)
 119         self.mkdir_nonascii(self.local_dir)
 120         self.collective_dircap = ""
 121
 122         alice_clock = task.Clock()
 123         bob_clock = task.Clock()
 124         d = self.setup_alice_and_bob(alice_clock, bob_clock)
 125
 126         def upload_stuff(ignore):
 127             uploadable = Data("", self.alice_magicfolder._client.convergence)
 128             return self.alice_magicfolder._client.upload(uploadable)
 129         d.addCallback(upload_stuff)
 130         def check_is_upload(ignore):
 131             alice_clock.advance(99)
 132             d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 0, magic=self.alice_magicfolder))
 133             d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0, magic=self.alice_magicfolder))
 134             d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
 135             d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 0, magic=self.alice_magicfolder))
 136             d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
 137             d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 0, magic=self.alice_magicfolder))
 138
 139         d.addCallback(check_is_upload)
 140         def _cleanup(ign, magicfolder, clock):
 141             if magicfolder is not None:
 142                 d2 = magicfolder.finish()
 143                 clock.advance(0)
 144                 return d2
 145         def cleanup_Alice_and_Bob(result):
 146             print "cleanup alice bob test\n"
 147             d = defer.succeed(None)
 148             d.addCallback(_cleanup, self.alice_magicfolder, alice_clock)
 149             d.addCallback(_cleanup, self.bob_magicfolder, bob_clock)
 150             d.addCallback(lambda ign: result)
 151             return d
 152
 153         d.addBoth(cleanup_Alice_and_Bob)
 154         return d
 155
 156     def test_move_tree(self):
 157         self.set_up_grid()
 158
 159         self.local_dir = abspath_expanduser_unicode(self.unicode_or_fallback(u"l\u00F8cal_dir", u"local_dir"),
 160                                                     base=self.basedir)
 161         self.mkdir_nonascii(self.local_dir)
 162
 163         empty_tree_name = self.unicode_or_fallback(u"empty_tr\u00EAe", u"empty_tree")
 164         empty_tree_dir = abspath_expanduser_unicode(empty_tree_name, base=self.basedir)
 165         new_empty_tree_dir = abspath_expanduser_unicode(empty_tree_name, base=self.local_dir)
 166
 167         small_tree_name = self.unicode_or_fallback(u"small_tr\u00EAe", u"empty_tree")
 168         small_tree_dir = abspath_expanduser_unicode(small_tree_name, base=self.basedir)
 169         new_small_tree_dir = abspath_expanduser_unicode(small_tree_name, base=self.local_dir)
 170
 171         d = self.create_invite_join_magic_folder(u"Alice", self.local_dir)
 172         d.addCallback(self._restart_client)
 173
 174         def _check_move_empty_tree(res):
 175             print "_check_move_empty_tree"
 176             uploaded_d = self.magicfolder.uploader.set_hook('processed')
 177             self.mkdir_nonascii(empty_tree_dir)
 178             os.rename(empty_tree_dir, new_empty_tree_dir)
 179             self.notify(to_filepath(new_empty_tree_dir), self.inotify.IN_MOVED_TO)
 180
 181             return uploaded_d
 182         d.addCallback(_check_move_empty_tree)
 183         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 184         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 1))
 185         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.files_uploaded'), 0))
 186         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 187         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.directories_created'), 1))
 188
 189         def _check_move_small_tree(res):
 190             print "_check_move_small_tree"
 191             uploaded_d = self.magicfolder.uploader.set_hook('processed', ignore_count=1)
 192             self.mkdir_nonascii(small_tree_dir)
 193             what_path = abspath_expanduser_unicode(u"what", base=small_tree_dir)
 194             fileutil.write(what_path, "say when")
 195             os.rename(small_tree_dir, new_small_tree_dir)
 196             self.notify(to_filepath(new_small_tree_dir), self.inotify.IN_MOVED_TO)
 197
 198             return uploaded_d
 199         d.addCallback(_check_move_small_tree)
 200         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 201         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 3))
 202         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.files_uploaded'), 1))
 203         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 204         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.directories_created'), 2))
 205
 206         def _check_moved_tree_is_watched(res):
 207             print "_check_moved_tree_is_watched"
 208             uploaded_d = self.magicfolder.uploader.set_hook('processed')
 209             another_path = abspath_expanduser_unicode(u"another", base=new_small_tree_dir)
 210             fileutil.write(another_path, "file")
 211             self.notify(to_filepath(another_path), self.inotify.IN_CLOSE_WRITE)
 212
 213             return uploaded_d
 214         d.addCallback(_check_moved_tree_is_watched)
 215         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 216         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 4))
 217         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.files_uploaded'), 2))
 218         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 219         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.directories_created'), 2))
 220
 221         # Files that are moved out of the upload directory should no longer be watched.
 222         #def _move_dir_away(ign):
 223         #    os.rename(new_empty_tree_dir, empty_tree_dir)
 224         #    # Wuh? Why don't we get this event for the real test?
 225         #    #self.notify(to_filepath(new_empty_tree_dir), self.inotify.IN_MOVED_FROM)
 226         #d.addCallback(_move_dir_away)
 227         #def create_file(val):
 228         #    test_file = abspath_expanduser_unicode(u"what", base=empty_tree_dir)
 229         #    fileutil.write(test_file, "meow")
 230         #    #self.notify(...)
 231         #    return
 232         #d.addCallback(create_file)
 233         #d.addCallback(lambda ign: time.sleep(1))  # XXX ICK
 234         #d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 235         #d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 4))
 236         #d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.files_uploaded'), 2))
 237         #d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 238         #d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.directories_created'), 2))
 239
 240         d.addBoth(self.cleanup)
 241         return d
 242
 243     def test_persistence(self):
 244         """
 245         Perform an upload of a given file and then stop the client.
 246         Start a new client and magic-folder service... and verify that the file is NOT uploaded
 247         a second time. This test is meant to test the database persistence along with
 248         the startup and shutdown code paths of the magic-folder service.
 249         """
 250         self.set_up_grid()
 251         self.local_dir = abspath_expanduser_unicode(u"test_persistence", base=self.basedir)
 252         self.mkdir_nonascii(self.local_dir)
 253         self.collective_dircap = ""
 254
 255         d = defer.succeed(None)
 256         d.addCallback(lambda ign: self.create_invite_join_magic_folder(u"Alice", self.local_dir))
 257         d.addCallback(self._restart_client)
 258
 259         def create_test_file(filename):
 260             d2 = self.magicfolder.uploader.set_hook('processed')
 261             test_file = abspath_expanduser_unicode(filename, base=self.local_dir)
 262             fileutil.write(test_file, "meow %s" % filename)
 263             self.notify(to_filepath(test_file), self.inotify.IN_CLOSE_WRITE)
 264             return d2
 265         d.addCallback(lambda ign: create_test_file(u"what1"))
 266         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 267         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 1))
 268         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 269         d.addCallback(self.cleanup)
 270
 271         d.addCallback(self._restart_client)
 272         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 273         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 1))
 274         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 275         d.addCallback(lambda ign: create_test_file(u"what2"))
 276         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 277         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 2))
 278         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 279         d.addBoth(self.cleanup)
 280         return d
 281
 282     @defer.inlineCallbacks
 283     def test_delete(self):
 284         self.set_up_grid()
 285         self.local_dir = os.path.join(self.basedir, u"local_dir")
 286         self.mkdir_nonascii(self.local_dir)
 287
 288         yield self.create_invite_join_magic_folder(u"Alice\u0101", self.local_dir)
 289         yield self._restart_client(None)
 290
 291         try:
 292             # create a file
 293             up_proc = self.magicfolder.uploader.set_hook('processed')
 294             # down_proc = self.magicfolder.downloader.set_hook('processed')
 295             path = os.path.join(self.local_dir, u'foo')
 296             fileutil.write(path, 'foo\n')
 297             self.notify(to_filepath(path), self.inotify.IN_CLOSE_WRITE)
 298             yield up_proc
 299             self.assertTrue(os.path.exists(path))
 300
 301             # the real test part: delete the file
 302             up_proc = self.magicfolder.uploader.set_hook('processed')
 303             os.unlink(path)
 304             self.notify(to_filepath(path), self.inotify.IN_DELETE)
 305             yield up_proc
 306             self.assertFalse(os.path.exists(path))
 307
 308             # ensure we still have a DB entry, and that the version is 1
 309             node, metadata = yield self.magicfolder.downloader._get_collective_latest_file(u'foo')
 310             self.assertTrue(node is not None, "Failed to find %r in DMD" % (path,))
 311             self.failUnlessEqual(metadata['version'], 1)
 312
 313         finally:
 314             yield self.cleanup(None)
 315
 316     @defer.inlineCallbacks
 317     def test_delete_and_restore(self):
 318         self.set_up_grid()
 319         self.local_dir = os.path.join(self.basedir, u"local_dir")
 320         self.mkdir_nonascii(self.local_dir)
 321
 322         yield self.create_invite_join_magic_folder(u"Alice\u0101", self.local_dir)
 323         yield self._restart_client(None)
 324
 325         try:
 326             # create a file
 327             up_proc = self.magicfolder.uploader.set_hook('processed')
 328             # down_proc = self.magicfolder.downloader.set_hook('processed')
 329             path = os.path.join(self.local_dir, u'foo')
 330             fileutil.write(path, 'foo\n')
 331             self.notify(to_filepath(path), self.inotify.IN_CLOSE_WRITE)
 332             yield up_proc
 333             self.assertTrue(os.path.exists(path))
 334
 335             # delete the file
 336             up_proc = self.magicfolder.uploader.set_hook('processed')
 337             os.unlink(path)
 338             self.notify(to_filepath(path), self.inotify.IN_DELETE)
 339             yield up_proc
 340             self.assertFalse(os.path.exists(path))
 341
 342             # ensure we still have a DB entry, and that the version is 1
 343             node, metadata = yield self.magicfolder.downloader._get_collective_latest_file(u'foo')
 344             self.assertTrue(node is not None, "Failed to find %r in DMD" % (path,))
 345             self.failUnlessEqual(metadata['version'], 1)
 346
 347             # restore the file, with different contents
 348             up_proc = self.magicfolder.uploader.set_hook('processed')
 349             path = os.path.join(self.local_dir, u'foo')
 350             fileutil.write(path, 'bar\n')
 351             self.notify(to_filepath(path), self.inotify.IN_CLOSE_WRITE)
 352             yield up_proc
 353
 354             # ensure we still have a DB entry, and that the version is 2
 355             node, metadata = yield self.magicfolder.downloader._get_collective_latest_file(u'foo')
 356             self.assertTrue(node is not None, "Failed to find %r in DMD" % (path,))
 357             self.failUnlessEqual(metadata['version'], 2)
 358
 359         finally:
 360             yield self.cleanup(None)
 361
 362     @defer.inlineCallbacks
 363     def test_alice_delete_bob_restore(self):
 364         alice_clock = task.Clock()
 365         bob_clock = task.Clock()
 366         yield self.setup_alice_and_bob(alice_clock, bob_clock)
 367         alice_dir = self.alice_magicfolder.uploader._local_path_u
 368         bob_dir = self.bob_magicfolder.uploader._local_path_u
 369         alice_fname = os.path.join(alice_dir, 'blam')
 370         bob_fname = os.path.join(bob_dir, 'blam')
 371
 372         try:
 373             # alice creates a file, bob downloads it
 374             alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
 375             bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
 376
 377             fileutil.write(alice_fname, 'contents0\n')
 378             self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 379
 380             alice_clock.advance(0)
 381             yield alice_proc  # alice uploads
 382
 383             bob_clock.advance(0)
 384             yield bob_proc    # bob downloads
 385
 386             # check the state
 387             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
 388             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 0)
 389             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
 390             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 0)
 391             yield self.failUnlessReallyEqual(
 392                 self._get_count('downloader.objects_failed', client=self.bob_magicfolder._client),
 393                 0
 394             )
 395             yield self.failUnlessReallyEqual(
 396                 self._get_count('downloader.objects_downloaded', client=self.bob_magicfolder._client),
 397                 1
 398             )
 399
 400             print("BOB DELETE")
 401             # now bob deletes it (bob should upload, alice download)
 402             bob_proc = self.bob_magicfolder.uploader.set_hook('processed')
 403             alice_proc = self.alice_magicfolder.downloader.set_hook('processed')
 404             os.unlink(bob_fname)
 405             self.notify(to_filepath(bob_fname), self.inotify.IN_DELETE, magic=self.bob_magicfolder)
 406
 407             bob_clock.advance(0)
 408             yield bob_proc
 409             alice_clock.advance(0)
 410             yield alice_proc
 411
 412             # check versions
 413             node, metadata = yield self.alice_magicfolder.downloader._get_collective_latest_file(u'blam')
 414             self.assertTrue(metadata['deleted'])
 415             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
 416             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 1)
 417             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
 418             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 1)
 419
 420             print("ALICE RESTORE")
 421             # now alice restores it (alice should upload, bob download)
 422             alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
 423             bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
 424             fileutil.write(alice_fname, 'new contents\n')
 425             self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 426
 427             alice_clock.advance(0)
 428             yield alice_proc
 429             bob_clock.advance(0)
 430             yield bob_proc
 431
 432             # check versions
 433             node, metadata = yield self.alice_magicfolder.downloader._get_collective_latest_file(u'blam')
 434             self.assertTrue('deleted' not in metadata or not metadata['deleted'])
 435             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 2)
 436             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 2)
 437             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 2)
 438             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 2)
 439
 440         finally:
 441             # cleanup
 442             d0 = self.alice_magicfolder.finish()
 443             alice_clock.advance(0)
 444             yield d0
 445
 446             d1 = self.bob_magicfolder.finish()
 447             bob_clock.advance(0)
 448             yield d1
 449
 450     @defer.inlineCallbacks
 451     def test_alice_sees_bobs_delete_with_error(self):
 452         # alice creates a file, bob deletes it -- and we also arrange
 453         # for Alice's file to have "gone missing" as well.
 454         alice_clock = task.Clock()
 455         bob_clock = task.Clock()
 456         yield self.setup_alice_and_bob(alice_clock, bob_clock)
 457         alice_dir = self.alice_magicfolder.uploader._local_path_u
 458         bob_dir = self.bob_magicfolder.uploader._local_path_u
 459         alice_fname = os.path.join(alice_dir, 'blam')
 460         bob_fname = os.path.join(bob_dir, 'blam')
 461
 462         try:
 463             # alice creates a file, bob downloads it
 464             alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
 465             bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
 466
 467             fileutil.write(alice_fname, 'contents0\n')
 468             self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 469
 470             alice_clock.advance(0)
 471             yield alice_proc  # alice uploads
 472
 473             bob_clock.advance(0)
 474             yield bob_proc    # bob downloads
 475
 476             # check the state
 477             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
 478             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 0)
 479             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
 480             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 0)
 481             yield self.failUnlessReallyEqual(
 482                 self._get_count('downloader.objects_failed', client=self.bob_magicfolder._client),
 483                 0
 484             )
 485             yield self.failUnlessReallyEqual(
 486                 self._get_count('downloader.objects_downloaded', client=self.bob_magicfolder._client),
 487                 1
 488             )
 489
 490             # now bob deletes it (bob should upload, alice download)
 491             bob_proc = self.bob_magicfolder.uploader.set_hook('processed')
 492             alice_proc = self.alice_magicfolder.downloader.set_hook('processed')
 493             os.unlink(bob_fname)
 494             self.notify(to_filepath(bob_fname), self.inotify.IN_DELETE, magic=self.bob_magicfolder)
 495             # just after notifying bob, we also delete alice's,
 496             # covering the 'except' flow in _rename_deleted_file()
 497             os.unlink(alice_fname)
 498
 499             bob_clock.advance(0)
 500             yield bob_proc
 501             alice_clock.advance(0)
 502             yield alice_proc
 503
 504             # check versions
 505             node, metadata = yield self.alice_magicfolder.downloader._get_collective_latest_file(u'blam')
 506             self.assertTrue(metadata['deleted'])
 507             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
 508             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 1)
 509             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
 510             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 1)
 511
 512         finally:
 513             # cleanup
 514             d0 = self.alice_magicfolder.finish()
 515             alice_clock.advance(0)
 516             yield d0
 517
 518             d1 = self.bob_magicfolder.finish()
 519             bob_clock.advance(0)
 520             yield d1
 521
 522     @defer.inlineCallbacks
 523     def test_alice_create_bob_update(self):
 524         alice_clock = task.Clock()
 525         bob_clock = task.Clock()
 526         yield self.setup_alice_and_bob(alice_clock, bob_clock)
 527         alice_dir = self.alice_magicfolder.uploader._local_path_u
 528         bob_dir = self.bob_magicfolder.uploader._local_path_u
 529         alice_fname = os.path.join(alice_dir, 'blam')
 530         bob_fname = os.path.join(bob_dir, 'blam')
 531
 532         try:
 533             # alice creates a file, bob downloads it
 534             alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
 535             bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
 536
 537             fileutil.write(alice_fname, 'contents0\n')
 538             self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 539
 540             alice_clock.advance(0)
 541             yield alice_proc  # alice uploads
 542
 543             bob_clock.advance(0)
 544             yield bob_proc    # bob downloads
 545
 546             # check the state
 547             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
 548             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 0)
 549             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
 550             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 0)
 551             yield self.failUnlessReallyEqual(
 552                 self._get_count('downloader.objects_failed', client=self.bob_magicfolder._client),
 553                 0
 554             )
 555             yield self.failUnlessReallyEqual(
 556                 self._get_count('downloader.objects_downloaded', client=self.bob_magicfolder._client),
 557                 1
 558             )
 559
 560             # now bob updates it (bob should upload, alice download)
 561             bob_proc = self.bob_magicfolder.uploader.set_hook('processed')
 562             alice_proc = self.alice_magicfolder.downloader.set_hook('processed')
 563             fileutil.write(bob_fname, 'bob wuz here\n')
 564             self.notify(to_filepath(bob_fname), self.inotify.IN_CLOSE_WRITE, magic=self.bob_magicfolder)
 565
 566             bob_clock.advance(0)
 567             yield bob_proc
 568             alice_clock.advance(0)
 569             yield alice_proc
 570
 571             # check the state
 572             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
 573             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 1)
 574             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
 575             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 1)
 576
 577         finally:
 578             # cleanup
 579             d0 = self.alice_magicfolder.finish()
 580             alice_clock.advance(0)
 581             yield d0
 582
 583             d1 = self.bob_magicfolder.finish()
 584             bob_clock.advance(0)
 585             yield d1
 586
 587     @defer.inlineCallbacks
 588     def test_alice_delete_and_restore(self):
 589         alice_clock = task.Clock()
 590         bob_clock = task.Clock()
 591         yield self.setup_alice_and_bob(alice_clock, bob_clock)
 592         alice_dir = self.alice_magicfolder.uploader._local_path_u
 593         bob_dir = self.bob_magicfolder.uploader._local_path_u
 594         alice_fname = os.path.join(alice_dir, 'blam')
 595         bob_fname = os.path.join(bob_dir, 'blam')
 596
 597         try:
 598             # alice creates a file, bob downloads it
 599             alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
 600             bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
 601
 602             fileutil.write(alice_fname, 'contents0\n')
 603             self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 604
 605             alice_clock.advance(0)
 606             yield alice_proc  # alice uploads
 607
 608             bob_clock.advance(0)
 609             yield bob_proc    # bob downloads
 610
 611             # check the state
 612             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
 613             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 0)
 614             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
 615             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 0)
 616             yield self.failUnlessReallyEqual(
 617                 self._get_count('downloader.objects_failed', client=self.bob_magicfolder._client),
 618                 0
 619             )
 620             yield self.failUnlessReallyEqual(
 621                 self._get_count('downloader.objects_downloaded', client=self.bob_magicfolder._client),
 622                 1
 623             )
 624             self.failUnless(os.path.exists(bob_fname))
 625
 626             # now alice deletes it (alice should upload, bob download)
 627             alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
 628             bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
 629             os.unlink(alice_fname)
 630             self.notify(to_filepath(alice_fname), self.inotify.IN_DELETE, magic=self.alice_magicfolder)
 631
 632             alice_clock.advance(0)
 633             yield alice_proc
 634             bob_clock.advance(0)
 635             yield bob_proc
 636
 637             # check the state
 638             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
 639             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 1)
 640             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
 641             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 1)
 642             self.failIf(os.path.exists(bob_fname))
 643
 644             # now alice restores the file (with new contents)
 645             alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
 646             bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
 647             fileutil.write(alice_fname, 'alice wuz here\n')
 648             self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 649
 650             alice_clock.advance(0)
 651             yield alice_proc
 652             bob_clock.advance(0)
 653             yield bob_proc
 654
 655             # check the state
 656             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 2)
 657             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 2)
 658             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 2)
 659             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 2)
 660             self.failUnless(os.path.exists(bob_fname))
 661
 662         finally:
 663             # cleanup
 664             d0 = self.alice_magicfolder.finish()
 665             alice_clock.advance(0)
 666             yield d0
 667
 668             d1 = self.bob_magicfolder.finish()
 669             bob_clock.advance(0)
 670             yield d1
 671
 672     def test_magic_folder(self):
 673         self.set_up_grid()
 674         self.local_dir = os.path.join(self.basedir, self.unicode_or_fallback(u"loc\u0101l_dir", u"local_dir"))
 675         self.mkdir_nonascii(self.local_dir)
 676
 677         d = self.create_invite_join_magic_folder(u"Alice\u0101", self.local_dir)
 678         d.addCallback(self._restart_client)
 679
 680         # Write something short enough for a LIT file.
 681         d.addCallback(lambda ign: self._check_file(u"short", "test"))
 682
 683         # Write to the same file again with different data.
 684         d.addCallback(lambda ign: self._check_file(u"short", "different"))
 685
 686         # Test that temporary files are not uploaded.
 687         d.addCallback(lambda ign: self._check_file(u"tempfile", "test", temporary=True))
 688
 689         # Test creation of a subdirectory.
 690         d.addCallback(lambda ign: self._check_mkdir(u"directory"))
 691
 692         # Write something longer, and also try to test a Unicode name if the fs can represent it.
 693         name_u = self.unicode_or_fallback(u"l\u00F8ng", u"long")
 694         d.addCallback(lambda ign: self._check_file(name_u, "test"*100))
 695
 696         # TODO: test that causes an upload failure.
 697         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 698
 699         d.addBoth(self.cleanup)
 700         return d
 701
 702     def _check_mkdir(self, name_u):
 703         return self._check_file(name_u + u"/", "", directory=True)
 704
 705     def _check_file(self, name_u, data, temporary=False, directory=False):
 706         precondition(not (temporary and directory), temporary=temporary, directory=directory)
 707
 708         print "%r._check_file(%r, %r, temporary=%r, directory=%r)" % (self, name_u, data, temporary, directory)
 709         previously_uploaded = self._get_count('uploader.objects_succeeded')
 710         previously_disappeared = self._get_count('uploader.objects_disappeared')
 711
 712         d = self.magicfolder.uploader.set_hook('processed')
 713
 714         path_u = abspath_expanduser_unicode(name_u, base=self.local_dir)
 715         path = to_filepath(path_u)
 716
 717         if directory:
 718             os.mkdir(path_u)
 719             event_mask = self.inotify.IN_CREATE | self.inotify.IN_ISDIR
 720         else:
 721             # We don't use FilePath.setContent() here because it creates a temporary file that
 722             # is renamed into place, which causes events that the test is not expecting.
 723             f = open(path_u, "wb")
 724             try:
 725                 if temporary and sys.platform != "win32":
 726                     os.unlink(path_u)
 727                 f.write(data)
 728             finally:
 729                 f.close()
 730             if temporary and sys.platform == "win32":
 731                 os.unlink(path_u)
 732                 self.notify(path, self.inotify.IN_DELETE, flush=False)
 733             event_mask = self.inotify.IN_CLOSE_WRITE
 734
 735         self.notify(path, event_mask)
 736         encoded_name_u = magicpath.path2magic(name_u)
 737
 738         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 739         if temporary:
 740             d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_disappeared'),
 741                                                                  previously_disappeared + 1))
 742         else:
 743             def _here(res, n):
 744                 print "here %r %r" % (n, res)
 745                 return res
 746             d.addBoth(_here, 1)
 747             d.addCallback(lambda ign: self.upload_dirnode.list())
 748             d.addBoth(_here, 1.5)
 749             d.addCallback(lambda ign: self.upload_dirnode.get(encoded_name_u))
 750             d.addBoth(_here, 2)
 751             d.addCallback(download_to_data)
 752             d.addBoth(_here, 3)
 753             d.addCallback(lambda actual_data: self.failUnlessReallyEqual(actual_data, data))
 754             d.addBoth(_here, 4)
 755             d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'),
 756                                                                  previously_uploaded + 1))
 757
 758         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 759         return d
 760
 761     def _check_version_in_dmd(self, magicfolder, relpath_u, expected_version):
 762         encoded_name_u = magicpath.path2magic(relpath_u)
 763         d = magicfolder.downloader._get_collective_latest_file(encoded_name_u)
 764         def check_latest(result):
 765             if result[0] is not None:
 766                 node, metadata = result
 767                 d.addCallback(lambda ign: self.failUnlessEqual(metadata['version'], expected_version))
 768         d.addCallback(check_latest)
 769         return d
 770
 771     def _check_version_in_local_db(self, magicfolder, relpath_u, expected_version):
 772         db_entry = magicfolder._db.get_db_entry(relpath_u)
 773         if db_entry is not None:
 774             #print "_check_version_in_local_db: %r has version %s" % (relpath_u, version)
 775             self.failUnlessEqual(db_entry.version, expected_version)
 776
 777     def _check_file_gone(self, magicfolder, relpath_u):
 778         path = os.path.join(magicfolder.uploader._local_path_u, relpath_u)
 779         self.assertTrue(not os.path.exists(path))
 780
 781     def _check_uploader_count(self, name, expected, magic=None):
 782         self.failUnlessReallyEqual(self._get_count('uploader.'+name, client=(magic or self.alice_magicfolder)._client),
 783                                    expected)
 784
 785     def _check_downloader_count(self, name, expected, magic=None):
 786         self.failUnlessReallyEqual(self._get_count('downloader.'+name, client=(magic or self.bob_magicfolder)._client),
 787                                    expected)
 788
 789     def test_alice_bob(self):
 790         alice_clock = task.Clock()
 791         bob_clock = task.Clock()
 792         d = self.setup_alice_and_bob(alice_clock, bob_clock)
 793
 794         def _wait_for_Alice(ign, downloaded_d):
 795             print "Now waiting for Alice to download\n"
 796             alice_clock.advance(0)
 797             return downloaded_d
 798
 799         def _wait_for_Bob(ign, downloaded_d):
 800             print "Now waiting for Bob to download\n"
 801             bob_clock.advance(0)
 802             return downloaded_d
 803
 804         def _wait_for(ign, something_to_do, alice=True):
 805             if alice:
 806                 downloaded_d = self.bob_magicfolder.downloader.set_hook('processed')
 807                 uploaded_d = self.alice_magicfolder.uploader.set_hook('processed')
 808             else:
 809                 downloaded_d = self.alice_magicfolder.downloader.set_hook('processed')
 810                 uploaded_d = self.bob_magicfolder.uploader.set_hook('processed')
 811             something_to_do()
 812             if alice:
 813                 print "Waiting for Alice to upload\n"
 814                 alice_clock.advance(0)
 815                 uploaded_d.addCallback(_wait_for_Bob, downloaded_d)
 816             else:
 817                 print "Waiting for Bob to upload\n"
 818                 bob_clock.advance(0)
 819                 uploaded_d.addCallback(_wait_for_Alice, downloaded_d)
 820             return uploaded_d
 821
 822         def Alice_to_write_a_file():
 823             print "Alice writes a file\n"
 824             self.file_path = abspath_expanduser_unicode(u"file1", base=self.alice_magicfolder.uploader._local_path_u)
 825             fileutil.write(self.file_path, "meow, meow meow. meow? meow meow! meow.")
 826             self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 827         d.addCallback(_wait_for, Alice_to_write_a_file)
 828
 829         d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file1", 0))
 830         d.addCallback(lambda ign: self._check_version_in_local_db(self.alice_magicfolder, u"file1", 0))
 831         d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0))
 832         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 1))
 833         d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 1))
 834         d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0))
 835         d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0))
 836         d.addCallback(lambda ign: self._check_uploader_count('objects_conflicted', 0))
 837         d.addCallback(lambda ign: self._check_uploader_count('objects_conflicted', 0, magic=self.bob_magicfolder))
 838
 839         d.addCallback(lambda ign: self._check_version_in_local_db(self.bob_magicfolder, u"file1", 0))
 840         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0))
 841         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 1))
 842         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 0, magic=self.bob_magicfolder))
 843
 844         def check_delete_file(ign):
 845             d_bob = self.bob_magicfolder.uploader.set_hook('processed')
 846             def Alice_to_delete_file():
 847                 print "Alice deletes the file!\n"
 848                 os.unlink(self.file_path)
 849                 self.notify(to_filepath(self.file_path), self.inotify.IN_DELETE, magic=self.alice_magicfolder)
 850
 851             d_alice = defer.succeed(None)
 852             d_alice.addCallback(_wait_for, Alice_to_delete_file)
 853
 854             p = abspath_expanduser_unicode(u"file1", base=self.bob_magicfolder.uploader._local_path_u)
 855             if sys.platform == "win32":
 856                 self.notify(to_filepath(p), self.inotify.IN_MOVED_FROM, magic=self.bob_magicfolder, flush=False)
 857                 self.notify(to_filepath(p + u'.backup'), self.inotify.IN_MOVED_TO, magic=self.bob_magicfolder)
 858             else:
 859                 self.notify(to_filepath(p + u'.backup'), self.inotify.IN_CREATE, magic=self.bob_magicfolder, flush=False)
 860                 self.notify(to_filepath(p), self.inotify.IN_DELETE, magic=self.bob_magicfolder)
 861
 862             d_alice.addCallback(lambda ign: bob_clock.advance(0))
 863             return DeferredListShouldSucceed([d_alice, d_bob])
 864         d.addCallback(check_delete_file)
 865
 866         d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file1", 1))
 867         d.addCallback(lambda ign: self._check_version_in_local_db(self.alice_magicfolder, u"file1", 1))
 868         d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0))
 869         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 2))
 870         d.addCallback(lambda ign: self._check_uploader_count('objects_not_uploaded', 1, magic=self.bob_magicfolder))
 871         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 1, magic=self.bob_magicfolder))
 872
 873         d.addCallback(lambda ign: self._check_version_in_local_db(self.bob_magicfolder, u"file1", 1))
 874         d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file1", 1))
 875         d.addCallback(lambda ign: self._check_file_gone(self.bob_magicfolder, u"file1"))
 876         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0))
 877         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 2))
 878
 879         def Alice_to_rewrite_file():
 880             print "Alice rewrites file\n"
 881             self.file_path = abspath_expanduser_unicode(u"file1", base=self.alice_magicfolder.uploader._local_path_u)
 882             fileutil.write(self.file_path, "Alice suddenly sees the white rabbit running into the forest.")
 883             self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 884         d.addCallback(_wait_for, Alice_to_rewrite_file)
 885
 886         d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file1", 2))
 887         d.addCallback(lambda ign: self._check_version_in_local_db(self.alice_magicfolder, u"file1", 2))
 888         d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0))
 889         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 3))
 890         d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 3))
 891         d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0))
 892         d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0))
 893         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
 894         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0))
 895
 896         d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file1", 2))
 897         d.addCallback(lambda ign: self._check_version_in_local_db(self.bob_magicfolder, u"file1", 2))
 898         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0))
 899         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 3))
 900
 901         path_u = u"/tmp/magic_folder_test"
 902         encoded_path_u = magicpath.path2magic(u"/tmp/magic_folder_test")
 903
 904         def Alice_tries_to_p0wn_Bob(ign):
 905             print "Alice tries to p0wn Bob\n"
 906             processed_d = self.bob_magicfolder.downloader.set_hook('processed')
 907
 908             # upload a file that would provoke the security bug from #2506
 909             uploadable = Data("", self.alice_magicfolder._client.convergence)
 910             alice_dmd = self.alice_magicfolder.uploader._upload_dirnode
 911
 912             d2 = alice_dmd.add_file(encoded_path_u, uploadable, metadata={"version": 0}, overwrite=True)
 913             d2.addCallback(lambda ign: self.failUnless(alice_dmd.has_child(encoded_path_u)))
 914             d2.addCallback(_wait_for_Bob, processed_d)
 915             return d2
 916         d.addCallback(Alice_tries_to_p0wn_Bob)
 917
 918         d.addCallback(lambda ign: self.failIf(os.path.exists(path_u)))
 919         d.addCallback(lambda ign: self._check_version_in_local_db(self.bob_magicfolder, encoded_path_u, None))
 920         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 3))
 921         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
 922         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0))
 923
 924         def Bob_to_rewrite_file():
 925             print "Bob rewrites file\n"
 926             self.file_path = abspath_expanduser_unicode(u"file1", base=self.bob_magicfolder.uploader._local_path_u)
 927             print "---- bob's file is %r" % (self.file_path,)
 928             fileutil.write(self.file_path, "No white rabbit to be found.")
 929             self.magicfolder = self.bob_magicfolder
 930             self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE)
 931         d.addCallback(lambda ign: _wait_for(None, Bob_to_rewrite_file, alice=False))
 932
 933         d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file1", 3))
 934         d.addCallback(lambda ign: self._check_version_in_local_db(self.bob_magicfolder, u"file1", 3))
 935         d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0, magic=self.bob_magicfolder))
 936         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 2, magic=self.bob_magicfolder))
 937         d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 1, magic=self.bob_magicfolder))
 938         d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0, magic=self.bob_magicfolder))
 939         d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0, magic=self.bob_magicfolder))
 940         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0))
 941
 942         d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file1", 3))
 943         d.addCallback(lambda ign: self._check_version_in_local_db(self.alice_magicfolder, u"file1", 3))
 944         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
 945         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 1, magic=self.alice_magicfolder))
 946         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
 947
 948         def Alice_conflicts_with_Bobs_last_downloaded_uri():
 949             print "Alice conflicts with Bob\n"
 950             downloaded_d = self.bob_magicfolder.downloader.set_hook('processed')
 951             uploadable = Data("do not follow the white rabbit", self.alice_magicfolder._client.convergence)
 952             alice_dmd = self.alice_magicfolder.uploader._upload_dirnode
 953             d2 = alice_dmd.add_file(u"file1", uploadable,
 954                                     metadata={"version": 5,
 955                                               "last_downloaded_uri" : "URI:LIT:" },
 956                                     overwrite=True)
 957             print "Waiting for Alice to upload\n"
 958             d2.addCallback(lambda ign: bob_clock.advance(6))
 959             d2.addCallback(lambda ign: downloaded_d)
 960             d2.addCallback(lambda ign: self.failUnless(alice_dmd.has_child(encoded_path_u)))
 961             return d2
 962
 963         d.addCallback(lambda ign: Alice_conflicts_with_Bobs_last_downloaded_uri())
 964         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 4))
 965         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 1))
 966         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 1, magic=self.alice_magicfolder))
 967         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
 968         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
 969
 970         # prepare to perform another conflict test
 971         def Alice_to_write_file2():
 972             print "Alice writes a file\n"
 973             self.file_path = abspath_expanduser_unicode(u"file2", base=self.alice_magicfolder.uploader._local_path_u)
 974             fileutil.write(self.file_path, "something")
 975             self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 976         d.addCallback(_wait_for, Alice_to_write_file2)
 977         d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file2", 0))
 978         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
 979         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
 980
 981         def Bob_to_rewrite_file2():
 982             print "Bob rewrites file\n"
 983             self.file_path = abspath_expanduser_unicode(u"file2", base=self.bob_magicfolder.uploader._local_path_u)
 984             print "---- bob's file is %r" % (self.file_path,)
 985             fileutil.write(self.file_path, "roger roger. what vector?")
 986             self.magicfolder = self.bob_magicfolder
 987             self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE)
 988         d.addCallback(lambda ign: _wait_for(None, Bob_to_rewrite_file2, alice=False))
 989         d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file2", 1))
 990         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 5))
 991         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 1))
 992         d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0, magic=self.bob_magicfolder))
 993         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 3, magic=self.bob_magicfolder))
 994         d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 2, magic=self.bob_magicfolder))
 995         d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0, magic=self.bob_magicfolder))
 996         d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0, magic=self.bob_magicfolder))
 997         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
 998         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
 999         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 2, magic=self.alice_magicfolder))
1000
1001         # XXX here we advance the clock and then test again to make sure no values are monotonically increasing
1002         # with each queue turn ;-p
1003         alice_clock.advance(6)
1004         bob_clock.advance(6)
1005         d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file2", 1))
1006         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 5))
1007         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 1))
1008         d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0, magic=self.bob_magicfolder))
1009         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 3, magic=self.bob_magicfolder))
1010         d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 2, magic=self.bob_magicfolder))
1011         d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0, magic=self.bob_magicfolder))
1012         d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0, magic=self.bob_magicfolder))
1013         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
1014         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
1015         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 2, magic=self.alice_magicfolder))
1016
1017         def Alice_conflicts_with_Bobs_last_uploaded_uri():
1018             print "Alice conflicts with Bob\n"
1019             encoded_path_u = magicpath.path2magic(u"file2")
1020             downloaded_d = self.bob_magicfolder.downloader.set_hook('processed')
1021             uploadable = Data("rabbits with sharp fangs", self.alice_magicfolder._client.convergence)
1022             alice_dmd = self.alice_magicfolder.uploader._upload_dirnode
1023             d2 = alice_dmd.add_file(u"file2", uploadable,
1024                                     metadata={"version": 5,
1025                                               "last_uploaded_uri" : "URI:LIT:" },
1026                                     overwrite=True)
1027             print "Waiting for Alice to upload\n"
1028             d2.addCallback(lambda ign: bob_clock.advance(6))
1029             d2.addCallback(lambda ign: downloaded_d)
1030             d2.addCallback(lambda ign: self.failUnless(alice_dmd.has_child(encoded_path_u)))
1031             return d2
1032         d.addCallback(lambda ign: Alice_conflicts_with_Bobs_last_uploaded_uri())
1033         d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file2", 5))
1034         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 6))
1035         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 2))
1036         d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0, magic=self.bob_magicfolder))
1037         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 3, magic=self.bob_magicfolder))
1038         d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 2, magic=self.bob_magicfolder))
1039         d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0, magic=self.bob_magicfolder))
1040         d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0, magic=self.bob_magicfolder))
1041         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
1042         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
1043         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 2, magic=self.alice_magicfolder))
1044
1045         alice_clock.advance(6)
1046         bob_clock.advance(6)
1047         alice_clock.advance(6)
1048         bob_clock.advance(6)
1049
1050         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 2, magic=self.alice_magicfolder))
1051         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
1052         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 2))
1053         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 6))
1054
1055         # prepare to perform another conflict test
1056         def Alice_to_write_file3():
1057             print "Alice writes a file\n"
1058             self.file_path = abspath_expanduser_unicode(u"file3", base=self.alice_magicfolder.uploader._local_path_u)
1059             fileutil.write(self.file_path, "something")
1060             self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
1061         d.addCallback(_wait_for, Alice_to_write_file3)
1062         d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file3", 0))
1063         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
1064         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 7))
1065         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 2, magic=self.alice_magicfolder))
1066         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 2))
1067         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
1068
1069         def Bob_to_rewrite_file3():
1070             print "Bob rewrites file\n"
1071             self.file_path = abspath_expanduser_unicode(u"file3", base=self.bob_magicfolder.uploader._local_path_u)
1072             print "---- bob's file is %r" % (self.file_path,)
1073             fileutil.write(self.file_path, "roger roger")
1074             self.magicfolder = self.bob_magicfolder
1075             self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE)
1076         d.addCallback(lambda ign: _wait_for(None, Bob_to_rewrite_file3, alice=False))
1077         d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file3", 1))
1078         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 7))
1079         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 2))
1080         d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0, magic=self.bob_magicfolder))
1081         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 4, magic=self.bob_magicfolder))
1082         d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 3, magic=self.bob_magicfolder))
1083         d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0, magic=self.bob_magicfolder))
1084         d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0, magic=self.bob_magicfolder))
1085         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
1086         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
1087         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 3, magic=self.alice_magicfolder))
1088
1089
1090
1091         def _cleanup(ign, magicfolder, clock):
1092             if magicfolder is not None:
1093                 d2 = magicfolder.finish()
1094                 clock.advance(0)
1095                 return d2
1096
1097         def cleanup_Alice_and_Bob(result):
1098             print "cleanup alice bob test\n"
1099             d = defer.succeed(None)
1100             d.addCallback(_cleanup, self.alice_magicfolder, alice_clock)
1101             d.addCallback(_cleanup, self.bob_magicfolder, bob_clock)
1102             d.addCallback(lambda ign: result)
1103             return d
1104         d.addBoth(cleanup_Alice_and_Bob)
1105         return d
1106
1107
1108 class MockTest(MagicFolderTestMixin, unittest.TestCase):
1109     """This can run on any platform, and even if twisted.internet.inotify can't be imported."""
1110
1111     def setUp(self):
1112         MagicFolderTestMixin.setUp(self)
1113         self.inotify = fake_inotify
1114         self.patch(magic_folder, 'get_inotify_module', lambda: self.inotify)
1115
1116     def notify(self, path, mask, magic=None, flush=True):
1117         if magic is None:
1118             magic = self.magicfolder
1119         magic.uploader._notifier.event(path, mask)
1120         # no flush for the mock test.
1121
1122     def test_errors(self):
1123         self.set_up_grid()
1124
1125         errors_dir = abspath_expanduser_unicode(u"errors_dir", base=self.basedir)
1126         os.mkdir(errors_dir)
1127         not_a_dir = abspath_expanduser_unicode(u"NOT_A_DIR", base=self.basedir)
1128         fileutil.write(not_a_dir, "")
1129         magicfolderdb = abspath_expanduser_unicode(u"magicfolderdb", base=self.basedir)
1130         doesnotexist  = abspath_expanduser_unicode(u"doesnotexist", base=self.basedir)
1131
1132         client = self.g.clients[0]
1133         d = client.create_dirnode()
1134         def _check_errors(n):
1135             self.failUnless(IDirectoryNode.providedBy(n))
1136             upload_dircap = n.get_uri()
1137             readonly_dircap = n.get_readonly_uri()
1138
1139             self.shouldFail(AssertionError, 'nonexistent local.directory', 'there is no directory',
1140                             MagicFolder, client, upload_dircap, '', doesnotexist, magicfolderdb, 0077)
1141             self.shouldFail(AssertionError, 'non-directory local.directory', 'is not a directory',
1142                             MagicFolder, client, upload_dircap, '', not_a_dir, magicfolderdb, 0077)
1143             self.shouldFail(AssertionError, 'bad upload.dircap', 'does not refer to a directory',
1144                             MagicFolder, client, 'bad', '', errors_dir, magicfolderdb, 0077)
1145             self.shouldFail(AssertionError, 'non-directory upload.dircap', 'does not refer to a directory',
1146                             MagicFolder, client, 'URI:LIT:foo', '', errors_dir, magicfolderdb, 0077)
1147             self.shouldFail(AssertionError, 'readonly upload.dircap', 'is not a writecap to a directory',
1148                             MagicFolder, client, readonly_dircap, '', errors_dir, magicfolderdb, 0077)
1149             self.shouldFail(AssertionError, 'collective dircap', 'is not a readonly cap to a directory',
1150                             MagicFolder, client, upload_dircap, upload_dircap, errors_dir, magicfolderdb, 0077)
1151
1152             def _not_implemented():
1153                 raise NotImplementedError("blah")
1154             self.patch(magic_folder, 'get_inotify_module', _not_implemented)
1155             self.shouldFail(NotImplementedError, 'unsupported', 'blah',
1156                             MagicFolder, client, upload_dircap, '', errors_dir, magicfolderdb, 0077)
1157         d.addCallback(_check_errors)
1158         return d
1159
1160     def test_write_downloaded_file(self):
1161         workdir = u"cli/MagicFolder/write-downloaded-file"
1162         local_file = fileutil.abspath_expanduser_unicode(os.path.join(workdir, "foobar"))
1163
1164         class TestWriteFileMixin(WriteFileMixin):
1165             def _log(self, msg):
1166                 pass
1167
1168         writefile = TestWriteFileMixin()
1169         writefile._umask = 0077
1170
1171         # create a file with name "foobar" with content "foo"
1172         # write downloaded file content "bar" into "foobar" with is_conflict = False
1173         fileutil.make_dirs(workdir)
1174         fileutil.write(local_file, "foo")
1175
1176         # if is_conflict is False, then the .conflict file shouldn't exist.
1177         writefile._write_downloaded_file(local_file, "bar", False, None)
1178         conflicted_path = local_file + u".conflict"
1179         self.failIf(os.path.exists(conflicted_path))
1180
1181         # At this point, the backup file should exist with content "foo"
1182         backup_path = local_file + u".backup"
1183         self.failUnless(os.path.exists(backup_path))
1184         self.failUnlessEqual(fileutil.read(backup_path), "foo")
1185
1186         # .tmp file shouldn't exist
1187         self.failIf(os.path.exists(local_file + u".tmp"))
1188
1189         # .. and the original file should have the new content
1190         self.failUnlessEqual(fileutil.read(local_file), "bar")
1191
1192         # now a test for conflicted case
1193         writefile._write_downloaded_file(local_file, "bar", True, None)
1194         self.failUnless(os.path.exists(conflicted_path))
1195
1196         # .tmp file shouldn't exist
1197         self.failIf(os.path.exists(local_file + u".tmp"))
1198
1199     def test_periodic_full_scan(self):
1200         self.set_up_grid()
1201         self.local_dir = abspath_expanduser_unicode(u"test_periodic_full_scan",base=self.basedir)
1202         self.mkdir_nonascii(self.local_dir)
1203
1204         alice_clock = task.Clock()
1205         d = self.do_create_magic_folder(0)
1206         d.addCallback(lambda ign: self.do_invite(0, u"Alice\u00F8"))
1207         def get_invite_code(result):
1208             self.invite_code = result[1].strip()
1209         d.addCallback(get_invite_code)
1210         d.addCallback(lambda ign: self.do_join(0, self.local_dir, self.invite_code))
1211         def get_alice_caps(ign):
1212             self.alice_collective_dircap, self.alice_upload_dircap = self.get_caps_from_files(0)
1213         d.addCallback(get_alice_caps)
1214         d.addCallback(lambda ign: self.check_joined_config(0, self.alice_upload_dircap))
1215         d.addCallback(lambda ign: self.check_config(0, self.local_dir))
1216         def get_Alice_magicfolder(result):
1217             self.magicfolder = self.init_magicfolder(0, self.alice_upload_dircap,
1218                                                            self.alice_collective_dircap,
1219                                                            self.local_dir, alice_clock)
1220             return result
1221         d.addCallback(get_Alice_magicfolder)
1222         empty_tree_name = self.unicode_or_fallback(u"empty_tr\u00EAe", u"empty_tree")
1223         empty_tree_dir = abspath_expanduser_unicode(empty_tree_name, base=self.basedir)
1224         new_empty_tree_dir = abspath_expanduser_unicode(empty_tree_name, base=self.local_dir)
1225
1226         def _check_move_empty_tree(res):
1227             print "CHECK MOVE EMPTY TREE"
1228             uploaded_d = self.magicfolder.uploader.set_hook('processed')
1229             self.mkdir_nonascii(empty_tree_dir)
1230             os.rename(empty_tree_dir, new_empty_tree_dir)
1231             self.notify(to_filepath(new_empty_tree_dir), self.inotify.IN_MOVED_TO)
1232             return uploaded_d
1233         d.addCallback(_check_move_empty_tree)
1234         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
1235         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 1))
1236         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.files_uploaded'), 0))
1237         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
1238         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.directories_created'), 1))
1239
1240         def _create_file_without_event(res):
1241             print "CREATE FILE WITHOUT EMITTING EVENT"
1242             processed_d = self.magicfolder.uploader.set_hook('processed')
1243             what_path = abspath_expanduser_unicode(u"what", base=new_empty_tree_dir)
1244             fileutil.write(what_path, "say when")
1245             print "ADVANCE CLOCK"
1246             alice_clock.advance(self.magicfolder.uploader._periodic_full_scan_duration + 1)
1247             return processed_d
1248         d.addCallback(_create_file_without_event)
1249         def _advance_clock(res):
1250             print "_advance_clock"
1251             processed_d = self.magicfolder.uploader.set_hook('processed')
1252             alice_clock.advance(0)
1253             return processed_d
1254         d.addCallback(_advance_clock)
1255         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.files_uploaded'), 1))
1256         def cleanup(res):
1257             d2 = self.magicfolder.finish()
1258             alice_clock.advance(0)
1259             return d2
1260         d.addCallback(cleanup)
1261         return d
1262
1263 class RealTest(MagicFolderTestMixin, unittest.TestCase):
1264     """This is skipped unless both Twisted and the platform support inotify."""
1265
1266     def setUp(self):
1267         MagicFolderTestMixin.setUp(self)
1268         self.inotify = magic_folder.get_inotify_module()
1269
1270     def notify(self, path, mask, magic=None, flush=True):
1271         # Writing to the filesystem causes the notification.
1272         # However, flushing filesystem buffers may be necessary on Windows.
1273         if flush:
1274             fileutil.flush_volume(path.path)
1275
1276 try:
1277     magic_folder.get_inotify_module()
1278 except NotImplementedError:
1279     RealTest.skip = "Magic Folder support can only be tested for-real on an OS that supports inotify or equivalent."