src/allmydata/test/test_magic_folder.py

   1
   2 import os, sys
   3
   4 from twisted.trial import unittest
   5 from twisted.internet import defer, task
   6
   7 from allmydata.interfaces import IDirectoryNode
   8 from allmydata.util.assertutil import precondition
   9
  10 from allmydata.util import fake_inotify, fileutil
  11 from allmydata.util.encodingutil import get_filesystem_encoding, to_filepath
  12 from allmydata.util.consumer import download_to_data
  13 from allmydata.test.no_network import GridTestMixin
  14 from allmydata.test.common_util import ReallyEqualMixin, NonASCIIPathMixin
  15 from allmydata.test.common import ShouldFailMixin
  16 from .test_cli_magic_folder import MagicFolderCLITestMixin
  17
  18 from allmydata.frontends import magic_folder
  19 from allmydata.frontends.magic_folder import MagicFolder, Downloader, WriteFileMixin
  20 from allmydata import magicfolderdb, magicpath
  21 from allmydata.util.fileutil import abspath_expanduser_unicode
  22 from allmydata.immutable.upload import Data
  23
  24
  25 class MagicFolderTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, ReallyEqualMixin, NonASCIIPathMixin):
  26     """
  27     These tests will be run both with a mock notifier, and (on platforms that support it)
  28     with the real INotify.
  29     """
  30
  31     def setUp(self):
  32         GridTestMixin.setUp(self)
  33         temp = self.mktemp()
  34         self.basedir = abspath_expanduser_unicode(temp.decode(get_filesystem_encoding()))
  35         self.magicfolder = None
  36         self.patch(Downloader, 'REMOTE_SCAN_INTERVAL', 0)
  37
  38     def _get_count(self, name, client=None):
  39         counters = (client or self.get_client()).stats_provider.get_stats()["counters"]
  40         return counters.get('magic_folder.%s' % (name,), 0)
  41
  42     def _createdb(self):
  43         dbfile = abspath_expanduser_unicode(u"magicfolderdb.sqlite", base=self.basedir)
  44         mdb = magicfolderdb.get_magicfolderdb(dbfile, create_version=(magicfolderdb.SCHEMA_v1, 1))
  45         self.failUnless(mdb, "unable to create magicfolderdb from %r" % (dbfile,))
  46         self.failUnlessEqual(mdb.VERSION, 1)
  47         return mdb
  48
  49     def _restart_client(self, ign):
  50         #print "_restart_client"
  51         d = self.restart_client()
  52         d.addCallback(self._wait_until_started)
  53         return d
  54
  55     def _wait_until_started(self, ign):
  56         #print "_wait_until_started"
  57         self.magicfolder = self.get_client().getServiceNamed('magic-folder')
  58         return self.magicfolder.ready()
  59
  60     def test_db_basic(self):
  61         fileutil.make_dirs(self.basedir)
  62         self._createdb()
  63
  64     def test_db_persistence(self):
  65         """Test that a file upload creates an entry in the database."""
  66
  67         fileutil.make_dirs(self.basedir)
  68         db = self._createdb()
  69
  70         relpath1 = u"myFile1"
  71         pathinfo = fileutil.PathInfo(isdir=False, isfile=True, islink=False,
  72                                      exists=True, size=1, mtime=123, ctime=456)
  73         db.did_upload_version(relpath1, 0, 'URI:LIT:1', 'URI:LIT:0', 0, pathinfo)
  74
  75         c = db.cursor
  76         c.execute("SELECT size, mtime, ctime"
  77                   " FROM local_files"
  78                   " WHERE path=?",
  79                   (relpath1,))
  80         row = c.fetchone()
  81         self.failUnlessEqual(row, (pathinfo.size, pathinfo.mtime, pathinfo.ctime))
  82
  83         # Second test uses magic_folder.is_new_file instead of SQL query directly
  84         # to confirm the previous upload entry in the db.
  85         relpath2 = u"myFile2"
  86         path2 = os.path.join(self.basedir, relpath2)
  87         fileutil.write(path2, "meow\n")
  88         pathinfo = fileutil.get_pathinfo(path2)
  89         db.did_upload_version(relpath2, 0, 'URI:LIT:2', 'URI:LIT:1', 0, pathinfo)
  90         db_entry = db.get_db_entry(relpath2)
  91         self.failUnlessFalse(magic_folder.is_new_file(pathinfo, db_entry))
  92
  93         different_pathinfo = fileutil.PathInfo(isdir=False, isfile=True, islink=False,
  94                                                exists=True, size=0, mtime=pathinfo.mtime, ctime=pathinfo.ctime)
  95         self.failUnlessTrue(magic_folder.is_new_file(different_pathinfo, db_entry))
  96
  97     def test_magicfolder_start_service(self):
  98         self.set_up_grid()
  99
 100         self.local_dir = abspath_expanduser_unicode(self.unicode_or_fallback(u"l\u00F8cal_dir", u"local_dir"),
 101                                                     base=self.basedir)
 102         self.mkdir_nonascii(self.local_dir)
 103
 104         d = defer.succeed(None)
 105         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.dirs_monitored'), 0))
 106
 107         d.addCallback(lambda ign: self.create_invite_join_magic_folder(u"Alice", self.local_dir))
 108         d.addCallback(self._restart_client)
 109
 110         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.dirs_monitored'), 1))
 111         d.addBoth(self.cleanup)
 112         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.dirs_monitored'), 0))
 113         return d
 114
 115     def test_scan_once_on_startup(self):
 116         self.set_up_grid()
 117         self.local_dir = abspath_expanduser_unicode(u"test_scan_once_on_startup", base=self.basedir)
 118         self.mkdir_nonascii(self.local_dir)
 119         self.collective_dircap = ""
 120
 121         alice_clock = task.Clock()
 122         bob_clock = task.Clock()
 123         d = self.setup_alice_and_bob(alice_clock, bob_clock)
 124
 125         def upload_stuff(ignore):
 126             uploadable = Data("", self.alice_magicfolder._client.convergence)
 127             return self.alice_magicfolder._client.upload(uploadable)
 128         d.addCallback(upload_stuff)
 129         def check_is_upload(ignore):
 130             alice_clock.advance(99)
 131             d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 0, magic=self.alice_magicfolder))
 132             d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0, magic=self.alice_magicfolder))
 133             d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
 134             d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 0, magic=self.alice_magicfolder))
 135             d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
 136             d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 0, magic=self.alice_magicfolder))
 137
 138         d.addCallback(check_is_upload)
 139         def _cleanup(ign, magicfolder, clock):
 140             if magicfolder is not None:
 141                 d2 = magicfolder.finish()
 142                 clock.advance(0)
 143                 return d2
 144         def cleanup_Alice_and_Bob(result):
 145             print "cleanup alice bob test\n"
 146             d = defer.succeed(None)
 147             d.addCallback(_cleanup, self.alice_magicfolder, alice_clock)
 148             d.addCallback(_cleanup, self.bob_magicfolder, bob_clock)
 149             d.addCallback(lambda ign: result)
 150             return d
 151
 152         d.addBoth(cleanup_Alice_and_Bob)
 153         return d
 154
 155     def test_move_tree(self):
 156         self.set_up_grid()
 157
 158         self.local_dir = abspath_expanduser_unicode(self.unicode_or_fallback(u"l\u00F8cal_dir", u"local_dir"),
 159                                                     base=self.basedir)
 160         self.mkdir_nonascii(self.local_dir)
 161
 162         empty_tree_name = self.unicode_or_fallback(u"empty_tr\u00EAe", u"empty_tree")
 163         empty_tree_dir = abspath_expanduser_unicode(empty_tree_name, base=self.basedir)
 164         new_empty_tree_dir = abspath_expanduser_unicode(empty_tree_name, base=self.local_dir)
 165
 166         small_tree_name = self.unicode_or_fallback(u"small_tr\u00EAe", u"empty_tree")
 167         small_tree_dir = abspath_expanduser_unicode(small_tree_name, base=self.basedir)
 168         new_small_tree_dir = abspath_expanduser_unicode(small_tree_name, base=self.local_dir)
 169
 170         d = self.create_invite_join_magic_folder(u"Alice", self.local_dir)
 171         d.addCallback(self._restart_client)
 172
 173         def _check_move_empty_tree(res):
 174             print "_check_move_empty_tree"
 175             uploaded_d = self.magicfolder.uploader.set_hook('processed')
 176             self.mkdir_nonascii(empty_tree_dir)
 177             os.rename(empty_tree_dir, new_empty_tree_dir)
 178             self.notify(to_filepath(new_empty_tree_dir), self.inotify.IN_MOVED_TO)
 179
 180             return uploaded_d
 181         d.addCallback(_check_move_empty_tree)
 182         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 183         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 1))
 184         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.files_uploaded'), 0))
 185         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 186         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.directories_created'), 1))
 187
 188         def _check_move_small_tree(res):
 189             print "_check_move_small_tree"
 190             uploaded_d = self.magicfolder.uploader.set_hook('processed', ignore_count=1)
 191             self.mkdir_nonascii(small_tree_dir)
 192             what_path = abspath_expanduser_unicode(u"what", base=small_tree_dir)
 193             fileutil.write(what_path, "say when")
 194             os.rename(small_tree_dir, new_small_tree_dir)
 195             self.notify(to_filepath(new_small_tree_dir), self.inotify.IN_MOVED_TO)
 196
 197             return uploaded_d
 198         d.addCallback(_check_move_small_tree)
 199         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 200         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 3))
 201         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.files_uploaded'), 1))
 202         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 203         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.directories_created'), 2))
 204
 205         def _check_moved_tree_is_watched(res):
 206             print "_check_moved_tree_is_watched"
 207             uploaded_d = self.magicfolder.uploader.set_hook('processed')
 208             another_path = abspath_expanduser_unicode(u"another", base=new_small_tree_dir)
 209             fileutil.write(another_path, "file")
 210             self.notify(to_filepath(another_path), self.inotify.IN_CLOSE_WRITE)
 211
 212             return uploaded_d
 213         d.addCallback(_check_moved_tree_is_watched)
 214         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 215         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 4))
 216         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.files_uploaded'), 2))
 217         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 218         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.directories_created'), 2))
 219
 220         # Files that are moved out of the upload directory should no longer be watched.
 221         #def _move_dir_away(ign):
 222         #    os.rename(new_empty_tree_dir, empty_tree_dir)
 223         #    # Wuh? Why don't we get this event for the real test?
 224         #    #self.notify(to_filepath(new_empty_tree_dir), self.inotify.IN_MOVED_FROM)
 225         #d.addCallback(_move_dir_away)
 226         #def create_file(val):
 227         #    test_file = abspath_expanduser_unicode(u"what", base=empty_tree_dir)
 228         #    fileutil.write(test_file, "meow")
 229         #    #self.notify(...)
 230         #    return
 231         #d.addCallback(create_file)
 232         #d.addCallback(lambda ign: time.sleep(1))  # XXX ICK
 233         #d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 234         #d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 4))
 235         #d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.files_uploaded'), 2))
 236         #d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 237         #d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.directories_created'), 2))
 238
 239         d.addBoth(self.cleanup)
 240         return d
 241
 242     def test_persistence(self):
 243         """
 244         Perform an upload of a given file and then stop the client.
 245         Start a new client and magic-folder service... and verify that the file is NOT uploaded
 246         a second time. This test is meant to test the database persistence along with
 247         the startup and shutdown code paths of the magic-folder service.
 248         """
 249         self.set_up_grid()
 250         self.local_dir = abspath_expanduser_unicode(u"test_persistence", base=self.basedir)
 251         self.mkdir_nonascii(self.local_dir)
 252         self.collective_dircap = ""
 253
 254         d = defer.succeed(None)
 255         d.addCallback(lambda ign: self.create_invite_join_magic_folder(u"Alice", self.local_dir))
 256         d.addCallback(self._restart_client)
 257
 258         def create_test_file(filename):
 259             d2 = self.magicfolder.uploader.set_hook('processed')
 260             test_file = abspath_expanduser_unicode(filename, base=self.local_dir)
 261             fileutil.write(test_file, "meow %s" % filename)
 262             self.notify(to_filepath(test_file), self.inotify.IN_CLOSE_WRITE)
 263             return d2
 264         d.addCallback(lambda ign: create_test_file(u"what1"))
 265         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 266         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 1))
 267         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 268         d.addCallback(self.cleanup)
 269
 270         d.addCallback(self._restart_client)
 271         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 272         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 1))
 273         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 274         d.addCallback(lambda ign: create_test_file(u"what2"))
 275         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 276         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 2))
 277         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 278         d.addBoth(self.cleanup)
 279         return d
 280
 281     @defer.inlineCallbacks
 282     def test_delete(self):
 283         self.set_up_grid()
 284         self.local_dir = os.path.join(self.basedir, u"local_dir")
 285         self.mkdir_nonascii(self.local_dir)
 286
 287         yield self.create_invite_join_magic_folder(u"Alice\u0101", self.local_dir)
 288         yield self._restart_client(None)
 289
 290         try:
 291             # create a file
 292             up_proc = self.magicfolder.uploader.set_hook('processed')
 293             # down_proc = self.magicfolder.downloader.set_hook('processed')
 294             path = os.path.join(self.local_dir, u'foo')
 295             fileutil.write(path, 'foo\n')
 296             self.notify(to_filepath(path), self.inotify.IN_CLOSE_WRITE)
 297             yield up_proc
 298             self.assertTrue(os.path.exists(path))
 299
 300             # the real test part: delete the file
 301             up_proc = self.magicfolder.uploader.set_hook('processed')
 302             os.unlink(path)
 303             self.notify(to_filepath(path), self.inotify.IN_DELETE)
 304             yield up_proc
 305             self.assertFalse(os.path.exists(path))
 306
 307             # ensure we still have a DB entry, and that the version is 1
 308             node, metadata = yield self.magicfolder.downloader._get_collective_latest_file(u'foo')
 309             self.assertTrue(node is not None, "Failed to find %r in DMD" % (path,))
 310             self.failUnlessEqual(metadata['version'], 1)
 311
 312         finally:
 313             yield self.cleanup(None)
 314
 315     @defer.inlineCallbacks
 316     def test_delete_and_restore(self):
 317         self.set_up_grid()
 318         self.local_dir = os.path.join(self.basedir, u"local_dir")
 319         self.mkdir_nonascii(self.local_dir)
 320
 321         yield self.create_invite_join_magic_folder(u"Alice\u0101", self.local_dir)
 322         yield self._restart_client(None)
 323
 324         try:
 325             # create a file
 326             up_proc = self.magicfolder.uploader.set_hook('processed')
 327             # down_proc = self.magicfolder.downloader.set_hook('processed')
 328             path = os.path.join(self.local_dir, u'foo')
 329             fileutil.write(path, 'foo\n')
 330             self.notify(to_filepath(path), self.inotify.IN_CLOSE_WRITE)
 331             yield up_proc
 332             self.assertTrue(os.path.exists(path))
 333
 334             # delete the file
 335             up_proc = self.magicfolder.uploader.set_hook('processed')
 336             os.unlink(path)
 337             self.notify(to_filepath(path), self.inotify.IN_DELETE)
 338             yield up_proc
 339             self.assertFalse(os.path.exists(path))
 340
 341             # ensure we still have a DB entry, and that the version is 1
 342             node, metadata = yield self.magicfolder.downloader._get_collective_latest_file(u'foo')
 343             self.assertTrue(node is not None, "Failed to find %r in DMD" % (path,))
 344             self.failUnlessEqual(metadata['version'], 1)
 345
 346             # restore the file, with different contents
 347             up_proc = self.magicfolder.uploader.set_hook('processed')
 348             path = os.path.join(self.local_dir, u'foo')
 349             fileutil.write(path, 'bar\n')
 350             self.notify(to_filepath(path), self.inotify.IN_CLOSE_WRITE)
 351             yield up_proc
 352
 353             # ensure we still have a DB entry, and that the version is 2
 354             node, metadata = yield self.magicfolder.downloader._get_collective_latest_file(u'foo')
 355             self.assertTrue(node is not None, "Failed to find %r in DMD" % (path,))
 356             self.failUnlessEqual(metadata['version'], 2)
 357
 358         finally:
 359             yield self.cleanup(None)
 360
 361     @defer.inlineCallbacks
 362     def test_alice_delete_bob_restore(self):
 363         alice_clock = task.Clock()
 364         bob_clock = task.Clock()
 365         yield self.setup_alice_and_bob(alice_clock, bob_clock)
 366         alice_dir = self.alice_magicfolder.uploader._local_path_u
 367         bob_dir = self.bob_magicfolder.uploader._local_path_u
 368         alice_fname = os.path.join(alice_dir, 'blam')
 369         bob_fname = os.path.join(bob_dir, 'blam')
 370
 371         try:
 372             # alice creates a file, bob downloads it
 373             alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
 374             bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
 375
 376             fileutil.write(alice_fname, 'contents0\n')
 377             self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 378
 379             alice_clock.advance(0)
 380             yield alice_proc  # alice uploads
 381
 382             bob_clock.advance(0)
 383             yield bob_proc    # bob downloads
 384
 385             # check the state
 386             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
 387             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 0)
 388             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
 389             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 0)
 390             yield self.failUnlessReallyEqual(
 391                 self._get_count('downloader.objects_failed', client=self.bob_magicfolder._client),
 392                 0
 393             )
 394             yield self.failUnlessReallyEqual(
 395                 self._get_count('downloader.objects_downloaded', client=self.bob_magicfolder._client),
 396                 1
 397             )
 398
 399             print("BOB DELETE")
 400             # now bob deletes it (bob should upload, alice download)
 401             bob_proc = self.bob_magicfolder.uploader.set_hook('processed')
 402             alice_proc = self.alice_magicfolder.downloader.set_hook('processed')
 403             os.unlink(bob_fname)
 404             self.notify(to_filepath(bob_fname), self.inotify.IN_DELETE, magic=self.bob_magicfolder)
 405
 406             bob_clock.advance(0)
 407             yield bob_proc
 408             alice_clock.advance(0)
 409             yield alice_proc
 410
 411             # check versions
 412             node, metadata = yield self.alice_magicfolder.downloader._get_collective_latest_file(u'blam')
 413             self.assertTrue(metadata['deleted'])
 414             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
 415             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 1)
 416             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
 417             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 1)
 418
 419             print("ALICE RESTORE")
 420             # now alice restores it (alice should upload, bob download)
 421             alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
 422             bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
 423             fileutil.write(alice_fname, 'new contents\n')
 424             self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 425
 426             alice_clock.advance(0)
 427             yield alice_proc
 428             bob_clock.advance(0)
 429             yield bob_proc
 430
 431             # check versions
 432             node, metadata = yield self.alice_magicfolder.downloader._get_collective_latest_file(u'blam')
 433             self.assertTrue('deleted' not in metadata or not metadata['deleted'])
 434             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 2)
 435             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 2)
 436             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 2)
 437             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 2)
 438
 439         finally:
 440             # cleanup
 441             d0 = self.alice_magicfolder.finish()
 442             alice_clock.advance(0)
 443             yield d0
 444
 445             d1 = self.bob_magicfolder.finish()
 446             bob_clock.advance(0)
 447             yield d1
 448
 449     @defer.inlineCallbacks
 450     def test_alice_sees_bobs_delete_with_error(self):
 451         # alice creates a file, bob deletes it -- and we also arrange
 452         # for Alice's file to have "gone missing" as well.
 453         alice_clock = task.Clock()
 454         bob_clock = task.Clock()
 455         yield self.setup_alice_and_bob(alice_clock, bob_clock)
 456         alice_dir = self.alice_magicfolder.uploader._local_path_u
 457         bob_dir = self.bob_magicfolder.uploader._local_path_u
 458         alice_fname = os.path.join(alice_dir, 'blam')
 459         bob_fname = os.path.join(bob_dir, 'blam')
 460
 461         try:
 462             # alice creates a file, bob downloads it
 463             alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
 464             bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
 465
 466             fileutil.write(alice_fname, 'contents0\n')
 467             self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 468
 469             alice_clock.advance(0)
 470             yield alice_proc  # alice uploads
 471
 472             bob_clock.advance(0)
 473             yield bob_proc    # bob downloads
 474
 475             # check the state
 476             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
 477             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 0)
 478             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
 479             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 0)
 480             yield self.failUnlessReallyEqual(
 481                 self._get_count('downloader.objects_failed', client=self.bob_magicfolder._client),
 482                 0
 483             )
 484             yield self.failUnlessReallyEqual(
 485                 self._get_count('downloader.objects_downloaded', client=self.bob_magicfolder._client),
 486                 1
 487             )
 488
 489             # now bob deletes it (bob should upload, alice download)
 490             bob_proc = self.bob_magicfolder.uploader.set_hook('processed')
 491             alice_proc = self.alice_magicfolder.downloader.set_hook('processed')
 492             os.unlink(bob_fname)
 493             self.notify(to_filepath(bob_fname), self.inotify.IN_DELETE, magic=self.bob_magicfolder)
 494             # just after notifying bob, we also delete alice's,
 495             # covering the 'except' flow in _rename_deleted_file()
 496             os.unlink(alice_fname)
 497
 498             bob_clock.advance(0)
 499             yield bob_proc
 500             alice_clock.advance(0)
 501             yield alice_proc
 502
 503             # check versions
 504             node, metadata = yield self.alice_magicfolder.downloader._get_collective_latest_file(u'blam')
 505             self.assertTrue(metadata['deleted'])
 506             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
 507             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 1)
 508             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
 509             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 1)
 510
 511         finally:
 512             # cleanup
 513             d0 = self.alice_magicfolder.finish()
 514             alice_clock.advance(0)
 515             yield d0
 516
 517             d1 = self.bob_magicfolder.finish()
 518             bob_clock.advance(0)
 519             yield d1
 520
 521     @defer.inlineCallbacks
 522     def test_alice_create_bob_update(self):
 523         alice_clock = task.Clock()
 524         bob_clock = task.Clock()
 525         yield self.setup_alice_and_bob(alice_clock, bob_clock)
 526         alice_dir = self.alice_magicfolder.uploader._local_path_u
 527         bob_dir = self.bob_magicfolder.uploader._local_path_u
 528         alice_fname = os.path.join(alice_dir, 'blam')
 529         bob_fname = os.path.join(bob_dir, 'blam')
 530
 531         try:
 532             # alice creates a file, bob downloads it
 533             alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
 534             bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
 535
 536             fileutil.write(alice_fname, 'contents0\n')
 537             self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 538
 539             alice_clock.advance(0)
 540             yield alice_proc  # alice uploads
 541
 542             bob_clock.advance(0)
 543             yield bob_proc    # bob downloads
 544
 545             # check the state
 546             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
 547             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 0)
 548             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
 549             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 0)
 550             yield self.failUnlessReallyEqual(
 551                 self._get_count('downloader.objects_failed', client=self.bob_magicfolder._client),
 552                 0
 553             )
 554             yield self.failUnlessReallyEqual(
 555                 self._get_count('downloader.objects_downloaded', client=self.bob_magicfolder._client),
 556                 1
 557             )
 558
 559             # now bob updates it (bob should upload, alice download)
 560             bob_proc = self.bob_magicfolder.uploader.set_hook('processed')
 561             alice_proc = self.alice_magicfolder.downloader.set_hook('processed')
 562             fileutil.write(bob_fname, 'bob wuz here\n')
 563             self.notify(to_filepath(bob_fname), self.inotify.IN_CLOSE_WRITE, magic=self.bob_magicfolder)
 564
 565             bob_clock.advance(0)
 566             yield bob_proc
 567             alice_clock.advance(0)
 568             yield alice_proc
 569
 570             # check the state
 571             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
 572             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 1)
 573             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
 574             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 1)
 575
 576         finally:
 577             # cleanup
 578             d0 = self.alice_magicfolder.finish()
 579             alice_clock.advance(0)
 580             yield d0
 581
 582             d1 = self.bob_magicfolder.finish()
 583             bob_clock.advance(0)
 584             yield d1
 585
 586     @defer.inlineCallbacks
 587     def test_alice_delete_and_restore(self):
 588         alice_clock = task.Clock()
 589         bob_clock = task.Clock()
 590         yield self.setup_alice_and_bob(alice_clock, bob_clock)
 591         alice_dir = self.alice_magicfolder.uploader._local_path_u
 592         bob_dir = self.bob_magicfolder.uploader._local_path_u
 593         alice_fname = os.path.join(alice_dir, 'blam')
 594         bob_fname = os.path.join(bob_dir, 'blam')
 595
 596         try:
 597             # alice creates a file, bob downloads it
 598             alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
 599             bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
 600
 601             fileutil.write(alice_fname, 'contents0\n')
 602             self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 603
 604             alice_clock.advance(0)
 605             yield alice_proc  # alice uploads
 606
 607             bob_clock.advance(0)
 608             yield bob_proc    # bob downloads
 609
 610             # check the state
 611             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
 612             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 0)
 613             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
 614             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 0)
 615             yield self.failUnlessReallyEqual(
 616                 self._get_count('downloader.objects_failed', client=self.bob_magicfolder._client),
 617                 0
 618             )
 619             yield self.failUnlessReallyEqual(
 620                 self._get_count('downloader.objects_downloaded', client=self.bob_magicfolder._client),
 621                 1
 622             )
 623             self.failUnless(os.path.exists(bob_fname))
 624
 625             # now alice deletes it (alice should upload, bob download)
 626             alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
 627             bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
 628             os.unlink(alice_fname)
 629             self.notify(to_filepath(alice_fname), self.inotify.IN_DELETE, magic=self.alice_magicfolder)
 630
 631             alice_clock.advance(0)
 632             yield alice_proc
 633             bob_clock.advance(0)
 634             yield bob_proc
 635
 636             # check the state
 637             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
 638             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 1)
 639             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
 640             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 1)
 641             self.failIf(os.path.exists(bob_fname))
 642
 643             # now alice restores the file (with new contents)
 644             alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
 645             bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
 646             fileutil.write(alice_fname, 'alice wuz here\n')
 647             self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 648
 649             alice_clock.advance(0)
 650             yield alice_proc
 651             bob_clock.advance(0)
 652             yield bob_proc
 653
 654             # check the state
 655             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 2)
 656             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 2)
 657             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 2)
 658             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 2)
 659             self.failUnless(os.path.exists(bob_fname))
 660
 661         finally:
 662             # cleanup
 663             d0 = self.alice_magicfolder.finish()
 664             alice_clock.advance(0)
 665             yield d0
 666
 667             d1 = self.bob_magicfolder.finish()
 668             bob_clock.advance(0)
 669             yield d1
 670
 671     def test_magic_folder(self):
 672         self.set_up_grid()
 673         self.local_dir = os.path.join(self.basedir, self.unicode_or_fallback(u"loc\u0101l_dir", u"local_dir"))
 674         self.mkdir_nonascii(self.local_dir)
 675
 676         d = self.create_invite_join_magic_folder(u"Alice\u0101", self.local_dir)
 677         d.addCallback(self._restart_client)
 678
 679         # Write something short enough for a LIT file.
 680         d.addCallback(lambda ign: self._check_file(u"short", "test"))
 681
 682         # Write to the same file again with different data.
 683         d.addCallback(lambda ign: self._check_file(u"short", "different"))
 684
 685         # Test that temporary files are not uploaded.
 686         d.addCallback(lambda ign: self._check_file(u"tempfile", "test", temporary=True))
 687
 688         # Test creation of a subdirectory.
 689         d.addCallback(lambda ign: self._check_mkdir(u"directory"))
 690
 691         # Write something longer, and also try to test a Unicode name if the fs can represent it.
 692         name_u = self.unicode_or_fallback(u"l\u00F8ng", u"long")
 693         d.addCallback(lambda ign: self._check_file(name_u, "test"*100))
 694
 695         # TODO: test that causes an upload failure.
 696         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 697
 698         d.addBoth(self.cleanup)
 699         return d
 700
 701     def _check_mkdir(self, name_u):
 702         return self._check_file(name_u + u"/", "", directory=True)
 703
 704     def _check_file(self, name_u, data, temporary=False, directory=False):
 705         precondition(not (temporary and directory), temporary=temporary, directory=directory)
 706
 707         print "%r._check_file(%r, %r, temporary=%r, directory=%r)" % (self, name_u, data, temporary, directory)
 708         previously_uploaded = self._get_count('uploader.objects_succeeded')
 709         previously_disappeared = self._get_count('uploader.objects_disappeared')
 710
 711         d = self.magicfolder.uploader.set_hook('processed')
 712
 713         path_u = abspath_expanduser_unicode(name_u, base=self.local_dir)
 714         path = to_filepath(path_u)
 715
 716         if directory:
 717             os.mkdir(path_u)
 718             event_mask = self.inotify.IN_CREATE | self.inotify.IN_ISDIR
 719         else:
 720             # We don't use FilePath.setContent() here because it creates a temporary file that
 721             # is renamed into place, which causes events that the test is not expecting.
 722             f = open(path_u, "wb")
 723             try:
 724                 if temporary and sys.platform != "win32":
 725                     os.unlink(path_u)
 726                 f.write(data)
 727             finally:
 728                 f.close()
 729             if temporary and sys.platform == "win32":
 730                 os.unlink(path_u)
 731                 self.notify(path, self.inotify.IN_DELETE, flush=False)
 732             event_mask = self.inotify.IN_CLOSE_WRITE
 733
 734         self.notify(path, event_mask)
 735         encoded_name_u = magicpath.path2magic(name_u)
 736
 737         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 738         if temporary:
 739             d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_disappeared'),
 740                                                                  previously_disappeared + 1))
 741         else:
 742             def _here(res, n):
 743                 print "here %r %r" % (n, res)
 744                 return res
 745             d.addBoth(_here, 1)
 746             d.addCallback(lambda ign: self.upload_dirnode.list())
 747             d.addBoth(_here, 1.5)
 748             d.addCallback(lambda ign: self.upload_dirnode.get(encoded_name_u))
 749             d.addBoth(_here, 2)
 750             d.addCallback(download_to_data)
 751             d.addBoth(_here, 3)
 752             d.addCallback(lambda actual_data: self.failUnlessReallyEqual(actual_data, data))
 753             d.addBoth(_here, 4)
 754             d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'),
 755                                                                  previously_uploaded + 1))
 756
 757         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 758         return d
 759
 760     def _check_version_in_dmd(self, magicfolder, relpath_u, expected_version):
 761         encoded_name_u = magicpath.path2magic(relpath_u)
 762         d = magicfolder.downloader._get_collective_latest_file(encoded_name_u)
 763         def check_latest(result):
 764             if result[0] is not None:
 765                 node, metadata = result
 766                 d.addCallback(lambda ign: self.failUnlessEqual(metadata['version'], expected_version))
 767         d.addCallback(check_latest)
 768         return d
 769
 770     def _check_version_in_local_db(self, magicfolder, relpath_u, expected_version):
 771         db_entry = magicfolder._db.get_db_entry(relpath_u)
 772         if db_entry is not None:
 773             #print "_check_version_in_local_db: %r has version %s" % (relpath_u, version)
 774             self.failUnlessEqual(db_entry.version, expected_version)
 775
 776     def _check_file_gone(self, magicfolder, relpath_u):
 777         path = os.path.join(magicfolder.uploader._local_path_u, relpath_u)
 778         self.assertTrue(not os.path.exists(path))
 779
 780     def _check_uploader_count(self, name, expected, magic=None):
 781         self.failUnlessReallyEqual(self._get_count('uploader.'+name, client=(magic or self.alice_magicfolder)._client),
 782                                    expected)
 783
 784     def _check_downloader_count(self, name, expected, magic=None):
 785         self.failUnlessReallyEqual(self._get_count('downloader.'+name, client=(magic or self.bob_magicfolder)._client),
 786                                    expected)
 787
 788     def test_alice_bob(self):
 789         alice_clock = task.Clock()
 790         bob_clock = task.Clock()
 791         d = self.setup_alice_and_bob(alice_clock, bob_clock)
 792
 793         def _wait_for_Alice(ign, downloaded_d):
 794             print "Now waiting for Alice to download\n"
 795             alice_clock.advance(0)
 796             return downloaded_d
 797
 798         def _wait_for_Bob(ign, downloaded_d):
 799             print "Now waiting for Bob to download\n"
 800             bob_clock.advance(0)
 801             return downloaded_d
 802
 803         def _wait_for(ign, something_to_do, alice=True):
 804             if alice:
 805                 downloaded_d = self.bob_magicfolder.downloader.set_hook('processed')
 806                 uploaded_d = self.alice_magicfolder.uploader.set_hook('processed')
 807             else:
 808                 downloaded_d = self.alice_magicfolder.downloader.set_hook('processed')
 809                 uploaded_d = self.bob_magicfolder.uploader.set_hook('processed')
 810             something_to_do()
 811             if alice:
 812                 print "Waiting for Alice to upload\n"
 813                 alice_clock.advance(0)
 814                 uploaded_d.addCallback(_wait_for_Bob, downloaded_d)
 815             else:
 816                 print "Waiting for Bob to upload\n"
 817                 bob_clock.advance(0)
 818                 uploaded_d.addCallback(_wait_for_Alice, downloaded_d)
 819             return uploaded_d
 820
 821         def Alice_to_write_a_file():
 822             print "Alice writes a file\n"
 823             self.file_path = abspath_expanduser_unicode(u"file1", base=self.alice_magicfolder.uploader._local_path_u)
 824             fileutil.write(self.file_path, "meow, meow meow. meow? meow meow! meow.")
 825             self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 826         d.addCallback(_wait_for, Alice_to_write_a_file)
 827
 828         d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file1", 0))
 829         d.addCallback(lambda ign: self._check_version_in_local_db(self.alice_magicfolder, u"file1", 0))
 830         d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0))
 831         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 1))
 832         d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 1))
 833         d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0))
 834         d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0))
 835         d.addCallback(lambda ign: self._check_uploader_count('objects_conflicted', 0))
 836         d.addCallback(lambda ign: self._check_uploader_count('objects_conflicted', 0, magic=self.bob_magicfolder))
 837
 838         d.addCallback(lambda ign: self._check_version_in_local_db(self.bob_magicfolder, u"file1", 0))
 839         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0))
 840         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 1))
 841         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 0, magic=self.bob_magicfolder))
 842
 843         def Alice_to_delete_file():
 844             print "Alice deletes the file!\n"
 845             os.unlink(self.file_path)
 846             self.notify(to_filepath(self.file_path), self.inotify.IN_DELETE, magic=self.alice_magicfolder)
 847         d.addCallback(_wait_for, Alice_to_delete_file)
 848
 849         def notify_bob_moved(ign):
 850             d0 = self.bob_magicfolder.uploader.set_hook('processed')
 851             p = abspath_expanduser_unicode(u"file1", base=self.bob_magicfolder.uploader._local_path_u)
 852             self.notify(to_filepath(p), self.inotify.IN_MOVED_FROM, magic=self.bob_magicfolder, flush=False)
 853             self.notify(to_filepath(p + u'.backup'), self.inotify.IN_MOVED_TO, magic=self.bob_magicfolder)
 854             bob_clock.advance(0)
 855             return d0
 856         d.addCallback(notify_bob_moved)
 857
 858         d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file1", 1))
 859         d.addCallback(lambda ign: self._check_version_in_local_db(self.alice_magicfolder, u"file1", 1))
 860         d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0))
 861         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 2))
 862         d.addCallback(lambda ign: self._check_uploader_count('objects_not_uploaded', 1, magic=self.bob_magicfolder))
 863         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 1, magic=self.bob_magicfolder))
 864
 865         d.addCallback(lambda ign: self._check_version_in_local_db(self.bob_magicfolder, u"file1", 1))
 866         d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file1", 1))
 867         d.addCallback(lambda ign: self._check_file_gone(self.bob_magicfolder, u"file1"))
 868         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0))
 869         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 2))
 870
 871         def Alice_to_rewrite_file():
 872             print "Alice rewrites file\n"
 873             self.file_path = abspath_expanduser_unicode(u"file1", base=self.alice_magicfolder.uploader._local_path_u)
 874             fileutil.write(self.file_path, "Alice suddenly sees the white rabbit running into the forest.")
 875             self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 876         d.addCallback(_wait_for, Alice_to_rewrite_file)
 877
 878         d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file1", 2))
 879         d.addCallback(lambda ign: self._check_version_in_local_db(self.alice_magicfolder, u"file1", 2))
 880         d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0))
 881         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 3))
 882         d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 3))
 883         d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0))
 884         d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0))
 885         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
 886         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0))
 887
 888         d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file1", 2))
 889         d.addCallback(lambda ign: self._check_version_in_local_db(self.bob_magicfolder, u"file1", 2))
 890         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0))
 891         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 3))
 892
 893         path_u = u"/tmp/magic_folder_test"
 894         encoded_path_u = magicpath.path2magic(u"/tmp/magic_folder_test")
 895
 896         def Alice_tries_to_p0wn_Bob(ign):
 897             print "Alice tries to p0wn Bob\n"
 898             processed_d = self.bob_magicfolder.downloader.set_hook('processed')
 899
 900             # upload a file that would provoke the security bug from #2506
 901             uploadable = Data("", self.alice_magicfolder._client.convergence)
 902             alice_dmd = self.alice_magicfolder.uploader._upload_dirnode
 903
 904             d2 = alice_dmd.add_file(encoded_path_u, uploadable, metadata={"version": 0}, overwrite=True)
 905             d2.addCallback(lambda ign: self.failUnless(alice_dmd.has_child(encoded_path_u)))
 906             d2.addCallback(_wait_for_Bob, processed_d)
 907             return d2
 908         d.addCallback(Alice_tries_to_p0wn_Bob)
 909
 910         d.addCallback(lambda ign: self.failIf(os.path.exists(path_u)))
 911         d.addCallback(lambda ign: self._check_version_in_local_db(self.bob_magicfolder, encoded_path_u, None))
 912         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 3))
 913         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
 914         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0))
 915
 916         def Bob_to_rewrite_file():
 917             print "Bob rewrites file\n"
 918             self.file_path = abspath_expanduser_unicode(u"file1", base=self.bob_magicfolder.uploader._local_path_u)
 919             print "---- bob's file is %r" % (self.file_path,)
 920             fileutil.write(self.file_path, "No white rabbit to be found.")
 921             self.magicfolder = self.bob_magicfolder
 922             self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE)
 923         d.addCallback(lambda ign: _wait_for(None, Bob_to_rewrite_file, alice=False))
 924
 925         d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file1", 3))
 926         d.addCallback(lambda ign: self._check_version_in_local_db(self.bob_magicfolder, u"file1", 3))
 927         d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0, magic=self.bob_magicfolder))
 928         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 2, magic=self.bob_magicfolder))
 929         d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 1, magic=self.bob_magicfolder))
 930         d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0, magic=self.bob_magicfolder))
 931         d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0, magic=self.bob_magicfolder))
 932         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0))
 933
 934         d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file1", 3))
 935         d.addCallback(lambda ign: self._check_version_in_local_db(self.alice_magicfolder, u"file1", 3))
 936         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
 937         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 1, magic=self.alice_magicfolder))
 938         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
 939
 940         def Alice_conflicts_with_Bobs_last_downloaded_uri():
 941             print "Alice conflicts with Bob\n"
 942             downloaded_d = self.bob_magicfolder.downloader.set_hook('processed')
 943             uploadable = Data("do not follow the white rabbit", self.alice_magicfolder._client.convergence)
 944             alice_dmd = self.alice_magicfolder.uploader._upload_dirnode
 945             d2 = alice_dmd.add_file(u"file1", uploadable,
 946                                     metadata={"version": 5,
 947                                               "last_downloaded_uri" : "URI:LIT:" },
 948                                     overwrite=True)
 949             print "Waiting for Alice to upload\n"
 950             d2.addCallback(lambda ign: bob_clock.advance(6))
 951             d2.addCallback(lambda ign: downloaded_d)
 952             d2.addCallback(lambda ign: self.failUnless(alice_dmd.has_child(encoded_path_u)))
 953             return d2
 954
 955         d.addCallback(lambda ign: Alice_conflicts_with_Bobs_last_downloaded_uri())
 956         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 4))
 957         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 1))
 958         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 1, magic=self.alice_magicfolder))
 959         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
 960         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
 961
 962         # prepare to perform another conflict test
 963         def Alice_to_write_file2():
 964             print "Alice writes a file\n"
 965             self.file_path = abspath_expanduser_unicode(u"file2", base=self.alice_magicfolder.uploader._local_path_u)
 966             fileutil.write(self.file_path, "something")
 967             self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 968         d.addCallback(_wait_for, Alice_to_write_file2)
 969         d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file2", 0))
 970         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
 971         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
 972
 973         def Bob_to_rewrite_file2():
 974             print "Bob rewrites file\n"
 975             self.file_path = abspath_expanduser_unicode(u"file2", base=self.bob_magicfolder.uploader._local_path_u)
 976             print "---- bob's file is %r" % (self.file_path,)
 977             fileutil.write(self.file_path, "roger roger. what vector?")
 978             self.magicfolder = self.bob_magicfolder
 979             self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE)
 980         d.addCallback(lambda ign: _wait_for(None, Bob_to_rewrite_file2, alice=False))
 981         d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file2", 1))
 982         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 5))
 983         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 1))
 984         d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0, magic=self.bob_magicfolder))
 985         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 3, magic=self.bob_magicfolder))
 986         d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 2, magic=self.bob_magicfolder))
 987         d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0, magic=self.bob_magicfolder))
 988         d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0, magic=self.bob_magicfolder))
 989         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
 990         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
 991         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 2, magic=self.alice_magicfolder))
 992
 993         # XXX here we advance the clock and then test again to make sure no values are monotonically increasing
 994         # with each queue turn ;-p
 995         alice_clock.advance(6)
 996         bob_clock.advance(6)
 997         d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file2", 1))
 998         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 5))
 999         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 1))
1000         d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0, magic=self.bob_magicfolder))
1001         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 3, magic=self.bob_magicfolder))
1002         d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 2, magic=self.bob_magicfolder))
1003         d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0, magic=self.bob_magicfolder))
1004         d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0, magic=self.bob_magicfolder))
1005         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
1006         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
1007         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 2, magic=self.alice_magicfolder))
1008
1009         def Alice_conflicts_with_Bobs_last_uploaded_uri():
1010             print "Alice conflicts with Bob\n"
1011             encoded_path_u = magicpath.path2magic(u"file2")
1012             downloaded_d = self.bob_magicfolder.downloader.set_hook('processed')
1013             uploadable = Data("rabbits with sharp fangs", self.alice_magicfolder._client.convergence)
1014             alice_dmd = self.alice_magicfolder.uploader._upload_dirnode
1015             d2 = alice_dmd.add_file(u"file2", uploadable,
1016                                     metadata={"version": 5,
1017                                               "last_uploaded_uri" : "URI:LIT:" },
1018                                     overwrite=True)
1019             print "Waiting for Alice to upload\n"
1020             d2.addCallback(lambda ign: bob_clock.advance(6))
1021             d2.addCallback(lambda ign: downloaded_d)
1022             d2.addCallback(lambda ign: self.failUnless(alice_dmd.has_child(encoded_path_u)))
1023             return d2
1024         d.addCallback(lambda ign: Alice_conflicts_with_Bobs_last_uploaded_uri())
1025         d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file2", 5))
1026         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 6))
1027         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 2))
1028         d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0, magic=self.bob_magicfolder))
1029         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 3, magic=self.bob_magicfolder))
1030         d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 2, magic=self.bob_magicfolder))
1031         d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0, magic=self.bob_magicfolder))
1032         d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0, magic=self.bob_magicfolder))
1033         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
1034         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
1035         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 2, magic=self.alice_magicfolder))
1036
1037         alice_clock.advance(6)
1038         bob_clock.advance(6)
1039         alice_clock.advance(6)
1040         bob_clock.advance(6)
1041
1042         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 2, magic=self.alice_magicfolder))
1043         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
1044         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 2))
1045         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 6))
1046
1047         # prepare to perform another conflict test
1048         def Alice_to_write_file3():
1049             print "Alice writes a file\n"
1050             self.file_path = abspath_expanduser_unicode(u"file3", base=self.alice_magicfolder.uploader._local_path_u)
1051             fileutil.write(self.file_path, "something")
1052             self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
1053         d.addCallback(_wait_for, Alice_to_write_file3)
1054         d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file3", 0))
1055         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
1056         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 7))
1057         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 2, magic=self.alice_magicfolder))
1058         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 2))
1059         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
1060
1061         def Bob_to_rewrite_file3():
1062             print "Bob rewrites file\n"
1063             self.file_path = abspath_expanduser_unicode(u"file3", base=self.bob_magicfolder.uploader._local_path_u)
1064             print "---- bob's file is %r" % (self.file_path,)
1065             fileutil.write(self.file_path, "roger roger")
1066             self.magicfolder = self.bob_magicfolder
1067             self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE)
1068         d.addCallback(lambda ign: _wait_for(None, Bob_to_rewrite_file3, alice=False))
1069         d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file3", 1))
1070         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 7))
1071         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 2))
1072         d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0, magic=self.bob_magicfolder))
1073         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 4, magic=self.bob_magicfolder))
1074         d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 3, magic=self.bob_magicfolder))
1075         d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0, magic=self.bob_magicfolder))
1076         d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0, magic=self.bob_magicfolder))
1077         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
1078         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
1079         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 3, magic=self.alice_magicfolder))
1080
1081
1082
1083         def _cleanup(ign, magicfolder, clock):
1084             if magicfolder is not None:
1085                 d2 = magicfolder.finish()
1086                 clock.advance(0)
1087                 return d2
1088
1089         def cleanup_Alice_and_Bob(result):
1090             print "cleanup alice bob test\n"
1091             d = defer.succeed(None)
1092             d.addCallback(_cleanup, self.alice_magicfolder, alice_clock)
1093             d.addCallback(_cleanup, self.bob_magicfolder, bob_clock)
1094             d.addCallback(lambda ign: result)
1095             return d
1096         d.addBoth(cleanup_Alice_and_Bob)
1097         return d
1098
1099
1100 class MockTest(MagicFolderTestMixin, unittest.TestCase):
1101     """This can run on any platform, and even if twisted.internet.inotify can't be imported."""
1102
1103     def setUp(self):
1104         MagicFolderTestMixin.setUp(self)
1105         self.inotify = fake_inotify
1106         self.patch(magic_folder, 'get_inotify_module', lambda: self.inotify)
1107
1108     def notify(self, path, mask, magic=None, flush=True):
1109         if magic is None:
1110             magic = self.magicfolder
1111         magic.uploader._notifier.event(path, mask)
1112         # no flush for the mock test.
1113
1114     def test_errors(self):
1115         self.set_up_grid()
1116
1117         errors_dir = abspath_expanduser_unicode(u"errors_dir", base=self.basedir)
1118         os.mkdir(errors_dir)
1119         not_a_dir = abspath_expanduser_unicode(u"NOT_A_DIR", base=self.basedir)
1120         fileutil.write(not_a_dir, "")
1121         magicfolderdb = abspath_expanduser_unicode(u"magicfolderdb", base=self.basedir)
1122         doesnotexist  = abspath_expanduser_unicode(u"doesnotexist", base=self.basedir)
1123
1124         client = self.g.clients[0]
1125         d = client.create_dirnode()
1126         def _check_errors(n):
1127             self.failUnless(IDirectoryNode.providedBy(n))
1128             upload_dircap = n.get_uri()
1129             readonly_dircap = n.get_readonly_uri()
1130
1131             self.shouldFail(AssertionError, 'nonexistent local.directory', 'there is no directory',
1132                             MagicFolder, client, upload_dircap, '', doesnotexist, magicfolderdb, 0077)
1133             self.shouldFail(AssertionError, 'non-directory local.directory', 'is not a directory',
1134                             MagicFolder, client, upload_dircap, '', not_a_dir, magicfolderdb, 0077)
1135             self.shouldFail(AssertionError, 'bad upload.dircap', 'does not refer to a directory',
1136                             MagicFolder, client, 'bad', '', errors_dir, magicfolderdb, 0077)
1137             self.shouldFail(AssertionError, 'non-directory upload.dircap', 'does not refer to a directory',
1138                             MagicFolder, client, 'URI:LIT:foo', '', errors_dir, magicfolderdb, 0077)
1139             self.shouldFail(AssertionError, 'readonly upload.dircap', 'is not a writecap to a directory',
1140                             MagicFolder, client, readonly_dircap, '', errors_dir, magicfolderdb, 0077)
1141             self.shouldFail(AssertionError, 'collective dircap', 'is not a readonly cap to a directory',
1142                             MagicFolder, client, upload_dircap, upload_dircap, errors_dir, magicfolderdb, 0077)
1143
1144             def _not_implemented():
1145                 raise NotImplementedError("blah")
1146             self.patch(magic_folder, 'get_inotify_module', _not_implemented)
1147             self.shouldFail(NotImplementedError, 'unsupported', 'blah',
1148                             MagicFolder, client, upload_dircap, '', errors_dir, magicfolderdb, 0077)
1149         d.addCallback(_check_errors)
1150         return d
1151
1152     def test_write_downloaded_file(self):
1153         workdir = u"cli/MagicFolder/write-downloaded-file"
1154         local_file = fileutil.abspath_expanduser_unicode(os.path.join(workdir, "foobar"))
1155
1156         class TestWriteFileMixin(WriteFileMixin):
1157             def _log(self, msg):
1158                 pass
1159
1160         writefile = TestWriteFileMixin()
1161         writefile._umask = 0077
1162
1163         # create a file with name "foobar" with content "foo"
1164         # write downloaded file content "bar" into "foobar" with is_conflict = False
1165         fileutil.make_dirs(workdir)
1166         fileutil.write(local_file, "foo")
1167
1168         # if is_conflict is False, then the .conflict file shouldn't exist.
1169         writefile._write_downloaded_file(local_file, "bar", False, None)
1170         conflicted_path = local_file + u".conflict"
1171         self.failIf(os.path.exists(conflicted_path))
1172
1173         # At this point, the backup file should exist with content "foo"
1174         backup_path = local_file + u".backup"
1175         self.failUnless(os.path.exists(backup_path))
1176         self.failUnlessEqual(fileutil.read(backup_path), "foo")
1177
1178         # .tmp file shouldn't exist
1179         self.failIf(os.path.exists(local_file + u".tmp"))
1180
1181         # .. and the original file should have the new content
1182         self.failUnlessEqual(fileutil.read(local_file), "bar")
1183
1184         # now a test for conflicted case
1185         writefile._write_downloaded_file(local_file, "bar", True, None)
1186         self.failUnless(os.path.exists(conflicted_path))
1187
1188         # .tmp file shouldn't exist
1189         self.failIf(os.path.exists(local_file + u".tmp"))
1190
1191     def test_periodic_full_scan(self):
1192         self.set_up_grid()
1193         self.local_dir = abspath_expanduser_unicode(u"test_periodic_full_scan",base=self.basedir)
1194         self.mkdir_nonascii(self.local_dir)
1195
1196         alice_clock = task.Clock()
1197         d = self.do_create_magic_folder(0)
1198         d.addCallback(lambda ign: self.do_invite(0, u"Alice\u00F8"))
1199         def get_invite_code(result):
1200             self.invite_code = result[1].strip()
1201         d.addCallback(get_invite_code)
1202         d.addCallback(lambda ign: self.do_join(0, self.local_dir, self.invite_code))
1203         def get_alice_caps(ign):
1204             self.alice_collective_dircap, self.alice_upload_dircap = self.get_caps_from_files(0)
1205         d.addCallback(get_alice_caps)
1206         d.addCallback(lambda ign: self.check_joined_config(0, self.alice_upload_dircap))
1207         d.addCallback(lambda ign: self.check_config(0, self.local_dir))
1208         def get_Alice_magicfolder(result):
1209             self.magicfolder = self.init_magicfolder(0, self.alice_upload_dircap,
1210                                                            self.alice_collective_dircap,
1211                                                            self.local_dir, alice_clock)
1212             return result
1213         d.addCallback(get_Alice_magicfolder)
1214         empty_tree_name = self.unicode_or_fallback(u"empty_tr\u00EAe", u"empty_tree")
1215         empty_tree_dir = abspath_expanduser_unicode(empty_tree_name, base=self.basedir)
1216         new_empty_tree_dir = abspath_expanduser_unicode(empty_tree_name, base=self.local_dir)
1217
1218         def _check_move_empty_tree(res):
1219             print "CHECK MOVE EMPTY TREE"
1220             uploaded_d = self.magicfolder.uploader.set_hook('processed')
1221             self.mkdir_nonascii(empty_tree_dir)
1222             os.rename(empty_tree_dir, new_empty_tree_dir)
1223             self.notify(to_filepath(new_empty_tree_dir), self.inotify.IN_MOVED_TO)
1224             return uploaded_d
1225         d.addCallback(_check_move_empty_tree)
1226         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
1227         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 1))
1228         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.files_uploaded'), 0))
1229         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
1230         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.directories_created'), 1))
1231
1232         def _create_file_without_event(res):
1233             print "CREATE FILE WITHOUT EMITTING EVENT"
1234             what_path = abspath_expanduser_unicode(u"what", base=new_empty_tree_dir)
1235             fileutil.write(what_path, "say when")
1236         d.addCallback(_create_file_without_event)
1237         def advance_clock(res):
1238             alice_clock.advance(20)
1239         d.addCallback(advance_clock)
1240         def hook(res):
1241             uploaded_d = self.magicfolder.uploader.set_hook('processed')
1242             return uploaded_d
1243         d.addCallback(hook)
1244         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.files_uploaded'), 1))
1245         d.addCallback(lambda ign: self.magicfolder.finish())
1246         return d
1247
1248 class RealTest(MagicFolderTestMixin, unittest.TestCase):
1249     """This is skipped unless both Twisted and the platform support inotify."""
1250
1251     def setUp(self):
1252         MagicFolderTestMixin.setUp(self)
1253         self.inotify = magic_folder.get_inotify_module()
1254
1255     def notify(self, path, mask, magic=None, flush=True):
1256         # Writing to the filesystem causes the notification.
1257         # However, flushing filesystem buffers may be necessary on Windows.
1258         if flush:
1259             fileutil.flush_volume(path.path)
1260
1261 try:
1262     magic_folder.get_inotify_module()
1263 except NotImplementedError:
1264     RealTest.skip = "Magic Folder support can only be tested for-real on an OS that supports inotify or equivalent."