src/allmydata/test/test_magic_folder.py

   1
   2 import os, sys
   3 import shutil
   4
   5 from twisted.trial import unittest
   6 from twisted.internet import defer, task
   7
   8 from allmydata.interfaces import IDirectoryNode
   9 from allmydata.util.assertutil import precondition
  10
  11 from allmydata.util import fake_inotify, fileutil
  12 from allmydata.util.deferredutil import DeferredListShouldSucceed
  13 from allmydata.util.encodingutil import get_filesystem_encoding, to_filepath
  14 from allmydata.util.consumer import download_to_data
  15 from allmydata.test.no_network import GridTestMixin
  16 from allmydata.test.common_util import ReallyEqualMixin, NonASCIIPathMixin
  17 from allmydata.test.common import ShouldFailMixin
  18 from .test_cli_magic_folder import MagicFolderCLITestMixin
  19
  20 from allmydata.frontends import magic_folder
  21 from allmydata.frontends.magic_folder import MagicFolder, Downloader, WriteFileMixin
  22 from allmydata import magicfolderdb, magicpath
  23 from allmydata.util.fileutil import abspath_expanduser_unicode
  24 from allmydata.immutable.upload import Data
  25
  26
  27 class MagicFolderTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, ReallyEqualMixin, NonASCIIPathMixin):
  28     """
  29     These tests will be run both with a mock notifier, and (on platforms that support it)
  30     with the real INotify.
  31     """
  32
  33     def setUp(self):
  34         GridTestMixin.setUp(self)
  35         temp = self.mktemp()
  36         self.basedir = abspath_expanduser_unicode(temp.decode(get_filesystem_encoding()))
  37         self.magicfolder = None
  38         self.patch(Downloader, 'REMOTE_SCAN_INTERVAL', 0)
  39
  40     def _get_count(self, name, client=None):
  41         counters = (client or self.get_client()).stats_provider.get_stats()["counters"]
  42         return counters.get('magic_folder.%s' % (name,), 0)
  43
  44     def _createdb(self):
  45         dbfile = abspath_expanduser_unicode(u"magicfolderdb.sqlite", base=self.basedir)
  46         mdb = magicfolderdb.get_magicfolderdb(dbfile, create_version=(magicfolderdb.SCHEMA_v1, 1))
  47         self.failUnless(mdb, "unable to create magicfolderdb from %r" % (dbfile,))
  48         self.failUnlessEqual(mdb.VERSION, 1)
  49         return mdb
  50
  51     def _restart_client(self, ign):
  52         #print "_restart_client"
  53         d = self.restart_client()
  54         d.addCallback(self._wait_until_started)
  55         return d
  56
  57     def _wait_until_started(self, ign):
  58         #print "_wait_until_started"
  59         self.magicfolder = self.get_client().getServiceNamed('magic-folder')
  60         return self.magicfolder.ready()
  61
  62     def test_db_basic(self):
  63         fileutil.make_dirs(self.basedir)
  64         self._createdb()
  65
  66     def test_db_persistence(self):
  67         """Test that a file upload creates an entry in the database."""
  68
  69         fileutil.make_dirs(self.basedir)
  70         db = self._createdb()
  71
  72         relpath1 = u"myFile1"
  73         pathinfo = fileutil.PathInfo(isdir=False, isfile=True, islink=False,
  74                                      exists=True, size=1, mtime=123, ctime=456)
  75         db.did_upload_version(relpath1, 0, 'URI:LIT:1', 'URI:LIT:0', 0, pathinfo)
  76
  77         c = db.cursor
  78         c.execute("SELECT size, mtime, ctime"
  79                   " FROM local_files"
  80                   " WHERE path=?",
  81                   (relpath1,))
  82         row = c.fetchone()
  83         self.failUnlessEqual(row, (pathinfo.size, pathinfo.mtime, pathinfo.ctime))
  84
  85         # Second test uses magic_folder.is_new_file instead of SQL query directly
  86         # to confirm the previous upload entry in the db.
  87         relpath2 = u"myFile2"
  88         path2 = os.path.join(self.basedir, relpath2)
  89         fileutil.write(path2, "meow\n")
  90         pathinfo = fileutil.get_pathinfo(path2)
  91         db.did_upload_version(relpath2, 0, 'URI:LIT:2', 'URI:LIT:1', 0, pathinfo)
  92         db_entry = db.get_db_entry(relpath2)
  93         self.failUnlessFalse(magic_folder.is_new_file(pathinfo, db_entry))
  94
  95         different_pathinfo = fileutil.PathInfo(isdir=False, isfile=True, islink=False,
  96                                                exists=True, size=0, mtime=pathinfo.mtime, ctime=pathinfo.ctime)
  97         self.failUnlessTrue(magic_folder.is_new_file(different_pathinfo, db_entry))
  98
  99     def test_magicfolder_start_service(self):
 100         self.set_up_grid()
 101
 102         self.local_dir = abspath_expanduser_unicode(self.unicode_or_fallback(u"l\u00F8cal_dir", u"local_dir"),
 103                                                     base=self.basedir)
 104         self.mkdir_nonascii(self.local_dir)
 105
 106         d = defer.succeed(None)
 107         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.dirs_monitored'), 0))
 108
 109         d.addCallback(lambda ign: self.create_invite_join_magic_folder(u"Alice", self.local_dir))
 110         d.addCallback(self._restart_client)
 111
 112         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.dirs_monitored'), 1))
 113         d.addBoth(self.cleanup)
 114         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.dirs_monitored'), 0))
 115         return d
 116
 117     def test_scan_once_on_startup(self):
 118         self.set_up_grid()
 119         self.local_dir = abspath_expanduser_unicode(u"test_scan_once_on_startup", base=self.basedir)
 120         self.mkdir_nonascii(self.local_dir)
 121         self.collective_dircap = ""
 122
 123         alice_clock = task.Clock()
 124         bob_clock = task.Clock()
 125         d = self.setup_alice_and_bob(alice_clock, bob_clock)
 126
 127         def upload_stuff(ignore):
 128             uploadable = Data("", self.alice_magicfolder._client.convergence)
 129             return self.alice_magicfolder._client.upload(uploadable)
 130         d.addCallback(upload_stuff)
 131         def check_is_upload(ignore):
 132             alice_clock.advance(99)
 133             d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 0, magic=self.alice_magicfolder))
 134             d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0, magic=self.alice_magicfolder))
 135             d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
 136             d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 0, magic=self.alice_magicfolder))
 137             d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
 138             d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 0, magic=self.alice_magicfolder))
 139
 140         d.addCallback(check_is_upload)
 141         def _cleanup(ign, magicfolder, clock):
 142             if magicfolder is not None:
 143                 d2 = magicfolder.finish()
 144                 clock.advance(0)
 145                 return d2
 146         def cleanup_Alice_and_Bob(result):
 147             print "cleanup alice bob test\n"
 148             d = defer.succeed(None)
 149             d.addCallback(_cleanup, self.alice_magicfolder, alice_clock)
 150             d.addCallback(_cleanup, self.bob_magicfolder, bob_clock)
 151             d.addCallback(lambda ign: result)
 152             return d
 153
 154         d.addBoth(cleanup_Alice_and_Bob)
 155         return d
 156
 157     def test_move_tree(self):
 158         self.set_up_grid()
 159
 160         self.local_dir = abspath_expanduser_unicode(self.unicode_or_fallback(u"l\u00F8cal_dir", u"local_dir"),
 161                                                     base=self.basedir)
 162         self.mkdir_nonascii(self.local_dir)
 163
 164         empty_tree_name = self.unicode_or_fallback(u"empty_tr\u00EAe", u"empty_tree")
 165         empty_tree_dir = abspath_expanduser_unicode(empty_tree_name, base=self.basedir)
 166         new_empty_tree_dir = abspath_expanduser_unicode(empty_tree_name, base=self.local_dir)
 167
 168         small_tree_name = self.unicode_or_fallback(u"small_tr\u00EAe", u"empty_tree")
 169         small_tree_dir = abspath_expanduser_unicode(small_tree_name, base=self.basedir)
 170         new_small_tree_dir = abspath_expanduser_unicode(small_tree_name, base=self.local_dir)
 171
 172         d = self.create_invite_join_magic_folder(u"Alice", self.local_dir)
 173         d.addCallback(self._restart_client)
 174
 175         def _check_move_empty_tree(res):
 176             print "_check_move_empty_tree"
 177             uploaded_d = self.magicfolder.uploader.set_hook('processed')
 178             self.mkdir_nonascii(empty_tree_dir)
 179             os.rename(empty_tree_dir, new_empty_tree_dir)
 180             self.notify(to_filepath(new_empty_tree_dir), self.inotify.IN_MOVED_TO)
 181
 182             return uploaded_d
 183         d.addCallback(_check_move_empty_tree)
 184         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 185         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 1))
 186         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.files_uploaded'), 0))
 187         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 188         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.directories_created'), 1))
 189
 190         def _check_move_small_tree(res):
 191             print "_check_move_small_tree"
 192             uploaded_d = self.magicfolder.uploader.set_hook('processed', ignore_count=1)
 193             self.mkdir_nonascii(small_tree_dir)
 194             what_path = abspath_expanduser_unicode(u"what", base=small_tree_dir)
 195             fileutil.write(what_path, "say when")
 196             os.rename(small_tree_dir, new_small_tree_dir)
 197             self.notify(to_filepath(new_small_tree_dir), self.inotify.IN_MOVED_TO)
 198
 199             return uploaded_d
 200         d.addCallback(_check_move_small_tree)
 201         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 202         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 3))
 203         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.files_uploaded'), 1))
 204         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 205         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.directories_created'), 2))
 206
 207         def _check_moved_tree_is_watched(res):
 208             print "_check_moved_tree_is_watched"
 209             uploaded_d = self.magicfolder.uploader.set_hook('processed')
 210             another_path = abspath_expanduser_unicode(u"another", base=new_small_tree_dir)
 211             fileutil.write(another_path, "file")
 212             self.notify(to_filepath(another_path), self.inotify.IN_CLOSE_WRITE)
 213
 214             return uploaded_d
 215         d.addCallback(_check_moved_tree_is_watched)
 216         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 217         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 4))
 218         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.files_uploaded'), 2))
 219         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 220         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.directories_created'), 2))
 221
 222         # Files that are moved out of the upload directory should no longer be watched.
 223         #def _move_dir_away(ign):
 224         #    os.rename(new_empty_tree_dir, empty_tree_dir)
 225         #    # Wuh? Why don't we get this event for the real test?
 226         #    #self.notify(to_filepath(new_empty_tree_dir), self.inotify.IN_MOVED_FROM)
 227         #d.addCallback(_move_dir_away)
 228         #def create_file(val):
 229         #    test_file = abspath_expanduser_unicode(u"what", base=empty_tree_dir)
 230         #    fileutil.write(test_file, "meow")
 231         #    #self.notify(...)
 232         #    return
 233         #d.addCallback(create_file)
 234         #d.addCallback(lambda ign: time.sleep(1))  # XXX ICK
 235         #d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 236         #d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 4))
 237         #d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.files_uploaded'), 2))
 238         #d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 239         #d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.directories_created'), 2))
 240
 241         d.addBoth(self.cleanup)
 242         return d
 243
 244     def test_persistence(self):
 245         """
 246         Perform an upload of a given file and then stop the client.
 247         Start a new client and magic-folder service... and verify that the file is NOT uploaded
 248         a second time. This test is meant to test the database persistence along with
 249         the startup and shutdown code paths of the magic-folder service.
 250         """
 251         self.set_up_grid()
 252         self.local_dir = abspath_expanduser_unicode(u"test_persistence", base=self.basedir)
 253         self.mkdir_nonascii(self.local_dir)
 254         self.collective_dircap = ""
 255
 256         d = defer.succeed(None)
 257         d.addCallback(lambda ign: self.create_invite_join_magic_folder(u"Alice", self.local_dir))
 258         d.addCallback(self._restart_client)
 259
 260         def create_test_file(filename):
 261             d2 = self.magicfolder.uploader.set_hook('processed')
 262             test_file = abspath_expanduser_unicode(filename, base=self.local_dir)
 263             fileutil.write(test_file, "meow %s" % filename)
 264             self.notify(to_filepath(test_file), self.inotify.IN_CLOSE_WRITE)
 265             return d2
 266         d.addCallback(lambda ign: create_test_file(u"what1"))
 267         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 268         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 1))
 269         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 270         d.addCallback(self.cleanup)
 271
 272         d.addCallback(self._restart_client)
 273         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 274         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 1))
 275         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 276         d.addCallback(lambda ign: create_test_file(u"what2"))
 277         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 278         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 2))
 279         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 280         d.addBoth(self.cleanup)
 281         return d
 282
 283     @defer.inlineCallbacks
 284     def test_delete(self):
 285         self.set_up_grid()
 286         self.local_dir = os.path.join(self.basedir, u"local_dir")
 287         self.mkdir_nonascii(self.local_dir)
 288
 289         yield self.create_invite_join_magic_folder(u"Alice\u0101", self.local_dir)
 290         yield self._restart_client(None)
 291
 292         try:
 293             # create a file
 294             up_proc = self.magicfolder.uploader.set_hook('processed')
 295             # down_proc = self.magicfolder.downloader.set_hook('processed')
 296             path = os.path.join(self.local_dir, u'foo')
 297             fileutil.write(path, 'foo\n')
 298             self.notify(to_filepath(path), self.inotify.IN_CLOSE_WRITE)
 299             yield up_proc
 300             self.assertTrue(os.path.exists(path))
 301
 302             # the real test part: delete the file
 303             up_proc = self.magicfolder.uploader.set_hook('processed')
 304             os.unlink(path)
 305             self.notify(to_filepath(path), self.inotify.IN_DELETE)
 306             yield up_proc
 307             self.assertFalse(os.path.exists(path))
 308
 309             # ensure we still have a DB entry, and that the version is 1
 310             node, metadata = yield self.magicfolder.downloader._get_collective_latest_file(u'foo')
 311             self.assertTrue(node is not None, "Failed to find %r in DMD" % (path,))
 312             self.failUnlessEqual(metadata['version'], 1)
 313
 314         finally:
 315             yield self.cleanup(None)
 316
 317     @defer.inlineCallbacks
 318     def test_delete_and_restore(self):
 319         self.set_up_grid()
 320         self.local_dir = os.path.join(self.basedir, u"local_dir")
 321         self.mkdir_nonascii(self.local_dir)
 322
 323         yield self.create_invite_join_magic_folder(u"Alice\u0101", self.local_dir)
 324         yield self._restart_client(None)
 325
 326         try:
 327             # create a file
 328             up_proc = self.magicfolder.uploader.set_hook('processed')
 329             # down_proc = self.magicfolder.downloader.set_hook('processed')
 330             path = os.path.join(self.local_dir, u'foo')
 331             fileutil.write(path, 'foo\n')
 332             self.notify(to_filepath(path), self.inotify.IN_CLOSE_WRITE)
 333             yield up_proc
 334             self.assertTrue(os.path.exists(path))
 335
 336             # delete the file
 337             up_proc = self.magicfolder.uploader.set_hook('processed')
 338             os.unlink(path)
 339             self.notify(to_filepath(path), self.inotify.IN_DELETE)
 340             yield up_proc
 341             self.assertFalse(os.path.exists(path))
 342
 343             # ensure we still have a DB entry, and that the version is 1
 344             node, metadata = yield self.magicfolder.downloader._get_collective_latest_file(u'foo')
 345             self.assertTrue(node is not None, "Failed to find %r in DMD" % (path,))
 346             self.failUnlessEqual(metadata['version'], 1)
 347
 348             # restore the file, with different contents
 349             up_proc = self.magicfolder.uploader.set_hook('processed')
 350             path = os.path.join(self.local_dir, u'foo')
 351             fileutil.write(path, 'bar\n')
 352             self.notify(to_filepath(path), self.inotify.IN_CLOSE_WRITE)
 353             yield up_proc
 354
 355             # ensure we still have a DB entry, and that the version is 2
 356             node, metadata = yield self.magicfolder.downloader._get_collective_latest_file(u'foo')
 357             self.assertTrue(node is not None, "Failed to find %r in DMD" % (path,))
 358             self.failUnlessEqual(metadata['version'], 2)
 359
 360         finally:
 361             yield self.cleanup(None)
 362
 363     @defer.inlineCallbacks
 364     def test_alice_delete_bob_restore(self):
 365         alice_clock = task.Clock()
 366         bob_clock = task.Clock()
 367         yield self.setup_alice_and_bob(alice_clock, bob_clock)
 368         alice_dir = self.alice_magicfolder.uploader._local_path_u
 369         bob_dir = self.bob_magicfolder.uploader._local_path_u
 370         alice_fname = os.path.join(alice_dir, 'blam')
 371         bob_fname = os.path.join(bob_dir, 'blam')
 372
 373         try:
 374             # alice creates a file, bob downloads it
 375             alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
 376             bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
 377
 378             fileutil.write(alice_fname, 'contents0\n')
 379             self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 380
 381             alice_clock.advance(0)
 382             yield alice_proc  # alice uploads
 383
 384             bob_clock.advance(0)
 385             yield bob_proc    # bob downloads
 386
 387             # check the state
 388             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
 389             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 0)
 390             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
 391             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 0)
 392             yield self.failUnlessReallyEqual(
 393                 self._get_count('downloader.objects_failed', client=self.bob_magicfolder._client),
 394                 0
 395             )
 396             yield self.failUnlessReallyEqual(
 397                 self._get_count('downloader.objects_downloaded', client=self.bob_magicfolder._client),
 398                 1
 399             )
 400
 401             print("BOB DELETE")
 402             # now bob deletes it (bob should upload, alice download)
 403             bob_proc = self.bob_magicfolder.uploader.set_hook('processed')
 404             alice_proc = self.alice_magicfolder.downloader.set_hook('processed')
 405             os.unlink(bob_fname)
 406             self.notify(to_filepath(bob_fname), self.inotify.IN_DELETE, magic=self.bob_magicfolder)
 407
 408             bob_clock.advance(0)
 409             yield bob_proc
 410             alice_clock.advance(0)
 411             yield alice_proc
 412
 413             # check versions
 414             node, metadata = yield self.alice_magicfolder.downloader._get_collective_latest_file(u'blam')
 415             self.assertTrue(metadata['deleted'])
 416             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
 417             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 1)
 418             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
 419             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 1)
 420
 421             print("ALICE RESTORE")
 422             # now alice restores it (alice should upload, bob download)
 423             alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
 424             bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
 425             fileutil.write(alice_fname, 'new contents\n')
 426             self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 427
 428             alice_clock.advance(0)
 429             yield alice_proc
 430             bob_clock.advance(0)
 431             yield bob_proc
 432
 433             # check versions
 434             node, metadata = yield self.alice_magicfolder.downloader._get_collective_latest_file(u'blam')
 435             self.assertTrue('deleted' not in metadata or not metadata['deleted'])
 436             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 2)
 437             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 2)
 438             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 2)
 439             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 2)
 440
 441         finally:
 442             # cleanup
 443             d0 = self.alice_magicfolder.finish()
 444             alice_clock.advance(0)
 445             yield d0
 446
 447             d1 = self.bob_magicfolder.finish()
 448             bob_clock.advance(0)
 449             yield d1
 450
 451     @defer.inlineCallbacks
 452     def test_alice_sees_bobs_delete_with_error(self):
 453         # alice creates a file, bob deletes it -- and we also arrange
 454         # for Alice's file to have "gone missing" as well.
 455         alice_clock = task.Clock()
 456         bob_clock = task.Clock()
 457         yield self.setup_alice_and_bob(alice_clock, bob_clock)
 458         alice_dir = self.alice_magicfolder.uploader._local_path_u
 459         bob_dir = self.bob_magicfolder.uploader._local_path_u
 460         alice_fname = os.path.join(alice_dir, 'blam')
 461         bob_fname = os.path.join(bob_dir, 'blam')
 462
 463         try:
 464             # alice creates a file, bob downloads it
 465             alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
 466             bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
 467
 468             fileutil.write(alice_fname, 'contents0\n')
 469             self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 470
 471             alice_clock.advance(0)
 472             yield alice_proc  # alice uploads
 473
 474             bob_clock.advance(0)
 475             yield bob_proc    # bob downloads
 476
 477             # check the state
 478             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
 479             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 0)
 480             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
 481             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 0)
 482             yield self.failUnlessReallyEqual(
 483                 self._get_count('downloader.objects_failed', client=self.bob_magicfolder._client),
 484                 0
 485             )
 486             yield self.failUnlessReallyEqual(
 487                 self._get_count('downloader.objects_downloaded', client=self.bob_magicfolder._client),
 488                 1
 489             )
 490
 491             # now bob deletes it (bob should upload, alice download)
 492             bob_proc = self.bob_magicfolder.uploader.set_hook('processed')
 493             alice_proc = self.alice_magicfolder.downloader.set_hook('processed')
 494             os.unlink(bob_fname)
 495             self.notify(to_filepath(bob_fname), self.inotify.IN_DELETE, magic=self.bob_magicfolder)
 496             # just after notifying bob, we also delete alice's,
 497             # covering the 'except' flow in _rename_deleted_file()
 498             os.unlink(alice_fname)
 499
 500             bob_clock.advance(0)
 501             yield bob_proc
 502             alice_clock.advance(0)
 503             yield alice_proc
 504
 505             # check versions
 506             node, metadata = yield self.alice_magicfolder.downloader._get_collective_latest_file(u'blam')
 507             self.assertTrue(metadata['deleted'])
 508             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
 509             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 1)
 510             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
 511             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 1)
 512
 513         finally:
 514             # cleanup
 515             d0 = self.alice_magicfolder.finish()
 516             alice_clock.advance(0)
 517             yield d0
 518
 519             d1 = self.bob_magicfolder.finish()
 520             bob_clock.advance(0)
 521             yield d1
 522
 523     @defer.inlineCallbacks
 524     def test_alice_create_bob_update(self):
 525         alice_clock = task.Clock()
 526         bob_clock = task.Clock()
 527         yield self.setup_alice_and_bob(alice_clock, bob_clock)
 528         alice_dir = self.alice_magicfolder.uploader._local_path_u
 529         bob_dir = self.bob_magicfolder.uploader._local_path_u
 530         alice_fname = os.path.join(alice_dir, 'blam')
 531         bob_fname = os.path.join(bob_dir, 'blam')
 532
 533         try:
 534             # alice creates a file, bob downloads it
 535             alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
 536             bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
 537
 538             fileutil.write(alice_fname, 'contents0\n')
 539             self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 540
 541             alice_clock.advance(0)
 542             yield alice_proc  # alice uploads
 543
 544             bob_clock.advance(0)
 545             yield bob_proc    # bob downloads
 546
 547             # check the state
 548             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
 549             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 0)
 550             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
 551             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 0)
 552             yield self.failUnlessReallyEqual(
 553                 self._get_count('downloader.objects_failed', client=self.bob_magicfolder._client),
 554                 0
 555             )
 556             yield self.failUnlessReallyEqual(
 557                 self._get_count('downloader.objects_downloaded', client=self.bob_magicfolder._client),
 558                 1
 559             )
 560
 561             # now bob updates it (bob should upload, alice download)
 562             bob_proc = self.bob_magicfolder.uploader.set_hook('processed')
 563             alice_proc = self.alice_magicfolder.downloader.set_hook('processed')
 564             fileutil.write(bob_fname, 'bob wuz here\n')
 565             self.notify(to_filepath(bob_fname), self.inotify.IN_CLOSE_WRITE, magic=self.bob_magicfolder)
 566
 567             bob_clock.advance(0)
 568             yield bob_proc
 569             alice_clock.advance(0)
 570             yield alice_proc
 571
 572             # check the state
 573             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
 574             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 1)
 575             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
 576             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 1)
 577
 578         finally:
 579             # cleanup
 580             d0 = self.alice_magicfolder.finish()
 581             alice_clock.advance(0)
 582             yield d0
 583
 584             d1 = self.bob_magicfolder.finish()
 585             bob_clock.advance(0)
 586             yield d1
 587
 588     @defer.inlineCallbacks
 589     def test_alice_delete_and_restore(self):
 590         alice_clock = task.Clock()
 591         bob_clock = task.Clock()
 592         yield self.setup_alice_and_bob(alice_clock, bob_clock)
 593         alice_dir = self.alice_magicfolder.uploader._local_path_u
 594         bob_dir = self.bob_magicfolder.uploader._local_path_u
 595         alice_fname = os.path.join(alice_dir, 'blam')
 596         bob_fname = os.path.join(bob_dir, 'blam')
 597
 598         try:
 599             # alice creates a file, bob downloads it
 600             alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
 601             bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
 602
 603             fileutil.write(alice_fname, 'contents0\n')
 604             self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 605
 606             alice_clock.advance(0)
 607             yield alice_proc  # alice uploads
 608
 609             bob_clock.advance(0)
 610             yield bob_proc    # bob downloads
 611
 612             # check the state
 613             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
 614             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 0)
 615             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
 616             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 0)
 617             yield self.failUnlessReallyEqual(
 618                 self._get_count('downloader.objects_failed', client=self.bob_magicfolder._client),
 619                 0
 620             )
 621             yield self.failUnlessReallyEqual(
 622                 self._get_count('downloader.objects_downloaded', client=self.bob_magicfolder._client),
 623                 1
 624             )
 625             self.failUnless(os.path.exists(bob_fname))
 626
 627             # now alice deletes it (alice should upload, bob download)
 628             alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
 629             bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
 630             os.unlink(alice_fname)
 631             self.notify(to_filepath(alice_fname), self.inotify.IN_DELETE, magic=self.alice_magicfolder)
 632
 633             alice_clock.advance(0)
 634             yield alice_proc
 635             bob_clock.advance(0)
 636             yield bob_proc
 637
 638             # check the state
 639             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
 640             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 1)
 641             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
 642             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 1)
 643             self.failIf(os.path.exists(bob_fname))
 644
 645             # now alice restores the file (with new contents)
 646             alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
 647             bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
 648             fileutil.write(alice_fname, 'alice wuz here\n')
 649             self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 650
 651             alice_clock.advance(0)
 652             yield alice_proc
 653             bob_clock.advance(0)
 654             yield bob_proc
 655
 656             # check the state
 657             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 2)
 658             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 2)
 659             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 2)
 660             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 2)
 661             self.failUnless(os.path.exists(bob_fname))
 662
 663         finally:
 664             # cleanup
 665             d0 = self.alice_magicfolder.finish()
 666             alice_clock.advance(0)
 667             yield d0
 668
 669             d1 = self.bob_magicfolder.finish()
 670             bob_clock.advance(0)
 671             yield d1
 672
 673     def test_magic_folder(self):
 674         self.set_up_grid()
 675         self.local_dir = os.path.join(self.basedir, self.unicode_or_fallback(u"loc\u0101l_dir", u"local_dir"))
 676         self.mkdir_nonascii(self.local_dir)
 677
 678         d = self.create_invite_join_magic_folder(u"Alice\u0101", self.local_dir)
 679         d.addCallback(self._restart_client)
 680
 681         # Write something short enough for a LIT file.
 682         d.addCallback(lambda ign: self._check_file(u"short", "test"))
 683
 684         # Write to the same file again with different data.
 685         d.addCallback(lambda ign: self._check_file(u"short", "different"))
 686
 687         # Test that temporary files are not uploaded.
 688         d.addCallback(lambda ign: self._check_file(u"tempfile", "test", temporary=True))
 689
 690         # Test creation of a subdirectory.
 691         d.addCallback(lambda ign: self._check_mkdir(u"directory"))
 692
 693         # Write something longer, and also try to test a Unicode name if the fs can represent it.
 694         name_u = self.unicode_or_fallback(u"l\u00F8ng", u"long")
 695         d.addCallback(lambda ign: self._check_file(name_u, "test"*100))
 696
 697         # TODO: test that causes an upload failure.
 698         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 699
 700         d.addBoth(self.cleanup)
 701         return d
 702
 703     def _check_mkdir(self, name_u):
 704         return self._check_file(name_u + u"/", "", directory=True)
 705
 706     def _check_file(self, name_u, data, temporary=False, directory=False):
 707         precondition(not (temporary and directory), temporary=temporary, directory=directory)
 708
 709         print "%r._check_file(%r, %r, temporary=%r, directory=%r)" % (self, name_u, data, temporary, directory)
 710         previously_uploaded = self._get_count('uploader.objects_succeeded')
 711         previously_disappeared = self._get_count('uploader.objects_disappeared')
 712
 713         d = self.magicfolder.uploader.set_hook('processed')
 714
 715         path_u = abspath_expanduser_unicode(name_u, base=self.local_dir)
 716         path = to_filepath(path_u)
 717
 718         if directory:
 719             os.mkdir(path_u)
 720             event_mask = self.inotify.IN_CREATE | self.inotify.IN_ISDIR
 721         else:
 722             # We don't use FilePath.setContent() here because it creates a temporary file that
 723             # is renamed into place, which causes events that the test is not expecting.
 724             f = open(path_u, "wb")
 725             try:
 726                 if temporary and sys.platform != "win32":
 727                     os.unlink(path_u)
 728                 f.write(data)
 729             finally:
 730                 f.close()
 731             if temporary and sys.platform == "win32":
 732                 os.unlink(path_u)
 733                 self.notify(path, self.inotify.IN_DELETE, flush=False)
 734             event_mask = self.inotify.IN_CLOSE_WRITE
 735
 736         self.notify(path, event_mask)
 737         encoded_name_u = magicpath.path2magic(name_u)
 738
 739         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 740         if temporary:
 741             d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_disappeared'),
 742                                                                  previously_disappeared + 1))
 743         else:
 744             def _here(res, n):
 745                 print "here %r %r" % (n, res)
 746                 return res
 747             d.addBoth(_here, 1)
 748             d.addCallback(lambda ign: self.upload_dirnode.list())
 749             d.addBoth(_here, 1.5)
 750             d.addCallback(lambda ign: self.upload_dirnode.get(encoded_name_u))
 751             d.addBoth(_here, 2)
 752             d.addCallback(download_to_data)
 753             d.addBoth(_here, 3)
 754             d.addCallback(lambda actual_data: self.failUnlessReallyEqual(actual_data, data))
 755             d.addBoth(_here, 4)
 756             d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'),
 757                                                                  previously_uploaded + 1))
 758
 759         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 760         return d
 761
 762     def _check_version_in_dmd(self, magicfolder, relpath_u, expected_version):
 763         encoded_name_u = magicpath.path2magic(relpath_u)
 764         d = magicfolder.downloader._get_collective_latest_file(encoded_name_u)
 765         def check_latest(result):
 766             if result[0] is not None:
 767                 node, metadata = result
 768                 d.addCallback(lambda ign: self.failUnlessEqual(metadata['version'], expected_version))
 769         d.addCallback(check_latest)
 770         return d
 771
 772     def _check_version_in_local_db(self, magicfolder, relpath_u, expected_version):
 773         db_entry = magicfolder._db.get_db_entry(relpath_u)
 774         if db_entry is not None:
 775             #print "_check_version_in_local_db: %r has version %s" % (relpath_u, version)
 776             self.failUnlessEqual(db_entry.version, expected_version)
 777
 778     def _check_file_gone(self, magicfolder, relpath_u):
 779         path = os.path.join(magicfolder.uploader._local_path_u, relpath_u)
 780         self.assertTrue(not os.path.exists(path))
 781
 782     def _check_uploader_count(self, name, expected, magic=None):
 783         self.failUnlessReallyEqual(self._get_count('uploader.'+name, client=(magic or self.alice_magicfolder)._client),
 784                                    expected)
 785
 786     def _check_downloader_count(self, name, expected, magic=None):
 787         self.failUnlessReallyEqual(self._get_count('downloader.'+name, client=(magic or self.bob_magicfolder)._client),
 788                                    expected)
 789
 790     def test_alice_bob(self):
 791         alice_clock = task.Clock()
 792         bob_clock = task.Clock()
 793         d = self.setup_alice_and_bob(alice_clock, bob_clock)
 794
 795         def _wait_for_Alice(ign, downloaded_d):
 796             print "Now waiting for Alice to download\n"
 797             alice_clock.advance(0)
 798             return downloaded_d
 799
 800         def _wait_for_Bob(ign, downloaded_d):
 801             print "Now waiting for Bob to download\n"
 802             bob_clock.advance(0)
 803             return downloaded_d
 804
 805         def _wait_for(ign, something_to_do, alice=True):
 806             if alice:
 807                 downloaded_d = self.bob_magicfolder.downloader.set_hook('processed')
 808                 uploaded_d = self.alice_magicfolder.uploader.set_hook('processed')
 809             else:
 810                 downloaded_d = self.alice_magicfolder.downloader.set_hook('processed')
 811                 uploaded_d = self.bob_magicfolder.uploader.set_hook('processed')
 812             something_to_do()
 813             if alice:
 814                 print "Waiting for Alice to upload\n"
 815                 alice_clock.advance(0)
 816                 uploaded_d.addCallback(_wait_for_Bob, downloaded_d)
 817             else:
 818                 print "Waiting for Bob to upload\n"
 819                 bob_clock.advance(0)
 820                 uploaded_d.addCallback(_wait_for_Alice, downloaded_d)
 821             return uploaded_d
 822
 823         def Alice_to_write_a_file():
 824             print "Alice writes a file\n"
 825             self.file_path = abspath_expanduser_unicode(u"file1", base=self.alice_magicfolder.uploader._local_path_u)
 826             fileutil.write(self.file_path, "meow, meow meow. meow? meow meow! meow.")
 827             self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 828         d.addCallback(_wait_for, Alice_to_write_a_file)
 829
 830         d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file1", 0))
 831         d.addCallback(lambda ign: self._check_version_in_local_db(self.alice_magicfolder, u"file1", 0))
 832         d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0))
 833         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 1))
 834         d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 1))
 835         d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0))
 836         d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0))
 837         d.addCallback(lambda ign: self._check_uploader_count('objects_conflicted', 0))
 838         d.addCallback(lambda ign: self._check_uploader_count('objects_conflicted', 0, magic=self.bob_magicfolder))
 839
 840         d.addCallback(lambda ign: self._check_version_in_local_db(self.bob_magicfolder, u"file1", 0))
 841         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0))
 842         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 1))
 843         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 0, magic=self.bob_magicfolder))
 844
 845         def check_delete_file(ign):
 846             d_bob = self.bob_magicfolder.uploader.set_hook('processed')
 847             def Alice_to_delete_file():
 848                 print "Alice deletes the file!\n"
 849                 os.unlink(self.file_path)
 850                 self.notify(to_filepath(self.file_path), self.inotify.IN_DELETE, magic=self.alice_magicfolder)
 851
 852             d_alice = defer.succeed(None)
 853             d_alice.addCallback(_wait_for, Alice_to_delete_file)
 854
 855             p = abspath_expanduser_unicode(u"file1", base=self.bob_magicfolder.uploader._local_path_u)
 856             if sys.platform == "win32":
 857                 self.notify(to_filepath(p), self.inotify.IN_MOVED_FROM, magic=self.bob_magicfolder, flush=False)
 858                 self.notify(to_filepath(p + u'.backup'), self.inotify.IN_MOVED_TO, magic=self.bob_magicfolder)
 859             else:
 860                 self.notify(to_filepath(p + u'.backup'), self.inotify.IN_CREATE, magic=self.bob_magicfolder, flush=False)
 861                 self.notify(to_filepath(p), self.inotify.IN_DELETE, magic=self.bob_magicfolder)
 862
 863             d_alice.addCallback(lambda ign: bob_clock.advance(0))
 864             return DeferredListShouldSucceed([d_alice, d_bob])
 865         d.addCallback(check_delete_file)
 866
 867         d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file1", 1))
 868         d.addCallback(lambda ign: self._check_version_in_local_db(self.alice_magicfolder, u"file1", 1))
 869         d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0))
 870         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 2))
 871         d.addCallback(lambda ign: self._check_uploader_count('objects_not_uploaded', 1, magic=self.bob_magicfolder))
 872         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 1, magic=self.bob_magicfolder))
 873
 874         d.addCallback(lambda ign: self._check_version_in_local_db(self.bob_magicfolder, u"file1", 1))
 875         d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file1", 1))
 876         d.addCallback(lambda ign: self._check_file_gone(self.bob_magicfolder, u"file1"))
 877         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0))
 878         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 2))
 879
 880         def Alice_to_rewrite_file():
 881             print "Alice rewrites file\n"
 882             self.file_path = abspath_expanduser_unicode(u"file1", base=self.alice_magicfolder.uploader._local_path_u)
 883             fileutil.write(self.file_path, "Alice suddenly sees the white rabbit running into the forest.")
 884             self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 885         d.addCallback(_wait_for, Alice_to_rewrite_file)
 886
 887         d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file1", 2))
 888         d.addCallback(lambda ign: self._check_version_in_local_db(self.alice_magicfolder, u"file1", 2))
 889         d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0))
 890         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 3))
 891         d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 3))
 892         d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0))
 893         d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0))
 894         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
 895         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0))
 896
 897         d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file1", 2))
 898         d.addCallback(lambda ign: self._check_version_in_local_db(self.bob_magicfolder, u"file1", 2))
 899         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0))
 900         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 3))
 901
 902         path_u = u"/tmp/magic_folder_test"
 903         encoded_path_u = magicpath.path2magic(u"/tmp/magic_folder_test")
 904
 905         def Alice_tries_to_p0wn_Bob(ign):
 906             print "Alice tries to p0wn Bob\n"
 907             processed_d = self.bob_magicfolder.downloader.set_hook('processed')
 908
 909             # upload a file that would provoke the security bug from #2506
 910             uploadable = Data("", self.alice_magicfolder._client.convergence)
 911             alice_dmd = self.alice_magicfolder.uploader._upload_dirnode
 912
 913             d2 = alice_dmd.add_file(encoded_path_u, uploadable, metadata={"version": 0}, overwrite=True)
 914             d2.addCallback(lambda ign: self.failUnless(alice_dmd.has_child(encoded_path_u)))
 915             d2.addCallback(_wait_for_Bob, processed_d)
 916             return d2
 917         d.addCallback(Alice_tries_to_p0wn_Bob)
 918
 919         d.addCallback(lambda ign: self.failIf(os.path.exists(path_u)))
 920         d.addCallback(lambda ign: self._check_version_in_local_db(self.bob_magicfolder, encoded_path_u, None))
 921         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 3))
 922         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
 923         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0))
 924
 925         def Bob_to_rewrite_file():
 926             print "Bob rewrites file\n"
 927             self.file_path = abspath_expanduser_unicode(u"file1", base=self.bob_magicfolder.uploader._local_path_u)
 928             print "---- bob's file is %r" % (self.file_path,)
 929             fileutil.write(self.file_path, "No white rabbit to be found.")
 930             self.magicfolder = self.bob_magicfolder
 931             self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE)
 932         d.addCallback(lambda ign: _wait_for(None, Bob_to_rewrite_file, alice=False))
 933
 934         d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file1", 3))
 935         d.addCallback(lambda ign: self._check_version_in_local_db(self.bob_magicfolder, u"file1", 3))
 936         d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0, magic=self.bob_magicfolder))
 937         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 2, magic=self.bob_magicfolder))
 938         d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 1, magic=self.bob_magicfolder))
 939         d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0, magic=self.bob_magicfolder))
 940         d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0, magic=self.bob_magicfolder))
 941         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0))
 942
 943         d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file1", 3))
 944         d.addCallback(lambda ign: self._check_version_in_local_db(self.alice_magicfolder, u"file1", 3))
 945         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
 946         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 1, magic=self.alice_magicfolder))
 947         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
 948
 949         def Alice_conflicts_with_Bobs_last_downloaded_uri():
 950             print "Alice conflicts with Bob\n"
 951             downloaded_d = self.bob_magicfolder.downloader.set_hook('processed')
 952             uploadable = Data("do not follow the white rabbit", self.alice_magicfolder._client.convergence)
 953             alice_dmd = self.alice_magicfolder.uploader._upload_dirnode
 954             d2 = alice_dmd.add_file(u"file1", uploadable,
 955                                     metadata={"version": 5,
 956                                               "last_downloaded_uri" : "URI:LIT:" },
 957                                     overwrite=True)
 958             print "Waiting for Alice to upload\n"
 959             d2.addCallback(lambda ign: bob_clock.advance(6))
 960             d2.addCallback(lambda ign: downloaded_d)
 961             d2.addCallback(lambda ign: self.failUnless(alice_dmd.has_child(encoded_path_u)))
 962             return d2
 963
 964         d.addCallback(lambda ign: Alice_conflicts_with_Bobs_last_downloaded_uri())
 965         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 4))
 966         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 1))
 967         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 1, magic=self.alice_magicfolder))
 968         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
 969         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
 970
 971         # prepare to perform another conflict test
 972         def Alice_to_write_file2():
 973             print "Alice writes a file\n"
 974             self.file_path = abspath_expanduser_unicode(u"file2", base=self.alice_magicfolder.uploader._local_path_u)
 975             fileutil.write(self.file_path, "something")
 976             self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 977         d.addCallback(_wait_for, Alice_to_write_file2)
 978         d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file2", 0))
 979         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
 980         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
 981
 982         def Bob_to_rewrite_file2():
 983             print "Bob rewrites file\n"
 984             self.file_path = abspath_expanduser_unicode(u"file2", base=self.bob_magicfolder.uploader._local_path_u)
 985             print "---- bob's file is %r" % (self.file_path,)
 986             fileutil.write(self.file_path, "roger roger. what vector?")
 987             self.magicfolder = self.bob_magicfolder
 988             self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE)
 989         d.addCallback(lambda ign: _wait_for(None, Bob_to_rewrite_file2, alice=False))
 990         d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file2", 1))
 991         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 5))
 992         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 1))
 993         d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0, magic=self.bob_magicfolder))
 994         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 3, magic=self.bob_magicfolder))
 995         d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 2, magic=self.bob_magicfolder))
 996         d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0, magic=self.bob_magicfolder))
 997         d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0, magic=self.bob_magicfolder))
 998         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
 999         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
1000         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 2, magic=self.alice_magicfolder))
1001
1002         # XXX here we advance the clock and then test again to make sure no values are monotonically increasing
1003         # with each queue turn ;-p
1004         alice_clock.advance(6)
1005         bob_clock.advance(6)
1006         d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file2", 1))
1007         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 5))
1008         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 1))
1009         d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0, magic=self.bob_magicfolder))
1010         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 3, magic=self.bob_magicfolder))
1011         d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 2, magic=self.bob_magicfolder))
1012         d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0, magic=self.bob_magicfolder))
1013         d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0, magic=self.bob_magicfolder))
1014         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
1015         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
1016         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 2, magic=self.alice_magicfolder))
1017
1018         def Alice_conflicts_with_Bobs_last_uploaded_uri():
1019             print "Alice conflicts with Bob\n"
1020             encoded_path_u = magicpath.path2magic(u"file2")
1021             downloaded_d = self.bob_magicfolder.downloader.set_hook('processed')
1022             uploadable = Data("rabbits with sharp fangs", self.alice_magicfolder._client.convergence)
1023             alice_dmd = self.alice_magicfolder.uploader._upload_dirnode
1024             d2 = alice_dmd.add_file(u"file2", uploadable,
1025                                     metadata={"version": 5,
1026                                               "last_uploaded_uri" : "URI:LIT:" },
1027                                     overwrite=True)
1028             print "Waiting for Alice to upload\n"
1029             d2.addCallback(lambda ign: bob_clock.advance(6))
1030             d2.addCallback(lambda ign: downloaded_d)
1031             d2.addCallback(lambda ign: self.failUnless(alice_dmd.has_child(encoded_path_u)))
1032             return d2
1033         d.addCallback(lambda ign: Alice_conflicts_with_Bobs_last_uploaded_uri())
1034         d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file2", 5))
1035         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 6))
1036         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 2))
1037         d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0, magic=self.bob_magicfolder))
1038         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 3, magic=self.bob_magicfolder))
1039         d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 2, magic=self.bob_magicfolder))
1040         d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0, magic=self.bob_magicfolder))
1041         d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0, magic=self.bob_magicfolder))
1042         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
1043         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
1044         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 2, magic=self.alice_magicfolder))
1045
1046         alice_clock.advance(6)
1047         bob_clock.advance(6)
1048         alice_clock.advance(6)
1049         bob_clock.advance(6)
1050
1051         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 2, magic=self.alice_magicfolder))
1052         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
1053         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 2))
1054         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 6))
1055
1056         # prepare to perform another conflict test
1057         def Alice_to_write_file3():
1058             print "Alice writes a file\n"
1059             self.file_path = abspath_expanduser_unicode(u"file3", base=self.alice_magicfolder.uploader._local_path_u)
1060             fileutil.write(self.file_path, "something")
1061             self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
1062         d.addCallback(_wait_for, Alice_to_write_file3)
1063         d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file3", 0))
1064         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
1065         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 7))
1066         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 2, magic=self.alice_magicfolder))
1067         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 2))
1068         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
1069
1070         def Bob_to_rewrite_file3():
1071             print "Bob rewrites file\n"
1072             self.file_path = abspath_expanduser_unicode(u"file3", base=self.bob_magicfolder.uploader._local_path_u)
1073             print "---- bob's file is %r" % (self.file_path,)
1074             fileutil.write(self.file_path, "roger roger")
1075             self.magicfolder = self.bob_magicfolder
1076             self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE)
1077         d.addCallback(lambda ign: _wait_for(None, Bob_to_rewrite_file3, alice=False))
1078         d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file3", 1))
1079         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 7))
1080         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 2))
1081         d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0, magic=self.bob_magicfolder))
1082         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 4, magic=self.bob_magicfolder))
1083         d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 3, magic=self.bob_magicfolder))
1084         d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0, magic=self.bob_magicfolder))
1085         d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0, magic=self.bob_magicfolder))
1086         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
1087         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
1088         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 3, magic=self.alice_magicfolder))
1089
1090
1091
1092         def _cleanup(ign, magicfolder, clock):
1093             if magicfolder is not None:
1094                 d2 = magicfolder.finish()
1095                 clock.advance(0)
1096                 return d2
1097
1098         def cleanup_Alice_and_Bob(result):
1099             print "cleanup alice bob test\n"
1100             d = defer.succeed(None)
1101             d.addCallback(_cleanup, self.alice_magicfolder, alice_clock)
1102             d.addCallback(_cleanup, self.bob_magicfolder, bob_clock)
1103             d.addCallback(lambda ign: result)
1104             return d
1105         d.addBoth(cleanup_Alice_and_Bob)
1106         return d
1107
1108
1109 class MockTest(MagicFolderTestMixin, unittest.TestCase):
1110     """This can run on any platform, and even if twisted.internet.inotify can't be imported."""
1111
1112     def setUp(self):
1113         MagicFolderTestMixin.setUp(self)
1114         self.inotify = fake_inotify
1115         self.patch(magic_folder, 'get_inotify_module', lambda: self.inotify)
1116
1117     def notify(self, path, mask, magic=None, flush=True):
1118         if magic is None:
1119             magic = self.magicfolder
1120         magic.uploader._notifier.event(path, mask)
1121         # no flush for the mock test.
1122
1123     def test_errors(self):
1124         self.set_up_grid()
1125
1126         errors_dir = abspath_expanduser_unicode(u"errors_dir", base=self.basedir)
1127         os.mkdir(errors_dir)
1128         not_a_dir = abspath_expanduser_unicode(u"NOT_A_DIR", base=self.basedir)
1129         fileutil.write(not_a_dir, "")
1130         magicfolderdb = abspath_expanduser_unicode(u"magicfolderdb", base=self.basedir)
1131         doesnotexist  = abspath_expanduser_unicode(u"doesnotexist", base=self.basedir)
1132
1133         client = self.g.clients[0]
1134         d = client.create_dirnode()
1135         def _check_errors(n):
1136             self.failUnless(IDirectoryNode.providedBy(n))
1137             upload_dircap = n.get_uri()
1138             readonly_dircap = n.get_readonly_uri()
1139
1140             self.shouldFail(AssertionError, 'nonexistent local.directory', 'there is no directory',
1141                             MagicFolder, client, upload_dircap, '', doesnotexist, magicfolderdb, 0077)
1142             self.shouldFail(AssertionError, 'non-directory local.directory', 'is not a directory',
1143                             MagicFolder, client, upload_dircap, '', not_a_dir, magicfolderdb, 0077)
1144             self.shouldFail(AssertionError, 'bad upload.dircap', 'does not refer to a directory',
1145                             MagicFolder, client, 'bad', '', errors_dir, magicfolderdb, 0077)
1146             self.shouldFail(AssertionError, 'non-directory upload.dircap', 'does not refer to a directory',
1147                             MagicFolder, client, 'URI:LIT:foo', '', errors_dir, magicfolderdb, 0077)
1148             self.shouldFail(AssertionError, 'readonly upload.dircap', 'is not a writecap to a directory',
1149                             MagicFolder, client, readonly_dircap, '', errors_dir, magicfolderdb, 0077)
1150             self.shouldFail(AssertionError, 'collective dircap', 'is not a readonly cap to a directory',
1151                             MagicFolder, client, upload_dircap, upload_dircap, errors_dir, magicfolderdb, 0077)
1152
1153             def _not_implemented():
1154                 raise NotImplementedError("blah")
1155             self.patch(magic_folder, 'get_inotify_module', _not_implemented)
1156             self.shouldFail(NotImplementedError, 'unsupported', 'blah',
1157                             MagicFolder, client, upload_dircap, '', errors_dir, magicfolderdb, 0077)
1158         d.addCallback(_check_errors)
1159         return d
1160
1161     def test_write_downloaded_file(self):
1162         workdir = u"cli/MagicFolder/write-downloaded-file"
1163         local_file = fileutil.abspath_expanduser_unicode(os.path.join(workdir, "foobar"))
1164
1165         class TestWriteFileMixin(WriteFileMixin):
1166             def _log(self, msg):
1167                 pass
1168
1169         writefile = TestWriteFileMixin()
1170         writefile._umask = 0077
1171
1172         # create a file with name "foobar" with content "foo"
1173         # write downloaded file content "bar" into "foobar" with is_conflict = False
1174         fileutil.make_dirs(workdir)
1175         fileutil.write(local_file, "foo")
1176
1177         # if is_conflict is False, then the .conflict file shouldn't exist.
1178         writefile._write_downloaded_file(local_file, "bar", False, None)
1179         conflicted_path = local_file + u".conflict"
1180         self.failIf(os.path.exists(conflicted_path))
1181
1182         # At this point, the backup file should exist with content "foo"
1183         backup_path = local_file + u".backup"
1184         self.failUnless(os.path.exists(backup_path))
1185         self.failUnlessEqual(fileutil.read(backup_path), "foo")
1186
1187         # .tmp file shouldn't exist
1188         self.failIf(os.path.exists(local_file + u".tmp"))
1189
1190         # .. and the original file should have the new content
1191         self.failUnlessEqual(fileutil.read(local_file), "bar")
1192
1193         # now a test for conflicted case
1194         writefile._write_downloaded_file(local_file, "bar", True, None)
1195         self.failUnless(os.path.exists(conflicted_path))
1196
1197         # .tmp file shouldn't exist
1198         self.failIf(os.path.exists(local_file + u".tmp"))
1199
1200     def test_periodic_full_scan(self):
1201         self.set_up_grid()
1202         self.local_dir = abspath_expanduser_unicode(u"test_periodic_full_scan",base=self.basedir)
1203         self.mkdir_nonascii(self.local_dir)
1204
1205         alice_clock = task.Clock()
1206         d = self.do_create_magic_folder(0)
1207         d.addCallback(lambda ign: self.do_invite(0, u"Alice\u00F8"))
1208         def get_invite_code(result):
1209             self.invite_code = result[1].strip()
1210         d.addCallback(get_invite_code)
1211         d.addCallback(lambda ign: self.do_join(0, self.local_dir, self.invite_code))
1212         def get_alice_caps(ign):
1213             self.alice_collective_dircap, self.alice_upload_dircap = self.get_caps_from_files(0)
1214         d.addCallback(get_alice_caps)
1215         d.addCallback(lambda ign: self.check_joined_config(0, self.alice_upload_dircap))
1216         d.addCallback(lambda ign: self.check_config(0, self.local_dir))
1217         def get_Alice_magicfolder(result):
1218             self.magicfolder = self.init_magicfolder(0, self.alice_upload_dircap,
1219                                                            self.alice_collective_dircap,
1220                                                            self.local_dir, alice_clock)
1221             return result
1222         d.addCallback(get_Alice_magicfolder)
1223         empty_tree_name = self.unicode_or_fallback(u"empty_tr\u00EAe", u"empty_tree")
1224         empty_tree_dir = abspath_expanduser_unicode(empty_tree_name, base=self.basedir)
1225         new_empty_tree_dir = abspath_expanduser_unicode(empty_tree_name, base=self.local_dir)
1226
1227         def _check_move_empty_tree(res):
1228             print "CHECK MOVE EMPTY TREE"
1229             uploaded_d = self.magicfolder.uploader.set_hook('processed')
1230             self.mkdir_nonascii(empty_tree_dir)
1231             os.rename(empty_tree_dir, new_empty_tree_dir)
1232             self.notify(to_filepath(new_empty_tree_dir), self.inotify.IN_MOVED_TO)
1233             return uploaded_d
1234         d.addCallback(_check_move_empty_tree)
1235         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
1236         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 1))
1237         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.files_uploaded'), 0))
1238         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
1239         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.directories_created'), 1))
1240
1241         def _create_file_without_event(res):
1242             print "CREATE FILE WITHOUT EMITTING EVENT"
1243             processed_d = self.magicfolder.uploader.set_hook('processed')
1244             what_path = abspath_expanduser_unicode(u"what", base=new_empty_tree_dir)
1245             fileutil.write(what_path, "say when")
1246             print "ADVANCE CLOCK"
1247             alice_clock.advance(self.magicfolder.uploader._periodic_full_scan_duration + 1)
1248             return processed_d
1249         d.addCallback(_create_file_without_event)
1250         def _advance_clock(res):
1251             print "_advance_clock"
1252             processed_d = self.magicfolder.uploader.set_hook('processed')
1253             alice_clock.advance(0)
1254             return processed_d
1255         d.addCallback(_advance_clock)
1256         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.files_uploaded'), 1))
1257         def cleanup(res):
1258             d2 = self.magicfolder.finish()
1259             alice_clock.advance(0)
1260             return d2
1261         d.addCallback(cleanup)
1262         return d
1263
1264 class RealTest(MagicFolderTestMixin, unittest.TestCase):
1265     """This is skipped unless both Twisted and the platform support inotify."""
1266
1267     def setUp(self):
1268         MagicFolderTestMixin.setUp(self)
1269         self.inotify = magic_folder.get_inotify_module()
1270
1271     def notify(self, path, mask, magic=None, flush=True):
1272         # Writing to the filesystem causes the notification.
1273         # However, flushing filesystem buffers may be necessary on Windows.
1274         if flush:
1275             fileutil.flush_volume(path.path)
1276
1277 try:
1278     magic_folder.get_inotify_module()
1279 except NotImplementedError:
1280     RealTest.skip = "Magic Folder support can only be tested for-real on an OS that supports inotify or equivalent."