src/allmydata/test/test_magic_folder.py

   1
   2 import os, sys
   3 import shutil
   4
   5 from twisted.trial import unittest
   6 from twisted.internet import defer, task
   7
   8 from allmydata.interfaces import IDirectoryNode
   9 from allmydata.util.assertutil import precondition
  10
  11 from allmydata.util import fake_inotify, fileutil
  12 from allmydata.util.encodingutil import get_filesystem_encoding, to_filepath
  13 from allmydata.util.consumer import download_to_data
  14 from allmydata.test.no_network import GridTestMixin
  15 from allmydata.test.common_util import ReallyEqualMixin, NonASCIIPathMixin
  16 from allmydata.test.common import ShouldFailMixin
  17 from .test_cli_magic_folder import MagicFolderCLITestMixin
  18
  19 from allmydata.frontends import magic_folder
  20 from allmydata.frontends.magic_folder import MagicFolder, Downloader, WriteFileMixin
  21 from allmydata import magicfolderdb, magicpath
  22 from allmydata.util.fileutil import abspath_expanduser_unicode
  23 from allmydata.immutable.upload import Data
  24
  25 from twisted.internet.base import DelayedCall
  26 #DelayedCall.debug = True
  27
  28 class MagicFolderTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, ReallyEqualMixin, NonASCIIPathMixin):
  29     """
  30     These tests will be run both with a mock notifier, and (on platforms that support it)
  31     with the real INotify.
  32     """
  33
  34     def setUp(self):
  35         GridTestMixin.setUp(self)
  36         temp = self.mktemp()
  37         self.basedir = abspath_expanduser_unicode(temp.decode(get_filesystem_encoding()))
  38         self.magicfolder = None
  39         self.patch(Downloader, 'REMOTE_SCAN_INTERVAL', 0)
  40
  41     def _get_count(self, name, client=None):
  42         counters = (client or self.get_client()).stats_provider.get_stats()["counters"]
  43         return counters.get('magic_folder.%s' % (name,), 0)
  44
  45     def _createdb(self):
  46         dbfile = abspath_expanduser_unicode(u"magicfolderdb.sqlite", base=self.basedir)
  47         mdb = magicfolderdb.get_magicfolderdb(dbfile, create_version=(magicfolderdb.SCHEMA_v1, 1))
  48         self.failUnless(mdb, "unable to create magicfolderdb from %r" % (dbfile,))
  49         self.failUnlessEqual(mdb.VERSION, 1)
  50         return mdb
  51
  52     def _restart_client(self, ign):
  53         #print "_restart_client"
  54         d = self.restart_client()
  55         d.addCallback(self._wait_until_started)
  56         return d
  57
  58     def _wait_until_started(self, ign):
  59         #print "_wait_until_started"
  60         self.magicfolder = self.get_client().getServiceNamed('magic-folder')
  61         return self.magicfolder.ready()
  62
  63     def test_db_basic(self):
  64         fileutil.make_dirs(self.basedir)
  65         self._createdb()
  66
  67     def test_db_persistence(self):
  68         """Test that a file upload creates an entry in the database."""
  69
  70         fileutil.make_dirs(self.basedir)
  71         db = self._createdb()
  72
  73         relpath1 = u"myFile1"
  74         pathinfo = fileutil.PathInfo(isdir=False, isfile=True, islink=False,
  75                                      exists=True, size=1, mtime=123, ctime=456)
  76         db.did_upload_version(relpath1, 0, 'URI:LIT:1', 'URI:LIT:0', 0, pathinfo)
  77
  78         c = db.cursor
  79         c.execute("SELECT size, mtime, ctime"
  80                   " FROM local_files"
  81                   " WHERE path=?",
  82                   (relpath1,))
  83         row = c.fetchone()
  84         self.failUnlessEqual(row, (pathinfo.size, pathinfo.mtime, pathinfo.ctime))
  85
  86         # Second test uses magic_folder.is_new_file instead of SQL query directly
  87         # to confirm the previous upload entry in the db.
  88         relpath2 = u"myFile2"
  89         path2 = os.path.join(self.basedir, relpath2)
  90         fileutil.write(path2, "meow\n")
  91         pathinfo = fileutil.get_pathinfo(path2)
  92         db.did_upload_version(relpath2, 0, 'URI:LIT:2', 'URI:LIT:1', 0, pathinfo)
  93         db_entry = db.get_db_entry(relpath2)
  94         self.failUnlessFalse(magic_folder.is_new_file(pathinfo, db_entry))
  95
  96         different_pathinfo = fileutil.PathInfo(isdir=False, isfile=True, islink=False,
  97                                                exists=True, size=0, mtime=pathinfo.mtime, ctime=pathinfo.ctime)
  98         self.failUnlessTrue(magic_folder.is_new_file(different_pathinfo, db_entry))
  99
 100     def test_magicfolder_start_service(self):
 101         self.set_up_grid()
 102
 103         self.local_dir = abspath_expanduser_unicode(self.unicode_or_fallback(u"l\u00F8cal_dir", u"local_dir"),
 104                                                     base=self.basedir)
 105         self.mkdir_nonascii(self.local_dir)
 106
 107         d = defer.succeed(None)
 108         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.dirs_monitored'), 0))
 109
 110         d.addCallback(lambda ign: self.create_invite_join_magic_folder(u"Alice", self.local_dir))
 111         d.addCallback(self._restart_client)
 112
 113         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.dirs_monitored'), 1))
 114         d.addBoth(self.cleanup)
 115         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.dirs_monitored'), 0))
 116         return d
 117
 118     def test_scan_once_on_startup(self):
 119         self.set_up_grid()
 120         self.local_dir = abspath_expanduser_unicode(u"test_scan_once_on_startup", base=self.basedir)
 121         self.mkdir_nonascii(self.local_dir)
 122         self.collective_dircap = ""
 123
 124         alice_clock = task.Clock()
 125         bob_clock = task.Clock()
 126         d = self.setup_alice_and_bob(alice_clock, bob_clock)
 127
 128         def upload_stuff(ignore):
 129             uploadable = Data("", self.alice_magicfolder._client.convergence)
 130             return self.alice_magicfolder._client.upload(uploadable)
 131         d.addCallback(upload_stuff)
 132         def check_is_upload(ignore):
 133             alice_clock.advance(99)
 134             d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 0, magic=self.alice_magicfolder))
 135             d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0, magic=self.alice_magicfolder))
 136             d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
 137             d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 0, magic=self.alice_magicfolder))
 138             d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
 139             d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 0, magic=self.alice_magicfolder))
 140
 141         d.addCallback(check_is_upload)
 142         def _cleanup(ign, magicfolder, clock):
 143             if magicfolder is not None:
 144                 d2 = magicfolder.finish()
 145                 clock.advance(0)
 146                 return d2
 147         def cleanup_Alice_and_Bob(result):
 148             print "cleanup alice bob test\n"
 149             d = defer.succeed(None)
 150             d.addCallback(_cleanup, self.alice_magicfolder, alice_clock)
 151             d.addCallback(_cleanup, self.bob_magicfolder, bob_clock)
 152             d.addCallback(lambda ign: result)
 153             return d
 154
 155         d.addBoth(cleanup_Alice_and_Bob)
 156         return d
 157
 158     def test_move_tree(self):
 159         self.set_up_grid()
 160
 161         self.local_dir = abspath_expanduser_unicode(self.unicode_or_fallback(u"l\u00F8cal_dir", u"local_dir"),
 162                                                     base=self.basedir)
 163         self.mkdir_nonascii(self.local_dir)
 164
 165         empty_tree_name = self.unicode_or_fallback(u"empty_tr\u00EAe", u"empty_tree")
 166         empty_tree_dir = abspath_expanduser_unicode(empty_tree_name, base=self.basedir)
 167         new_empty_tree_dir = abspath_expanduser_unicode(empty_tree_name, base=self.local_dir)
 168
 169         small_tree_name = self.unicode_or_fallback(u"small_tr\u00EAe", u"empty_tree")
 170         small_tree_dir = abspath_expanduser_unicode(small_tree_name, base=self.basedir)
 171         new_small_tree_dir = abspath_expanduser_unicode(small_tree_name, base=self.local_dir)
 172
 173         d = self.create_invite_join_magic_folder(u"Alice", self.local_dir)
 174         d.addCallback(self._restart_client)
 175
 176         def _check_move_empty_tree(res):
 177             print "_check_move_empty_tree"
 178             uploaded_d = self.magicfolder.uploader.set_hook('processed')
 179             self.mkdir_nonascii(empty_tree_dir)
 180             os.rename(empty_tree_dir, new_empty_tree_dir)
 181             self.notify(to_filepath(new_empty_tree_dir), self.inotify.IN_MOVED_TO)
 182
 183             return uploaded_d
 184         d.addCallback(_check_move_empty_tree)
 185         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 186         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 1))
 187         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.files_uploaded'), 0))
 188         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 189         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.directories_created'), 1))
 190
 191         def _check_move_small_tree(res):
 192             print "_check_move_small_tree"
 193             uploaded_d = self.magicfolder.uploader.set_hook('processed', ignore_count=1)
 194             self.mkdir_nonascii(small_tree_dir)
 195             what_path = abspath_expanduser_unicode(u"what", base=small_tree_dir)
 196             fileutil.write(what_path, "say when")
 197             os.rename(small_tree_dir, new_small_tree_dir)
 198             self.notify(to_filepath(new_small_tree_dir), self.inotify.IN_MOVED_TO)
 199
 200             return uploaded_d
 201         d.addCallback(_check_move_small_tree)
 202         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 203         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 3))
 204         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.files_uploaded'), 1))
 205         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 206         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.directories_created'), 2))
 207
 208         def _check_moved_tree_is_watched(res):
 209             print "_check_moved_tree_is_watched"
 210             uploaded_d = self.magicfolder.uploader.set_hook('processed')
 211             another_path = abspath_expanduser_unicode(u"another", base=new_small_tree_dir)
 212             fileutil.write(another_path, "file")
 213             self.notify(to_filepath(another_path), self.inotify.IN_CLOSE_WRITE)
 214
 215             return uploaded_d
 216         d.addCallback(_check_moved_tree_is_watched)
 217         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 218         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 4))
 219         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.files_uploaded'), 2))
 220         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 221         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.directories_created'), 2))
 222
 223         # Files that are moved out of the upload directory should no longer be watched.
 224         #def _move_dir_away(ign):
 225         #    os.rename(new_empty_tree_dir, empty_tree_dir)
 226         #    # Wuh? Why don't we get this event for the real test?
 227         #    #self.notify(to_filepath(new_empty_tree_dir), self.inotify.IN_MOVED_FROM)
 228         #d.addCallback(_move_dir_away)
 229         #def create_file(val):
 230         #    test_file = abspath_expanduser_unicode(u"what", base=empty_tree_dir)
 231         #    fileutil.write(test_file, "meow")
 232         #    #self.notify(...)
 233         #    return
 234         #d.addCallback(create_file)
 235         #d.addCallback(lambda ign: time.sleep(1))  # XXX ICK
 236         #d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 237         #d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 4))
 238         #d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.files_uploaded'), 2))
 239         #d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 240         #d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.directories_created'), 2))
 241
 242         d.addBoth(self.cleanup)
 243         return d
 244
 245     def test_persistence(self):
 246         """
 247         Perform an upload of a given file and then stop the client.
 248         Start a new client and magic-folder service... and verify that the file is NOT uploaded
 249         a second time. This test is meant to test the database persistence along with
 250         the startup and shutdown code paths of the magic-folder service.
 251         """
 252         self.set_up_grid()
 253         self.local_dir = abspath_expanduser_unicode(u"test_persistence", base=self.basedir)
 254         self.mkdir_nonascii(self.local_dir)
 255         self.collective_dircap = ""
 256
 257         d = defer.succeed(None)
 258         d.addCallback(lambda ign: self.create_invite_join_magic_folder(u"Alice", self.local_dir))
 259         d.addCallback(self._restart_client)
 260
 261         def create_test_file(filename):
 262             d2 = self.magicfolder.uploader.set_hook('processed')
 263             test_file = abspath_expanduser_unicode(filename, base=self.local_dir)
 264             fileutil.write(test_file, "meow %s" % filename)
 265             self.notify(to_filepath(test_file), self.inotify.IN_CLOSE_WRITE)
 266             return d2
 267         d.addCallback(lambda ign: create_test_file(u"what1"))
 268         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 269         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 1))
 270         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 271         d.addCallback(self.cleanup)
 272
 273         d.addCallback(self._restart_client)
 274         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 275         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 1))
 276         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 277         d.addCallback(lambda ign: create_test_file(u"what2"))
 278         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 279         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 2))
 280         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 281         d.addBoth(self.cleanup)
 282         return d
 283
 284     @defer.inlineCallbacks
 285     def test_delete(self):
 286         self.set_up_grid()
 287         self.local_dir = os.path.join(self.basedir, u"local_dir")
 288         self.mkdir_nonascii(self.local_dir)
 289
 290         yield self.create_invite_join_magic_folder(u"Alice\u0101", self.local_dir)
 291         yield self._restart_client(None)
 292
 293         try:
 294             # create a file
 295             up_proc = self.magicfolder.uploader.set_hook('processed')
 296             # down_proc = self.magicfolder.downloader.set_hook('processed')
 297             path = os.path.join(self.local_dir, u'foo')
 298             fileutil.write(path, 'foo\n')
 299             self.notify(to_filepath(path), self.inotify.IN_CLOSE_WRITE)
 300             yield up_proc
 301             self.assertTrue(os.path.exists(path))
 302
 303             # the real test part: delete the file
 304             up_proc = self.magicfolder.uploader.set_hook('processed')
 305             os.unlink(path)
 306             self.notify(to_filepath(path), self.inotify.IN_DELETE)
 307             yield up_proc
 308             self.assertFalse(os.path.exists(path))
 309
 310             # ensure we still have a DB entry, and that the version is 1
 311             node, metadata = yield self.magicfolder.downloader._get_collective_latest_file(u'foo')
 312             self.assertTrue(node is not None, "Failed to find %r in DMD" % (path,))
 313             self.failUnlessEqual(metadata['version'], 1)
 314
 315         finally:
 316             yield self.cleanup(None)
 317
 318     @defer.inlineCallbacks
 319     def test_delete_and_restore(self):
 320         self.set_up_grid()
 321         self.local_dir = os.path.join(self.basedir, u"local_dir")
 322         self.mkdir_nonascii(self.local_dir)
 323
 324         yield self.create_invite_join_magic_folder(u"Alice\u0101", self.local_dir)
 325         yield self._restart_client(None)
 326
 327         try:
 328             # create a file
 329             up_proc = self.magicfolder.uploader.set_hook('processed')
 330             # down_proc = self.magicfolder.downloader.set_hook('processed')
 331             path = os.path.join(self.local_dir, u'foo')
 332             fileutil.write(path, 'foo\n')
 333             self.notify(to_filepath(path), self.inotify.IN_CLOSE_WRITE)
 334             yield up_proc
 335             self.assertTrue(os.path.exists(path))
 336
 337             # delete the file
 338             up_proc = self.magicfolder.uploader.set_hook('processed')
 339             os.unlink(path)
 340             self.notify(to_filepath(path), self.inotify.IN_DELETE)
 341             yield up_proc
 342             self.assertFalse(os.path.exists(path))
 343
 344             # ensure we still have a DB entry, and that the version is 1
 345             node, metadata = yield self.magicfolder.downloader._get_collective_latest_file(u'foo')
 346             self.assertTrue(node is not None, "Failed to find %r in DMD" % (path,))
 347             self.failUnlessEqual(metadata['version'], 1)
 348
 349             # restore the file, with different contents
 350             up_proc = self.magicfolder.uploader.set_hook('processed')
 351             path = os.path.join(self.local_dir, u'foo')
 352             fileutil.write(path, 'bar\n')
 353             self.notify(to_filepath(path), self.inotify.IN_CLOSE_WRITE)
 354             yield up_proc
 355
 356             # ensure we still have a DB entry, and that the version is 2
 357             node, metadata = yield self.magicfolder.downloader._get_collective_latest_file(u'foo')
 358             self.assertTrue(node is not None, "Failed to find %r in DMD" % (path,))
 359             self.failUnlessEqual(metadata['version'], 2)
 360
 361         finally:
 362             yield self.cleanup(None)
 363
 364     @defer.inlineCallbacks
 365     def test_alice_delete_bob_restore(self):
 366         alice_clock = task.Clock()
 367         bob_clock = task.Clock()
 368         yield self.setup_alice_and_bob(alice_clock, bob_clock)
 369         alice_dir = self.alice_magicfolder.uploader._local_path_u
 370         bob_dir = self.bob_magicfolder.uploader._local_path_u
 371         alice_fname = os.path.join(alice_dir, 'blam')
 372         bob_fname = os.path.join(bob_dir, 'blam')
 373
 374         try:
 375             # alice creates a file, bob downloads it
 376             alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
 377             bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
 378
 379             fileutil.write(alice_fname, 'contents0\n')
 380             self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 381
 382             alice_clock.advance(0)
 383             yield alice_proc  # alice uploads
 384
 385             bob_clock.advance(0)
 386             yield bob_proc    # bob downloads
 387
 388             # check the state
 389             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
 390             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 0)
 391             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
 392             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 0)
 393             yield self.failUnlessReallyEqual(
 394                 self._get_count('downloader.objects_failed', client=self.bob_magicfolder._client),
 395                 0
 396             )
 397             yield self.failUnlessReallyEqual(
 398                 self._get_count('downloader.objects_downloaded', client=self.bob_magicfolder._client),
 399                 1
 400             )
 401
 402             print("BOB DELETE")
 403             # now bob deletes it (bob should upload, alice download)
 404             bob_proc = self.bob_magicfolder.uploader.set_hook('processed')
 405             alice_proc = self.alice_magicfolder.downloader.set_hook('processed')
 406             os.unlink(bob_fname)
 407             self.notify(to_filepath(bob_fname), self.inotify.IN_DELETE, magic=self.bob_magicfolder)
 408
 409             bob_clock.advance(0)
 410             yield bob_proc
 411             alice_clock.advance(0)
 412             yield alice_proc
 413
 414             # check versions
 415             node, metadata = yield self.alice_magicfolder.downloader._get_collective_latest_file(u'blam')
 416             self.assertTrue(metadata['deleted'])
 417             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
 418             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 1)
 419             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
 420             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 1)
 421
 422             print("ALICE RESTORE")
 423             # now alice restores it (alice should upload, bob download)
 424             alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
 425             bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
 426             fileutil.write(alice_fname, 'new contents\n')
 427             self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 428
 429             alice_clock.advance(0)
 430             yield alice_proc
 431             bob_clock.advance(0)
 432             yield bob_proc
 433
 434             # check versions
 435             node, metadata = yield self.alice_magicfolder.downloader._get_collective_latest_file(u'blam')
 436             self.assertTrue('deleted' not in metadata or not metadata['deleted'])
 437             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 2)
 438             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 2)
 439             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 2)
 440             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 2)
 441
 442         finally:
 443             # cleanup
 444             d0 = self.alice_magicfolder.finish()
 445             alice_clock.advance(0)
 446             yield d0
 447
 448             d1 = self.bob_magicfolder.finish()
 449             bob_clock.advance(0)
 450             yield d1
 451
 452     @defer.inlineCallbacks
 453     def test_alice_sees_bobs_delete_with_error(self):
 454         # alice creates a file, bob deletes it -- and we also arrange
 455         # for Alice's file to have "gone missing" as well.
 456         alice_clock = task.Clock()
 457         bob_clock = task.Clock()
 458         yield self.setup_alice_and_bob(alice_clock, bob_clock)
 459         alice_dir = self.alice_magicfolder.uploader._local_path_u
 460         bob_dir = self.bob_magicfolder.uploader._local_path_u
 461         alice_fname = os.path.join(alice_dir, 'blam')
 462         bob_fname = os.path.join(bob_dir, 'blam')
 463
 464         try:
 465             # alice creates a file, bob downloads it
 466             alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
 467             bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
 468
 469             fileutil.write(alice_fname, 'contents0\n')
 470             self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 471
 472             alice_clock.advance(0)
 473             yield alice_proc  # alice uploads
 474
 475             bob_clock.advance(0)
 476             yield bob_proc    # bob downloads
 477
 478             # check the state
 479             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
 480             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 0)
 481             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
 482             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 0)
 483             yield self.failUnlessReallyEqual(
 484                 self._get_count('downloader.objects_failed', client=self.bob_magicfolder._client),
 485                 0
 486             )
 487             yield self.failUnlessReallyEqual(
 488                 self._get_count('downloader.objects_downloaded', client=self.bob_magicfolder._client),
 489                 1
 490             )
 491
 492             # now bob deletes it (bob should upload, alice download)
 493             bob_proc = self.bob_magicfolder.uploader.set_hook('processed')
 494             alice_proc = self.alice_magicfolder.downloader.set_hook('processed')
 495             os.unlink(bob_fname)
 496             self.notify(to_filepath(bob_fname), self.inotify.IN_DELETE, magic=self.bob_magicfolder)
 497             # just after notifying bob, we also delete alice's,
 498             # covering the 'except' flow in _rename_deleted_file()
 499             os.unlink(alice_fname)
 500
 501             bob_clock.advance(0)
 502             yield bob_proc
 503             alice_clock.advance(0)
 504             yield alice_proc
 505
 506             # check versions
 507             node, metadata = yield self.alice_magicfolder.downloader._get_collective_latest_file(u'blam')
 508             self.assertTrue(metadata['deleted'])
 509             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
 510             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 1)
 511             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
 512             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 1)
 513
 514         finally:
 515             # cleanup
 516             d0 = self.alice_magicfolder.finish()
 517             alice_clock.advance(0)
 518             yield d0
 519
 520             d1 = self.bob_magicfolder.finish()
 521             bob_clock.advance(0)
 522             yield d1
 523
 524     @defer.inlineCallbacks
 525     def test_alice_create_bob_update(self):
 526         alice_clock = task.Clock()
 527         bob_clock = task.Clock()
 528         yield self.setup_alice_and_bob(alice_clock, bob_clock)
 529         alice_dir = self.alice_magicfolder.uploader._local_path_u
 530         bob_dir = self.bob_magicfolder.uploader._local_path_u
 531         alice_fname = os.path.join(alice_dir, 'blam')
 532         bob_fname = os.path.join(bob_dir, 'blam')
 533
 534         try:
 535             # alice creates a file, bob downloads it
 536             alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
 537             bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
 538
 539             fileutil.write(alice_fname, 'contents0\n')
 540             self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 541
 542             alice_clock.advance(0)
 543             yield alice_proc  # alice uploads
 544
 545             bob_clock.advance(0)
 546             yield bob_proc    # bob downloads
 547
 548             # check the state
 549             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
 550             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 0)
 551             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
 552             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 0)
 553             yield self.failUnlessReallyEqual(
 554                 self._get_count('downloader.objects_failed', client=self.bob_magicfolder._client),
 555                 0
 556             )
 557             yield self.failUnlessReallyEqual(
 558                 self._get_count('downloader.objects_downloaded', client=self.bob_magicfolder._client),
 559                 1
 560             )
 561
 562             # now bob updates it (bob should upload, alice download)
 563             bob_proc = self.bob_magicfolder.uploader.set_hook('processed')
 564             alice_proc = self.alice_magicfolder.downloader.set_hook('processed')
 565             fileutil.write(bob_fname, 'bob wuz here\n')
 566             self.notify(to_filepath(bob_fname), self.inotify.IN_CLOSE_WRITE, magic=self.bob_magicfolder)
 567
 568             bob_clock.advance(0)
 569             yield bob_proc
 570             alice_clock.advance(0)
 571             yield alice_proc
 572
 573             # check the state
 574             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
 575             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 1)
 576             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
 577             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 1)
 578
 579         finally:
 580             # cleanup
 581             d0 = self.alice_magicfolder.finish()
 582             alice_clock.advance(0)
 583             try:
 584                 yield d0
 585             except Exception as e:
 586                 print("WARNING:", e)
 587
 588             d1 = self.bob_magicfolder.finish()
 589             bob_clock.advance(0)
 590             try:
 591                 yield d1
 592             except Exception as e:
 593                 print("WARNING:", e)
 594
 595     @defer.inlineCallbacks
 596     def test_alice_delete_and_restore(self):
 597         alice_clock = task.Clock()
 598         bob_clock = task.Clock()
 599         yield self.setup_alice_and_bob(alice_clock, bob_clock)
 600         alice_dir = self.alice_magicfolder.uploader._local_path_u
 601         bob_dir = self.bob_magicfolder.uploader._local_path_u
 602         alice_fname = os.path.join(alice_dir, 'blam')
 603         bob_fname = os.path.join(bob_dir, 'blam')
 604
 605         try:
 606             # alice creates a file, bob downloads it
 607             alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
 608             bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
 609
 610             fileutil.write(alice_fname, 'contents0\n')
 611             self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 612
 613             alice_clock.advance(0)
 614             yield alice_proc  # alice uploads
 615
 616             bob_clock.advance(0)
 617             yield bob_proc    # bob downloads
 618
 619             # check the state
 620             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
 621             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 0)
 622             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
 623             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 0)
 624             yield self.failUnlessReallyEqual(
 625                 self._get_count('downloader.objects_failed', client=self.bob_magicfolder._client),
 626                 0
 627             )
 628             yield self.failUnlessReallyEqual(
 629                 self._get_count('downloader.objects_downloaded', client=self.bob_magicfolder._client),
 630                 1
 631             )
 632             self.failUnless(os.path.exists(bob_fname))
 633
 634             # now alice deletes it (alice should upload, bob download)
 635             alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
 636             bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
 637             os.unlink(alice_fname)
 638             self.notify(to_filepath(alice_fname), self.inotify.IN_DELETE, magic=self.alice_magicfolder)
 639
 640             alice_clock.advance(0)
 641             yield alice_proc
 642             bob_clock.advance(0)
 643             yield bob_proc
 644
 645             # check the state
 646             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
 647             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 1)
 648             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
 649             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 1)
 650             self.failIf(os.path.exists(bob_fname))
 651
 652             # now alice restores the file (with new contents)
 653             alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
 654             bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
 655             fileutil.write(alice_fname, 'alice wuz here\n')
 656             self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 657
 658             alice_clock.advance(0)
 659             yield alice_proc
 660             bob_clock.advance(0)
 661             yield bob_proc
 662
 663             # check the state
 664             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 2)
 665             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 2)
 666             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 2)
 667             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 2)
 668             self.failUnless(os.path.exists(bob_fname))
 669
 670         finally:
 671             # cleanup
 672             d0 = self.alice_magicfolder.finish()
 673             alice_clock.advance(0)
 674             yield d0
 675
 676             d1 = self.bob_magicfolder.finish()
 677             bob_clock.advance(0)
 678             yield d1
 679
 680     def test_magic_folder(self):
 681         self.set_up_grid()
 682         self.local_dir = os.path.join(self.basedir, self.unicode_or_fallback(u"loc\u0101l_dir", u"local_dir"))
 683         self.mkdir_nonascii(self.local_dir)
 684
 685         d = self.create_invite_join_magic_folder(u"Alice\u0101", self.local_dir)
 686         d.addCallback(self._restart_client)
 687
 688         # Write something short enough for a LIT file.
 689         d.addCallback(lambda ign: self._check_file(u"short", "test"))
 690
 691         # Write to the same file again with different data.
 692         d.addCallback(lambda ign: self._check_file(u"short", "different"))
 693
 694         # Test that temporary files are not uploaded.
 695         d.addCallback(lambda ign: self._check_file(u"tempfile", "test", temporary=True))
 696
 697         # Test creation of a subdirectory.
 698         d.addCallback(lambda ign: self._check_mkdir(u"directory"))
 699
 700         # Write something longer, and also try to test a Unicode name if the fs can represent it.
 701         name_u = self.unicode_or_fallback(u"l\u00F8ng", u"long")
 702         d.addCallback(lambda ign: self._check_file(name_u, "test"*100))
 703
 704         # TODO: test that causes an upload failure.
 705         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 706
 707         d.addBoth(self.cleanup)
 708         return d
 709
 710     def _check_mkdir(self, name_u):
 711         return self._check_file(name_u + u"/", "", directory=True)
 712
 713     def _check_file(self, name_u, data, temporary=False, directory=False):
 714         precondition(not (temporary and directory), temporary=temporary, directory=directory)
 715
 716         print "%r._check_file(%r, %r, temporary=%r, directory=%r)" % (self, name_u, data, temporary, directory)
 717         previously_uploaded = self._get_count('uploader.objects_succeeded')
 718         previously_disappeared = self._get_count('uploader.objects_disappeared')
 719
 720         d = self.magicfolder.uploader.set_hook('processed')
 721
 722         path_u = abspath_expanduser_unicode(name_u, base=self.local_dir)
 723         path = to_filepath(path_u)
 724
 725         if directory:
 726             os.mkdir(path_u)
 727             event_mask = self.inotify.IN_CREATE | self.inotify.IN_ISDIR
 728         else:
 729             # We don't use FilePath.setContent() here because it creates a temporary file that
 730             # is renamed into place, which causes events that the test is not expecting.
 731             f = open(path_u, "wb")
 732             try:
 733                 if temporary and sys.platform != "win32":
 734                     os.unlink(path_u)
 735                 f.write(data)
 736             finally:
 737                 f.close()
 738             if temporary and sys.platform == "win32":
 739                 os.unlink(path_u)
 740                 self.notify(path, self.inotify.IN_DELETE, flush=False)
 741             event_mask = self.inotify.IN_CLOSE_WRITE
 742
 743         self.notify(path, event_mask)
 744         encoded_name_u = magicpath.path2magic(name_u)
 745
 746         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 747         if temporary:
 748             d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_disappeared'),
 749                                                                  previously_disappeared + 1))
 750         else:
 751             def _here(res, n):
 752                 print "here %r %r" % (n, res)
 753                 return res
 754             d.addBoth(_here, 1)
 755             d.addCallback(lambda ign: self.upload_dirnode.list())
 756             d.addBoth(_here, 1.5)
 757             d.addCallback(lambda ign: self.upload_dirnode.get(encoded_name_u))
 758             d.addBoth(_here, 2)
 759             d.addCallback(download_to_data)
 760             d.addBoth(_here, 3)
 761             d.addCallback(lambda actual_data: self.failUnlessReallyEqual(actual_data, data))
 762             d.addBoth(_here, 4)
 763             d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'),
 764                                                                  previously_uploaded + 1))
 765
 766         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 767         return d
 768
 769     def _check_version_in_dmd(self, magicfolder, relpath_u, expected_version):
 770         encoded_name_u = magicpath.path2magic(relpath_u)
 771         d = magicfolder.downloader._get_collective_latest_file(encoded_name_u)
 772         def check_latest(result):
 773             if result[0] is not None:
 774                 node, metadata = result
 775                 d.addCallback(lambda ign: self.failUnlessEqual(metadata['version'], expected_version))
 776         d.addCallback(check_latest)
 777         return d
 778
 779     def _check_version_in_local_db(self, magicfolder, relpath_u, expected_version):
 780         db_entry = magicfolder._db.get_db_entry(relpath_u)
 781         if db_entry is not None:
 782             #print "_check_version_in_local_db: %r has version %s" % (relpath_u, version)
 783             self.failUnlessEqual(db_entry.version, expected_version)
 784
 785     def _check_file_gone(self, magicfolder, relpath_u):
 786         path = os.path.join(magicfolder.uploader._local_path_u, relpath_u)
 787         self.assertTrue(not os.path.exists(path))
 788
 789     def _check_uploader_count(self, name, expected, magic=None):
 790         self.failUnlessReallyEqual(self._get_count('uploader.'+name, client=(magic or self.alice_magicfolder)._client),
 791                                    expected)
 792
 793     def _check_downloader_count(self, name, expected, magic=None):
 794         self.failUnlessReallyEqual(self._get_count('downloader.'+name, client=(magic or self.bob_magicfolder)._client),
 795                                    expected)
 796
 797     def test_alice_bob(self):
 798         alice_clock = task.Clock()
 799         bob_clock = task.Clock()
 800         d = self.setup_alice_and_bob(alice_clock, bob_clock)
 801         def _cleanup(ign, magicfolder, clock):
 802             if magicfolder is not None:
 803                 d2 = magicfolder.finish()
 804                 clock.advance(0)
 805                 return d2
 806
 807         def cleanup_Alice_and_Bob(result):
 808             print "cleanup alice bob test\n"
 809             d = defer.succeed(None)
 810             d.addCallback(_cleanup, self.alice_magicfolder, alice_clock)
 811             d.addCallback(_cleanup, self.bob_magicfolder, bob_clock)
 812             d.addCallback(lambda ign: result)
 813             return d
 814         # callback version is added at the end...
 815         d.addErrback(cleanup_Alice_and_Bob)
 816
 817         def _wait_for_Alice(ign, downloaded_d):
 818             print "Now waiting for Alice to download\n"
 819             alice_clock.advance(0)
 820             return downloaded_d
 821
 822         def _wait_for_Bob(ign, downloaded_d):
 823             print "Now waiting for Bob to download\n"
 824             bob_clock.advance(0)
 825             return downloaded_d
 826
 827         def _wait_for(ign, something_to_do, alice=True):
 828             if alice:
 829                 downloaded_d = self.bob_magicfolder.downloader.set_hook('processed')
 830                 uploaded_d = self.alice_magicfolder.uploader.set_hook('processed')
 831             else:
 832                 downloaded_d = self.alice_magicfolder.downloader.set_hook('processed')
 833                 uploaded_d = self.bob_magicfolder.uploader.set_hook('processed')
 834             something_to_do()
 835             if alice:
 836                 print "Waiting for Alice to upload\n"
 837                 alice_clock.advance(0)
 838                 uploaded_d.addCallback(_wait_for_Bob, downloaded_d)
 839             else:
 840                 print "Waiting for Bob to upload\n"
 841                 bob_clock.advance(0)
 842                 uploaded_d.addCallback(_wait_for_Alice, downloaded_d)
 843             return uploaded_d
 844
 845         def Alice_to_write_a_file():
 846             print "Alice writes a file\n"
 847             self.file_path = abspath_expanduser_unicode(u"file1", base=self.alice_magicfolder.uploader._local_path_u)
 848             fileutil.write(self.file_path, "meow, meow meow. meow? meow meow! meow.")
 849             self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 850         d.addCallback(_wait_for, Alice_to_write_a_file)
 851
 852         d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file1", 0))
 853         d.addCallback(lambda ign: self._check_version_in_local_db(self.alice_magicfolder, u"file1", 0))
 854         d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0))
 855         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 1))
 856         d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 1))
 857         d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0))
 858         d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0))
 859         d.addCallback(lambda ign: self._check_uploader_count('objects_conflicted', 0))
 860         d.addCallback(lambda ign: self._check_uploader_count('objects_conflicted', 0, magic=self.bob_magicfolder))
 861
 862         d.addCallback(lambda ign: self._check_version_in_local_db(self.bob_magicfolder, u"file1", 0))
 863         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0))
 864         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 1))
 865         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 0, magic=self.bob_magicfolder))
 866
 867         def Alice_to_delete_file():
 868             print "Alice deletes the file!\n"
 869             shutil.move(self.file_path, self.file_path + u'.backup')
 870             self.notify(to_filepath(self.file_path), self.inotify.IN_DELETE, magic=self.alice_magicfolder)
 871         d.addCallback(_wait_for, Alice_to_delete_file)
 872
 873         def notify_bob_moved(ign):
 874             d0 = self.bob_magicfolder.uploader.set_hook('processed')
 875             p = abspath_expanduser_unicode(u"file1", base=self.bob_magicfolder.uploader._local_path_u)
 876             self.notify(to_filepath(p), self.inotify.IN_MOVED_FROM, magic=self.bob_magicfolder, flush=False)
 877             self.notify(to_filepath(p + u'.backup'), self.inotify.IN_MOVED_TO, magic=self.bob_magicfolder)
 878             bob_clock.advance(0)
 879             return d0
 880         d.addCallback(notify_bob_moved)
 881
 882         # so on just-windows the above doesn't do IN_MOVED_FROM nor
 883         # simulate same in RealTest; trying to compare to unix
 884
 885         d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file1", 1))
 886         d.addCallback(lambda ign: self._check_version_in_local_db(self.alice_magicfolder, u"file1", 1))
 887         d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0))
 888         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 2))
 889         d.addCallback(lambda ign: self._check_uploader_count('objects_not_uploaded', 1, magic=self.bob_magicfolder))
 890         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 1, magic=self.bob_magicfolder))
 891
 892         d.addCallback(lambda ign: self._check_version_in_local_db(self.bob_magicfolder, u"file1", 1))
 893         d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file1", 1))
 894         d.addCallback(lambda ign: self._check_file_gone(self.bob_magicfolder, u"file1"))
 895         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0))
 896         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 2))
 897
 898         def Alice_to_rewrite_file():
 899             print "Alice rewrites file\n"
 900             self.file_path = abspath_expanduser_unicode(u"file1", base=self.alice_magicfolder.uploader._local_path_u)
 901             fileutil.write(self.file_path, "Alice suddenly sees the white rabbit running into the forest.")
 902             self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 903         d.addCallback(_wait_for, Alice_to_rewrite_file)
 904
 905         d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file1", 2))
 906         d.addCallback(lambda ign: self._check_version_in_local_db(self.alice_magicfolder, u"file1", 2))
 907         d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0))
 908         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 3))
 909         d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 3))
 910         d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0))
 911         d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0))
 912         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
 913         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0))
 914
 915         d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file1", 2))
 916         d.addCallback(lambda ign: self._check_version_in_local_db(self.bob_magicfolder, u"file1", 2))
 917         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0))
 918         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 3))
 919
 920         path_u = u"/tmp/magic_folder_test"
 921         encoded_path_u = magicpath.path2magic(u"/tmp/magic_folder_test")
 922
 923         def Alice_tries_to_p0wn_Bob(ign):
 924             print "Alice tries to p0wn Bob\n"
 925             processed_d = self.bob_magicfolder.downloader.set_hook('processed')
 926
 927             # upload a file that would provoke the security bug from #2506
 928             uploadable = Data("", self.alice_magicfolder._client.convergence)
 929             alice_dmd = self.alice_magicfolder.uploader._upload_dirnode
 930
 931             d2 = alice_dmd.add_file(encoded_path_u, uploadable, metadata={"version": 0}, overwrite=True)
 932             d2.addCallback(lambda ign: self.failUnless(alice_dmd.has_child(encoded_path_u)))
 933             d2.addCallback(_wait_for_Bob, processed_d)
 934             return d2
 935         d.addCallback(Alice_tries_to_p0wn_Bob)
 936
 937         d.addCallback(lambda ign: self.failIf(os.path.exists(path_u)))
 938         d.addCallback(lambda ign: self._check_version_in_local_db(self.bob_magicfolder, encoded_path_u, None))
 939         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 3))
 940         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
 941         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0))
 942
 943         def Bob_to_rewrite_file():
 944             print "Bob rewrites file\n"
 945             self.file_path = abspath_expanduser_unicode(u"file1", base=self.bob_magicfolder.uploader._local_path_u)
 946             print "---- bob's file is %r" % (self.file_path,)
 947             fileutil.write(self.file_path, "No white rabbit to be found.")
 948             self.magicfolder = self.bob_magicfolder
 949             self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE)
 950         d.addCallback(lambda ign: _wait_for(None, Bob_to_rewrite_file, alice=False))
 951
 952         d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file1", 3))
 953         d.addCallback(lambda ign: self._check_version_in_local_db(self.bob_magicfolder, u"file1", 3))
 954         d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0, magic=self.bob_magicfolder))
 955         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 2, magic=self.bob_magicfolder))
 956         d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 1, magic=self.bob_magicfolder))
 957         d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0, magic=self.bob_magicfolder))
 958         d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0, magic=self.bob_magicfolder))
 959         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0))
 960
 961         d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file1", 3))
 962         d.addCallback(lambda ign: self._check_version_in_local_db(self.alice_magicfolder, u"file1", 3))
 963         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
 964         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 1, magic=self.alice_magicfolder))
 965         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
 966
 967         def Alice_conflicts_with_Bobs_last_downloaded_uri():
 968             print "Alice conflicts with Bob\n"
 969             downloaded_d = self.bob_magicfolder.downloader.set_hook('processed')
 970             uploadable = Data("do not follow the white rabbit", self.alice_magicfolder._client.convergence)
 971             alice_dmd = self.alice_magicfolder.uploader._upload_dirnode
 972             d2 = alice_dmd.add_file(u"file1", uploadable,
 973                                     metadata={"version": 5,
 974                                               "last_downloaded_uri" : "URI:LIT:" },
 975                                     overwrite=True)
 976             print "Waiting for Alice to upload\n"
 977             d2.addCallback(lambda ign: bob_clock.advance(6))
 978             d2.addCallback(lambda ign: downloaded_d)
 979             d2.addCallback(lambda ign: self.failUnless(alice_dmd.has_child(encoded_path_u)))
 980             return d2
 981
 982         d.addCallback(lambda ign: Alice_conflicts_with_Bobs_last_downloaded_uri())
 983         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 4))
 984         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 1))
 985         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 1, magic=self.alice_magicfolder))
 986         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
 987         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
 988
 989         # prepare to perform another conflict test
 990         def Alice_to_write_file2():
 991             print "Alice writes a file\n"
 992             self.file_path = abspath_expanduser_unicode(u"file2", base=self.alice_magicfolder.uploader._local_path_u)
 993             fileutil.write(self.file_path, "something")
 994             self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 995         d.addCallback(_wait_for, Alice_to_write_file2)
 996         d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file2", 0))
 997         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
 998         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
 999
1000         def Bob_to_rewrite_file2():
1001             print "Bob rewrites file\n"
1002             self.file_path = abspath_expanduser_unicode(u"file2", base=self.bob_magicfolder.uploader._local_path_u)
1003             print "---- bob's file is %r" % (self.file_path,)
1004             fileutil.write(self.file_path, "roger roger. what vector?")
1005             self.magicfolder = self.bob_magicfolder
1006             self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE)
1007         d.addCallback(lambda ign: _wait_for(None, Bob_to_rewrite_file2, alice=False))
1008         d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file2", 1))
1009         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 5))
1010         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 1))
1011         d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0, magic=self.bob_magicfolder))
1012         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 3, magic=self.bob_magicfolder))
1013         d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 2, magic=self.bob_magicfolder))
1014         d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0, magic=self.bob_magicfolder))
1015         d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0, magic=self.bob_magicfolder))
1016         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
1017         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
1018         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 2, magic=self.alice_magicfolder))
1019
1020         # XXX here we advance the clock and then test again to make sure no values are monotonically increasing
1021         # with each queue turn ;-p
1022         alice_clock.advance(6)
1023         bob_clock.advance(6)
1024         d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file2", 1))
1025         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 5))
1026         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 1))
1027         d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0, magic=self.bob_magicfolder))
1028         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 3, magic=self.bob_magicfolder))
1029         d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 2, magic=self.bob_magicfolder))
1030         d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0, magic=self.bob_magicfolder))
1031         d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0, magic=self.bob_magicfolder))
1032         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
1033         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
1034         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 2, magic=self.alice_magicfolder))
1035
1036         def Alice_conflicts_with_Bobs_last_uploaded_uri():
1037             print "Alice conflicts with Bob\n"
1038             encoded_path_u = magicpath.path2magic(u"file2")
1039             downloaded_d = self.bob_magicfolder.downloader.set_hook('processed')
1040             uploadable = Data("rabbits with sharp fangs", self.alice_magicfolder._client.convergence)
1041             alice_dmd = self.alice_magicfolder.uploader._upload_dirnode
1042             d2 = alice_dmd.add_file(u"file2", uploadable,
1043                                     metadata={"version": 5,
1044                                               "last_uploaded_uri" : "URI:LIT:" },
1045                                     overwrite=True)
1046             print "Waiting for Alice to upload\n"
1047             d2.addCallback(lambda ign: bob_clock.advance(6))
1048             d2.addCallback(lambda ign: downloaded_d)
1049             d2.addCallback(lambda ign: self.failUnless(alice_dmd.has_child(encoded_path_u)))
1050             return d2
1051         d.addCallback(lambda ign: Alice_conflicts_with_Bobs_last_uploaded_uri())
1052         d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file2", 5))
1053         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 6))
1054         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 2))
1055         d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0, magic=self.bob_magicfolder))
1056         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 3, magic=self.bob_magicfolder))
1057         d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 2, magic=self.bob_magicfolder))
1058         d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0, magic=self.bob_magicfolder))
1059         d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0, magic=self.bob_magicfolder))
1060         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
1061         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
1062         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 2, magic=self.alice_magicfolder))
1063
1064         alice_clock.advance(6)
1065         bob_clock.advance(6)
1066         alice_clock.advance(6)
1067         bob_clock.advance(6)
1068
1069         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 2, magic=self.alice_magicfolder))
1070         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
1071         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 2))
1072         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 6))
1073
1074         # prepare to perform another conflict test
1075         def Alice_to_write_file3():
1076             print "Alice writes a file\n"
1077             self.file_path = abspath_expanduser_unicode(u"file3", base=self.alice_magicfolder.uploader._local_path_u)
1078             fileutil.write(self.file_path, "something")
1079             self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
1080         d.addCallback(_wait_for, Alice_to_write_file3)
1081         d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file3", 0))
1082         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
1083         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 7))
1084         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 2, magic=self.alice_magicfolder))
1085         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 2))
1086         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
1087
1088         def Bob_to_rewrite_file3():
1089             print "Bob rewrites file\n"
1090             self.file_path = abspath_expanduser_unicode(u"file3", base=self.bob_magicfolder.uploader._local_path_u)
1091             print "---- bob's file is %r" % (self.file_path,)
1092             fileutil.write(self.file_path, "roger roger")
1093             self.magicfolder = self.bob_magicfolder
1094             self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE)
1095         d.addCallback(lambda ign: _wait_for(None, Bob_to_rewrite_file3, alice=False))
1096         d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file3", 1))
1097         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 7))
1098         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 2))
1099         d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0, magic=self.bob_magicfolder))
1100         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 4, magic=self.bob_magicfolder))
1101         d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 3, magic=self.bob_magicfolder))
1102         d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0, magic=self.bob_magicfolder))
1103         d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0, magic=self.bob_magicfolder))
1104         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
1105         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
1106         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 3, magic=self.alice_magicfolder))
1107
1108         d.addCallback(cleanup_Alice_and_Bob)
1109         return d
1110
1111
1112 class MockTest(MagicFolderTestMixin, unittest.TestCase):
1113     """This can run on any platform, and even if twisted.internet.inotify can't be imported."""
1114
1115     def setUp(self):
1116         MagicFolderTestMixin.setUp(self)
1117         self.inotify = fake_inotify
1118         self.patch(magic_folder, 'get_inotify_module', lambda: self.inotify)
1119
1120     def notify(self, path, mask, magic=None, flush=True):
1121         if magic is None:
1122             magic = self.magicfolder
1123         magic.uploader._notifier.event(path, mask)
1124         # no flush for the mock test.
1125
1126     def test_errors(self):
1127         self.set_up_grid()
1128
1129         errors_dir = abspath_expanduser_unicode(u"errors_dir", base=self.basedir)
1130         os.mkdir(errors_dir)
1131         not_a_dir = abspath_expanduser_unicode(u"NOT_A_DIR", base=self.basedir)
1132         fileutil.write(not_a_dir, "")
1133         magicfolderdb = abspath_expanduser_unicode(u"magicfolderdb", base=self.basedir)
1134         doesnotexist  = abspath_expanduser_unicode(u"doesnotexist", base=self.basedir)
1135
1136         client = self.g.clients[0]
1137         d = client.create_dirnode()
1138         def _check_errors(n):
1139             self.failUnless(IDirectoryNode.providedBy(n))
1140             upload_dircap = n.get_uri()
1141             readonly_dircap = n.get_readonly_uri()
1142
1143             self.shouldFail(AssertionError, 'nonexistent local.directory', 'there is no directory',
1144                             MagicFolder, client, upload_dircap, '', doesnotexist, magicfolderdb, 0077)
1145             self.shouldFail(AssertionError, 'non-directory local.directory', 'is not a directory',
1146                             MagicFolder, client, upload_dircap, '', not_a_dir, magicfolderdb, 0077)
1147             self.shouldFail(AssertionError, 'bad upload.dircap', 'does not refer to a directory',
1148                             MagicFolder, client, 'bad', '', errors_dir, magicfolderdb, 0077)
1149             self.shouldFail(AssertionError, 'non-directory upload.dircap', 'does not refer to a directory',
1150                             MagicFolder, client, 'URI:LIT:foo', '', errors_dir, magicfolderdb, 0077)
1151             self.shouldFail(AssertionError, 'readonly upload.dircap', 'is not a writecap to a directory',
1152                             MagicFolder, client, readonly_dircap, '', errors_dir, magicfolderdb, 0077)
1153             self.shouldFail(AssertionError, 'collective dircap', 'is not a readonly cap to a directory',
1154                             MagicFolder, client, upload_dircap, upload_dircap, errors_dir, magicfolderdb, 0077)
1155
1156             def _not_implemented():
1157                 raise NotImplementedError("blah")
1158             self.patch(magic_folder, 'get_inotify_module', _not_implemented)
1159             self.shouldFail(NotImplementedError, 'unsupported', 'blah',
1160                             MagicFolder, client, upload_dircap, '', errors_dir, magicfolderdb, 0077)
1161         d.addCallback(_check_errors)
1162         return d
1163
1164     def test_write_downloaded_file(self):
1165         workdir = u"cli/MagicFolder/write-downloaded-file"
1166         local_file = fileutil.abspath_expanduser_unicode(os.path.join(workdir, "foobar"))
1167
1168         class TestWriteFileMixin(WriteFileMixin):
1169             def _log(self, msg):
1170                 pass
1171
1172         writefile = TestWriteFileMixin()
1173         writefile._umask = 0077
1174
1175         # create a file with name "foobar" with content "foo"
1176         # write downloaded file content "bar" into "foobar" with is_conflict = False
1177         fileutil.make_dirs(workdir)
1178         fileutil.write(local_file, "foo")
1179
1180         # if is_conflict is False, then the .conflict file shouldn't exist.
1181         writefile._write_downloaded_file(local_file, "bar", False, None)
1182         conflicted_path = local_file + u".conflict"
1183         self.failIf(os.path.exists(conflicted_path))
1184
1185         # At this point, the backup file should exist with content "foo"
1186         backup_path = local_file + u".backup"
1187         self.failUnless(os.path.exists(backup_path))
1188         self.failUnlessEqual(fileutil.read(backup_path), "foo")
1189
1190         # .tmp file shouldn't exist
1191         self.failIf(os.path.exists(local_file + u".tmp"))
1192
1193         # .. and the original file should have the new content
1194         self.failUnlessEqual(fileutil.read(local_file), "bar")
1195
1196         # now a test for conflicted case
1197         writefile._write_downloaded_file(local_file, "bar", True, None)
1198         self.failUnless(os.path.exists(conflicted_path))
1199
1200         # .tmp file shouldn't exist
1201         self.failIf(os.path.exists(local_file + u".tmp"))
1202
1203
1204 class RealTest(MagicFolderTestMixin, unittest.TestCase):
1205     """This is skipped unless both Twisted and the platform support inotify."""
1206
1207     def setUp(self):
1208         MagicFolderTestMixin.setUp(self)
1209         self.inotify = magic_folder.get_inotify_module()
1210
1211     def notify(self, path, mask, magic=None, flush=True):
1212         # Writing to the filesystem causes the notification.
1213         # However, flushing filesystem buffers may be necessary on Windows.
1214         if flush:
1215             try:
1216                 pass#fileutil.flush_volume(path.path)
1217             except Exception:
1218                 pass  # e.g. permission errors
1219
1220 try:
1221     magic_folder.get_inotify_module()
1222 except NotImplementedError:
1223     RealTest.skip = "Magic Folder support can only be tested for-real on an OS that supports inotify or equivalent."