src/allmydata/test/test_magic_folder.py

   1
   2 import os, sys
   3
   4 from twisted.trial import unittest
   5 from twisted.internet import defer, task
   6
   7 from allmydata.interfaces import IDirectoryNode
   8 from allmydata.util.assertutil import precondition
   9
  10 from allmydata.util import fake_inotify, fileutil
  11 from allmydata.util.encodingutil import get_filesystem_encoding, to_filepath
  12 from allmydata.util.consumer import download_to_data
  13 from allmydata.test.no_network import GridTestMixin
  14 from allmydata.test.common_util import ReallyEqualMixin, NonASCIIPathMixin
  15 from allmydata.test.common import ShouldFailMixin
  16 from .test_cli_magic_folder import MagicFolderCLITestMixin
  17
  18 from allmydata.frontends import magic_folder
  19 from allmydata.frontends.magic_folder import MagicFolder, Downloader, WriteFileMixin
  20 from allmydata import magicfolderdb, magicpath
  21 from allmydata.util.fileutil import abspath_expanduser_unicode
  22 from allmydata.immutable.upload import Data
  23
  24
  25 class MagicFolderTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, ReallyEqualMixin, NonASCIIPathMixin):
  26     """
  27     These tests will be run both with a mock notifier, and (on platforms that support it)
  28     with the real INotify.
  29     """
  30
  31     def setUp(self):
  32         GridTestMixin.setUp(self)
  33         temp = self.mktemp()
  34         self.basedir = abspath_expanduser_unicode(temp.decode(get_filesystem_encoding()))
  35         self.magicfolder = None
  36         self.patch(Downloader, 'REMOTE_SCAN_INTERVAL', 0)
  37
  38     def _get_count(self, name, client=None):
  39         counters = (client or self.get_client()).stats_provider.get_stats()["counters"]
  40         return counters.get('magic_folder.%s' % (name,), 0)
  41
  42     def _createdb(self):
  43         dbfile = abspath_expanduser_unicode(u"magicfolderdb.sqlite", base=self.basedir)
  44         mdb = magicfolderdb.get_magicfolderdb(dbfile, create_version=(magicfolderdb.SCHEMA_v1, 1))
  45         self.failUnless(mdb, "unable to create magicfolderdb from %r" % (dbfile,))
  46         self.failUnlessEqual(mdb.VERSION, 1)
  47         return mdb
  48
  49     def _restart_client(self, ign):
  50         #print "_restart_client"
  51         d = self.restart_client()
  52         d.addCallback(self._wait_until_started)
  53         return d
  54
  55     def _wait_until_started(self, ign):
  56         #print "_wait_until_started"
  57         self.magicfolder = self.get_client().getServiceNamed('magic-folder')
  58         return self.magicfolder.ready()
  59
  60     def test_db_basic(self):
  61         fileutil.make_dirs(self.basedir)
  62         self._createdb()
  63
  64     def test_db_persistence(self):
  65         """Test that a file upload creates an entry in the database."""
  66
  67         fileutil.make_dirs(self.basedir)
  68         db = self._createdb()
  69
  70         relpath1 = u"myFile1"
  71         pathinfo = fileutil.PathInfo(isdir=False, isfile=True, islink=False,
  72                                      exists=True, size=1, mtime=123, ctime=456)
  73         db.did_upload_version(relpath1, 0, 'URI:LIT:1', 'URI:LIT:0', 0, pathinfo)
  74
  75         c = db.cursor
  76         c.execute("SELECT size, mtime, ctime"
  77                   " FROM local_files"
  78                   " WHERE path=?",
  79                   (relpath1,))
  80         row = c.fetchone()
  81         self.failUnlessEqual(row, (pathinfo.size, pathinfo.mtime, pathinfo.ctime))
  82
  83         # Second test uses db.is_new_file instead of SQL query directly
  84         # to confirm the previous upload entry in the db.
  85         relpath2 = u"myFile2"
  86         path2 = os.path.join(self.basedir, relpath2)
  87         fileutil.write(path2, "meow\n")
  88         pathinfo = fileutil.get_pathinfo(path2)
  89         db.did_upload_version(relpath2, 0, 'URI:LIT:2', 'URI:LIT:1', 0, pathinfo)
  90         self.failUnlessFalse(db.is_new_file(pathinfo, relpath2))
  91
  92         different_pathinfo = fileutil.PathInfo(isdir=False, isfile=True, islink=False,
  93                                                exists=True, size=0, mtime=pathinfo.mtime, ctime=pathinfo.ctime)
  94         self.failUnlessTrue(db.is_new_file(different_pathinfo, relpath2))
  95
  96     def test_magicfolder_start_service(self):
  97         self.set_up_grid()
  98
  99         self.local_dir = abspath_expanduser_unicode(self.unicode_or_fallback(u"l\u00F8cal_dir", u"local_dir"),
 100                                                     base=self.basedir)
 101         self.mkdir_nonascii(self.local_dir)
 102
 103         d = defer.succeed(None)
 104         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.dirs_monitored'), 0))
 105
 106         d.addCallback(lambda ign: self.create_invite_join_magic_folder(u"Alice", self.local_dir))
 107         d.addCallback(self._restart_client)
 108
 109         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.dirs_monitored'), 1))
 110         d.addBoth(self.cleanup)
 111         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.dirs_monitored'), 0))
 112         return d
 113
 114     def test_move_tree(self):
 115         self.set_up_grid()
 116
 117         self.local_dir = abspath_expanduser_unicode(self.unicode_or_fallback(u"l\u00F8cal_dir", u"local_dir"),
 118                                                     base=self.basedir)
 119         self.mkdir_nonascii(self.local_dir)
 120
 121         empty_tree_name = self.unicode_or_fallback(u"empty_tr\u00EAe", u"empty_tree")
 122         empty_tree_dir = abspath_expanduser_unicode(empty_tree_name, base=self.basedir)
 123         new_empty_tree_dir = abspath_expanduser_unicode(empty_tree_name, base=self.local_dir)
 124
 125         small_tree_name = self.unicode_or_fallback(u"small_tr\u00EAe", u"empty_tree")
 126         small_tree_dir = abspath_expanduser_unicode(small_tree_name, base=self.basedir)
 127         new_small_tree_dir = abspath_expanduser_unicode(small_tree_name, base=self.local_dir)
 128
 129         d = self.create_invite_join_magic_folder(u"Alice", self.local_dir)
 130         d.addCallback(self._restart_client)
 131
 132         def _check_move_empty_tree(res):
 133             print "_check_move_empty_tree"
 134             uploaded_d = self.magicfolder.uploader.set_hook('processed')
 135             self.mkdir_nonascii(empty_tree_dir)
 136             os.rename(empty_tree_dir, new_empty_tree_dir)
 137             self.notify(to_filepath(new_empty_tree_dir), self.inotify.IN_MOVED_TO)
 138
 139             return uploaded_d
 140         d.addCallback(_check_move_empty_tree)
 141         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 142         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 1))
 143         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.files_uploaded'), 0))
 144         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 145         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.directories_created'), 1))
 146
 147         def _check_move_small_tree(res):
 148             print "_check_move_small_tree"
 149             uploaded_d = self.magicfolder.uploader.set_hook('processed', ignore_count=1)
 150             self.mkdir_nonascii(small_tree_dir)
 151             what_path = abspath_expanduser_unicode(u"what", base=small_tree_dir)
 152             fileutil.write(what_path, "say when")
 153             os.rename(small_tree_dir, new_small_tree_dir)
 154             self.notify(to_filepath(new_small_tree_dir), self.inotify.IN_MOVED_TO)
 155
 156             return uploaded_d
 157         d.addCallback(_check_move_small_tree)
 158         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 159         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 3))
 160         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.files_uploaded'), 1))
 161         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 162         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.directories_created'), 2))
 163
 164         def _check_moved_tree_is_watched(res):
 165             print "_check_moved_tree_is_watched"
 166             uploaded_d = self.magicfolder.uploader.set_hook('processed')
 167             another_path = abspath_expanduser_unicode(u"another", base=new_small_tree_dir)
 168             fileutil.write(another_path, "file")
 169             self.notify(to_filepath(another_path), self.inotify.IN_CLOSE_WRITE)
 170
 171             return uploaded_d
 172         d.addCallback(_check_moved_tree_is_watched)
 173         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 174         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 4))
 175         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.files_uploaded'), 2))
 176         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 177         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.directories_created'), 2))
 178
 179         # Files that are moved out of the upload directory should no longer be watched.
 180         #def _move_dir_away(ign):
 181         #    os.rename(new_empty_tree_dir, empty_tree_dir)
 182         #    # Wuh? Why don't we get this event for the real test?
 183         #    #self.notify(to_filepath(new_empty_tree_dir), self.inotify.IN_MOVED_FROM)
 184         #d.addCallback(_move_dir_away)
 185         #def create_file(val):
 186         #    test_file = abspath_expanduser_unicode(u"what", base=empty_tree_dir)
 187         #    fileutil.write(test_file, "meow")
 188         #    #self.notify(...)
 189         #    return
 190         #d.addCallback(create_file)
 191         #d.addCallback(lambda ign: time.sleep(1))  # XXX ICK
 192         #d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 193         #d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 4))
 194         #d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.files_uploaded'), 2))
 195         #d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 196         #d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.directories_created'), 2))
 197
 198         d.addBoth(self.cleanup)
 199         return d
 200
 201     def test_persistence(self):
 202         """
 203         Perform an upload of a given file and then stop the client.
 204         Start a new client and magic-folder service... and verify that the file is NOT uploaded
 205         a second time. This test is meant to test the database persistence along with
 206         the startup and shutdown code paths of the magic-folder service.
 207         """
 208         self.set_up_grid()
 209         self.local_dir = abspath_expanduser_unicode(u"test_persistence", base=self.basedir)
 210         self.mkdir_nonascii(self.local_dir)
 211         self.collective_dircap = ""
 212
 213         d = defer.succeed(None)
 214         d.addCallback(lambda ign: self.create_invite_join_magic_folder(u"Alice", self.local_dir))
 215         d.addCallback(self._restart_client)
 216
 217         def create_test_file(filename):
 218             d2 = self.magicfolder.uploader.set_hook('processed')
 219             test_file = abspath_expanduser_unicode(filename, base=self.local_dir)
 220             fileutil.write(test_file, "meow %s" % filename)
 221             self.notify(to_filepath(test_file), self.inotify.IN_CLOSE_WRITE)
 222             return d2
 223         d.addCallback(lambda ign: create_test_file(u"what1"))
 224         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 225         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 1))
 226         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 227         d.addCallback(self.cleanup)
 228
 229         d.addCallback(self._restart_client)
 230         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 231         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 1))
 232         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 233         d.addCallback(lambda ign: create_test_file(u"what2"))
 234         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 235         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 2))
 236         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 237         d.addBoth(self.cleanup)
 238         return d
 239
 240     @defer.inlineCallbacks
 241     def test_delete(self):
 242         self.set_up_grid()
 243         self.local_dir = os.path.join(self.basedir, u"local_dir")
 244         self.mkdir_nonascii(self.local_dir)
 245
 246         yield self.create_invite_join_magic_folder(u"Alice\u0101", self.local_dir)
 247         yield self._restart_client(None)
 248
 249         try:
 250             # create a file
 251             up_proc = self.magicfolder.uploader.set_hook('processed')
 252             # down_proc = self.magicfolder.downloader.set_hook('processed')
 253             path = os.path.join(self.local_dir, u'foo')
 254             fileutil.write(path, 'foo\n')
 255             self.notify(to_filepath(path), self.inotify.IN_CLOSE_WRITE)
 256             yield up_proc
 257             self.assertTrue(os.path.exists(path))
 258
 259             # the real test part: delete the file
 260             up_proc = self.magicfolder.uploader.set_hook('processed')
 261             os.unlink(path)
 262             self.notify(to_filepath(path), self.inotify.IN_DELETE)
 263             yield up_proc
 264             self.assertFalse(os.path.exists(path))
 265
 266             # ensure we still have a DB entry, and that the version is 1
 267             node, metadata = yield self.magicfolder.downloader._get_collective_latest_file(u'foo')
 268             self.assertTrue(node is not None, "Failed to find %r in DMD" % (path,))
 269             self.failUnlessEqual(metadata['version'], 1)
 270
 271         finally:
 272             yield self.cleanup(None)
 273
 274     @defer.inlineCallbacks
 275     def test_delete_and_restore(self):
 276         self.set_up_grid()
 277         self.local_dir = os.path.join(self.basedir, u"local_dir")
 278         self.mkdir_nonascii(self.local_dir)
 279
 280         yield self.create_invite_join_magic_folder(u"Alice\u0101", self.local_dir)
 281         yield self._restart_client(None)
 282
 283         try:
 284             # create a file
 285             up_proc = self.magicfolder.uploader.set_hook('processed')
 286             # down_proc = self.magicfolder.downloader.set_hook('processed')
 287             path = os.path.join(self.local_dir, u'foo')
 288             fileutil.write(path, 'foo\n')
 289             self.notify(to_filepath(path), self.inotify.IN_CLOSE_WRITE)
 290             yield up_proc
 291             self.assertTrue(os.path.exists(path))
 292
 293             # delete the file
 294             up_proc = self.magicfolder.uploader.set_hook('processed')
 295             os.unlink(path)
 296             self.notify(to_filepath(path), self.inotify.IN_DELETE)
 297             yield up_proc
 298             self.assertFalse(os.path.exists(path))
 299
 300             # ensure we still have a DB entry, and that the version is 1
 301             node, metadata = yield self.magicfolder.downloader._get_collective_latest_file(u'foo')
 302             self.assertTrue(node is not None, "Failed to find %r in DMD" % (path,))
 303             self.failUnlessEqual(metadata['version'], 1)
 304
 305             # restore the file, with different contents
 306             up_proc = self.magicfolder.uploader.set_hook('processed')
 307             path = os.path.join(self.local_dir, u'foo')
 308             fileutil.write(path, 'bar\n')
 309             self.notify(to_filepath(path), self.inotify.IN_CLOSE_WRITE)
 310             yield up_proc
 311
 312             # ensure we still have a DB entry, and that the version is 2
 313             node, metadata = yield self.magicfolder.downloader._get_collective_latest_file(u'foo')
 314             self.assertTrue(node is not None, "Failed to find %r in DMD" % (path,))
 315             self.failUnlessEqual(metadata['version'], 2)
 316
 317         finally:
 318             yield self.cleanup(None)
 319
 320     @defer.inlineCallbacks
 321     def test_alice_delete_bob_restore(self):
 322         alice_clock = task.Clock()
 323         bob_clock = task.Clock()
 324         yield self.setup_alice_and_bob(alice_clock, bob_clock)
 325         alice_dir = self.alice_magicfolder.uploader._local_path_u
 326         bob_dir = self.bob_magicfolder.uploader._local_path_u
 327         alice_fname = os.path.join(alice_dir, 'blam')
 328         bob_fname = os.path.join(bob_dir, 'blam')
 329
 330         try:
 331             # alice creates a file, bob downloads it
 332             alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
 333             bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
 334
 335             fileutil.write(alice_fname, 'contents0\n')
 336             self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 337
 338             alice_clock.advance(0)
 339             yield alice_proc  # alice uploads
 340
 341             bob_clock.advance(0)
 342             yield bob_proc    # bob downloads
 343
 344             # check the state
 345             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
 346             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 0)
 347             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
 348             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 0)
 349             yield self.failUnlessReallyEqual(
 350                 self._get_count('downloader.objects_failed', client=self.bob_magicfolder._client),
 351                 0
 352             )
 353             yield self.failUnlessReallyEqual(
 354                 self._get_count('downloader.objects_downloaded', client=self.bob_magicfolder._client),
 355                 1
 356             )
 357
 358             print("BOB DELETE")
 359             # now bob deletes it (bob should upload, alice download)
 360             bob_proc = self.bob_magicfolder.uploader.set_hook('processed')
 361             alice_proc = self.alice_magicfolder.downloader.set_hook('processed')
 362             os.unlink(bob_fname)
 363             self.notify(to_filepath(bob_fname), self.inotify.IN_DELETE, magic=self.bob_magicfolder)
 364
 365             bob_clock.advance(0)
 366             yield bob_proc
 367             alice_clock.advance(0)
 368             yield alice_proc
 369
 370             # check versions
 371             node, metadata = yield self.alice_magicfolder.downloader._get_collective_latest_file(u'blam')
 372             self.assertTrue(metadata['deleted'])
 373             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
 374             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 1)
 375             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
 376             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 1)
 377
 378             print("ALICE RESTORE")
 379             # now alice restores it (alice should upload, bob download)
 380             alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
 381             bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
 382             fileutil.write(alice_fname, 'new contents\n')
 383             self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 384
 385             alice_clock.advance(0)
 386             yield alice_proc
 387             bob_clock.advance(0)
 388             yield bob_proc
 389
 390             # check versions
 391             node, metadata = yield self.alice_magicfolder.downloader._get_collective_latest_file(u'blam')
 392             self.assertTrue('deleted' not in metadata or not metadata['deleted'])
 393             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 2)
 394             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 2)
 395             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 2)
 396             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 2)
 397
 398         finally:
 399             # cleanup
 400             d0 = self.alice_magicfolder.finish()
 401             alice_clock.advance(0)
 402             yield d0
 403
 404             d1 = self.bob_magicfolder.finish()
 405             bob_clock.advance(0)
 406             yield d1
 407
 408     @defer.inlineCallbacks
 409     def test_alice_sees_bobs_delete_with_error(self):
 410         # alice creates a file, bob deletes it -- and we also arrange
 411         # for Alice's file to have "gone missing" as well.
 412         alice_clock = task.Clock()
 413         bob_clock = task.Clock()
 414         yield self.setup_alice_and_bob(alice_clock, bob_clock)
 415         alice_dir = self.alice_magicfolder.uploader._local_path_u
 416         bob_dir = self.bob_magicfolder.uploader._local_path_u
 417         alice_fname = os.path.join(alice_dir, 'blam')
 418         bob_fname = os.path.join(bob_dir, 'blam')
 419
 420         try:
 421             # alice creates a file, bob downloads it
 422             alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
 423             bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
 424
 425             fileutil.write(alice_fname, 'contents0\n')
 426             self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 427
 428             alice_clock.advance(0)
 429             yield alice_proc  # alice uploads
 430
 431             bob_clock.advance(0)
 432             yield bob_proc    # bob downloads
 433
 434             # check the state
 435             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
 436             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 0)
 437             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
 438             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 0)
 439             yield self.failUnlessReallyEqual(
 440                 self._get_count('downloader.objects_failed', client=self.bob_magicfolder._client),
 441                 0
 442             )
 443             yield self.failUnlessReallyEqual(
 444                 self._get_count('downloader.objects_downloaded', client=self.bob_magicfolder._client),
 445                 1
 446             )
 447
 448             print("BOB DELETE")
 449             # now bob deletes it (bob should upload, alice download)
 450             bob_proc = self.bob_magicfolder.uploader.set_hook('processed')
 451             alice_proc = self.alice_magicfolder.downloader.set_hook('processed')
 452             os.unlink(bob_fname)
 453             self.notify(to_filepath(bob_fname), self.inotify.IN_DELETE, magic=self.bob_magicfolder)
 454             # just after notifying bob, we also delete alice's,
 455             # covering the 'except' flow in _rename_deleted_file()
 456             os.unlink(alice_fname)
 457
 458             bob_clock.advance(0)
 459             yield bob_proc
 460             alice_clock.advance(0)
 461             yield alice_proc
 462
 463             # check versions
 464             node, metadata = yield self.alice_magicfolder.downloader._get_collective_latest_file(u'blam')
 465             self.assertTrue(metadata['deleted'])
 466             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
 467             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 1)
 468             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
 469             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 1)
 470
 471         finally:
 472             # cleanup
 473             d0 = self.alice_magicfolder.finish()
 474             alice_clock.advance(0)
 475             yield d0
 476
 477             d1 = self.bob_magicfolder.finish()
 478             bob_clock.advance(0)
 479             yield d1
 480
 481     @defer.inlineCallbacks
 482     def test_alice_create_bob_update(self):
 483         alice_clock = task.Clock()
 484         bob_clock = task.Clock()
 485         yield self.setup_alice_and_bob(alice_clock, bob_clock)
 486         alice_dir = self.alice_magicfolder.uploader._local_path_u
 487         bob_dir = self.bob_magicfolder.uploader._local_path_u
 488         alice_fname = os.path.join(alice_dir, 'blam')
 489         bob_fname = os.path.join(bob_dir, 'blam')
 490
 491         try:
 492             # alice creates a file, bob downloads it
 493             alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
 494             bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
 495
 496             fileutil.write(alice_fname, 'contents0\n')
 497             self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 498
 499             alice_clock.advance(0)
 500             yield alice_proc  # alice uploads
 501
 502             bob_clock.advance(0)
 503             yield bob_proc    # bob downloads
 504
 505             # check the state
 506             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
 507             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 0)
 508             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
 509             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 0)
 510             yield self.failUnlessReallyEqual(
 511                 self._get_count('downloader.objects_failed', client=self.bob_magicfolder._client),
 512                 0
 513             )
 514             yield self.failUnlessReallyEqual(
 515                 self._get_count('downloader.objects_downloaded', client=self.bob_magicfolder._client),
 516                 1
 517             )
 518
 519             # now bob updates it (bob should upload, alice download)
 520             bob_proc = self.bob_magicfolder.uploader.set_hook('processed')
 521             alice_proc = self.alice_magicfolder.downloader.set_hook('processed')
 522             fileutil.write(bob_fname, 'bob wuz here\n')
 523             self.notify(to_filepath(bob_fname), self.inotify.IN_CLOSE_WRITE, magic=self.bob_magicfolder)
 524
 525             bob_clock.advance(0)
 526             yield bob_proc
 527             alice_clock.advance(0)
 528             yield alice_proc
 529
 530             # check the state
 531             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
 532             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 1)
 533             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
 534             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 1)
 535
 536         finally:
 537             # cleanup
 538             d0 = self.alice_magicfolder.finish()
 539             alice_clock.advance(0)
 540             yield d0
 541
 542             d1 = self.bob_magicfolder.finish()
 543             bob_clock.advance(0)
 544             yield d1
 545
 546     @defer.inlineCallbacks
 547     def test_alice_delete_and_restore(self):
 548         alice_clock = task.Clock()
 549         bob_clock = task.Clock()
 550         yield self.setup_alice_and_bob(alice_clock, bob_clock)
 551         alice_dir = self.alice_magicfolder.uploader._local_path_u
 552         bob_dir = self.bob_magicfolder.uploader._local_path_u
 553         alice_fname = os.path.join(alice_dir, 'blam')
 554         bob_fname = os.path.join(bob_dir, 'blam')
 555
 556         try:
 557             # alice creates a file, bob downloads it
 558             alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
 559             bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
 560
 561             fileutil.write(alice_fname, 'contents0\n')
 562             self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 563
 564             alice_clock.advance(0)
 565             yield alice_proc  # alice uploads
 566
 567             bob_clock.advance(0)
 568             yield bob_proc    # bob downloads
 569
 570             # check the state
 571             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
 572             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 0)
 573             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
 574             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 0)
 575             yield self.failUnlessReallyEqual(
 576                 self._get_count('downloader.objects_failed', client=self.bob_magicfolder._client),
 577                 0
 578             )
 579             yield self.failUnlessReallyEqual(
 580                 self._get_count('downloader.objects_downloaded', client=self.bob_magicfolder._client),
 581                 1
 582             )
 583             self.failUnless(os.path.exists(bob_fname))
 584
 585             # now alice deletes it (alice should upload, bob download)
 586             alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
 587             bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
 588             os.unlink(alice_fname)
 589             self.notify(to_filepath(alice_fname), self.inotify.IN_DELETE, magic=self.alice_magicfolder)
 590
 591             alice_clock.advance(0)
 592             yield alice_proc
 593             bob_clock.advance(0)
 594             yield bob_proc
 595
 596             # check the state
 597             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
 598             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 1)
 599             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
 600             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 1)
 601             self.failIf(os.path.exists(bob_fname))
 602
 603             # now alice restores the file (with new contents)
 604             alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
 605             bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
 606             fileutil.write(alice_fname, 'alice wuz here\n')
 607             self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 608
 609             alice_clock.advance(0)
 610             yield alice_proc
 611             bob_clock.advance(0)
 612             yield bob_proc
 613
 614             # check the state
 615             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 2)
 616             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 2)
 617             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 2)
 618             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 2)
 619             self.failUnless(os.path.exists(bob_fname))
 620
 621         finally:
 622             # cleanup
 623             d0 = self.alice_magicfolder.finish()
 624             alice_clock.advance(0)
 625             yield d0
 626
 627             d1 = self.bob_magicfolder.finish()
 628             bob_clock.advance(0)
 629             yield d1
 630
 631     def test_magic_folder(self):
 632         self.set_up_grid()
 633         self.local_dir = os.path.join(self.basedir, self.unicode_or_fallback(u"loc\u0101l_dir", u"local_dir"))
 634         self.mkdir_nonascii(self.local_dir)
 635
 636         d = self.create_invite_join_magic_folder(u"Alice\u0101", self.local_dir)
 637         d.addCallback(self._restart_client)
 638
 639         # Write something short enough for a LIT file.
 640         d.addCallback(lambda ign: self._check_file(u"short", "test"))
 641
 642         # Write to the same file again with different data.
 643         d.addCallback(lambda ign: self._check_file(u"short", "different"))
 644
 645         # Test that temporary files are not uploaded.
 646         d.addCallback(lambda ign: self._check_file(u"tempfile", "test", temporary=True))
 647
 648         # Test creation of a subdirectory.
 649         d.addCallback(lambda ign: self._check_mkdir(u"directory"))
 650
 651         # Write something longer, and also try to test a Unicode name if the fs can represent it.
 652         name_u = self.unicode_or_fallback(u"l\u00F8ng", u"long")
 653         d.addCallback(lambda ign: self._check_file(name_u, "test"*100))
 654
 655         # TODO: test that causes an upload failure.
 656         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 657
 658         d.addBoth(self.cleanup)
 659         return d
 660
 661     def _check_mkdir(self, name_u):
 662         return self._check_file(name_u + u"/", "", directory=True)
 663
 664     def _check_file(self, name_u, data, temporary=False, directory=False):
 665         precondition(not (temporary and directory), temporary=temporary, directory=directory)
 666
 667         print "%r._check_file(%r, %r, temporary=%r, directory=%r)" % (self, name_u, data, temporary, directory)
 668         previously_uploaded = self._get_count('uploader.objects_succeeded')
 669         previously_disappeared = self._get_count('uploader.objects_disappeared')
 670
 671         d = self.magicfolder.uploader.set_hook('processed')
 672
 673         path_u = abspath_expanduser_unicode(name_u, base=self.local_dir)
 674         path = to_filepath(path_u)
 675
 676         if directory:
 677             os.mkdir(path_u)
 678             event_mask = self.inotify.IN_CREATE | self.inotify.IN_ISDIR
 679         else:
 680             # We don't use FilePath.setContent() here because it creates a temporary file that
 681             # is renamed into place, which causes events that the test is not expecting.
 682             f = open(path_u, "wb")
 683             try:
 684                 if temporary and sys.platform != "win32":
 685                     os.unlink(path_u)
 686                 f.write(data)
 687             finally:
 688                 f.close()
 689             if temporary and sys.platform == "win32":
 690                 os.unlink(path_u)
 691                 self.notify(path, self.inotify.IN_DELETE, flush=False)
 692             event_mask = self.inotify.IN_CLOSE_WRITE
 693
 694         self.notify(path, event_mask)
 695         encoded_name_u = magicpath.path2magic(name_u)
 696
 697         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 698         if temporary:
 699             d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_disappeared'),
 700                                                                  previously_disappeared + 1))
 701         else:
 702             def _here(res, n):
 703                 print "here %r %r" % (n, res)
 704                 return res
 705             d.addBoth(_here, 1)
 706             d.addCallback(lambda ign: self.upload_dirnode.list())
 707             d.addBoth(_here, 1.5)
 708             d.addCallback(lambda ign: self.upload_dirnode.get(encoded_name_u))
 709             d.addBoth(_here, 2)
 710             d.addCallback(download_to_data)
 711             d.addBoth(_here, 3)
 712             d.addCallback(lambda actual_data: self.failUnlessReallyEqual(actual_data, data))
 713             d.addBoth(_here, 4)
 714             d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'),
 715                                                                  previously_uploaded + 1))
 716
 717         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 718         return d
 719
 720     def _check_version_in_dmd(self, magicfolder, relpath_u, expected_version):
 721         encoded_name_u = magicpath.path2magic(relpath_u)
 722         d = magicfolder.downloader._get_collective_latest_file(encoded_name_u)
 723         def check_latest(result):
 724             if result[0] is not None:
 725                 node, metadata = result
 726                 d.addCallback(lambda ign: self.failUnlessEqual(metadata['version'], expected_version))
 727         d.addCallback(check_latest)
 728         return d
 729
 730     def _check_version_in_local_db(self, magicfolder, relpath_u, expected_version):
 731         db_entry = magicfolder._db.get_db_entry(relpath_u)
 732         if db_entry is not None:
 733             #print "_check_version_in_local_db: %r has version %s" % (relpath_u, version)
 734             self.failUnlessEqual(db_entry.version, expected_version)
 735
 736     def _check_file_gone(self, magicfolder, relpath_u):
 737         path = os.path.join(magicfolder.uploader._local_path_u, relpath_u)
 738         self.assertTrue(not os.path.exists(path))
 739
 740     def _check_uploader_count(self, name, expected, magic=None):
 741         self.failUnlessReallyEqual(self._get_count('uploader.'+name, client=(magic or self.alice_magicfolder)._client),
 742                                    expected)
 743
 744     def _check_downloader_count(self, name, expected, magic=None):
 745         self.failUnlessReallyEqual(self._get_count('downloader.'+name, client=(magic or self.bob_magicfolder)._client),
 746                                    expected)
 747
 748     def test_alice_bob(self):
 749         alice_clock = task.Clock()
 750         bob_clock = task.Clock()
 751         d = self.setup_alice_and_bob(alice_clock, bob_clock)
 752
 753         def _wait_for_Alice(ign, downloaded_d):
 754             print "Now waiting for Alice to download\n"
 755             alice_clock.advance(0)
 756             return downloaded_d
 757
 758         def _wait_for_Bob(ign, downloaded_d):
 759             print "Now waiting for Bob to download\n"
 760             bob_clock.advance(0)
 761             return downloaded_d
 762
 763         def _wait_for(ign, something_to_do, alice=True):
 764             if alice:
 765                 downloaded_d = self.bob_magicfolder.downloader.set_hook('processed')
 766                 uploaded_d = self.alice_magicfolder.uploader.set_hook('processed')
 767             else:
 768                 downloaded_d = self.alice_magicfolder.downloader.set_hook('processed')
 769                 uploaded_d = self.bob_magicfolder.uploader.set_hook('processed')
 770             something_to_do()
 771             if alice:
 772                 print "Waiting for Alice to upload\n"
 773                 alice_clock.advance(0)
 774                 uploaded_d.addCallback(_wait_for_Bob, downloaded_d)
 775             else:
 776                 print "Waiting for Bob to upload\n"
 777                 bob_clock.advance(0)
 778                 uploaded_d.addCallback(_wait_for_Alice, downloaded_d)
 779             return uploaded_d
 780
 781         def Alice_to_write_a_file():
 782             print "Alice writes a file\n"
 783             self.file_path = abspath_expanduser_unicode(u"file1", base=self.alice_magicfolder.uploader._local_path_u)
 784             fileutil.write(self.file_path, "meow, meow meow. meow? meow meow! meow.")
 785             self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 786         d.addCallback(_wait_for, Alice_to_write_a_file)
 787
 788         d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file1", 0))
 789         d.addCallback(lambda ign: self._check_version_in_local_db(self.alice_magicfolder, u"file1", 0))
 790         d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0))
 791         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 1))
 792         d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 1))
 793         d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0))
 794         d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0))
 795         d.addCallback(lambda ign: self._check_uploader_count('objects_conflicted', 0))
 796         d.addCallback(lambda ign: self._check_uploader_count('objects_conflicted', 0, magic=self.bob_magicfolder))
 797
 798         d.addCallback(lambda ign: self._check_version_in_local_db(self.bob_magicfolder, u"file1", 0))
 799         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0))
 800         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 1))
 801         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 0, magic=self.bob_magicfolder))
 802
 803         def Alice_to_delete_file():
 804             print "Alice deletes the file!\n"
 805             os.unlink(self.file_path)
 806             self.notify(to_filepath(self.file_path), self.inotify.IN_DELETE, magic=self.alice_magicfolder)
 807         d.addCallback(_wait_for, Alice_to_delete_file)
 808
 809         def notify_bob_moved(ign):
 810             d0 = self.bob_magicfolder.uploader.set_hook('processed')
 811             p = abspath_expanduser_unicode(u"file1", base=self.bob_magicfolder.uploader._local_path_u)
 812             self.notify(to_filepath(p), self.inotify.IN_MOVED_FROM, magic=self.bob_magicfolder, flush=False)
 813             self.notify(to_filepath(p + u'.backup'), self.inotify.IN_MOVED_TO, magic=self.bob_magicfolder)
 814             bob_clock.advance(0)
 815             return d0
 816         d.addCallback(notify_bob_moved)
 817
 818         d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file1", 1))
 819         d.addCallback(lambda ign: self._check_version_in_local_db(self.alice_magicfolder, u"file1", 1))
 820         d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0))
 821         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 2))
 822         d.addCallback(lambda ign: self._check_uploader_count('objects_not_uploaded', 1, magic=self.bob_magicfolder))
 823         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 1, magic=self.bob_magicfolder))
 824
 825         d.addCallback(lambda ign: self._check_version_in_local_db(self.bob_magicfolder, u"file1", 1))
 826         d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file1", 1))
 827         d.addCallback(lambda ign: self._check_file_gone(self.bob_magicfolder, u"file1"))
 828         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0))
 829         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 2))
 830
 831         def Alice_to_rewrite_file():
 832             print "Alice rewrites file\n"
 833             self.file_path = abspath_expanduser_unicode(u"file1", base=self.alice_magicfolder.uploader._local_path_u)
 834             fileutil.write(self.file_path, "Alice suddenly sees the white rabbit running into the forest.")
 835             self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 836         d.addCallback(_wait_for, Alice_to_rewrite_file)
 837
 838         d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file1", 2))
 839         d.addCallback(lambda ign: self._check_version_in_local_db(self.alice_magicfolder, u"file1", 2))
 840         d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0))
 841         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 3))
 842         d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 3))
 843         d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0))
 844         d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0))
 845         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
 846         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0))
 847
 848         d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file1", 2))
 849         d.addCallback(lambda ign: self._check_version_in_local_db(self.bob_magicfolder, u"file1", 2))
 850         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0))
 851         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 3))
 852
 853         path_u = u"/tmp/magic_folder_test"
 854         encoded_path_u = magicpath.path2magic(u"/tmp/magic_folder_test")
 855
 856         def Alice_tries_to_p0wn_Bob(ign):
 857             print "Alice tries to p0wn Bob\n"
 858             self.objects_excluded = self._get_count('downloader.objects_excluded', client=self.bob_magicfolder._client)
 859             processed_d = self.bob_magicfolder.downloader.set_hook('processed')
 860
 861             # upload a file that would provoke the security bug from #2506
 862             uploadable = Data("", self.alice_magicfolder._client.convergence)
 863             alice_dmd = self.alice_magicfolder.uploader._upload_dirnode
 864
 865             d2 = alice_dmd.add_file(encoded_path_u, uploadable, metadata={"version": 0}, overwrite=True)
 866             d2.addCallback(lambda ign: self.failUnless(alice_dmd.has_child(encoded_path_u)))
 867             d2.addCallback(_wait_for_Bob, processed_d)
 868             return d2
 869         d.addCallback(Alice_tries_to_p0wn_Bob)
 870
 871         d.addCallback(lambda ign: self.failIf(os.path.exists(path_u)))
 872         d.addCallback(lambda ign: self._check_version_in_local_db(self.bob_magicfolder, encoded_path_u, None))
 873         d.addCallback(lambda ign: self._check_downloader_count('objects_excluded', self.objects_excluded+1))
 874         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 3))
 875         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
 876         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0))
 877
 878         def Bob_to_rewrite_file():
 879             print "Bob rewrites file\n"
 880             self.file_path = abspath_expanduser_unicode(u"file1", base=self.bob_magicfolder.uploader._local_path_u)
 881             print "---- bob's file is %r" % (self.file_path,)
 882             fileutil.write(self.file_path, "No white rabbit to be found.")
 883             self.magicfolder = self.bob_magicfolder
 884             self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE)
 885         d.addCallback(lambda ign: _wait_for(None, Bob_to_rewrite_file, alice=False))
 886
 887         d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file1", 3))
 888         d.addCallback(lambda ign: self._check_version_in_local_db(self.bob_magicfolder, u"file1", 3))
 889         d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0, magic=self.bob_magicfolder))
 890         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 2, magic=self.bob_magicfolder))
 891         d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 1, magic=self.bob_magicfolder))
 892         d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0, magic=self.bob_magicfolder))
 893         d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0, magic=self.bob_magicfolder))
 894         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0))
 895
 896         d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file1", 3))
 897         d.addCallback(lambda ign: self._check_version_in_local_db(self.alice_magicfolder, u"file1", 3))
 898         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
 899         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 1, magic=self.alice_magicfolder))
 900         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
 901
 902         def Alice_conflicts_with_Bobs_last_downloaded_uri():
 903             print "Alice conflicts with Bob\n"
 904             downloaded_d = self.bob_magicfolder.downloader.set_hook('processed')
 905             uploadable = Data("do not follow the white rabbit", self.alice_magicfolder._client.convergence)
 906             alice_dmd = self.alice_magicfolder.uploader._upload_dirnode
 907             d2 = alice_dmd.add_file(u"file1", uploadable,
 908                                     metadata={"version": 5,
 909                                               "last_downloaded_uri" : "URI:LIT:" },
 910                                     overwrite=True)
 911             print "Waiting for Alice to upload\n"
 912             d2.addCallback(lambda ign: bob_clock.advance(6))
 913             d2.addCallback(lambda ign: downloaded_d)
 914             d2.addCallback(lambda ign: self.failUnless(alice_dmd.has_child(encoded_path_u)))
 915             return d2
 916
 917         d.addCallback(lambda ign: Alice_conflicts_with_Bobs_last_downloaded_uri())
 918         # XXX fix the code so that it doesn't increment objects_excluded each turn
 919         #d.addCallback(lambda ign: self._check_downloader_count('objects_excluded', 1))
 920         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 4))
 921         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 1))
 922         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 1, magic=self.alice_magicfolder))
 923         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
 924         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
 925
 926         # prepare to perform another conflict test
 927         def Alice_to_write_file2():
 928             print "Alice writes a file\n"
 929             self.file_path = abspath_expanduser_unicode(u"file2", base=self.alice_magicfolder.uploader._local_path_u)
 930             fileutil.write(self.file_path, "something")
 931             self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 932         d.addCallback(_wait_for, Alice_to_write_file2)
 933         d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file2", 0))
 934         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
 935         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
 936
 937         def Bob_to_rewrite_file2():
 938             print "Bob rewrites file\n"
 939             self.file_path = abspath_expanduser_unicode(u"file2", base=self.bob_magicfolder.uploader._local_path_u)
 940             print "---- bob's file is %r" % (self.file_path,)
 941             fileutil.write(self.file_path, "roger roger. what vector?")
 942             self.magicfolder = self.bob_magicfolder
 943             self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE)
 944         d.addCallback(lambda ign: _wait_for(None, Bob_to_rewrite_file2, alice=False))
 945         d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file2", 1))
 946         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 5))
 947         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 1))
 948         d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0, magic=self.bob_magicfolder))
 949         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 3, magic=self.bob_magicfolder))
 950         d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 2, magic=self.bob_magicfolder))
 951         d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0, magic=self.bob_magicfolder))
 952         d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0, magic=self.bob_magicfolder))
 953         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
 954         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
 955         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 2, magic=self.alice_magicfolder))
 956
 957         # XXX here we advance the clock and then test again to make sure no values are monotonically increasing
 958         # with each queue turn ;-p
 959         alice_clock.advance(6)
 960         bob_clock.advance(6)
 961         d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file2", 1))
 962         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 5))
 963         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 1))
 964         d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0, magic=self.bob_magicfolder))
 965         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 3, magic=self.bob_magicfolder))
 966         d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 2, magic=self.bob_magicfolder))
 967         d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0, magic=self.bob_magicfolder))
 968         d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0, magic=self.bob_magicfolder))
 969         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
 970         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
 971         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 2, magic=self.alice_magicfolder))
 972
 973         def Alice_conflicts_with_Bobs_last_uploaded_uri():
 974             print "Alice conflicts with Bob\n"
 975             encoded_path_u = magicpath.path2magic(u"file2")
 976             downloaded_d = self.bob_magicfolder.downloader.set_hook('processed')
 977             uploadable = Data("rabbits with sharp fangs", self.alice_magicfolder._client.convergence)
 978             alice_dmd = self.alice_magicfolder.uploader._upload_dirnode
 979             d2 = alice_dmd.add_file(u"file2", uploadable,
 980                                     metadata={"version": 5,
 981                                               "last_uploaded_uri" : "URI:LIT:" },
 982                                     overwrite=True)
 983             print "Waiting for Alice to upload\n"
 984             d2.addCallback(lambda ign: bob_clock.advance(6))
 985             d2.addCallback(lambda ign: downloaded_d)
 986             d2.addCallback(lambda ign: self.failUnless(alice_dmd.has_child(encoded_path_u)))
 987             return d2
 988         d.addCallback(lambda ign: Alice_conflicts_with_Bobs_last_uploaded_uri())
 989         d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file2", 5))
 990         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 6))
 991         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 2))
 992         d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0, magic=self.bob_magicfolder))
 993         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 3, magic=self.bob_magicfolder))
 994         d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 2, magic=self.bob_magicfolder))
 995         d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0, magic=self.bob_magicfolder))
 996         d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0, magic=self.bob_magicfolder))
 997         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
 998         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
 999         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 2, magic=self.alice_magicfolder))
1000
1001         alice_clock.advance(6)
1002         bob_clock.advance(6)
1003         alice_clock.advance(6)
1004         bob_clock.advance(6)
1005
1006         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 2, magic=self.alice_magicfolder))
1007         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
1008         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 2))
1009         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 6))
1010
1011         # prepare to perform another conflict test
1012         def Alice_to_write_file3():
1013             print "Alice writes a file\n"
1014             self.file_path = abspath_expanduser_unicode(u"file3", base=self.alice_magicfolder.uploader._local_path_u)
1015             fileutil.write(self.file_path, "something")
1016             self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
1017         d.addCallback(_wait_for, Alice_to_write_file3)
1018         d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file3", 0))
1019         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
1020         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 7))
1021         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 2, magic=self.alice_magicfolder))
1022         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 2))
1023         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
1024
1025         def Bob_to_rewrite_file3():
1026             print "Bob rewrites file\n"
1027             self.file_path = abspath_expanduser_unicode(u"file3", base=self.bob_magicfolder.uploader._local_path_u)
1028             print "---- bob's file is %r" % (self.file_path,)
1029             fileutil.write(self.file_path, "roger roger")
1030             self.magicfolder = self.bob_magicfolder
1031             self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE)
1032         d.addCallback(lambda ign: _wait_for(None, Bob_to_rewrite_file3, alice=False))
1033         d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file3", 1))
1034         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 7))
1035         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 2))
1036         d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0, magic=self.bob_magicfolder))
1037         d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 4, magic=self.bob_magicfolder))
1038         d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 3, magic=self.bob_magicfolder))
1039         d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0, magic=self.bob_magicfolder))
1040         d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0, magic=self.bob_magicfolder))
1041         d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
1042         d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
1043         d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 3, magic=self.alice_magicfolder))
1044
1045
1046
1047         def _cleanup(ign, magicfolder, clock):
1048             if magicfolder is not None:
1049                 d2 = magicfolder.finish()
1050                 clock.advance(0)
1051                 return d2
1052
1053         def cleanup_Alice_and_Bob(result):
1054             print "cleanup alice bob test\n"
1055             d = defer.succeed(None)
1056             d.addCallback(_cleanup, self.alice_magicfolder, alice_clock)
1057             d.addCallback(_cleanup, self.bob_magicfolder, bob_clock)
1058             d.addCallback(lambda ign: result)
1059             return d
1060         d.addBoth(cleanup_Alice_and_Bob)
1061         return d
1062
1063
1064 class MockTest(MagicFolderTestMixin, unittest.TestCase):
1065     """This can run on any platform, and even if twisted.internet.inotify can't be imported."""
1066
1067     def setUp(self):
1068         MagicFolderTestMixin.setUp(self)
1069         self.inotify = fake_inotify
1070         self.patch(magic_folder, 'get_inotify_module', lambda: self.inotify)
1071
1072     def notify(self, path, mask, magic=None, flush=True):
1073         if magic is None:
1074             magic = self.magicfolder
1075         magic.uploader._notifier.event(path, mask)
1076         # no flush for the mock test.
1077
1078     def test_errors(self):
1079         self.set_up_grid()
1080
1081         errors_dir = abspath_expanduser_unicode(u"errors_dir", base=self.basedir)
1082         os.mkdir(errors_dir)
1083         not_a_dir = abspath_expanduser_unicode(u"NOT_A_DIR", base=self.basedir)
1084         fileutil.write(not_a_dir, "")
1085         magicfolderdb = abspath_expanduser_unicode(u"magicfolderdb", base=self.basedir)
1086         doesnotexist  = abspath_expanduser_unicode(u"doesnotexist", base=self.basedir)
1087
1088         client = self.g.clients[0]
1089         d = client.create_dirnode()
1090         def _check_errors(n):
1091             self.failUnless(IDirectoryNode.providedBy(n))
1092             upload_dircap = n.get_uri()
1093             readonly_dircap = n.get_readonly_uri()
1094
1095             self.shouldFail(AssertionError, 'nonexistent local.directory', 'there is no directory',
1096                             MagicFolder, client, upload_dircap, '', doesnotexist, magicfolderdb)
1097             self.shouldFail(AssertionError, 'non-directory local.directory', 'is not a directory',
1098                             MagicFolder, client, upload_dircap, '', not_a_dir, magicfolderdb)
1099             self.shouldFail(AssertionError, 'bad upload.dircap', 'does not refer to a directory',
1100                             MagicFolder, client, 'bad', '', errors_dir, magicfolderdb)
1101             self.shouldFail(AssertionError, 'non-directory upload.dircap', 'does not refer to a directory',
1102                             MagicFolder, client, 'URI:LIT:foo', '', errors_dir, magicfolderdb)
1103             self.shouldFail(AssertionError, 'readonly upload.dircap', 'is not a writecap to a directory',
1104                             MagicFolder, client, readonly_dircap, '', errors_dir, magicfolderdb)
1105             self.shouldFail(AssertionError, 'collective dircap', 'is not a readonly cap to a directory',
1106                             MagicFolder, client, upload_dircap, upload_dircap, errors_dir, magicfolderdb)
1107
1108             def _not_implemented():
1109                 raise NotImplementedError("blah")
1110             self.patch(magic_folder, 'get_inotify_module', _not_implemented)
1111             self.shouldFail(NotImplementedError, 'unsupported', 'blah',
1112                             MagicFolder, client, upload_dircap, '', errors_dir, magicfolderdb)
1113         d.addCallback(_check_errors)
1114         return d
1115
1116     def test_write_downloaded_file(self):
1117         workdir = u"cli/MagicFolder/write-downloaded-file"
1118         local_file = fileutil.abspath_expanduser_unicode(os.path.join(workdir, "foobar"))
1119
1120         class TestWriteFileMixin(WriteFileMixin):
1121             def _log(self, msg):
1122                 pass
1123
1124         writefile = TestWriteFileMixin()
1125
1126         # create a file with name "foobar" with content "foo"
1127         # write downloaded file content "bar" into "foobar" with is_conflict = False
1128         fileutil.make_dirs(workdir)
1129         fileutil.write(local_file, "foo")
1130
1131         # if is_conflict is False, then the .conflict file shouldn't exist.
1132         writefile._write_downloaded_file(local_file, "bar", False, None)
1133         conflicted_path = local_file + u".conflict"
1134         self.failIf(os.path.exists(conflicted_path))
1135
1136         # At this point, the backup file should exist with content "foo"
1137         backup_path = local_file + u".backup"
1138         self.failUnless(os.path.exists(backup_path))
1139         self.failUnlessEqual(fileutil.read(backup_path), "foo")
1140
1141         # .tmp file shouldn't exist
1142         self.failIf(os.path.exists(local_file + u".tmp"))
1143
1144         # .. and the original file should have the new content
1145         self.failUnlessEqual(fileutil.read(local_file), "bar")
1146
1147         # now a test for conflicted case
1148         writefile._write_downloaded_file(local_file, "bar", True, None)
1149         self.failUnless(os.path.exists(conflicted_path))
1150
1151         # .tmp file shouldn't exist
1152         self.failIf(os.path.exists(local_file + u".tmp"))
1153
1154
1155 class RealTest(MagicFolderTestMixin, unittest.TestCase):
1156     """This is skipped unless both Twisted and the platform support inotify."""
1157
1158     def setUp(self):
1159         MagicFolderTestMixin.setUp(self)
1160         self.inotify = magic_folder.get_inotify_module()
1161
1162     def notify(self, path, mask, magic=None, flush=True):
1163         # Writing to the filesystem causes the notification.
1164         # However, flushing filesystem buffers may be necessary on Windows.
1165         if flush:
1166             fileutil.flush_volume(path.path)
1167
1168 try:
1169     magic_folder.get_inotify_module()
1170 except NotImplementedError:
1171     RealTest.skip = "Magic Folder support can only be tested for-real on an OS that supports inotify or equivalent."