src/allmydata/test/test_magic_folder.py

   1
   2 import os, sys
   3
   4 from twisted.trial import unittest
   5 from twisted.internet import defer, task
   6
   7 from allmydata.interfaces import IDirectoryNode
   8 from allmydata.util.assertutil import precondition
   9
  10 from allmydata.util import fake_inotify, fileutil
  11 from allmydata.util.deferredutil import DeferredListShouldSucceed
  12 from allmydata.util.encodingutil import get_filesystem_encoding, to_filepath
  13 from allmydata.util.consumer import download_to_data
  14 from allmydata.test.no_network import GridTestMixin
  15 from allmydata.test.common_util import ReallyEqualMixin, NonASCIIPathMixin
  16 from allmydata.test.common import ShouldFailMixin
  17 from .test_cli_magic_folder import MagicFolderCLITestMixin
  18
  19 from allmydata.frontends import magic_folder
  20 from allmydata.frontends.magic_folder import MagicFolder, Downloader, WriteFileMixin
  21 from allmydata import magicfolderdb, magicpath
  22 from allmydata.util.fileutil import abspath_expanduser_unicode
  23 from allmydata.immutable.upload import Data
  24
  25
  26 class MagicFolderTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, ReallyEqualMixin, NonASCIIPathMixin):
  27     """
  28     These tests will be run both with a mock notifier, and (on platforms that support it)
  29     with the real INotify.
  30     """
  31
  32     def setUp(self):
  33         GridTestMixin.setUp(self)
  34         temp = self.mktemp()
  35         self.basedir = abspath_expanduser_unicode(temp.decode(get_filesystem_encoding()))
  36         self.magicfolder = None
  37         self.patch(Downloader, 'REMOTE_SCAN_INTERVAL', 0)
  38
  39     def _get_count(self, name, client=None):
  40         counters = (client or self.get_client()).stats_provider.get_stats()["counters"]
  41         return counters.get('magic_folder.%s' % (name,), 0)
  42
  43     def _createdb(self):
  44         dbfile = abspath_expanduser_unicode(u"magicfolderdb.sqlite", base=self.basedir)
  45         mdb = magicfolderdb.get_magicfolderdb(dbfile, create_version=(magicfolderdb.SCHEMA_v1, 1))
  46         self.failUnless(mdb, "unable to create magicfolderdb from %r" % (dbfile,))
  47         self.failUnlessEqual(mdb.VERSION, 1)
  48         return mdb
  49
  50     def _restart_client(self, ign):
  51         #print "_restart_client"
  52         d = self.restart_client()
  53         d.addCallback(self._wait_until_started)
  54         return d
  55
  56     def _wait_until_started(self, ign):
  57         #print "_wait_until_started"
  58         self.magicfolder = self.get_client().getServiceNamed('magic-folder')
  59         return self.magicfolder.ready()
  60
  61     def test_db_basic(self):
  62         fileutil.make_dirs(self.basedir)
  63         self._createdb()
  64
  65     def test_db_persistence(self):
  66         """Test that a file upload creates an entry in the database."""
  67
  68         fileutil.make_dirs(self.basedir)
  69         db = self._createdb()
  70
  71         relpath1 = u"myFile1"
  72         pathinfo = fileutil.PathInfo(isdir=False, isfile=True, islink=False,
  73                                      exists=True, size=1, mtime=123, ctime=456)
  74         db.did_upload_version(relpath1, 0, 'URI:LIT:1', 'URI:LIT:0', 0, pathinfo)
  75
  76         c = db.cursor
  77         c.execute("SELECT size, mtime, ctime"
  78                   " FROM local_files"
  79                   " WHERE path=?",
  80                   (relpath1,))
  81         row = c.fetchone()
  82         self.failUnlessEqual(row, (pathinfo.size, pathinfo.mtime, pathinfo.ctime))
  83
  84         # Second test uses db.is_new_file instead of SQL query directly
  85         # to confirm the previous upload entry in the db.
  86         relpath2 = u"myFile2"
  87         path2 = os.path.join(self.basedir, relpath2)
  88         fileutil.write(path2, "meow\n")
  89         pathinfo = fileutil.get_pathinfo(path2)
  90         db.did_upload_version(relpath2, 0, 'URI:LIT:2', 'URI:LIT:1', 0, pathinfo)
  91         self.failUnlessFalse(db.is_new_file(pathinfo, relpath2))
  92
  93         different_pathinfo = fileutil.PathInfo(isdir=False, isfile=True, islink=False,
  94                                                exists=True, size=0, mtime=pathinfo.mtime, ctime=pathinfo.ctime)
  95         self.failUnlessTrue(db.is_new_file(different_pathinfo, relpath2))
  96
  97     def test_magicfolder_start_service(self):
  98         self.set_up_grid()
  99
 100         self.local_dir = abspath_expanduser_unicode(self.unicode_or_fallback(u"l\u00F8cal_dir", u"local_dir"),
 101                                                     base=self.basedir)
 102         self.mkdir_nonascii(self.local_dir)
 103
 104         d = defer.succeed(None)
 105         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.dirs_monitored'), 0))
 106
 107         d.addCallback(lambda ign: self.create_invite_join_magic_folder(u"Alice", self.local_dir))
 108         d.addCallback(self._restart_client)
 109
 110         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.dirs_monitored'), 1))
 111         d.addBoth(self.cleanup)
 112         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.dirs_monitored'), 0))
 113         return d
 114
 115     def test_move_tree(self):
 116         self.set_up_grid()
 117
 118         self.local_dir = abspath_expanduser_unicode(self.unicode_or_fallback(u"l\u00F8cal_dir", u"local_dir"),
 119                                                     base=self.basedir)
 120         self.mkdir_nonascii(self.local_dir)
 121
 122         empty_tree_name = self.unicode_or_fallback(u"empty_tr\u00EAe", u"empty_tree")
 123         empty_tree_dir = abspath_expanduser_unicode(empty_tree_name, base=self.basedir)
 124         new_empty_tree_dir = abspath_expanduser_unicode(empty_tree_name, base=self.local_dir)
 125
 126         small_tree_name = self.unicode_or_fallback(u"small_tr\u00EAe", u"empty_tree")
 127         small_tree_dir = abspath_expanduser_unicode(small_tree_name, base=self.basedir)
 128         new_small_tree_dir = abspath_expanduser_unicode(small_tree_name, base=self.local_dir)
 129
 130         d = self.create_invite_join_magic_folder(u"Alice", self.local_dir)
 131         d.addCallback(self._restart_client)
 132
 133         def _check_move_empty_tree(res):
 134             print "_check_move_empty_tree"
 135             downloaded_d = self.magicfolder.downloader.set_hook('processed')
 136             uploaded_d = self.magicfolder.uploader.set_hook('processed')
 137             self.mkdir_nonascii(empty_tree_dir)
 138             os.rename(empty_tree_dir, new_empty_tree_dir)
 139             self.notify(to_filepath(new_empty_tree_dir), self.inotify.IN_MOVED_TO)
 140
 141             return DeferredListShouldSucceed([downloaded_d, uploaded_d])
 142         d.addCallback(_check_move_empty_tree)
 143         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 144         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 1))
 145         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.files_uploaded'), 0))
 146         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 147         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.directories_created'), 1))
 148
 149         # FIXME check that Bob downloaded/created the empty tree.
 150
 151         def _check_move_small_tree(res):
 152             print "_check_move_small_tree"
 153             downloaded_d = self.magicfolder.downloader.set_hook('processed', ignore_count=1)
 154             uploaded_d = self.magicfolder.uploader.set_hook('processed', ignore_count=1)
 155             self.mkdir_nonascii(small_tree_dir)
 156             fileutil.write(abspath_expanduser_unicode(u"what", base=small_tree_dir), "say when")
 157             os.rename(small_tree_dir, new_small_tree_dir)
 158             self.notify(to_filepath(new_small_tree_dir), self.inotify.IN_MOVED_TO)
 159
 160             return DeferredListShouldSucceed([downloaded_d, uploaded_d])
 161         d.addCallback(_check_move_small_tree)
 162         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 163         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 3))
 164         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.files_uploaded'), 1))
 165         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 166         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.directories_created'), 2))
 167
 168         def _check_moved_tree_is_watched(res):
 169             print "_check_moved_tree_is_watched"
 170             downloaded_d = self.magicfolder.downloader.set_hook('processed', ignore_count=1)
 171             uploaded_d = self.magicfolder.uploader.set_hook('processed')
 172             fileutil.write(abspath_expanduser_unicode(u"another", base=new_small_tree_dir), "file")
 173             self.notify(to_filepath(abspath_expanduser_unicode(u"another", base=new_small_tree_dir)), self.inotify.IN_CLOSE_WRITE)
 174
 175             return DeferredListShouldSucceed([downloaded_d, uploaded_d])
 176         d.addCallback(_check_moved_tree_is_watched)
 177         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 178         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 4))
 179         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.files_uploaded'), 2))
 180         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 181         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.directories_created'), 2))
 182
 183         # Files that are moved out of the upload directory should no longer be watched.
 184         #def _move_dir_away(ign):
 185         #    os.rename(new_empty_tree_dir, empty_tree_dir)
 186         #    # Wuh? Why don't we get this event for the real test?
 187         #    #self.notify(to_filepath(new_empty_tree_dir), self.inotify.IN_MOVED_FROM)
 188         #d.addCallback(_move_dir_away)
 189         #def create_file(val):
 190         #    test_file = abspath_expanduser_unicode(u"what", base=empty_tree_dir)
 191         #    fileutil.write(test_file, "meow")
 192         #    #self.notify(...)
 193         #    return
 194         #d.addCallback(create_file)
 195         #d.addCallback(lambda ign: time.sleep(1))  # XXX ICK
 196         #d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 197         #d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 4))
 198         #d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.files_uploaded'), 2))
 199         #d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 200         #d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.directories_created'), 2))
 201
 202         d.addBoth(self.cleanup)
 203         return d
 204
 205     def test_persistence(self):
 206         """
 207         Perform an upload of a given file and then stop the client.
 208         Start a new client and magic-folder service... and verify that the file is NOT uploaded
 209         a second time. This test is meant to test the database persistence along with
 210         the startup and shutdown code paths of the magic-folder service.
 211         """
 212         self.set_up_grid()
 213         self.local_dir = abspath_expanduser_unicode(u"test_persistence", base=self.basedir)
 214         self.mkdir_nonascii(self.local_dir)
 215         self.collective_dircap = ""
 216
 217         d = defer.succeed(None)
 218         d.addCallback(lambda ign: self.create_invite_join_magic_folder(u"Alice", self.local_dir))
 219         d.addCallback(self._restart_client)
 220
 221         def create_test_file(filename):
 222             d2 = self.magicfolder.uploader.set_hook('processed')
 223             test_file = abspath_expanduser_unicode(filename, base=self.local_dir)
 224             fileutil.write(test_file, "meow %s" % filename)
 225             self.notify(to_filepath(test_file), self.inotify.IN_CLOSE_WRITE)
 226             return d2
 227         d.addCallback(lambda ign: create_test_file(u"what1"))
 228         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 229         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 1))
 230         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 231         d.addCallback(self.cleanup)
 232
 233         d.addCallback(self._restart_client)
 234         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 235         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 1))
 236         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 237         d.addCallback(lambda ign: create_test_file(u"what2"))
 238         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 239         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 2))
 240         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 241         d.addBoth(self.cleanup)
 242         return d
 243
 244     @defer.inlineCallbacks
 245     def test_delete(self):
 246         self.set_up_grid()
 247         self.local_dir = os.path.join(self.basedir, u"local_dir")
 248         self.mkdir_nonascii(self.local_dir)
 249
 250         yield self.create_invite_join_magic_folder(u"Alice\u0101", self.local_dir)
 251         yield self._restart_client(None)
 252
 253         try:
 254             # create a file
 255             up_proc = self.magicfolder.uploader.set_hook('processed')
 256             # down_proc = self.magicfolder.downloader.set_hook('processed')
 257             path = os.path.join(self.local_dir, u'foo')
 258             with open(path, 'w') as f:
 259                 f.write('foo\n')
 260             self.notify(to_filepath(path), self.inotify.IN_CLOSE_WRITE)
 261             yield up_proc
 262             self.assertTrue(os.path.exists(path))
 263
 264             # the real test part: delete the file
 265             up_proc = self.magicfolder.uploader.set_hook('processed')
 266             os.unlink(path)
 267             self.notify(to_filepath(path), self.inotify.IN_DELETE)
 268             yield up_proc
 269             self.assertFalse(os.path.exists(path))
 270
 271             # ensure we still have a DB entry, and that the version is 1
 272             node, metadata = yield self.magicfolder.downloader._get_collective_latest_file(u'foo')
 273             self.assertTrue(node is not None, "Failed to find '{}' in DMD".format(path))
 274             self.failUnlessEqual(metadata['version'], 1)
 275
 276         finally:
 277             yield self.cleanup(None)
 278
 279     @defer.inlineCallbacks
 280     def test_delete_and_restore(self):
 281         self.set_up_grid()
 282         self.local_dir = os.path.join(self.basedir, u"local_dir")
 283         self.mkdir_nonascii(self.local_dir)
 284
 285         yield self.create_invite_join_magic_folder(u"Alice\u0101", self.local_dir)
 286         yield self._restart_client(None)
 287
 288         try:
 289             # create a file
 290             up_proc = self.magicfolder.uploader.set_hook('processed')
 291             # down_proc = self.magicfolder.downloader.set_hook('processed')
 292             path = os.path.join(self.local_dir, u'foo')
 293             with open(path, 'w') as f:
 294                 f.write('foo\n')
 295             self.notify(to_filepath(path), self.inotify.IN_CLOSE_WRITE)
 296             yield up_proc
 297             self.assertTrue(os.path.exists(path))
 298
 299             # delete the file
 300             up_proc = self.magicfolder.uploader.set_hook('processed')
 301             os.unlink(path)
 302             self.notify(to_filepath(path), self.inotify.IN_DELETE)
 303             yield up_proc
 304             self.assertFalse(os.path.exists(path))
 305
 306             # ensure we still have a DB entry, and that the version is 1
 307             node, metadata = yield self.magicfolder.downloader._get_collective_latest_file(u'foo')
 308             self.assertTrue(node is not None, "Failed to find '{}' in DMD".format(path))
 309             self.failUnlessEqual(metadata['version'], 1)
 310
 311             # restore the file, with different contents
 312             up_proc = self.magicfolder.uploader.set_hook('processed')
 313             path = os.path.join(self.local_dir, u'foo')
 314             with open(path, 'w') as f:
 315                 f.write('bar\n')
 316             self.notify(to_filepath(path), self.inotify.IN_CLOSE_WRITE)
 317             yield up_proc
 318
 319             # ensure we still have a DB entry, and that the version is 2
 320             node, metadata = yield self.magicfolder.downloader._get_collective_latest_file(u'foo')
 321             self.assertTrue(node is not None, "Failed to find '{}' in DMD".format(path))
 322             self.failUnlessEqual(metadata['version'], 2)
 323
 324         finally:
 325             yield self.cleanup(None)
 326
 327
 328     @defer.inlineCallbacks
 329     def test_alice_delete_bob_restore(self):
 330         alice_clock = task.Clock()
 331         bob_clock = task.Clock()
 332         yield self.setup_alice_and_bob(alice_clock, bob_clock)
 333         alice_dir = self.alice_magicfolder.uploader._local_path_u
 334         bob_dir = self.bob_magicfolder.uploader._local_path_u
 335         alice_fname = os.path.join(alice_dir, 'blam')
 336         bob_fname = os.path.join(bob_dir, 'blam')
 337
 338         try:
 339             # alice creates a file, bob downloads it
 340             alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
 341             bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
 342
 343             with open(alice_fname, 'wb') as f:
 344                 f.write('contents0\n')
 345             self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 346
 347             alice_clock.advance(0)
 348             yield alice_proc  # alice uploads
 349
 350             bob_clock.advance(0)
 351             yield bob_proc    # bob downloads
 352
 353             # check the state
 354             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
 355             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 0)
 356             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
 357             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 0)
 358             yield self.failUnlessReallyEqual(
 359                 self._get_count('downloader.objects_failed', client=self.bob_magicfolder._client),
 360                 0
 361             )
 362             yield self.failUnlessReallyEqual(
 363                 self._get_count('downloader.objects_downloaded', client=self.bob_magicfolder._client),
 364                 1
 365             )
 366
 367             print("BOB DELETE")
 368             # now bob deletes it (bob should upload, alice download)
 369             bob_proc = self.bob_magicfolder.uploader.set_hook('processed')
 370             alice_proc = self.alice_magicfolder.downloader.set_hook('processed')
 371             os.unlink(bob_fname)
 372             self.notify(to_filepath(bob_fname), self.inotify.IN_DELETE, magic=self.bob_magicfolder)
 373
 374             bob_clock.advance(0)
 375             yield bob_proc
 376             alice_clock.advance(0)
 377             yield alice_proc
 378
 379             # check versions
 380             node, metadata = yield self.alice_magicfolder.downloader._get_collective_latest_file(u'blam')
 381             self.assertTrue(metadata['deleted'])
 382             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
 383             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 1)
 384             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
 385             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 1)
 386
 387             print("ALICE RESTORE")
 388             # now alice restores it (alice should upload, bob download)
 389             alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
 390             bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
 391             with open(alice_fname, 'wb') as f:
 392                 f.write('new contents\n')
 393             self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 394
 395             alice_clock.advance(0)
 396             yield alice_proc
 397             bob_clock.advance(0)
 398             yield bob_proc
 399
 400             # check versions
 401             node, metadata = yield self.alice_magicfolder.downloader._get_collective_latest_file(u'blam')
 402             self.assertTrue('deleted' not in metadata or not metadata['deleted'])
 403             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 2)
 404             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 2)
 405             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 2)
 406             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 2)
 407
 408         finally:
 409             # cleanup
 410             d0 = self.alice_magicfolder.finish()
 411             alice_clock.advance(0)
 412             yield d0
 413
 414             d1 = self.bob_magicfolder.finish()
 415             bob_clock.advance(0)
 416             yield d1
 417
 418     @defer.inlineCallbacks
 419     def test_alice_create_bob_update(self):
 420         alice_clock = task.Clock()
 421         bob_clock = task.Clock()
 422         yield self.setup_alice_and_bob(alice_clock, bob_clock)
 423         alice_dir = self.alice_magicfolder.uploader._local_path_u
 424         bob_dir = self.bob_magicfolder.uploader._local_path_u
 425         alice_fname = os.path.join(alice_dir, 'blam')
 426         bob_fname = os.path.join(bob_dir, 'blam')
 427
 428         try:
 429             # alice creates a file, bob downloads it
 430             alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
 431             bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
 432
 433             with open(alice_fname, 'wb') as f:
 434                 f.write('contents0\n')
 435             self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 436
 437             alice_clock.advance(0)
 438             yield alice_proc  # alice uploads
 439
 440             bob_clock.advance(0)
 441             yield bob_proc    # bob downloads
 442
 443             # check the state
 444             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
 445             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 0)
 446             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
 447             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 0)
 448             yield self.failUnlessReallyEqual(
 449                 self._get_count('downloader.objects_failed', client=self.bob_magicfolder._client),
 450                 0
 451             )
 452             yield self.failUnlessReallyEqual(
 453                 self._get_count('downloader.objects_downloaded', client=self.bob_magicfolder._client),
 454                 1
 455             )
 456
 457             # now bob updates it (bob should upload, alice download)
 458             bob_proc = self.bob_magicfolder.uploader.set_hook('processed')
 459             alice_proc = self.alice_magicfolder.downloader.set_hook('processed')
 460             with open(bob_fname, 'wb') as f:
 461                 f.write('bob wuz here\n')
 462             self.notify(to_filepath(bob_fname), self.inotify.IN_CLOSE_WRITE, magic=self.bob_magicfolder)
 463
 464             bob_clock.advance(0)
 465             yield bob_proc
 466             alice_clock.advance(0)
 467             yield alice_proc
 468
 469             # check the state
 470             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
 471             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 1)
 472             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
 473             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 1)
 474
 475         finally:
 476             # cleanup
 477             d0 = self.alice_magicfolder.finish()
 478             alice_clock.advance(0)
 479             yield d0
 480
 481             d1 = self.bob_magicfolder.finish()
 482             bob_clock.advance(0)
 483             yield d1
 484
 485     @defer.inlineCallbacks
 486     def test_alice_delete_and_restore(self):
 487         alice_clock = task.Clock()
 488         bob_clock = task.Clock()
 489         yield self.setup_alice_and_bob(alice_clock, bob_clock)
 490         alice_dir = self.alice_magicfolder.uploader._local_path_u
 491         bob_dir = self.bob_magicfolder.uploader._local_path_u
 492         alice_fname = os.path.join(alice_dir, 'blam')
 493         bob_fname = os.path.join(bob_dir, 'blam')
 494
 495         try:
 496             # alice creates a file, bob downloads it
 497             alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
 498             bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
 499
 500             with open(alice_fname, 'wb') as f:
 501                 f.write('contents0\n')
 502             self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 503
 504             alice_clock.advance(0)
 505             yield alice_proc  # alice uploads
 506
 507             bob_clock.advance(0)
 508             yield bob_proc    # bob downloads
 509
 510             # check the state
 511             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
 512             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 0)
 513             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
 514             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 0)
 515             yield self.failUnlessReallyEqual(
 516                 self._get_count('downloader.objects_failed', client=self.bob_magicfolder._client),
 517                 0
 518             )
 519             yield self.failUnlessReallyEqual(
 520                 self._get_count('downloader.objects_downloaded', client=self.bob_magicfolder._client),
 521                 1
 522             )
 523             self.failUnless(os.path.exists(bob_fname))
 524
 525             # now alice deletes it (alice should upload, bob download)
 526             alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
 527             bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
 528             os.unlink(alice_fname)
 529             self.notify(to_filepath(alice_fname), self.inotify.IN_DELETE, magic=self.alice_magicfolder)
 530
 531             alice_clock.advance(0)
 532             yield alice_proc
 533             bob_clock.advance(0)
 534             yield bob_proc
 535
 536             # check the state
 537             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
 538             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 1)
 539             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
 540             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 1)
 541             self.failIf(os.path.exists(bob_fname))
 542
 543             # now alice restores the file (with new contents)
 544             alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
 545             bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
 546             with open(alice_fname, 'wb') as f:
 547                 f.write('alice wuz here\n')
 548             self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 549
 550             alice_clock.advance(0)
 551             yield alice_proc
 552             bob_clock.advance(0)
 553             yield bob_proc
 554
 555             # check the state
 556             yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 2)
 557             yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 2)
 558             yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 2)
 559             yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 2)
 560             self.failUnless(os.path.exists(bob_fname))
 561
 562         finally:
 563             # cleanup
 564             d0 = self.alice_magicfolder.finish()
 565             alice_clock.advance(0)
 566             yield d0
 567
 568             d1 = self.bob_magicfolder.finish()
 569             bob_clock.advance(0)
 570             yield d1
 571
 572     def test_magic_folder(self):
 573         self.set_up_grid()
 574         self.local_dir = os.path.join(self.basedir, self.unicode_or_fallback(u"loc\u0101l_dir", u"local_dir"))
 575         self.mkdir_nonascii(self.local_dir)
 576
 577         d = self.create_invite_join_magic_folder(u"Alice\u0101", self.local_dir)
 578         d.addCallback(self._restart_client)
 579
 580         # Write something short enough for a LIT file.
 581         d.addCallback(lambda ign: self._check_file(u"short", "test"))
 582
 583         # Write to the same file again with different data.
 584         d.addCallback(lambda ign: self._check_file(u"short", "different"))
 585
 586         # Test that temporary files are not uploaded.
 587         d.addCallback(lambda ign: self._check_file(u"tempfile", "test", temporary=True))
 588
 589         # Test creation of a subdirectory.
 590         d.addCallback(lambda ign: self._check_mkdir(u"directory"))
 591
 592         # Write something longer, and also try to test a Unicode name if the fs can represent it.
 593         name_u = self.unicode_or_fallback(u"l\u00F8ng", u"long")
 594         d.addCallback(lambda ign: self._check_file(name_u, "test"*100))
 595
 596         # TODO: test that causes an upload failure.
 597         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 598
 599         d.addBoth(self.cleanup)
 600         return d
 601
 602     def _check_mkdir(self, name_u):
 603         return self._check_file(name_u + u"/", "", directory=True)
 604
 605     def _check_file(self, name_u, data, temporary=False, directory=False):
 606         precondition(not (temporary and directory), temporary=temporary, directory=directory)
 607
 608         previously_uploaded = self._get_count('uploader.objects_succeeded')
 609         previously_disappeared = self._get_count('uploader.objects_disappeared')
 610
 611         d = self.magicfolder.uploader.set_hook('processed')
 612
 613         path_u = abspath_expanduser_unicode(name_u, base=self.local_dir)
 614         path = to_filepath(path_u)
 615
 616         if directory:
 617             os.mkdir(path_u)
 618             event_mask = self.inotify.IN_CREATE | self.inotify.IN_ISDIR
 619         else:
 620             # We don't use FilePath.setContent() here because it creates a temporary file that
 621             # is renamed into place, which causes events that the test is not expecting.
 622             f = open(path_u, "wb")
 623             try:
 624                 if temporary and sys.platform != "win32":
 625                     os.unlink(path_u)
 626                 f.write(data)
 627             finally:
 628                 f.close()
 629             if temporary and sys.platform == "win32":
 630                 os.unlink(path_u)
 631                 self.notify(path, self.inotify.IN_DELETE)
 632             event_mask = self.inotify.IN_CLOSE_WRITE
 633
 634         fileutil.flush_volume(path_u)
 635         self.notify(path, event_mask)
 636         encoded_name_u = magicpath.path2magic(name_u)
 637
 638         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
 639         if temporary:
 640             d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_disappeared'),
 641                                                                  previously_disappeared + 1))
 642         else:
 643             d.addCallback(lambda ign: self.upload_dirnode.get(encoded_name_u))
 644             d.addCallback(download_to_data)
 645             d.addCallback(lambda actual_data: self.failUnlessReallyEqual(actual_data, data))
 646             d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'),
 647                                                                  previously_uploaded + 1))
 648
 649         d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
 650         return d
 651
 652     def _check_version_in_dmd(self, magicfolder, relpath_u, expected_version):
 653         encoded_name_u = magicpath.path2magic(relpath_u)
 654         d = magicfolder.downloader._get_collective_latest_file(encoded_name_u)
 655         def check_latest(result):
 656             if result[0] is not None:
 657                 node, metadata = result
 658                 d.addCallback(lambda ign: self.failUnlessEqual(metadata['version'], expected_version))
 659         d.addCallback(check_latest)
 660         return d
 661
 662     def _check_version_in_local_db(self, magicfolder, relpath_u, expected_version):
 663         version = magicfolder._db.get_local_file_version(relpath_u)
 664         #print "_check_version_in_local_db: %r has version %s" % (relpath_u, version)
 665         self.failUnlessEqual(version, expected_version)
 666
 667     def _check_file_gone(self, magicfolder, relpath_u):
 668         path = os.path.join(magicfolder.uploader._local_path_u, relpath_u)
 669         self.assertTrue(not os.path.exists(path))
 670
 671     def test_alice_bob(self):
 672         alice_clock = task.Clock()
 673         bob_clock = task.Clock()
 674         d = self.setup_alice_and_bob(alice_clock, bob_clock)
 675
 676         def _check_uploader_count(ign, name, expected):
 677             self.failUnlessReallyEqual(self._get_count('uploader.'+name, client=self.alice_magicfolder._client),
 678                                        expected)
 679         def _check_downloader_count(ign, name, expected):
 680             self.failUnlessReallyEqual(self._get_count('downloader.'+name, client=self.bob_magicfolder._client),
 681                                        expected)
 682
 683         def _wait_for_Bob(ign, downloaded_d):
 684             print "Now waiting for Bob to download\n"
 685             bob_clock.advance(0)
 686             return downloaded_d
 687
 688         def _wait_for(ign, something_to_do):
 689             downloaded_d = self.bob_magicfolder.downloader.set_hook('processed')
 690             uploaded_d = self.alice_magicfolder.uploader.set_hook('processed')
 691             something_to_do()
 692             print "Waiting for Alice to upload\n"
 693             alice_clock.advance(0)
 694             uploaded_d.addCallback(_wait_for_Bob, downloaded_d)
 695             return uploaded_d
 696
 697         def Alice_to_write_a_file():
 698             print "Alice writes a file\n"
 699             self.file_path = abspath_expanduser_unicode(u"file1", base=self.alice_magicfolder.uploader._local_path_u)
 700             fileutil.write(self.file_path, "meow, meow meow. meow? meow meow! meow.")
 701             self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 702         d.addCallback(_wait_for, Alice_to_write_a_file)
 703
 704         d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file1", 0))
 705         d.addCallback(lambda ign: self._check_version_in_local_db(self.alice_magicfolder, u"file1", 0))
 706         d.addCallback(_check_uploader_count, 'objects_failed', 0)
 707         d.addCallback(_check_uploader_count, 'objects_succeeded', 1)
 708         d.addCallback(_check_uploader_count, 'files_uploaded', 1)
 709         d.addCallback(_check_uploader_count, 'objects_queued', 0)
 710         d.addCallback(_check_uploader_count, 'directories_created', 0)
 711
 712         d.addCallback(lambda ign: self._check_version_in_local_db(self.bob_magicfolder, u"file1", 0))
 713         d.addCallback(_check_downloader_count, 'objects_failed', 0)
 714         d.addCallback(_check_downloader_count, 'objects_downloaded', 1)
 715
 716         def Alice_to_delete_file():
 717             print "Alice deletes the file!\n"
 718             os.unlink(self.file_path)
 719             self.notify(to_filepath(self.file_path), self.inotify.IN_DELETE, magic=self.alice_magicfolder)
 720         d.addCallback(_wait_for, Alice_to_delete_file)
 721
 722         def notify_bob_moved(ign):
 723             d0 = self.bob_magicfolder.uploader.set_hook('processed')
 724             p = abspath_expanduser_unicode(u"file1", base=self.bob_magicfolder.uploader._local_path_u)
 725             self.notify(to_filepath(p), self.inotify.IN_MOVED_FROM, magic=self.bob_magicfolder)
 726             self.notify(to_filepath(p + u'.backup'), self.inotify.IN_MOVED_TO, magic=self.bob_magicfolder)
 727             bob_clock.advance(0)
 728             return d0
 729         d.addCallback(notify_bob_moved)
 730
 731         d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file1", 1))
 732         d.addCallback(lambda ign: self._check_version_in_local_db(self.alice_magicfolder, u"file1", 1))
 733         d.addCallback(_check_uploader_count, 'objects_failed', 0)
 734         d.addCallback(_check_uploader_count, 'objects_succeeded', 2)
 735
 736         d.addCallback(lambda ign: self._check_version_in_local_db(self.bob_magicfolder, u"file1", 1))
 737         d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file1", 1))
 738         d.addCallback(lambda ign: self._check_file_gone(self.bob_magicfolder, u"file1"))
 739         d.addCallback(_check_downloader_count, 'objects_failed', 0)
 740         d.addCallback(_check_downloader_count, 'objects_excluded', 1)
 741         d.addCallback(_check_downloader_count, 'objects_downloaded', 2)
 742
 743         def Alice_to_rewrite_file():
 744             print "Alice rewrites file\n"
 745             self.file_path = abspath_expanduser_unicode(u"file1", base=self.alice_magicfolder.uploader._local_path_u)
 746             fileutil.write(self.file_path, "Alice suddenly sees the white rabbit running into the forest.")
 747             self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
 748         d.addCallback(_wait_for, Alice_to_rewrite_file)
 749
 750         d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file1", 2))
 751         d.addCallback(lambda ign: self._check_version_in_local_db(self.alice_magicfolder, u"file1", 2))
 752         d.addCallback(_check_uploader_count, 'objects_failed', 0)
 753         d.addCallback(_check_uploader_count, 'objects_succeeded', 3)
 754         d.addCallback(_check_uploader_count, 'files_uploaded', 3)
 755         d.addCallback(_check_uploader_count, 'objects_queued', 0)
 756         d.addCallback(_check_uploader_count, 'directories_created', 0)
 757
 758         d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file1", 2))
 759         d.addCallback(lambda ign: self._check_version_in_local_db(self.bob_magicfolder, u"file1", 2))
 760         d.addCallback(_check_downloader_count, 'objects_failed', 0)
 761         d.addCallback(_check_downloader_count, 'objects_downloaded', 3)
 762
 763         path_u = u"/tmp/magic_folder_test"
 764         encoded_path_u = magicpath.path2magic(u"/tmp/magic_folder_test")
 765
 766         def Alice_tries_to_p0wn_Bob(ign):
 767             print "Alice tries to p0wn Bob\n"
 768             processed_d = self.bob_magicfolder.downloader.set_hook('processed')
 769
 770             # upload a file that would provoke the security bug from #2506
 771             uploadable = Data("", self.alice_magicfolder._client.convergence)
 772             alice_dmd = self.alice_magicfolder.uploader._upload_dirnode
 773
 774             d2 = alice_dmd.add_file(encoded_path_u, uploadable, metadata={"version": 0}, overwrite=True)
 775             d2.addCallback(lambda ign: self.failUnless(alice_dmd.has_child(encoded_path_u)))
 776             d2.addCallback(_wait_for_Bob, processed_d)
 777             return d2
 778         d.addCallback(Alice_tries_to_p0wn_Bob)
 779
 780         d.addCallback(lambda ign: self.failIf(os.path.exists(path_u)))
 781         d.addCallback(lambda ign: self._check_version_in_local_db(self.bob_magicfolder, encoded_path_u, None))
 782         d.addCallback(_check_downloader_count, 'objects_excluded', 2)
 783         d.addCallback(_check_downloader_count, 'objects_downloaded', 3)
 784
 785         def _cleanup(ign, magicfolder, clock):
 786             if magicfolder is not None:
 787                 d2 = magicfolder.finish()
 788                 clock.advance(0)
 789                 return d2
 790
 791         def cleanup_Alice_and_Bob(result):
 792             print "cleanup alice bob test\n"
 793             d = defer.succeed(None)
 794             d.addCallback(_cleanup, self.alice_magicfolder, alice_clock)
 795             d.addCallback(_cleanup, self.bob_magicfolder, bob_clock)
 796             d.addCallback(lambda ign: result)
 797             return d
 798         d.addBoth(cleanup_Alice_and_Bob)
 799         return d
 800
 801
 802 class MockTest(MagicFolderTestMixin, unittest.TestCase):
 803     """This can run on any platform, and even if twisted.internet.inotify can't be imported."""
 804
 805     def setUp(self):
 806         MagicFolderTestMixin.setUp(self)
 807         self.inotify = fake_inotify
 808         self.patch(magic_folder, 'get_inotify_module', lambda: self.inotify)
 809
 810     def notify(self, path, mask, magic=None):
 811         if magic is None:
 812             magic = self.magicfolder
 813         magic.uploader._notifier.event(path, mask)
 814
 815     def test_errors(self):
 816         self.set_up_grid()
 817
 818         errors_dir = abspath_expanduser_unicode(u"errors_dir", base=self.basedir)
 819         os.mkdir(errors_dir)
 820         not_a_dir = abspath_expanduser_unicode(u"NOT_A_DIR", base=self.basedir)
 821         fileutil.write(not_a_dir, "")
 822         magicfolderdb = abspath_expanduser_unicode(u"magicfolderdb", base=self.basedir)
 823         doesnotexist  = abspath_expanduser_unicode(u"doesnotexist", base=self.basedir)
 824
 825         client = self.g.clients[0]
 826         d = client.create_dirnode()
 827         def _check_errors(n):
 828             self.failUnless(IDirectoryNode.providedBy(n))
 829             upload_dircap = n.get_uri()
 830             readonly_dircap = n.get_readonly_uri()
 831
 832             self.shouldFail(AssertionError, 'nonexistent local.directory', 'there is no directory',
 833                             MagicFolder, client, upload_dircap, '', doesnotexist, magicfolderdb)
 834             self.shouldFail(AssertionError, 'non-directory local.directory', 'is not a directory',
 835                             MagicFolder, client, upload_dircap, '', not_a_dir, magicfolderdb)
 836             self.shouldFail(AssertionError, 'bad upload.dircap', 'does not refer to a directory',
 837                             MagicFolder, client, 'bad', '', errors_dir, magicfolderdb)
 838             self.shouldFail(AssertionError, 'non-directory upload.dircap', 'does not refer to a directory',
 839                             MagicFolder, client, 'URI:LIT:foo', '', errors_dir, magicfolderdb)
 840             self.shouldFail(AssertionError, 'readonly upload.dircap', 'is not a writecap to a directory',
 841                             MagicFolder, client, readonly_dircap, '', errors_dir, magicfolderdb,)
 842             self.shouldFail(AssertionError, 'collective dircap',
 843                             "The URI in 'private/collective_dircap' is not a readonly cap to a directory.",
 844                             MagicFolder, client, upload_dircap, upload_dircap, errors_dir, magicfolderdb)
 845
 846             def _not_implemented():
 847                 raise NotImplementedError("blah")
 848             self.patch(magic_folder, 'get_inotify_module', _not_implemented)
 849             self.shouldFail(NotImplementedError, 'unsupported', 'blah',
 850                             MagicFolder, client, upload_dircap, '', errors_dir, magicfolderdb)
 851         d.addCallback(_check_errors)
 852         return d
 853
 854     def test_write_downloaded_file(self):
 855         workdir = u"cli/MagicFolder/write-downloaded-file"
 856         local_file = fileutil.abspath_expanduser_unicode(os.path.join(workdir, "foobar"))
 857
 858         class TestWriteFileMixin(WriteFileMixin):
 859             def _log(self, msg):
 860                 pass
 861
 862         writefile = TestWriteFileMixin()
 863
 864         # create a file with name "foobar" with content "foo"
 865         # write downloaded file content "bar" into "foobar" with is_conflict = False
 866         fileutil.make_dirs(workdir)
 867         fileutil.write(local_file, "foo")
 868
 869         # if is_conflict is False, then the .conflict file shouldn't exist.
 870         writefile._write_downloaded_file(local_file, "bar", False, None)
 871         conflicted_path = local_file + u".conflict"
 872         self.failIf(os.path.exists(conflicted_path))
 873
 874         # At this point, the backup file should exist with content "foo"
 875         backup_path = local_file + u".backup"
 876         self.failUnless(os.path.exists(backup_path))
 877         self.failUnlessEqual(fileutil.read(backup_path), "foo")
 878
 879         # .tmp file shouldn't exist
 880         self.failIf(os.path.exists(local_file + u".tmp"))
 881
 882         # .. and the original file should have the new content
 883         self.failUnlessEqual(fileutil.read(local_file), "bar")
 884
 885         # now a test for conflicted case
 886         writefile._write_downloaded_file(local_file, "bar", True, None)
 887         self.failUnless(os.path.exists(conflicted_path))
 888
 889         # .tmp file shouldn't exist
 890         self.failIf(os.path.exists(local_file + u".tmp"))
 891
 892
 893 class RealTest(MagicFolderTestMixin, unittest.TestCase):
 894     """This is skipped unless both Twisted and the platform support inotify."""
 895
 896     def setUp(self):
 897         MagicFolderTestMixin.setUp(self)
 898         self.inotify = magic_folder.get_inotify_module()
 899
 900     def notify(self, path, mask, **kw):
 901         # Writing to the filesystem causes the notification.
 902         pass
 903
 904 try:
 905     magic_folder.get_inotify_module()
 906 except NotImplementedError:
 907     RealTest.skip = "Magic Folder support can only be tested for-real on an OS that supports inotify or equivalent."