4 from twisted.trial import unittest
5 from twisted.internet import defer, task
7 from allmydata.interfaces import IDirectoryNode
8 from allmydata.util.assertutil import precondition
10 from allmydata.util import fake_inotify, fileutil
11 from allmydata.util.encodingutil import get_filesystem_encoding, to_filepath
12 from allmydata.util.consumer import download_to_data
13 from allmydata.test.no_network import GridTestMixin
14 from allmydata.test.common_util import ReallyEqualMixin, NonASCIIPathMixin
15 from allmydata.test.common import ShouldFailMixin
16 from .test_cli_magic_folder import MagicFolderCLITestMixin
18 from allmydata.frontends import magic_folder
19 from allmydata.frontends.magic_folder import MagicFolder, Downloader, WriteFileMixin
20 from allmydata import magicfolderdb, magicpath
21 from allmydata.util.fileutil import abspath_expanduser_unicode
22 from allmydata.immutable.upload import Data
25 class MagicFolderTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, ReallyEqualMixin, NonASCIIPathMixin):
27 These tests will be run both with a mock notifier, and (on platforms that support it)
28 with the real INotify.
32 GridTestMixin.setUp(self)
34 self.basedir = abspath_expanduser_unicode(temp.decode(get_filesystem_encoding()))
35 self.magicfolder = None
36 self.patch(Downloader, 'REMOTE_SCAN_INTERVAL', 0)
38 def _get_count(self, name, client=None):
39 counters = (client or self.get_client()).stats_provider.get_stats()["counters"]
40 return counters.get('magic_folder.%s' % (name,), 0)
43 dbfile = abspath_expanduser_unicode(u"magicfolderdb.sqlite", base=self.basedir)
44 mdb = magicfolderdb.get_magicfolderdb(dbfile, create_version=(magicfolderdb.SCHEMA_v1, 1))
45 self.failUnless(mdb, "unable to create magicfolderdb from %r" % (dbfile,))
46 self.failUnlessEqual(mdb.VERSION, 1)
49 def _restart_client(self, ign):
50 #print "_restart_client"
51 d = self.restart_client()
52 d.addCallback(self._wait_until_started)
55 def _wait_until_started(self, ign):
56 #print "_wait_until_started"
57 self.magicfolder = self.get_client().getServiceNamed('magic-folder')
58 return self.magicfolder.ready()
60 def test_db_basic(self):
61 fileutil.make_dirs(self.basedir)
64 def test_db_persistence(self):
65 """Test that a file upload creates an entry in the database."""
67 fileutil.make_dirs(self.basedir)
71 pathinfo = fileutil.PathInfo(isdir=False, isfile=True, islink=False,
72 exists=True, size=1, mtime=123, ctime=456)
73 db.did_upload_version(relpath1, 0, 'URI:LIT:1', 'URI:LIT:0', 0, pathinfo)
76 c.execute("SELECT size, mtime, ctime"
81 self.failUnlessEqual(row, (pathinfo.size, pathinfo.mtime, pathinfo.ctime))
83 # Second test uses magic_folder.is_new_file instead of SQL query directly
84 # to confirm the previous upload entry in the db.
86 path2 = os.path.join(self.basedir, relpath2)
87 fileutil.write(path2, "meow\n")
88 pathinfo = fileutil.get_pathinfo(path2)
89 db.did_upload_version(relpath2, 0, 'URI:LIT:2', 'URI:LIT:1', 0, pathinfo)
90 db_entry = db.get_db_entry(relpath2)
91 self.failUnlessFalse(magic_folder.is_new_file(pathinfo, db_entry))
93 different_pathinfo = fileutil.PathInfo(isdir=False, isfile=True, islink=False,
94 exists=True, size=0, mtime=pathinfo.mtime, ctime=pathinfo.ctime)
95 self.failUnlessTrue(magic_folder.is_new_file(different_pathinfo, db_entry))
97 def test_magicfolder_start_service(self):
100 self.local_dir = abspath_expanduser_unicode(self.unicode_or_fallback(u"l\u00F8cal_dir", u"local_dir"),
102 self.mkdir_nonascii(self.local_dir)
104 d = defer.succeed(None)
105 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.dirs_monitored'), 0))
107 d.addCallback(lambda ign: self.create_invite_join_magic_folder(u"Alice", self.local_dir))
108 d.addCallback(self._restart_client)
110 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.dirs_monitored'), 1))
111 d.addBoth(self.cleanup)
112 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.dirs_monitored'), 0))
115 def test_scan_once_on_startup(self):
117 self.local_dir = abspath_expanduser_unicode(u"test_scan_once_on_startup", base=self.basedir)
118 self.mkdir_nonascii(self.local_dir)
119 self.collective_dircap = ""
121 alice_clock = task.Clock()
122 bob_clock = task.Clock()
123 d = self.setup_alice_and_bob(alice_clock, bob_clock)
125 def upload_stuff(ignore):
126 uploadable = Data("", self.alice_magicfolder._client.convergence)
127 return self.alice_magicfolder._client.upload(uploadable)
128 d.addCallback(upload_stuff)
129 def check_is_upload(ignore):
130 alice_clock.advance(99)
131 d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 0, magic=self.alice_magicfolder))
132 d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0, magic=self.alice_magicfolder))
133 d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
134 d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 0, magic=self.alice_magicfolder))
135 d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
136 d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 0, magic=self.alice_magicfolder))
138 d.addCallback(check_is_upload)
139 def _cleanup(ign, magicfolder, clock):
140 if magicfolder is not None:
141 d2 = magicfolder.finish()
144 def cleanup_Alice_and_Bob(result):
145 print "cleanup alice bob test\n"
146 d = defer.succeed(None)
147 d.addCallback(_cleanup, self.alice_magicfolder, alice_clock)
148 d.addCallback(_cleanup, self.bob_magicfolder, bob_clock)
149 d.addCallback(lambda ign: result)
152 d.addBoth(cleanup_Alice_and_Bob)
155 def test_move_tree(self):
158 self.local_dir = abspath_expanduser_unicode(self.unicode_or_fallback(u"l\u00F8cal_dir", u"local_dir"),
160 self.mkdir_nonascii(self.local_dir)
162 empty_tree_name = self.unicode_or_fallback(u"empty_tr\u00EAe", u"empty_tree")
163 empty_tree_dir = abspath_expanduser_unicode(empty_tree_name, base=self.basedir)
164 new_empty_tree_dir = abspath_expanduser_unicode(empty_tree_name, base=self.local_dir)
166 small_tree_name = self.unicode_or_fallback(u"small_tr\u00EAe", u"empty_tree")
167 small_tree_dir = abspath_expanduser_unicode(small_tree_name, base=self.basedir)
168 new_small_tree_dir = abspath_expanduser_unicode(small_tree_name, base=self.local_dir)
170 d = self.create_invite_join_magic_folder(u"Alice", self.local_dir)
171 d.addCallback(self._restart_client)
173 def _check_move_empty_tree(res):
174 print "_check_move_empty_tree"
175 uploaded_d = self.magicfolder.uploader.set_hook('processed')
176 self.mkdir_nonascii(empty_tree_dir)
177 os.rename(empty_tree_dir, new_empty_tree_dir)
178 self.notify(to_filepath(new_empty_tree_dir), self.inotify.IN_MOVED_TO)
181 d.addCallback(_check_move_empty_tree)
182 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
183 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 1))
184 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.files_uploaded'), 0))
185 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
186 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.directories_created'), 1))
188 def _check_move_small_tree(res):
189 print "_check_move_small_tree"
190 uploaded_d = self.magicfolder.uploader.set_hook('processed', ignore_count=1)
191 self.mkdir_nonascii(small_tree_dir)
192 what_path = abspath_expanduser_unicode(u"what", base=small_tree_dir)
193 fileutil.write(what_path, "say when")
194 os.rename(small_tree_dir, new_small_tree_dir)
195 self.notify(to_filepath(new_small_tree_dir), self.inotify.IN_MOVED_TO)
198 d.addCallback(_check_move_small_tree)
199 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
200 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 3))
201 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.files_uploaded'), 1))
202 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
203 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.directories_created'), 2))
205 def _check_moved_tree_is_watched(res):
206 print "_check_moved_tree_is_watched"
207 uploaded_d = self.magicfolder.uploader.set_hook('processed')
208 another_path = abspath_expanduser_unicode(u"another", base=new_small_tree_dir)
209 fileutil.write(another_path, "file")
210 self.notify(to_filepath(another_path), self.inotify.IN_CLOSE_WRITE)
213 d.addCallback(_check_moved_tree_is_watched)
214 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
215 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 4))
216 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.files_uploaded'), 2))
217 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
218 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.directories_created'), 2))
220 # Files that are moved out of the upload directory should no longer be watched.
221 #def _move_dir_away(ign):
222 # os.rename(new_empty_tree_dir, empty_tree_dir)
223 # # Wuh? Why don't we get this event for the real test?
224 # #self.notify(to_filepath(new_empty_tree_dir), self.inotify.IN_MOVED_FROM)
225 #d.addCallback(_move_dir_away)
226 #def create_file(val):
227 # test_file = abspath_expanduser_unicode(u"what", base=empty_tree_dir)
228 # fileutil.write(test_file, "meow")
231 #d.addCallback(create_file)
232 #d.addCallback(lambda ign: time.sleep(1)) # XXX ICK
233 #d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
234 #d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 4))
235 #d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.files_uploaded'), 2))
236 #d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
237 #d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.directories_created'), 2))
239 d.addBoth(self.cleanup)
242 def test_persistence(self):
244 Perform an upload of a given file and then stop the client.
245 Start a new client and magic-folder service... and verify that the file is NOT uploaded
246 a second time. This test is meant to test the database persistence along with
247 the startup and shutdown code paths of the magic-folder service.
250 self.local_dir = abspath_expanduser_unicode(u"test_persistence", base=self.basedir)
251 self.mkdir_nonascii(self.local_dir)
252 self.collective_dircap = ""
254 d = defer.succeed(None)
255 d.addCallback(lambda ign: self.create_invite_join_magic_folder(u"Alice", self.local_dir))
256 d.addCallback(self._restart_client)
258 def create_test_file(filename):
259 d2 = self.magicfolder.uploader.set_hook('processed')
260 test_file = abspath_expanduser_unicode(filename, base=self.local_dir)
261 fileutil.write(test_file, "meow %s" % filename)
262 self.notify(to_filepath(test_file), self.inotify.IN_CLOSE_WRITE)
264 d.addCallback(lambda ign: create_test_file(u"what1"))
265 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
266 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 1))
267 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
268 d.addCallback(self.cleanup)
270 d.addCallback(self._restart_client)
271 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
272 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 1))
273 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
274 d.addCallback(lambda ign: create_test_file(u"what2"))
275 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
276 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 2))
277 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
278 d.addBoth(self.cleanup)
281 @defer.inlineCallbacks
282 def test_delete(self):
284 self.local_dir = os.path.join(self.basedir, u"local_dir")
285 self.mkdir_nonascii(self.local_dir)
287 yield self.create_invite_join_magic_folder(u"Alice\u0101", self.local_dir)
288 yield self._restart_client(None)
292 up_proc = self.magicfolder.uploader.set_hook('processed')
293 # down_proc = self.magicfolder.downloader.set_hook('processed')
294 path = os.path.join(self.local_dir, u'foo')
295 fileutil.write(path, 'foo\n')
296 self.notify(to_filepath(path), self.inotify.IN_CLOSE_WRITE)
298 self.assertTrue(os.path.exists(path))
300 # the real test part: delete the file
301 up_proc = self.magicfolder.uploader.set_hook('processed')
303 self.notify(to_filepath(path), self.inotify.IN_DELETE)
305 self.assertFalse(os.path.exists(path))
307 # ensure we still have a DB entry, and that the version is 1
308 node, metadata = yield self.magicfolder.downloader._get_collective_latest_file(u'foo')
309 self.assertTrue(node is not None, "Failed to find %r in DMD" % (path,))
310 self.failUnlessEqual(metadata['version'], 1)
313 yield self.cleanup(None)
315 @defer.inlineCallbacks
316 def test_delete_and_restore(self):
318 self.local_dir = os.path.join(self.basedir, u"local_dir")
319 self.mkdir_nonascii(self.local_dir)
321 yield self.create_invite_join_magic_folder(u"Alice\u0101", self.local_dir)
322 yield self._restart_client(None)
326 up_proc = self.magicfolder.uploader.set_hook('processed')
327 # down_proc = self.magicfolder.downloader.set_hook('processed')
328 path = os.path.join(self.local_dir, u'foo')
329 fileutil.write(path, 'foo\n')
330 self.notify(to_filepath(path), self.inotify.IN_CLOSE_WRITE)
332 self.assertTrue(os.path.exists(path))
335 up_proc = self.magicfolder.uploader.set_hook('processed')
337 self.notify(to_filepath(path), self.inotify.IN_DELETE)
339 self.assertFalse(os.path.exists(path))
341 # ensure we still have a DB entry, and that the version is 1
342 node, metadata = yield self.magicfolder.downloader._get_collective_latest_file(u'foo')
343 self.assertTrue(node is not None, "Failed to find %r in DMD" % (path,))
344 self.failUnlessEqual(metadata['version'], 1)
346 # restore the file, with different contents
347 up_proc = self.magicfolder.uploader.set_hook('processed')
348 path = os.path.join(self.local_dir, u'foo')
349 fileutil.write(path, 'bar\n')
350 self.notify(to_filepath(path), self.inotify.IN_CLOSE_WRITE)
353 # ensure we still have a DB entry, and that the version is 2
354 node, metadata = yield self.magicfolder.downloader._get_collective_latest_file(u'foo')
355 self.assertTrue(node is not None, "Failed to find %r in DMD" % (path,))
356 self.failUnlessEqual(metadata['version'], 2)
359 yield self.cleanup(None)
361 @defer.inlineCallbacks
362 def test_alice_delete_bob_restore(self):
363 alice_clock = task.Clock()
364 bob_clock = task.Clock()
365 yield self.setup_alice_and_bob(alice_clock, bob_clock)
366 alice_dir = self.alice_magicfolder.uploader._local_path_u
367 bob_dir = self.bob_magicfolder.uploader._local_path_u
368 alice_fname = os.path.join(alice_dir, 'blam')
369 bob_fname = os.path.join(bob_dir, 'blam')
372 # alice creates a file, bob downloads it
373 alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
374 bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
376 fileutil.write(alice_fname, 'contents0\n')
377 self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
379 alice_clock.advance(0)
380 yield alice_proc # alice uploads
383 yield bob_proc # bob downloads
386 yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
387 yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 0)
388 yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
389 yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 0)
390 yield self.failUnlessReallyEqual(
391 self._get_count('downloader.objects_failed', client=self.bob_magicfolder._client),
394 yield self.failUnlessReallyEqual(
395 self._get_count('downloader.objects_downloaded', client=self.bob_magicfolder._client),
400 # now bob deletes it (bob should upload, alice download)
401 bob_proc = self.bob_magicfolder.uploader.set_hook('processed')
402 alice_proc = self.alice_magicfolder.downloader.set_hook('processed')
404 self.notify(to_filepath(bob_fname), self.inotify.IN_DELETE, magic=self.bob_magicfolder)
408 alice_clock.advance(0)
412 node, metadata = yield self.alice_magicfolder.downloader._get_collective_latest_file(u'blam')
413 self.assertTrue(metadata['deleted'])
414 yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
415 yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 1)
416 yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
417 yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 1)
419 print("ALICE RESTORE")
420 # now alice restores it (alice should upload, bob download)
421 alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
422 bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
423 fileutil.write(alice_fname, 'new contents\n')
424 self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
426 alice_clock.advance(0)
432 node, metadata = yield self.alice_magicfolder.downloader._get_collective_latest_file(u'blam')
433 self.assertTrue('deleted' not in metadata or not metadata['deleted'])
434 yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 2)
435 yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 2)
436 yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 2)
437 yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 2)
441 d0 = self.alice_magicfolder.finish()
442 alice_clock.advance(0)
445 d1 = self.bob_magicfolder.finish()
449 @defer.inlineCallbacks
450 def test_alice_sees_bobs_delete_with_error(self):
451 # alice creates a file, bob deletes it -- and we also arrange
452 # for Alice's file to have "gone missing" as well.
453 alice_clock = task.Clock()
454 bob_clock = task.Clock()
455 yield self.setup_alice_and_bob(alice_clock, bob_clock)
456 alice_dir = self.alice_magicfolder.uploader._local_path_u
457 bob_dir = self.bob_magicfolder.uploader._local_path_u
458 alice_fname = os.path.join(alice_dir, 'blam')
459 bob_fname = os.path.join(bob_dir, 'blam')
462 # alice creates a file, bob downloads it
463 alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
464 bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
466 fileutil.write(alice_fname, 'contents0\n')
467 self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
469 alice_clock.advance(0)
470 yield alice_proc # alice uploads
473 yield bob_proc # bob downloads
476 yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
477 yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 0)
478 yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
479 yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 0)
480 yield self.failUnlessReallyEqual(
481 self._get_count('downloader.objects_failed', client=self.bob_magicfolder._client),
484 yield self.failUnlessReallyEqual(
485 self._get_count('downloader.objects_downloaded', client=self.bob_magicfolder._client),
489 # now bob deletes it (bob should upload, alice download)
490 bob_proc = self.bob_magicfolder.uploader.set_hook('processed')
491 alice_proc = self.alice_magicfolder.downloader.set_hook('processed')
493 self.notify(to_filepath(bob_fname), self.inotify.IN_DELETE, magic=self.bob_magicfolder)
494 # just after notifying bob, we also delete alice's,
495 # covering the 'except' flow in _rename_deleted_file()
496 os.unlink(alice_fname)
500 alice_clock.advance(0)
504 node, metadata = yield self.alice_magicfolder.downloader._get_collective_latest_file(u'blam')
505 self.assertTrue(metadata['deleted'])
506 yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
507 yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 1)
508 yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
509 yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 1)
513 d0 = self.alice_magicfolder.finish()
514 alice_clock.advance(0)
517 d1 = self.bob_magicfolder.finish()
521 @defer.inlineCallbacks
522 def test_alice_create_bob_update(self):
523 alice_clock = task.Clock()
524 bob_clock = task.Clock()
525 yield self.setup_alice_and_bob(alice_clock, bob_clock)
526 alice_dir = self.alice_magicfolder.uploader._local_path_u
527 bob_dir = self.bob_magicfolder.uploader._local_path_u
528 alice_fname = os.path.join(alice_dir, 'blam')
529 bob_fname = os.path.join(bob_dir, 'blam')
532 # alice creates a file, bob downloads it
533 alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
534 bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
536 fileutil.write(alice_fname, 'contents0\n')
537 self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
539 alice_clock.advance(0)
540 yield alice_proc # alice uploads
543 yield bob_proc # bob downloads
546 yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
547 yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 0)
548 yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
549 yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 0)
550 yield self.failUnlessReallyEqual(
551 self._get_count('downloader.objects_failed', client=self.bob_magicfolder._client),
554 yield self.failUnlessReallyEqual(
555 self._get_count('downloader.objects_downloaded', client=self.bob_magicfolder._client),
559 # now bob updates it (bob should upload, alice download)
560 bob_proc = self.bob_magicfolder.uploader.set_hook('processed')
561 alice_proc = self.alice_magicfolder.downloader.set_hook('processed')
562 fileutil.write(bob_fname, 'bob wuz here\n')
563 self.notify(to_filepath(bob_fname), self.inotify.IN_CLOSE_WRITE, magic=self.bob_magicfolder)
567 alice_clock.advance(0)
571 yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
572 yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 1)
573 yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
574 yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 1)
578 d0 = self.alice_magicfolder.finish()
579 alice_clock.advance(0)
582 d1 = self.bob_magicfolder.finish()
586 @defer.inlineCallbacks
587 def test_alice_delete_and_restore(self):
588 alice_clock = task.Clock()
589 bob_clock = task.Clock()
590 yield self.setup_alice_and_bob(alice_clock, bob_clock)
591 alice_dir = self.alice_magicfolder.uploader._local_path_u
592 bob_dir = self.bob_magicfolder.uploader._local_path_u
593 alice_fname = os.path.join(alice_dir, 'blam')
594 bob_fname = os.path.join(bob_dir, 'blam')
597 # alice creates a file, bob downloads it
598 alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
599 bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
601 fileutil.write(alice_fname, 'contents0\n')
602 self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
604 alice_clock.advance(0)
605 yield alice_proc # alice uploads
608 yield bob_proc # bob downloads
611 yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
612 yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 0)
613 yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
614 yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 0)
615 yield self.failUnlessReallyEqual(
616 self._get_count('downloader.objects_failed', client=self.bob_magicfolder._client),
619 yield self.failUnlessReallyEqual(
620 self._get_count('downloader.objects_downloaded', client=self.bob_magicfolder._client),
623 self.failUnless(os.path.exists(bob_fname))
625 # now alice deletes it (alice should upload, bob download)
626 alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
627 bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
628 os.unlink(alice_fname)
629 self.notify(to_filepath(alice_fname), self.inotify.IN_DELETE, magic=self.alice_magicfolder)
631 alice_clock.advance(0)
637 yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
638 yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 1)
639 yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
640 yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 1)
641 self.failIf(os.path.exists(bob_fname))
643 # now alice restores the file (with new contents)
644 alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
645 bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
646 fileutil.write(alice_fname, 'alice wuz here\n')
647 self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
649 alice_clock.advance(0)
655 yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 2)
656 yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 2)
657 yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 2)
658 yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 2)
659 self.failUnless(os.path.exists(bob_fname))
663 d0 = self.alice_magicfolder.finish()
664 alice_clock.advance(0)
667 d1 = self.bob_magicfolder.finish()
671 def test_magic_folder(self):
673 self.local_dir = os.path.join(self.basedir, self.unicode_or_fallback(u"loc\u0101l_dir", u"local_dir"))
674 self.mkdir_nonascii(self.local_dir)
676 d = self.create_invite_join_magic_folder(u"Alice\u0101", self.local_dir)
677 d.addCallback(self._restart_client)
679 # Write something short enough for a LIT file.
680 d.addCallback(lambda ign: self._check_file(u"short", "test"))
682 # Write to the same file again with different data.
683 d.addCallback(lambda ign: self._check_file(u"short", "different"))
685 # Test that temporary files are not uploaded.
686 d.addCallback(lambda ign: self._check_file(u"tempfile", "test", temporary=True))
688 # Test creation of a subdirectory.
689 d.addCallback(lambda ign: self._check_mkdir(u"directory"))
691 # Write something longer, and also try to test a Unicode name if the fs can represent it.
692 name_u = self.unicode_or_fallback(u"l\u00F8ng", u"long")
693 d.addCallback(lambda ign: self._check_file(name_u, "test"*100))
695 # TODO: test that causes an upload failure.
696 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
698 d.addBoth(self.cleanup)
701 def _check_mkdir(self, name_u):
702 return self._check_file(name_u + u"/", "", directory=True)
704 def _check_file(self, name_u, data, temporary=False, directory=False):
705 precondition(not (temporary and directory), temporary=temporary, directory=directory)
707 print "%r._check_file(%r, %r, temporary=%r, directory=%r)" % (self, name_u, data, temporary, directory)
708 previously_uploaded = self._get_count('uploader.objects_succeeded')
709 previously_disappeared = self._get_count('uploader.objects_disappeared')
711 d = self.magicfolder.uploader.set_hook('processed')
713 path_u = abspath_expanduser_unicode(name_u, base=self.local_dir)
714 path = to_filepath(path_u)
718 event_mask = self.inotify.IN_CREATE | self.inotify.IN_ISDIR
720 # We don't use FilePath.setContent() here because it creates a temporary file that
721 # is renamed into place, which causes events that the test is not expecting.
722 f = open(path_u, "wb")
724 if temporary and sys.platform != "win32":
729 if temporary and sys.platform == "win32":
731 self.notify(path, self.inotify.IN_DELETE, flush=False)
732 event_mask = self.inotify.IN_CLOSE_WRITE
734 self.notify(path, event_mask)
735 encoded_name_u = magicpath.path2magic(name_u)
737 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
739 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_disappeared'),
740 previously_disappeared + 1))
743 print "here %r %r" % (n, res)
746 d.addCallback(lambda ign: self.upload_dirnode.list())
747 d.addBoth(_here, 1.5)
748 d.addCallback(lambda ign: self.upload_dirnode.get(encoded_name_u))
750 d.addCallback(download_to_data)
752 d.addCallback(lambda actual_data: self.failUnlessReallyEqual(actual_data, data))
754 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'),
755 previously_uploaded + 1))
757 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
760 def _check_version_in_dmd(self, magicfolder, relpath_u, expected_version):
761 encoded_name_u = magicpath.path2magic(relpath_u)
762 d = magicfolder.downloader._get_collective_latest_file(encoded_name_u)
763 def check_latest(result):
764 if result[0] is not None:
765 node, metadata = result
766 d.addCallback(lambda ign: self.failUnlessEqual(metadata['version'], expected_version))
767 d.addCallback(check_latest)
770 def _check_version_in_local_db(self, magicfolder, relpath_u, expected_version):
771 db_entry = magicfolder._db.get_db_entry(relpath_u)
772 if db_entry is not None:
773 #print "_check_version_in_local_db: %r has version %s" % (relpath_u, version)
774 self.failUnlessEqual(db_entry.version, expected_version)
776 def _check_file_gone(self, magicfolder, relpath_u):
777 path = os.path.join(magicfolder.uploader._local_path_u, relpath_u)
778 self.assertTrue(not os.path.exists(path))
780 def _check_uploader_count(self, name, expected, magic=None):
781 self.failUnlessReallyEqual(self._get_count('uploader.'+name, client=(magic or self.alice_magicfolder)._client),
784 def _check_downloader_count(self, name, expected, magic=None):
785 self.failUnlessReallyEqual(self._get_count('downloader.'+name, client=(magic or self.bob_magicfolder)._client),
788 def test_alice_bob(self):
789 alice_clock = task.Clock()
790 bob_clock = task.Clock()
791 d = self.setup_alice_and_bob(alice_clock, bob_clock)
793 def _wait_for_Alice(ign, downloaded_d):
794 print "Now waiting for Alice to download\n"
795 alice_clock.advance(0)
798 def _wait_for_Bob(ign, downloaded_d):
799 print "Now waiting for Bob to download\n"
803 def _wait_for(ign, something_to_do, alice=True):
805 downloaded_d = self.bob_magicfolder.downloader.set_hook('processed')
806 uploaded_d = self.alice_magicfolder.uploader.set_hook('processed')
808 downloaded_d = self.alice_magicfolder.downloader.set_hook('processed')
809 uploaded_d = self.bob_magicfolder.uploader.set_hook('processed')
812 print "Waiting for Alice to upload\n"
813 alice_clock.advance(0)
814 uploaded_d.addCallback(_wait_for_Bob, downloaded_d)
816 print "Waiting for Bob to upload\n"
818 uploaded_d.addCallback(_wait_for_Alice, downloaded_d)
821 def Alice_to_write_a_file():
822 print "Alice writes a file\n"
823 self.file_path = abspath_expanduser_unicode(u"file1", base=self.alice_magicfolder.uploader._local_path_u)
824 fileutil.write(self.file_path, "meow, meow meow. meow? meow meow! meow.")
825 self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
826 d.addCallback(_wait_for, Alice_to_write_a_file)
828 d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file1", 0))
829 d.addCallback(lambda ign: self._check_version_in_local_db(self.alice_magicfolder, u"file1", 0))
830 d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0))
831 d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 1))
832 d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 1))
833 d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0))
834 d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0))
835 d.addCallback(lambda ign: self._check_uploader_count('objects_conflicted', 0))
836 d.addCallback(lambda ign: self._check_uploader_count('objects_conflicted', 0, magic=self.bob_magicfolder))
838 d.addCallback(lambda ign: self._check_version_in_local_db(self.bob_magicfolder, u"file1", 0))
839 d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0))
840 d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 1))
841 d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 0, magic=self.bob_magicfolder))
843 def Alice_to_delete_file():
844 print "Alice deletes the file!\n"
845 os.unlink(self.file_path)
846 self.notify(to_filepath(self.file_path), self.inotify.IN_DELETE, magic=self.alice_magicfolder)
847 d.addCallback(_wait_for, Alice_to_delete_file)
849 def notify_bob_moved(ign):
850 d0 = self.bob_magicfolder.uploader.set_hook('processed')
851 p = abspath_expanduser_unicode(u"file1", base=self.bob_magicfolder.uploader._local_path_u)
852 self.notify(to_filepath(p), self.inotify.IN_MOVED_FROM, magic=self.bob_magicfolder, flush=False)
853 self.notify(to_filepath(p + u'.backup'), self.inotify.IN_MOVED_TO, magic=self.bob_magicfolder)
856 d.addCallback(notify_bob_moved)
858 d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file1", 1))
859 d.addCallback(lambda ign: self._check_version_in_local_db(self.alice_magicfolder, u"file1", 1))
860 d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0))
861 d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 2))
862 d.addCallback(lambda ign: self._check_uploader_count('objects_not_uploaded', 1, magic=self.bob_magicfolder))
863 d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 1, magic=self.bob_magicfolder))
865 d.addCallback(lambda ign: self._check_version_in_local_db(self.bob_magicfolder, u"file1", 1))
866 d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file1", 1))
867 d.addCallback(lambda ign: self._check_file_gone(self.bob_magicfolder, u"file1"))
868 d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0))
869 d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 2))
871 def Alice_to_rewrite_file():
872 print "Alice rewrites file\n"
873 self.file_path = abspath_expanduser_unicode(u"file1", base=self.alice_magicfolder.uploader._local_path_u)
874 fileutil.write(self.file_path, "Alice suddenly sees the white rabbit running into the forest.")
875 self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
876 d.addCallback(_wait_for, Alice_to_rewrite_file)
878 d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file1", 2))
879 d.addCallback(lambda ign: self._check_version_in_local_db(self.alice_magicfolder, u"file1", 2))
880 d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0))
881 d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 3))
882 d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 3))
883 d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0))
884 d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0))
885 d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
886 d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0))
888 d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file1", 2))
889 d.addCallback(lambda ign: self._check_version_in_local_db(self.bob_magicfolder, u"file1", 2))
890 d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0))
891 d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 3))
893 path_u = u"/tmp/magic_folder_test"
894 encoded_path_u = magicpath.path2magic(u"/tmp/magic_folder_test")
896 def Alice_tries_to_p0wn_Bob(ign):
897 print "Alice tries to p0wn Bob\n"
898 processed_d = self.bob_magicfolder.downloader.set_hook('processed')
900 # upload a file that would provoke the security bug from #2506
901 uploadable = Data("", self.alice_magicfolder._client.convergence)
902 alice_dmd = self.alice_magicfolder.uploader._upload_dirnode
904 d2 = alice_dmd.add_file(encoded_path_u, uploadable, metadata={"version": 0}, overwrite=True)
905 d2.addCallback(lambda ign: self.failUnless(alice_dmd.has_child(encoded_path_u)))
906 d2.addCallback(_wait_for_Bob, processed_d)
908 d.addCallback(Alice_tries_to_p0wn_Bob)
910 d.addCallback(lambda ign: self.failIf(os.path.exists(path_u)))
911 d.addCallback(lambda ign: self._check_version_in_local_db(self.bob_magicfolder, encoded_path_u, None))
912 d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 3))
913 d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
914 d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0))
916 def Bob_to_rewrite_file():
917 print "Bob rewrites file\n"
918 self.file_path = abspath_expanduser_unicode(u"file1", base=self.bob_magicfolder.uploader._local_path_u)
919 print "---- bob's file is %r" % (self.file_path,)
920 fileutil.write(self.file_path, "No white rabbit to be found.")
921 self.magicfolder = self.bob_magicfolder
922 self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE)
923 d.addCallback(lambda ign: _wait_for(None, Bob_to_rewrite_file, alice=False))
925 d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file1", 3))
926 d.addCallback(lambda ign: self._check_version_in_local_db(self.bob_magicfolder, u"file1", 3))
927 d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0, magic=self.bob_magicfolder))
928 d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 2, magic=self.bob_magicfolder))
929 d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 1, magic=self.bob_magicfolder))
930 d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0, magic=self.bob_magicfolder))
931 d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0, magic=self.bob_magicfolder))
932 d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0))
934 d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file1", 3))
935 d.addCallback(lambda ign: self._check_version_in_local_db(self.alice_magicfolder, u"file1", 3))
936 d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
937 d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 1, magic=self.alice_magicfolder))
938 d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
940 def Alice_conflicts_with_Bobs_last_downloaded_uri():
941 print "Alice conflicts with Bob\n"
942 downloaded_d = self.bob_magicfolder.downloader.set_hook('processed')
943 uploadable = Data("do not follow the white rabbit", self.alice_magicfolder._client.convergence)
944 alice_dmd = self.alice_magicfolder.uploader._upload_dirnode
945 d2 = alice_dmd.add_file(u"file1", uploadable,
946 metadata={"version": 5,
947 "last_downloaded_uri" : "URI:LIT:" },
949 print "Waiting for Alice to upload\n"
950 d2.addCallback(lambda ign: bob_clock.advance(6))
951 d2.addCallback(lambda ign: downloaded_d)
952 d2.addCallback(lambda ign: self.failUnless(alice_dmd.has_child(encoded_path_u)))
955 d.addCallback(lambda ign: Alice_conflicts_with_Bobs_last_downloaded_uri())
956 d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 4))
957 d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 1))
958 d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 1, magic=self.alice_magicfolder))
959 d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
960 d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
962 # prepare to perform another conflict test
963 def Alice_to_write_file2():
964 print "Alice writes a file\n"
965 self.file_path = abspath_expanduser_unicode(u"file2", base=self.alice_magicfolder.uploader._local_path_u)
966 fileutil.write(self.file_path, "something")
967 self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
968 d.addCallback(_wait_for, Alice_to_write_file2)
969 d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file2", 0))
970 d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
971 d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
973 def Bob_to_rewrite_file2():
974 print "Bob rewrites file\n"
975 self.file_path = abspath_expanduser_unicode(u"file2", base=self.bob_magicfolder.uploader._local_path_u)
976 print "---- bob's file is %r" % (self.file_path,)
977 fileutil.write(self.file_path, "roger roger. what vector?")
978 self.magicfolder = self.bob_magicfolder
979 self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE)
980 d.addCallback(lambda ign: _wait_for(None, Bob_to_rewrite_file2, alice=False))
981 d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file2", 1))
982 d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 5))
983 d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 1))
984 d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0, magic=self.bob_magicfolder))
985 d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 3, magic=self.bob_magicfolder))
986 d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 2, magic=self.bob_magicfolder))
987 d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0, magic=self.bob_magicfolder))
988 d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0, magic=self.bob_magicfolder))
989 d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
990 d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
991 d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 2, magic=self.alice_magicfolder))
993 # XXX here we advance the clock and then test again to make sure no values are monotonically increasing
994 # with each queue turn ;-p
995 alice_clock.advance(6)
997 d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file2", 1))
998 d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 5))
999 d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 1))
1000 d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0, magic=self.bob_magicfolder))
1001 d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 3, magic=self.bob_magicfolder))
1002 d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 2, magic=self.bob_magicfolder))
1003 d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0, magic=self.bob_magicfolder))
1004 d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0, magic=self.bob_magicfolder))
1005 d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
1006 d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
1007 d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 2, magic=self.alice_magicfolder))
1009 def Alice_conflicts_with_Bobs_last_uploaded_uri():
1010 print "Alice conflicts with Bob\n"
1011 encoded_path_u = magicpath.path2magic(u"file2")
1012 downloaded_d = self.bob_magicfolder.downloader.set_hook('processed')
1013 uploadable = Data("rabbits with sharp fangs", self.alice_magicfolder._client.convergence)
1014 alice_dmd = self.alice_magicfolder.uploader._upload_dirnode
1015 d2 = alice_dmd.add_file(u"file2", uploadable,
1016 metadata={"version": 5,
1017 "last_uploaded_uri" : "URI:LIT:" },
1019 print "Waiting for Alice to upload\n"
1020 d2.addCallback(lambda ign: bob_clock.advance(6))
1021 d2.addCallback(lambda ign: downloaded_d)
1022 d2.addCallback(lambda ign: self.failUnless(alice_dmd.has_child(encoded_path_u)))
1024 d.addCallback(lambda ign: Alice_conflicts_with_Bobs_last_uploaded_uri())
1025 d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file2", 5))
1026 d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 6))
1027 d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 2))
1028 d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0, magic=self.bob_magicfolder))
1029 d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 3, magic=self.bob_magicfolder))
1030 d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 2, magic=self.bob_magicfolder))
1031 d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0, magic=self.bob_magicfolder))
1032 d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0, magic=self.bob_magicfolder))
1033 d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
1034 d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
1035 d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 2, magic=self.alice_magicfolder))
1037 alice_clock.advance(6)
1038 bob_clock.advance(6)
1039 alice_clock.advance(6)
1040 bob_clock.advance(6)
1042 d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 2, magic=self.alice_magicfolder))
1043 d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
1044 d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 2))
1045 d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 6))
1047 # prepare to perform another conflict test
1048 def Alice_to_write_file3():
1049 print "Alice writes a file\n"
1050 self.file_path = abspath_expanduser_unicode(u"file3", base=self.alice_magicfolder.uploader._local_path_u)
1051 fileutil.write(self.file_path, "something")
1052 self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
1053 d.addCallback(_wait_for, Alice_to_write_file3)
1054 d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file3", 0))
1055 d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
1056 d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 7))
1057 d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 2, magic=self.alice_magicfolder))
1058 d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 2))
1059 d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
1061 def Bob_to_rewrite_file3():
1062 print "Bob rewrites file\n"
1063 self.file_path = abspath_expanduser_unicode(u"file3", base=self.bob_magicfolder.uploader._local_path_u)
1064 print "---- bob's file is %r" % (self.file_path,)
1065 fileutil.write(self.file_path, "roger roger")
1066 self.magicfolder = self.bob_magicfolder
1067 self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE)
1068 d.addCallback(lambda ign: _wait_for(None, Bob_to_rewrite_file3, alice=False))
1069 d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file3", 1))
1070 d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 7))
1071 d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 2))
1072 d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0, magic=self.bob_magicfolder))
1073 d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 4, magic=self.bob_magicfolder))
1074 d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 3, magic=self.bob_magicfolder))
1075 d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0, magic=self.bob_magicfolder))
1076 d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0, magic=self.bob_magicfolder))
1077 d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
1078 d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
1079 d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 3, magic=self.alice_magicfolder))
1083 def _cleanup(ign, magicfolder, clock):
1084 if magicfolder is not None:
1085 d2 = magicfolder.finish()
1089 def cleanup_Alice_and_Bob(result):
1090 print "cleanup alice bob test\n"
1091 d = defer.succeed(None)
1092 d.addCallback(_cleanup, self.alice_magicfolder, alice_clock)
1093 d.addCallback(_cleanup, self.bob_magicfolder, bob_clock)
1094 d.addCallback(lambda ign: result)
1096 d.addBoth(cleanup_Alice_and_Bob)
1100 class MockTest(MagicFolderTestMixin, unittest.TestCase):
1101 """This can run on any platform, and even if twisted.internet.inotify can't be imported."""
1104 MagicFolderTestMixin.setUp(self)
1105 self.inotify = fake_inotify
1106 self.patch(magic_folder, 'get_inotify_module', lambda: self.inotify)
1108 def notify(self, path, mask, magic=None, flush=True):
1110 magic = self.magicfolder
1111 magic.uploader._notifier.event(path, mask)
1112 # no flush for the mock test.
1114 def test_errors(self):
1117 errors_dir = abspath_expanduser_unicode(u"errors_dir", base=self.basedir)
1118 os.mkdir(errors_dir)
1119 not_a_dir = abspath_expanduser_unicode(u"NOT_A_DIR", base=self.basedir)
1120 fileutil.write(not_a_dir, "")
1121 magicfolderdb = abspath_expanduser_unicode(u"magicfolderdb", base=self.basedir)
1122 doesnotexist = abspath_expanduser_unicode(u"doesnotexist", base=self.basedir)
1124 client = self.g.clients[0]
1125 d = client.create_dirnode()
1126 def _check_errors(n):
1127 self.failUnless(IDirectoryNode.providedBy(n))
1128 upload_dircap = n.get_uri()
1129 readonly_dircap = n.get_readonly_uri()
1131 self.shouldFail(AssertionError, 'nonexistent local.directory', 'there is no directory',
1132 MagicFolder, client, upload_dircap, '', doesnotexist, magicfolderdb, 0077)
1133 self.shouldFail(AssertionError, 'non-directory local.directory', 'is not a directory',
1134 MagicFolder, client, upload_dircap, '', not_a_dir, magicfolderdb, 0077)
1135 self.shouldFail(AssertionError, 'bad upload.dircap', 'does not refer to a directory',
1136 MagicFolder, client, 'bad', '', errors_dir, magicfolderdb, 0077)
1137 self.shouldFail(AssertionError, 'non-directory upload.dircap', 'does not refer to a directory',
1138 MagicFolder, client, 'URI:LIT:foo', '', errors_dir, magicfolderdb, 0077)
1139 self.shouldFail(AssertionError, 'readonly upload.dircap', 'is not a writecap to a directory',
1140 MagicFolder, client, readonly_dircap, '', errors_dir, magicfolderdb, 0077)
1141 self.shouldFail(AssertionError, 'collective dircap', 'is not a readonly cap to a directory',
1142 MagicFolder, client, upload_dircap, upload_dircap, errors_dir, magicfolderdb, 0077)
1144 def _not_implemented():
1145 raise NotImplementedError("blah")
1146 self.patch(magic_folder, 'get_inotify_module', _not_implemented)
1147 self.shouldFail(NotImplementedError, 'unsupported', 'blah',
1148 MagicFolder, client, upload_dircap, '', errors_dir, magicfolderdb, 0077)
1149 d.addCallback(_check_errors)
1152 def test_write_downloaded_file(self):
1153 workdir = u"cli/MagicFolder/write-downloaded-file"
1154 local_file = fileutil.abspath_expanduser_unicode(os.path.join(workdir, "foobar"))
1156 class TestWriteFileMixin(WriteFileMixin):
1157 def _log(self, msg):
1160 writefile = TestWriteFileMixin()
1161 writefile._umask = 0077
1163 # create a file with name "foobar" with content "foo"
1164 # write downloaded file content "bar" into "foobar" with is_conflict = False
1165 fileutil.make_dirs(workdir)
1166 fileutil.write(local_file, "foo")
1168 # if is_conflict is False, then the .conflict file shouldn't exist.
1169 writefile._write_downloaded_file(local_file, "bar", False, None)
1170 conflicted_path = local_file + u".conflict"
1171 self.failIf(os.path.exists(conflicted_path))
1173 # At this point, the backup file should exist with content "foo"
1174 backup_path = local_file + u".backup"
1175 self.failUnless(os.path.exists(backup_path))
1176 self.failUnlessEqual(fileutil.read(backup_path), "foo")
1178 # .tmp file shouldn't exist
1179 self.failIf(os.path.exists(local_file + u".tmp"))
1181 # .. and the original file should have the new content
1182 self.failUnlessEqual(fileutil.read(local_file), "bar")
1184 # now a test for conflicted case
1185 writefile._write_downloaded_file(local_file, "bar", True, None)
1186 self.failUnless(os.path.exists(conflicted_path))
1188 # .tmp file shouldn't exist
1189 self.failIf(os.path.exists(local_file + u".tmp"))
1191 def test_periodic_full_scan(self):
1193 self.local_dir = abspath_expanduser_unicode(u"test_periodic_full_scan",base=self.basedir)
1194 self.mkdir_nonascii(self.local_dir)
1196 alice_clock = task.Clock()
1197 d = self.do_create_magic_folder(0)
1198 d.addCallback(lambda ign: self.do_invite(0, u"Alice\u00F8"))
1199 def get_invite_code(result):
1200 self.invite_code = result[1].strip()
1201 d.addCallback(get_invite_code)
1202 d.addCallback(lambda ign: self.do_join(0, self.local_dir, self.invite_code))
1203 def get_alice_caps(ign):
1204 self.alice_collective_dircap, self.alice_upload_dircap = self.get_caps_from_files(0)
1205 d.addCallback(get_alice_caps)
1206 d.addCallback(lambda ign: self.check_joined_config(0, self.alice_upload_dircap))
1207 d.addCallback(lambda ign: self.check_config(0, self.local_dir))
1208 def get_Alice_magicfolder(result):
1209 self.magicfolder = self.init_magicfolder(0, self.alice_upload_dircap,
1210 self.alice_collective_dircap,
1211 self.local_dir, alice_clock)
1213 d.addCallback(get_Alice_magicfolder)
1214 empty_tree_name = self.unicode_or_fallback(u"empty_tr\u00EAe", u"empty_tree")
1215 empty_tree_dir = abspath_expanduser_unicode(empty_tree_name, base=self.basedir)
1216 new_empty_tree_dir = abspath_expanduser_unicode(empty_tree_name, base=self.local_dir)
1218 def _check_move_empty_tree(res):
1219 print "CHECK MOVE EMPTY TREE"
1220 uploaded_d = self.magicfolder.uploader.set_hook('processed')
1221 self.mkdir_nonascii(empty_tree_dir)
1222 os.rename(empty_tree_dir, new_empty_tree_dir)
1223 self.notify(to_filepath(new_empty_tree_dir), self.inotify.IN_MOVED_TO)
1225 d.addCallback(_check_move_empty_tree)
1226 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
1227 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 1))
1228 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.files_uploaded'), 0))
1229 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
1230 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.directories_created'), 1))
1232 def _create_file_without_event(res):
1233 print "CREATE FILE WITHOUT EMITTING EVENT"
1234 uploaded_d = self.magicfolder.uploader.set_hook('processed')
1235 what_path = abspath_expanduser_unicode(u"what", base=new_empty_tree_dir)
1236 fileutil.write(what_path, "say when")
1237 print "ADVANCE CLOCK"
1238 alice_clock.advance(self.magicfolder.uploader._periodic_full_scan_duration + 1)
1240 d.addCallback(_create_file_without_event)
1241 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.files_uploaded'), 1))
1242 d.addCallback(lambda ign: self.magicfolder.finish())
1245 class RealTest(MagicFolderTestMixin, unittest.TestCase):
1246 """This is skipped unless both Twisted and the platform support inotify."""
1249 MagicFolderTestMixin.setUp(self)
1250 self.inotify = magic_folder.get_inotify_module()
1252 def notify(self, path, mask, magic=None, flush=True):
1253 # Writing to the filesystem causes the notification.
1254 # However, flushing filesystem buffers may be necessary on Windows.
1256 fileutil.flush_volume(path.path)
1259 magic_folder.get_inotify_module()
1260 except NotImplementedError:
1261 RealTest.skip = "Magic Folder support can only be tested for-real on an OS that supports inotify or equivalent."