4 from twisted.trial import unittest
5 from twisted.internet import defer, task
7 from allmydata.interfaces import IDirectoryNode
8 from allmydata.util.assertutil import precondition
10 from allmydata.util import fake_inotify, fileutil
11 from allmydata.util.deferredutil import DeferredListShouldSucceed
12 from allmydata.util.encodingutil import get_filesystem_encoding, to_filepath
13 from allmydata.util.consumer import download_to_data
14 from allmydata.test.no_network import GridTestMixin
15 from allmydata.test.common_util import ReallyEqualMixin, NonASCIIPathMixin
16 from allmydata.test.common import ShouldFailMixin
17 from .test_cli_magic_folder import MagicFolderCLITestMixin
19 from allmydata.frontends import magic_folder
20 from allmydata.frontends.magic_folder import MagicFolder, Downloader, WriteFileMixin
21 from allmydata import magicfolderdb, magicpath
22 from allmydata.util.fileutil import abspath_expanduser_unicode
23 from allmydata.immutable.upload import Data
26 class MagicFolderTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, ReallyEqualMixin, NonASCIIPathMixin):
28 These tests will be run both with a mock notifier, and (on platforms that support it)
29 with the real INotify.
33 GridTestMixin.setUp(self)
35 self.basedir = abspath_expanduser_unicode(temp.decode(get_filesystem_encoding()))
36 self.magicfolder = None
37 self.patch(Downloader, 'REMOTE_SCAN_INTERVAL', 0)
39 def _get_count(self, name, client=None):
40 counters = (client or self.get_client()).stats_provider.get_stats()["counters"]
41 return counters.get('magic_folder.%s' % (name,), 0)
44 dbfile = abspath_expanduser_unicode(u"magicfolderdb.sqlite", base=self.basedir)
45 mdb = magicfolderdb.get_magicfolderdb(dbfile, create_version=(magicfolderdb.SCHEMA_v1, 1))
46 self.failUnless(mdb, "unable to create magicfolderdb from %r" % (dbfile,))
47 self.failUnlessEqual(mdb.VERSION, 1)
50 def _restart_client(self, ign):
51 #print "_restart_client"
52 d = self.restart_client()
53 d.addCallback(self._wait_until_started)
56 def _wait_until_started(self, ign):
57 #print "_wait_until_started"
58 self.magicfolder = self.get_client().getServiceNamed('magic-folder')
59 return self.magicfolder.ready()
61 def test_db_basic(self):
62 fileutil.make_dirs(self.basedir)
65 def test_db_persistence(self):
66 """Test that a file upload creates an entry in the database."""
68 fileutil.make_dirs(self.basedir)
72 pathinfo = fileutil.PathInfo(isdir=False, isfile=True, islink=False,
73 exists=True, size=1, mtime=123, ctime=456)
74 db.did_upload_version(relpath1, 0, 'URI:LIT:1', 'URI:LIT:0', 0, pathinfo)
77 c.execute("SELECT size, mtime, ctime"
82 self.failUnlessEqual(row, (pathinfo.size, pathinfo.mtime, pathinfo.ctime))
84 # Second test uses db.is_new_file instead of SQL query directly
85 # to confirm the previous upload entry in the db.
87 path2 = os.path.join(self.basedir, relpath2)
88 fileutil.write(path2, "meow\n")
89 pathinfo = fileutil.get_pathinfo(path2)
90 db.did_upload_version(relpath2, 0, 'URI:LIT:2', 'URI:LIT:1', 0, pathinfo)
91 self.failUnlessFalse(db.is_new_file(pathinfo, relpath2))
93 different_pathinfo = fileutil.PathInfo(isdir=False, isfile=True, islink=False,
94 exists=True, size=0, mtime=pathinfo.mtime, ctime=pathinfo.ctime)
95 self.failUnlessTrue(db.is_new_file(different_pathinfo, relpath2))
97 def test_magicfolder_start_service(self):
100 self.local_dir = abspath_expanduser_unicode(self.unicode_or_fallback(u"l\u00F8cal_dir", u"local_dir"),
102 self.mkdir_nonascii(self.local_dir)
104 d = defer.succeed(None)
105 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.dirs_monitored'), 0))
107 d.addCallback(lambda ign: self.create_invite_join_magic_folder(u"Alice", self.local_dir))
108 d.addCallback(self._restart_client)
110 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.dirs_monitored'), 1))
111 d.addBoth(self.cleanup)
112 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.dirs_monitored'), 0))
115 def test_move_tree(self):
118 self.local_dir = abspath_expanduser_unicode(self.unicode_or_fallback(u"l\u00F8cal_dir", u"local_dir"),
120 self.mkdir_nonascii(self.local_dir)
122 empty_tree_name = self.unicode_or_fallback(u"empty_tr\u00EAe", u"empty_tree")
123 empty_tree_dir = abspath_expanduser_unicode(empty_tree_name, base=self.basedir)
124 new_empty_tree_dir = abspath_expanduser_unicode(empty_tree_name, base=self.local_dir)
126 small_tree_name = self.unicode_or_fallback(u"small_tr\u00EAe", u"empty_tree")
127 small_tree_dir = abspath_expanduser_unicode(small_tree_name, base=self.basedir)
128 new_small_tree_dir = abspath_expanduser_unicode(small_tree_name, base=self.local_dir)
130 d = self.create_invite_join_magic_folder(u"Alice", self.local_dir)
131 d.addCallback(self._restart_client)
133 def _check_move_empty_tree(res):
134 print "_check_move_empty_tree"
135 downloaded_d = self.magicfolder.downloader.set_hook('processed')
136 uploaded_d = self.magicfolder.uploader.set_hook('processed')
137 self.mkdir_nonascii(empty_tree_dir)
138 os.rename(empty_tree_dir, new_empty_tree_dir)
139 self.notify(to_filepath(new_empty_tree_dir), self.inotify.IN_MOVED_TO)
141 return DeferredListShouldSucceed([downloaded_d, uploaded_d])
142 d.addCallback(_check_move_empty_tree)
143 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
144 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 1))
145 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.files_uploaded'), 0))
146 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
147 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.directories_created'), 1))
149 # FIXME check that Bob downloaded/created the empty tree.
151 def _check_move_small_tree(res):
152 print "_check_move_small_tree"
153 downloaded_d = self.magicfolder.downloader.set_hook('processed', ignore_count=1)
154 uploaded_d = self.magicfolder.uploader.set_hook('processed', ignore_count=1)
155 self.mkdir_nonascii(small_tree_dir)
156 fileutil.write(abspath_expanduser_unicode(u"what", base=small_tree_dir), "say when")
157 os.rename(small_tree_dir, new_small_tree_dir)
158 self.notify(to_filepath(new_small_tree_dir), self.inotify.IN_MOVED_TO)
160 return DeferredListShouldSucceed([downloaded_d, uploaded_d])
161 d.addCallback(_check_move_small_tree)
162 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
163 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 3))
164 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.files_uploaded'), 1))
165 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
166 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.directories_created'), 2))
168 def _check_moved_tree_is_watched(res):
169 print "_check_moved_tree_is_watched"
170 downloaded_d = self.magicfolder.downloader.set_hook('processed', ignore_count=1)
171 uploaded_d = self.magicfolder.uploader.set_hook('processed')
172 fileutil.write(abspath_expanduser_unicode(u"another", base=new_small_tree_dir), "file")
173 self.notify(to_filepath(abspath_expanduser_unicode(u"another", base=new_small_tree_dir)), self.inotify.IN_CLOSE_WRITE)
175 return DeferredListShouldSucceed([downloaded_d, uploaded_d])
176 d.addCallback(_check_moved_tree_is_watched)
177 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
178 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 4))
179 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.files_uploaded'), 2))
180 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
181 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.directories_created'), 2))
183 # Files that are moved out of the upload directory should no longer be watched.
184 #def _move_dir_away(ign):
185 # os.rename(new_empty_tree_dir, empty_tree_dir)
186 # # Wuh? Why don't we get this event for the real test?
187 # #self.notify(to_filepath(new_empty_tree_dir), self.inotify.IN_MOVED_FROM)
188 #d.addCallback(_move_dir_away)
189 #def create_file(val):
190 # test_file = abspath_expanduser_unicode(u"what", base=empty_tree_dir)
191 # fileutil.write(test_file, "meow")
194 #d.addCallback(create_file)
195 #d.addCallback(lambda ign: time.sleep(1)) # XXX ICK
196 #d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
197 #d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 4))
198 #d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.files_uploaded'), 2))
199 #d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
200 #d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.directories_created'), 2))
202 d.addBoth(self.cleanup)
205 def test_persistence(self):
207 Perform an upload of a given file and then stop the client.
208 Start a new client and magic-folder service... and verify that the file is NOT uploaded
209 a second time. This test is meant to test the database persistence along with
210 the startup and shutdown code paths of the magic-folder service.
213 self.local_dir = abspath_expanduser_unicode(u"test_persistence", base=self.basedir)
214 self.mkdir_nonascii(self.local_dir)
215 self.collective_dircap = ""
217 d = defer.succeed(None)
218 d.addCallback(lambda ign: self.create_invite_join_magic_folder(u"Alice", self.local_dir))
219 d.addCallback(self._restart_client)
221 def create_test_file(filename):
222 d2 = self.magicfolder.uploader.set_hook('processed')
223 test_file = abspath_expanduser_unicode(filename, base=self.local_dir)
224 fileutil.write(test_file, "meow %s" % filename)
225 self.notify(to_filepath(test_file), self.inotify.IN_CLOSE_WRITE)
227 d.addCallback(lambda ign: create_test_file(u"what1"))
228 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
229 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 1))
230 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
231 d.addCallback(self.cleanup)
233 d.addCallback(self._restart_client)
234 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
235 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 1))
236 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
237 d.addCallback(lambda ign: create_test_file(u"what2"))
238 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
239 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 2))
240 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
241 d.addBoth(self.cleanup)
244 @defer.inlineCallbacks
245 def test_delete(self):
247 self.local_dir = os.path.join(self.basedir, u"local_dir")
248 self.mkdir_nonascii(self.local_dir)
250 yield self.create_invite_join_magic_folder(u"Alice\u0101", self.local_dir)
251 yield self._restart_client(None)
255 up_proc = self.magicfolder.uploader.set_hook('processed')
256 # down_proc = self.magicfolder.downloader.set_hook('processed')
257 path = os.path.join(self.local_dir, u'foo')
258 with open(path, 'w') as f:
260 self.notify(to_filepath(path), self.inotify.IN_CLOSE_WRITE)
262 self.assertTrue(os.path.exists(path))
264 # the real test part: delete the file
265 up_proc = self.magicfolder.uploader.set_hook('processed')
267 self.notify(to_filepath(path), self.inotify.IN_DELETE)
269 self.assertFalse(os.path.exists(path))
271 # ensure we still have a DB entry, and that the version is 1
272 node, metadata = yield self.magicfolder.downloader._get_collective_latest_file(u'foo')
273 self.assertTrue(node is not None, "Failed to find '{}' in DMD".format(path))
274 self.failUnlessEqual(metadata['version'], 1)
277 yield self.cleanup(None)
279 @defer.inlineCallbacks
280 def test_delete_and_restore(self):
282 self.local_dir = os.path.join(self.basedir, u"local_dir")
283 self.mkdir_nonascii(self.local_dir)
285 yield self.create_invite_join_magic_folder(u"Alice\u0101", self.local_dir)
286 yield self._restart_client(None)
290 up_proc = self.magicfolder.uploader.set_hook('processed')
291 # down_proc = self.magicfolder.downloader.set_hook('processed')
292 path = os.path.join(self.local_dir, u'foo')
293 with open(path, 'w') as f:
295 self.notify(to_filepath(path), self.inotify.IN_CLOSE_WRITE)
297 self.assertTrue(os.path.exists(path))
300 up_proc = self.magicfolder.uploader.set_hook('processed')
302 self.notify(to_filepath(path), self.inotify.IN_DELETE)
304 self.assertFalse(os.path.exists(path))
306 # ensure we still have a DB entry, and that the version is 1
307 node, metadata = yield self.magicfolder.downloader._get_collective_latest_file(u'foo')
308 self.assertTrue(node is not None, "Failed to find '{}' in DMD".format(path))
309 self.failUnlessEqual(metadata['version'], 1)
311 # restore the file, with different contents
312 up_proc = self.magicfolder.uploader.set_hook('processed')
313 path = os.path.join(self.local_dir, u'foo')
314 with open(path, 'w') as f:
316 self.notify(to_filepath(path), self.inotify.IN_CLOSE_WRITE)
319 # ensure we still have a DB entry, and that the version is 2
320 node, metadata = yield self.magicfolder.downloader._get_collective_latest_file(u'foo')
321 self.assertTrue(node is not None, "Failed to find '{}' in DMD".format(path))
322 self.failUnlessEqual(metadata['version'], 2)
325 yield self.cleanup(None)
328 @defer.inlineCallbacks
329 def test_alice_delete_bob_restore(self):
330 alice_clock = task.Clock()
331 bob_clock = task.Clock()
332 yield self.setup_alice_and_bob(alice_clock, bob_clock)
333 alice_dir = self.alice_magicfolder.uploader._local_path_u
334 bob_dir = self.bob_magicfolder.uploader._local_path_u
335 alice_fname = os.path.join(alice_dir, 'blam')
336 bob_fname = os.path.join(bob_dir, 'blam')
339 # alice creates a file, bob downloads it
340 alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
341 bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
343 with open(alice_fname, 'wb') as f:
344 f.write('contents0\n')
345 self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
347 alice_clock.advance(0)
348 yield alice_proc # alice uploads
351 yield bob_proc # bob downloads
354 yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
355 yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 0)
356 yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
357 yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 0)
358 yield self.failUnlessReallyEqual(
359 self._get_count('downloader.objects_failed', client=self.bob_magicfolder._client),
362 yield self.failUnlessReallyEqual(
363 self._get_count('downloader.objects_downloaded', client=self.bob_magicfolder._client),
368 # now bob deletes it (bob should upload, alice download)
369 bob_proc = self.bob_magicfolder.uploader.set_hook('processed')
370 alice_proc = self.alice_magicfolder.downloader.set_hook('processed')
372 self.notify(to_filepath(bob_fname), self.inotify.IN_DELETE, magic=self.bob_magicfolder)
376 alice_clock.advance(0)
380 node, metadata = yield self.alice_magicfolder.downloader._get_collective_latest_file(u'blam')
381 self.assertTrue(metadata['deleted'])
382 yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
383 yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 1)
384 yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
385 yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 1)
387 print("ALICE RESTORE")
388 # now alice restores it (alice should upload, bob download)
389 alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
390 bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
391 with open(alice_fname, 'wb') as f:
392 f.write('new contents\n')
393 self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
395 alice_clock.advance(0)
401 node, metadata = yield self.alice_magicfolder.downloader._get_collective_latest_file(u'blam')
402 self.assertTrue('deleted' not in metadata or not metadata['deleted'])
403 yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 2)
404 yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 2)
405 yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 2)
406 yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 2)
410 d0 = self.alice_magicfolder.finish()
411 alice_clock.advance(0)
414 d1 = self.bob_magicfolder.finish()
418 @defer.inlineCallbacks
419 def test_alice_create_bob_update(self):
420 alice_clock = task.Clock()
421 bob_clock = task.Clock()
422 yield self.setup_alice_and_bob(alice_clock, bob_clock)
423 alice_dir = self.alice_magicfolder.uploader._local_path_u
424 bob_dir = self.bob_magicfolder.uploader._local_path_u
425 alice_fname = os.path.join(alice_dir, 'blam')
426 bob_fname = os.path.join(bob_dir, 'blam')
429 # alice creates a file, bob downloads it
430 alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
431 bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
433 with open(alice_fname, 'wb') as f:
434 f.write('contents0\n')
435 self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
437 alice_clock.advance(0)
438 yield alice_proc # alice uploads
441 yield bob_proc # bob downloads
444 yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
445 yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 0)
446 yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
447 yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 0)
448 yield self.failUnlessReallyEqual(
449 self._get_count('downloader.objects_failed', client=self.bob_magicfolder._client),
452 yield self.failUnlessReallyEqual(
453 self._get_count('downloader.objects_downloaded', client=self.bob_magicfolder._client),
457 # now bob updates it (bob should upload, alice download)
458 bob_proc = self.bob_magicfolder.uploader.set_hook('processed')
459 alice_proc = self.alice_magicfolder.downloader.set_hook('processed')
460 with open(bob_fname, 'wb') as f:
461 f.write('bob wuz here\n')
462 self.notify(to_filepath(bob_fname), self.inotify.IN_CLOSE_WRITE, magic=self.bob_magicfolder)
466 alice_clock.advance(0)
470 yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
471 yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 1)
472 yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
473 yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 1)
477 d0 = self.alice_magicfolder.finish()
478 alice_clock.advance(0)
481 d1 = self.bob_magicfolder.finish()
485 @defer.inlineCallbacks
486 def test_alice_delete_and_restore(self):
487 alice_clock = task.Clock()
488 bob_clock = task.Clock()
489 yield self.setup_alice_and_bob(alice_clock, bob_clock)
490 alice_dir = self.alice_magicfolder.uploader._local_path_u
491 bob_dir = self.bob_magicfolder.uploader._local_path_u
492 alice_fname = os.path.join(alice_dir, 'blam')
493 bob_fname = os.path.join(bob_dir, 'blam')
496 # alice creates a file, bob downloads it
497 alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
498 bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
500 with open(alice_fname, 'wb') as f:
501 f.write('contents0\n')
502 self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
504 alice_clock.advance(0)
505 yield alice_proc # alice uploads
508 yield bob_proc # bob downloads
511 yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
512 yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 0)
513 yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
514 yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 0)
515 yield self.failUnlessReallyEqual(
516 self._get_count('downloader.objects_failed', client=self.bob_magicfolder._client),
519 yield self.failUnlessReallyEqual(
520 self._get_count('downloader.objects_downloaded', client=self.bob_magicfolder._client),
523 self.failUnless(os.path.exists(bob_fname))
525 # now alice deletes it (alice should upload, bob download)
526 alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
527 bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
528 os.unlink(alice_fname)
529 self.notify(to_filepath(alice_fname), self.inotify.IN_DELETE, magic=self.alice_magicfolder)
531 alice_clock.advance(0)
537 yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
538 yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 1)
539 yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
540 yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 1)
541 self.failIf(os.path.exists(bob_fname))
543 # now alice restores the file (with new contents)
544 alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
545 bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
546 with open(alice_fname, 'wb') as f:
547 f.write('alice wuz here\n')
548 self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
550 alice_clock.advance(0)
556 yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 2)
557 yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 2)
558 yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 2)
559 yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 2)
560 self.failUnless(os.path.exists(bob_fname))
564 d0 = self.alice_magicfolder.finish()
565 alice_clock.advance(0)
568 d1 = self.bob_magicfolder.finish()
572 def test_magic_folder(self):
574 self.local_dir = os.path.join(self.basedir, self.unicode_or_fallback(u"loc\u0101l_dir", u"local_dir"))
575 self.mkdir_nonascii(self.local_dir)
577 d = self.create_invite_join_magic_folder(u"Alice\u0101", self.local_dir)
578 d.addCallback(self._restart_client)
580 # Write something short enough for a LIT file.
581 d.addCallback(lambda ign: self._check_file(u"short", "test"))
583 # Write to the same file again with different data.
584 d.addCallback(lambda ign: self._check_file(u"short", "different"))
586 # Test that temporary files are not uploaded.
587 d.addCallback(lambda ign: self._check_file(u"tempfile", "test", temporary=True))
589 # Test creation of a subdirectory.
590 d.addCallback(lambda ign: self._check_mkdir(u"directory"))
592 # Write something longer, and also try to test a Unicode name if the fs can represent it.
593 name_u = self.unicode_or_fallback(u"l\u00F8ng", u"long")
594 d.addCallback(lambda ign: self._check_file(name_u, "test"*100))
596 # TODO: test that causes an upload failure.
597 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
599 d.addBoth(self.cleanup)
602 def _check_mkdir(self, name_u):
603 return self._check_file(name_u + u"/", "", directory=True)
605 def _check_file(self, name_u, data, temporary=False, directory=False):
606 precondition(not (temporary and directory), temporary=temporary, directory=directory)
608 previously_uploaded = self._get_count('uploader.objects_succeeded')
609 previously_disappeared = self._get_count('uploader.objects_disappeared')
611 d = self.magicfolder.uploader.set_hook('processed')
613 path_u = abspath_expanduser_unicode(name_u, base=self.local_dir)
614 path = to_filepath(path_u)
618 event_mask = self.inotify.IN_CREATE | self.inotify.IN_ISDIR
620 # We don't use FilePath.setContent() here because it creates a temporary file that
621 # is renamed into place, which causes events that the test is not expecting.
622 f = open(path_u, "wb")
624 if temporary and sys.platform != "win32":
629 if temporary and sys.platform == "win32":
631 self.notify(path, self.inotify.IN_DELETE)
632 event_mask = self.inotify.IN_CLOSE_WRITE
634 fileutil.flush_volume(path_u)
635 self.notify(path, event_mask)
636 encoded_name_u = magicpath.path2magic(name_u)
638 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
640 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_disappeared'),
641 previously_disappeared + 1))
643 d.addCallback(lambda ign: self.upload_dirnode.get(encoded_name_u))
644 d.addCallback(download_to_data)
645 d.addCallback(lambda actual_data: self.failUnlessReallyEqual(actual_data, data))
646 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'),
647 previously_uploaded + 1))
649 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
652 def _check_version_in_dmd(self, magicfolder, relpath_u, expected_version):
653 encoded_name_u = magicpath.path2magic(relpath_u)
654 d = magicfolder.downloader._get_collective_latest_file(encoded_name_u)
655 def check_latest(result):
656 if result[0] is not None:
657 node, metadata = result
658 d.addCallback(lambda ign: self.failUnlessEqual(metadata['version'], expected_version))
659 d.addCallback(check_latest)
662 def _check_version_in_local_db(self, magicfolder, relpath_u, expected_version):
663 version = magicfolder._db.get_local_file_version(relpath_u)
664 #print "_check_version_in_local_db: %r has version %s" % (relpath_u, version)
665 self.failUnlessEqual(version, expected_version)
667 def _check_file_gone(self, magicfolder, relpath_u):
668 path = os.path.join(magicfolder.uploader._local_path_u, relpath_u)
669 self.assertTrue(not os.path.exists(path))
671 def test_alice_bob(self):
672 alice_clock = task.Clock()
673 bob_clock = task.Clock()
674 d = self.setup_alice_and_bob(alice_clock, bob_clock)
676 def _check_uploader_count(ign, name, expected):
677 self.failUnlessReallyEqual(self._get_count('uploader.'+name, client=self.alice_magicfolder._client),
679 def _check_downloader_count(ign, name, expected):
680 self.failUnlessReallyEqual(self._get_count('downloader.'+name, client=self.bob_magicfolder._client),
683 def _wait_for_Bob(ign, downloaded_d):
684 print "Now waiting for Bob to download\n"
685 self.magicfolder = self.bob_magicfolder
689 def _wait_for(ign, something_to_do):
690 downloaded_d = self.bob_magicfolder.downloader.set_hook('processed')
691 uploaded_d = self.alice_magicfolder.uploader.set_hook('processed')
693 print "Waiting for Alice to upload\n"
694 alice_clock.advance(0)
695 uploaded_d.addCallback(_wait_for_Bob, downloaded_d)
698 def Alice_to_write_a_file():
699 print "Alice writes a file\n"
700 self.file_path = abspath_expanduser_unicode(u"file1", base=self.alice_magicfolder.uploader._local_path_u)
701 fileutil.write(self.file_path, "meow, meow meow. meow? meow meow! meow.")
702 self.magicfolder = self.alice_magicfolder
703 self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE)
704 d.addCallback(_wait_for, Alice_to_write_a_file)
706 d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file1", 0))
707 d.addCallback(lambda ign: self._check_version_in_local_db(self.alice_magicfolder, u"file1", 0))
708 d.addCallback(_check_uploader_count, 'objects_failed', 0)
709 d.addCallback(_check_uploader_count, 'objects_succeeded', 1)
710 d.addCallback(_check_uploader_count, 'files_uploaded', 1)
711 d.addCallback(_check_uploader_count, 'objects_queued', 0)
712 d.addCallback(_check_uploader_count, 'directories_created', 0)
714 d.addCallback(lambda ign: self._check_version_in_local_db(self.bob_magicfolder, u"file1", 0))
715 d.addCallback(_check_downloader_count, 'objects_failed', 0)
716 d.addCallback(_check_downloader_count, 'objects_downloaded', 1)
718 def Alice_to_delete_file():
719 print "Alice deletes the file!\n"
720 os.unlink(self.file_path)
721 self.magicfolder = self.alice_magicfolder
722 self.notify(to_filepath(self.file_path), self.inotify.IN_DELETE)
723 d.addCallback(_wait_for, Alice_to_delete_file)
725 def notify_bob_moved(ign):
726 d0 = self.bob_magicfolder.uploader.set_hook('processed')
727 self.magicfolder = self.bob_magicfolder
728 p = abspath_expanduser_unicode(u"file1", base=self.bob_magicfolder.uploader._local_path_u)
729 self.notify(to_filepath(p), self.inotify.IN_MOVED_FROM)
732 self.notify(to_filepath(p + u'.backup'), self.inotify.IN_MOVED_TO)
736 d.addCallback(notify_bob_moved)
738 d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file1", 1))
739 d.addCallback(lambda ign: self._check_version_in_local_db(self.alice_magicfolder, u"file1", 1))
740 d.addCallback(_check_uploader_count, 'objects_failed', 0)
741 d.addCallback(_check_uploader_count, 'objects_succeeded', 2)
743 d.addCallback(lambda ign: self._check_version_in_local_db(self.bob_magicfolder, u"file1", 1))
744 d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file1", 1))
745 d.addCallback(lambda ign: self._check_file_gone(self.bob_magicfolder, u"file1"))
746 d.addCallback(_check_downloader_count, 'objects_failed', 0)
747 d.addCallback(_check_downloader_count, 'objects_downloaded', 2)
749 def Alice_to_rewrite_file():
750 print "Alice rewrites file\n"
751 self.magicfolder = self.alice_magicfolder
752 self.file_path = abspath_expanduser_unicode(u"file1", base=self.alice_magicfolder.uploader._local_path_u)
753 fileutil.write(self.file_path, "Alice suddenly sees the white rabbit running into the forest.")
754 self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE)
755 d.addCallback(_wait_for, Alice_to_rewrite_file)
757 d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file1", 2))
758 d.addCallback(lambda ign: self._check_version_in_local_db(self.alice_magicfolder, u"file1", 2))
759 d.addCallback(_check_uploader_count, 'objects_failed', 0)
760 d.addCallback(_check_uploader_count, 'objects_succeeded', 3)
761 d.addCallback(_check_uploader_count, 'files_uploaded', 3)
762 d.addCallback(_check_uploader_count, 'objects_queued', 0)
763 d.addCallback(_check_uploader_count, 'directories_created', 0)
765 d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file1", 2))
766 d.addCallback(lambda ign: self._check_version_in_local_db(self.bob_magicfolder, u"file1", 2))
767 d.addCallback(_check_downloader_count, 'objects_failed', 0)
768 d.addCallback(_check_downloader_count, 'objects_downloaded', 3)
770 path_u = u"/tmp/magic_folder_test"
771 encoded_path_u = magicpath.path2magic(u"/tmp/magic_folder_test")
773 def Alice_tries_to_p0wn_Bob(ign):
774 print "Alice tries to p0wn Bob\n"
775 processed_d = self.bob_magicfolder.downloader.set_hook('processed')
777 # upload a file that would provoke the security bug from #2506
778 uploadable = Data("", self.alice_magicfolder._client.convergence)
779 alice_dmd = self.alice_magicfolder.uploader._upload_dirnode
781 d2 = alice_dmd.add_file(encoded_path_u, uploadable, metadata={"version": 0}, overwrite=True)
782 d2.addCallback(lambda ign: self.failUnless(alice_dmd.has_child(encoded_path_u)))
783 d2.addCallback(_wait_for_Bob, processed_d)
785 d.addCallback(Alice_tries_to_p0wn_Bob)
787 d.addCallback(lambda ign: self.failIf(os.path.exists(path_u)))
788 d.addCallback(lambda ign: self._check_version_in_local_db(self.bob_magicfolder, encoded_path_u, None))
789 d.addCallback(_check_downloader_count, 'objects_excluded', 2)
790 d.addCallback(_check_downloader_count, 'objects_downloaded', 3)
792 def _cleanup(ign, magicfolder, clock):
793 if magicfolder is not None:
794 d2 = magicfolder.finish()
798 def cleanup_Alice_and_Bob(result):
799 print "cleanup alice bob test\n"
800 d = defer.succeed(None)
801 d.addCallback(_cleanup, self.alice_magicfolder, alice_clock)
802 d.addCallback(_cleanup, self.bob_magicfolder, bob_clock)
803 d.addCallback(lambda ign: result)
805 d.addBoth(cleanup_Alice_and_Bob)
809 class MockTest(MagicFolderTestMixin, unittest.TestCase):
810 """This can run on any platform, and even if twisted.internet.inotify can't be imported."""
813 MagicFolderTestMixin.setUp(self)
814 self.inotify = fake_inotify
815 self.patch(magic_folder, 'get_inotify_module', lambda: self.inotify)
817 def notify(self, path, mask, magic=None):
819 magic = self.magicfolder
820 magic.uploader._notifier.event(path, mask)
822 def test_errors(self):
825 errors_dir = abspath_expanduser_unicode(u"errors_dir", base=self.basedir)
827 not_a_dir = abspath_expanduser_unicode(u"NOT_A_DIR", base=self.basedir)
828 fileutil.write(not_a_dir, "")
829 magicfolderdb = abspath_expanduser_unicode(u"magicfolderdb", base=self.basedir)
830 doesnotexist = abspath_expanduser_unicode(u"doesnotexist", base=self.basedir)
832 client = self.g.clients[0]
833 d = client.create_dirnode()
834 def _check_errors(n):
835 self.failUnless(IDirectoryNode.providedBy(n))
836 upload_dircap = n.get_uri()
837 readonly_dircap = n.get_readonly_uri()
839 self.shouldFail(AssertionError, 'nonexistent local.directory', 'there is no directory',
840 MagicFolder, client, upload_dircap, '', doesnotexist, magicfolderdb)
841 self.shouldFail(AssertionError, 'non-directory local.directory', 'is not a directory',
842 MagicFolder, client, upload_dircap, '', not_a_dir, magicfolderdb)
843 self.shouldFail(AssertionError, 'bad upload.dircap', 'does not refer to a directory',
844 MagicFolder, client, 'bad', '', errors_dir, magicfolderdb)
845 self.shouldFail(AssertionError, 'non-directory upload.dircap', 'does not refer to a directory',
846 MagicFolder, client, 'URI:LIT:foo', '', errors_dir, magicfolderdb)
847 self.shouldFail(AssertionError, 'readonly upload.dircap', 'is not a writecap to a directory',
848 MagicFolder, client, readonly_dircap, '', errors_dir, magicfolderdb,)
849 self.shouldFail(AssertionError, 'collective dircap',
850 "The URI in 'private/collective_dircap' is not a readonly cap to a directory.",
851 MagicFolder, client, upload_dircap, upload_dircap, errors_dir, magicfolderdb)
853 def _not_implemented():
854 raise NotImplementedError("blah")
855 self.patch(magic_folder, 'get_inotify_module', _not_implemented)
856 self.shouldFail(NotImplementedError, 'unsupported', 'blah',
857 MagicFolder, client, upload_dircap, '', errors_dir, magicfolderdb)
858 d.addCallback(_check_errors)
861 def test_write_downloaded_file(self):
862 workdir = u"cli/MagicFolder/write-downloaded-file"
863 local_file = fileutil.abspath_expanduser_unicode(os.path.join(workdir, "foobar"))
865 class TestWriteFileMixin(WriteFileMixin):
869 writefile = TestWriteFileMixin()
871 # create a file with name "foobar" with content "foo"
872 # write downloaded file content "bar" into "foobar" with is_conflict = False
873 fileutil.make_dirs(workdir)
874 fileutil.write(local_file, "foo")
876 # if is_conflict is False, then the .conflict file shouldn't exist.
877 writefile._write_downloaded_file(local_file, "bar", False, None)
878 conflicted_path = local_file + u".conflict"
879 self.failIf(os.path.exists(conflicted_path))
881 # At this point, the backup file should exist with content "foo"
882 backup_path = local_file + u".backup"
883 self.failUnless(os.path.exists(backup_path))
884 self.failUnlessEqual(fileutil.read(backup_path), "foo")
886 # .tmp file shouldn't exist
887 self.failIf(os.path.exists(local_file + u".tmp"))
889 # .. and the original file should have the new content
890 self.failUnlessEqual(fileutil.read(local_file), "bar")
892 # now a test for conflicted case
893 writefile._write_downloaded_file(local_file, "bar", True, None)
894 self.failUnless(os.path.exists(conflicted_path))
896 # .tmp file shouldn't exist
897 self.failIf(os.path.exists(local_file + u".tmp"))
900 class RealTest(MagicFolderTestMixin, unittest.TestCase):
901 """This is skipped unless both Twisted and the platform support inotify."""
904 MagicFolderTestMixin.setUp(self)
905 self.inotify = magic_folder.get_inotify_module()
907 def notify(self, path, mask, **kw):
908 # Writing to the filesystem causes the notification.
912 magic_folder.get_inotify_module()
913 except NotImplementedError:
914 RealTest.skip = "Magic Folder support can only be tested for-real on an OS that supports inotify or equivalent."