4 from twisted.trial import unittest
5 from twisted.internet import defer, task
7 from allmydata.interfaces import IDirectoryNode
8 from allmydata.util.assertutil import precondition
10 from allmydata.util import fake_inotify, fileutil
11 from allmydata.util.deferredutil import DeferredListShouldSucceed
12 from allmydata.util.encodingutil import get_filesystem_encoding, to_filepath
13 from allmydata.util.consumer import download_to_data
14 from allmydata.test.no_network import GridTestMixin
15 from allmydata.test.common_util import ReallyEqualMixin, NonASCIIPathMixin
16 from allmydata.test.common import ShouldFailMixin
17 from .test_cli_magic_folder import MagicFolderCLITestMixin
19 from allmydata.frontends import magic_folder
20 from allmydata.frontends.magic_folder import MagicFolder, Downloader, WriteFileMixin
21 from allmydata import magicfolderdb, magicpath
22 from allmydata.util.fileutil import abspath_expanduser_unicode
23 from allmydata.immutable.upload import Data
26 class MagicFolderTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, ReallyEqualMixin, NonASCIIPathMixin):
28 These tests will be run both with a mock notifier, and (on platforms that support it)
29 with the real INotify.
33 GridTestMixin.setUp(self)
35 self.basedir = abspath_expanduser_unicode(temp.decode(get_filesystem_encoding()))
36 self.magicfolder = None
37 self.patch(Downloader, 'REMOTE_SCAN_INTERVAL', 0)
39 def _get_count(self, name, client=None):
40 counters = (client or self.get_client()).stats_provider.get_stats()["counters"]
41 return counters.get('magic_folder.%s' % (name,), 0)
44 dbfile = abspath_expanduser_unicode(u"magicfolderdb.sqlite", base=self.basedir)
45 mdb = magicfolderdb.get_magicfolderdb(dbfile, create_version=(magicfolderdb.SCHEMA_v1, 1))
46 self.failUnless(mdb, "unable to create magicfolderdb from %r" % (dbfile,))
47 self.failUnlessEqual(mdb.VERSION, 1)
50 def _restart_client(self, ign):
51 #print "_restart_client"
52 d = self.restart_client()
53 d.addCallback(self._wait_until_started)
56 def _wait_until_started(self, ign):
57 #print "_wait_until_started"
58 self.magicfolder = self.get_client().getServiceNamed('magic-folder')
59 return self.magicfolder.ready()
61 def test_db_basic(self):
62 fileutil.make_dirs(self.basedir)
65 def test_db_persistence(self):
66 """Test that a file upload creates an entry in the database."""
68 fileutil.make_dirs(self.basedir)
72 pathinfo = fileutil.PathInfo(isdir=False, isfile=True, islink=False,
73 exists=True, size=1, mtime=123, ctime=456)
74 db.did_upload_version(relpath1, 0, 'URI:LIT:1', 'URI:LIT:0', 0, pathinfo)
77 c.execute("SELECT size, mtime, ctime"
82 self.failUnlessEqual(row, (pathinfo.size, pathinfo.mtime, pathinfo.ctime))
84 # Second test uses db.is_new_file instead of SQL query directly
85 # to confirm the previous upload entry in the db.
87 path2 = os.path.join(self.basedir, relpath2)
88 fileutil.write(path2, "meow\n")
89 pathinfo = fileutil.get_pathinfo(path2)
90 db.did_upload_version(relpath2, 0, 'URI:LIT:2', 'URI:LIT:1', 0, pathinfo)
91 self.failUnlessFalse(db.is_new_file(pathinfo, relpath2))
93 different_pathinfo = fileutil.PathInfo(isdir=False, isfile=True, islink=False,
94 exists=True, size=0, mtime=pathinfo.mtime, ctime=pathinfo.ctime)
95 self.failUnlessTrue(db.is_new_file(different_pathinfo, relpath2))
97 def test_magicfolder_start_service(self):
100 self.local_dir = abspath_expanduser_unicode(self.unicode_or_fallback(u"l\u00F8cal_dir", u"local_dir"),
102 self.mkdir_nonascii(self.local_dir)
104 d = defer.succeed(None)
105 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.dirs_monitored'), 0))
107 d.addCallback(lambda ign: self.create_invite_join_magic_folder(u"Alice", self.local_dir))
108 d.addCallback(self._restart_client)
110 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.dirs_monitored'), 1))
111 d.addBoth(self.cleanup)
112 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.dirs_monitored'), 0))
115 def test_move_tree(self):
118 self.local_dir = abspath_expanduser_unicode(self.unicode_or_fallback(u"l\u00F8cal_dir", u"local_dir"),
120 self.mkdir_nonascii(self.local_dir)
122 empty_tree_name = self.unicode_or_fallback(u"empty_tr\u00EAe", u"empty_tree")
123 empty_tree_dir = abspath_expanduser_unicode(empty_tree_name, base=self.basedir)
124 new_empty_tree_dir = abspath_expanduser_unicode(empty_tree_name, base=self.local_dir)
126 small_tree_name = self.unicode_or_fallback(u"small_tr\u00EAe", u"empty_tree")
127 small_tree_dir = abspath_expanduser_unicode(small_tree_name, base=self.basedir)
128 new_small_tree_dir = abspath_expanduser_unicode(small_tree_name, base=self.local_dir)
130 d = self.create_invite_join_magic_folder(u"Alice", self.local_dir)
131 d.addCallback(self._restart_client)
133 def _check_move_empty_tree(res):
134 print "_check_move_empty_tree"
135 uploaded_d = self.magicfolder.uploader.set_hook('processed')
136 self.mkdir_nonascii(empty_tree_dir)
137 os.rename(empty_tree_dir, new_empty_tree_dir)
138 self.notify(to_filepath(new_empty_tree_dir), self.inotify.IN_MOVED_TO)
141 d.addCallback(_check_move_empty_tree)
142 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
143 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 1))
144 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.files_uploaded'), 0))
145 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
146 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.directories_created'), 1))
148 def _check_move_small_tree(res):
149 print "_check_move_small_tree"
150 uploaded_d = self.magicfolder.uploader.set_hook('processed', ignore_count=1)
151 self.mkdir_nonascii(small_tree_dir)
152 what_path = abspath_expanduser_unicode(u"what", base=small_tree_dir)
153 fileutil.write(what_path, "say when")
154 os.rename(small_tree_dir, new_small_tree_dir)
155 self.notify(to_filepath(new_small_tree_dir), self.inotify.IN_MOVED_TO)
158 d.addCallback(_check_move_small_tree)
159 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
160 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 3))
161 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.files_uploaded'), 1))
162 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
163 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.directories_created'), 2))
165 def _check_moved_tree_is_watched(res):
166 print "_check_moved_tree_is_watched"
167 uploaded_d = self.magicfolder.uploader.set_hook('processed')
168 another_path = abspath_expanduser_unicode(u"another", base=new_small_tree_dir)
169 fileutil.write(another_path, "file")
170 self.notify(to_filepath(another_path), self.inotify.IN_CLOSE_WRITE)
173 d.addCallback(_check_moved_tree_is_watched)
174 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
175 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 4))
176 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.files_uploaded'), 2))
177 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
178 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.directories_created'), 2))
180 # Files that are moved out of the upload directory should no longer be watched.
181 #def _move_dir_away(ign):
182 # os.rename(new_empty_tree_dir, empty_tree_dir)
183 # # Wuh? Why don't we get this event for the real test?
184 # #self.notify(to_filepath(new_empty_tree_dir), self.inotify.IN_MOVED_FROM)
185 #d.addCallback(_move_dir_away)
186 #def create_file(val):
187 # test_file = abspath_expanduser_unicode(u"what", base=empty_tree_dir)
188 # fileutil.write(test_file, "meow")
191 #d.addCallback(create_file)
192 #d.addCallback(lambda ign: time.sleep(1)) # XXX ICK
193 #d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
194 #d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 4))
195 #d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.files_uploaded'), 2))
196 #d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
197 #d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.directories_created'), 2))
199 d.addBoth(self.cleanup)
202 def test_persistence(self):
204 Perform an upload of a given file and then stop the client.
205 Start a new client and magic-folder service... and verify that the file is NOT uploaded
206 a second time. This test is meant to test the database persistence along with
207 the startup and shutdown code paths of the magic-folder service.
210 self.local_dir = abspath_expanduser_unicode(u"test_persistence", base=self.basedir)
211 self.mkdir_nonascii(self.local_dir)
212 self.collective_dircap = ""
214 d = defer.succeed(None)
215 d.addCallback(lambda ign: self.create_invite_join_magic_folder(u"Alice", self.local_dir))
216 d.addCallback(self._restart_client)
218 def create_test_file(filename):
219 d2 = self.magicfolder.uploader.set_hook('processed')
220 test_file = abspath_expanduser_unicode(filename, base=self.local_dir)
221 fileutil.write(test_file, "meow %s" % filename)
222 self.notify(to_filepath(test_file), self.inotify.IN_CLOSE_WRITE)
224 d.addCallback(lambda ign: create_test_file(u"what1"))
225 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
226 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 1))
227 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
228 d.addCallback(self.cleanup)
230 d.addCallback(self._restart_client)
231 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
232 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 1))
233 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
234 d.addCallback(lambda ign: create_test_file(u"what2"))
235 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
236 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'), 2))
237 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
238 d.addBoth(self.cleanup)
241 @defer.inlineCallbacks
242 def test_delete(self):
244 self.local_dir = os.path.join(self.basedir, u"local_dir")
245 self.mkdir_nonascii(self.local_dir)
247 yield self.create_invite_join_magic_folder(u"Alice\u0101", self.local_dir)
248 yield self._restart_client(None)
252 up_proc = self.magicfolder.uploader.set_hook('processed')
253 # down_proc = self.magicfolder.downloader.set_hook('processed')
254 path = os.path.join(self.local_dir, u'foo')
255 fileutil.write(path, 'foo\n')
256 self.notify(to_filepath(path), self.inotify.IN_CLOSE_WRITE)
258 self.assertTrue(os.path.exists(path))
260 # the real test part: delete the file
261 up_proc = self.magicfolder.uploader.set_hook('processed')
263 self.notify(to_filepath(path), self.inotify.IN_DELETE)
265 self.assertFalse(os.path.exists(path))
267 # ensure we still have a DB entry, and that the version is 1
268 node, metadata = yield self.magicfolder.downloader._get_collective_latest_file(u'foo')
269 self.assertTrue(node is not None, "Failed to find %r in DMD" % (path,))
270 self.failUnlessEqual(metadata['version'], 1)
273 yield self.cleanup(None)
275 @defer.inlineCallbacks
276 def test_delete_and_restore(self):
278 self.local_dir = os.path.join(self.basedir, u"local_dir")
279 self.mkdir_nonascii(self.local_dir)
281 yield self.create_invite_join_magic_folder(u"Alice\u0101", self.local_dir)
282 yield self._restart_client(None)
286 up_proc = self.magicfolder.uploader.set_hook('processed')
287 # down_proc = self.magicfolder.downloader.set_hook('processed')
288 path = os.path.join(self.local_dir, u'foo')
289 fileutil.write(path, 'foo\n')
290 self.notify(to_filepath(path), self.inotify.IN_CLOSE_WRITE)
292 self.assertTrue(os.path.exists(path))
295 up_proc = self.magicfolder.uploader.set_hook('processed')
297 self.notify(to_filepath(path), self.inotify.IN_DELETE)
299 self.assertFalse(os.path.exists(path))
301 # ensure we still have a DB entry, and that the version is 1
302 node, metadata = yield self.magicfolder.downloader._get_collective_latest_file(u'foo')
303 self.assertTrue(node is not None, "Failed to find %r in DMD" % (path,))
304 self.failUnlessEqual(metadata['version'], 1)
306 # restore the file, with different contents
307 up_proc = self.magicfolder.uploader.set_hook('processed')
308 path = os.path.join(self.local_dir, u'foo')
309 fileutil.write(path, 'bar\n')
310 self.notify(to_filepath(path), self.inotify.IN_CLOSE_WRITE)
313 # ensure we still have a DB entry, and that the version is 2
314 node, metadata = yield self.magicfolder.downloader._get_collective_latest_file(u'foo')
315 self.assertTrue(node is not None, "Failed to find %r in DMD" % (path,))
316 self.failUnlessEqual(metadata['version'], 2)
319 yield self.cleanup(None)
322 @defer.inlineCallbacks
323 def test_alice_delete_bob_restore(self):
324 alice_clock = task.Clock()
325 bob_clock = task.Clock()
326 yield self.setup_alice_and_bob(alice_clock, bob_clock)
327 alice_dir = self.alice_magicfolder.uploader._local_path_u
328 bob_dir = self.bob_magicfolder.uploader._local_path_u
329 alice_fname = os.path.join(alice_dir, 'blam')
330 bob_fname = os.path.join(bob_dir, 'blam')
333 # alice creates a file, bob downloads it
334 alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
335 bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
337 fileutil.write(alice_fname, 'contents0\n')
338 self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
340 alice_clock.advance(0)
341 yield alice_proc # alice uploads
344 yield bob_proc # bob downloads
347 yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
348 yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 0)
349 yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
350 yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 0)
351 yield self.failUnlessReallyEqual(
352 self._get_count('downloader.objects_failed', client=self.bob_magicfolder._client),
355 yield self.failUnlessReallyEqual(
356 self._get_count('downloader.objects_downloaded', client=self.bob_magicfolder._client),
361 # now bob deletes it (bob should upload, alice download)
362 bob_proc = self.bob_magicfolder.uploader.set_hook('processed')
363 alice_proc = self.alice_magicfolder.downloader.set_hook('processed')
365 self.notify(to_filepath(bob_fname), self.inotify.IN_DELETE, magic=self.bob_magicfolder)
369 alice_clock.advance(0)
373 node, metadata = yield self.alice_magicfolder.downloader._get_collective_latest_file(u'blam')
374 self.assertTrue(metadata['deleted'])
375 yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
376 yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 1)
377 yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
378 yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 1)
380 print("ALICE RESTORE")
381 # now alice restores it (alice should upload, bob download)
382 alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
383 bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
384 fileutil.write(alice_fname, 'new contents\n')
385 self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
387 alice_clock.advance(0)
393 node, metadata = yield self.alice_magicfolder.downloader._get_collective_latest_file(u'blam')
394 self.assertTrue('deleted' not in metadata or not metadata['deleted'])
395 yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 2)
396 yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 2)
397 yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 2)
398 yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 2)
402 d0 = self.alice_magicfolder.finish()
403 alice_clock.advance(0)
406 d1 = self.bob_magicfolder.finish()
410 @defer.inlineCallbacks
411 def test_alice_create_bob_update(self):
412 alice_clock = task.Clock()
413 bob_clock = task.Clock()
414 yield self.setup_alice_and_bob(alice_clock, bob_clock)
415 alice_dir = self.alice_magicfolder.uploader._local_path_u
416 bob_dir = self.bob_magicfolder.uploader._local_path_u
417 alice_fname = os.path.join(alice_dir, 'blam')
418 bob_fname = os.path.join(bob_dir, 'blam')
421 # alice creates a file, bob downloads it
422 alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
423 bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
425 fileutil.write(alice_fname, 'contents0\n')
426 self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
428 alice_clock.advance(0)
429 yield alice_proc # alice uploads
432 yield bob_proc # bob downloads
435 yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
436 yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 0)
437 yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
438 yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 0)
439 yield self.failUnlessReallyEqual(
440 self._get_count('downloader.objects_failed', client=self.bob_magicfolder._client),
443 yield self.failUnlessReallyEqual(
444 self._get_count('downloader.objects_downloaded', client=self.bob_magicfolder._client),
448 # now bob updates it (bob should upload, alice download)
449 bob_proc = self.bob_magicfolder.uploader.set_hook('processed')
450 alice_proc = self.alice_magicfolder.downloader.set_hook('processed')
451 fileutil.write(bob_fname, 'bob wuz here\n')
452 self.notify(to_filepath(bob_fname), self.inotify.IN_CLOSE_WRITE, magic=self.bob_magicfolder)
456 alice_clock.advance(0)
460 yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
461 yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 1)
462 yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
463 yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 1)
467 d0 = self.alice_magicfolder.finish()
468 alice_clock.advance(0)
471 d1 = self.bob_magicfolder.finish()
475 @defer.inlineCallbacks
476 def test_alice_delete_and_restore(self):
477 alice_clock = task.Clock()
478 bob_clock = task.Clock()
479 yield self.setup_alice_and_bob(alice_clock, bob_clock)
480 alice_dir = self.alice_magicfolder.uploader._local_path_u
481 bob_dir = self.bob_magicfolder.uploader._local_path_u
482 alice_fname = os.path.join(alice_dir, 'blam')
483 bob_fname = os.path.join(bob_dir, 'blam')
486 # alice creates a file, bob downloads it
487 alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
488 bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
490 fileutil.write(alice_fname, 'contents0\n')
491 self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
493 alice_clock.advance(0)
494 yield alice_proc # alice uploads
497 yield bob_proc # bob downloads
500 yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
501 yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 0)
502 yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
503 yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 0)
504 yield self.failUnlessReallyEqual(
505 self._get_count('downloader.objects_failed', client=self.bob_magicfolder._client),
508 yield self.failUnlessReallyEqual(
509 self._get_count('downloader.objects_downloaded', client=self.bob_magicfolder._client),
512 self.failUnless(os.path.exists(bob_fname))
514 # now alice deletes it (alice should upload, bob download)
515 alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
516 bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
517 os.unlink(alice_fname)
518 self.notify(to_filepath(alice_fname), self.inotify.IN_DELETE, magic=self.alice_magicfolder)
520 alice_clock.advance(0)
526 yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 1)
527 yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 1)
528 yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 1)
529 yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 1)
530 self.failIf(os.path.exists(bob_fname))
532 # now alice restores the file (with new contents)
533 alice_proc = self.alice_magicfolder.uploader.set_hook('processed')
534 bob_proc = self.bob_magicfolder.downloader.set_hook('processed')
535 fileutil.write(alice_fname, 'alice wuz here\n')
536 self.notify(to_filepath(alice_fname), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
538 alice_clock.advance(0)
544 yield self._check_version_in_dmd(self.bob_magicfolder, u"blam", 2)
545 yield self._check_version_in_local_db(self.bob_magicfolder, u"blam", 2)
546 yield self._check_version_in_dmd(self.alice_magicfolder, u"blam", 2)
547 yield self._check_version_in_local_db(self.alice_magicfolder, u"blam", 2)
548 self.failUnless(os.path.exists(bob_fname))
552 d0 = self.alice_magicfolder.finish()
553 alice_clock.advance(0)
556 d1 = self.bob_magicfolder.finish()
560 def test_magic_folder(self):
562 self.local_dir = os.path.join(self.basedir, self.unicode_or_fallback(u"loc\u0101l_dir", u"local_dir"))
563 self.mkdir_nonascii(self.local_dir)
565 d = self.create_invite_join_magic_folder(u"Alice\u0101", self.local_dir)
566 d.addCallback(self._restart_client)
568 # Write something short enough for a LIT file.
569 d.addCallback(lambda ign: self._check_file(u"short", "test"))
571 # Write to the same file again with different data.
572 d.addCallback(lambda ign: self._check_file(u"short", "different"))
574 # Test that temporary files are not uploaded.
575 d.addCallback(lambda ign: self._check_file(u"tempfile", "test", temporary=True))
577 # Test creation of a subdirectory.
578 d.addCallback(lambda ign: self._check_mkdir(u"directory"))
580 # Write something longer, and also try to test a Unicode name if the fs can represent it.
581 name_u = self.unicode_or_fallback(u"l\u00F8ng", u"long")
582 d.addCallback(lambda ign: self._check_file(name_u, "test"*100))
584 # TODO: test that causes an upload failure.
585 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
587 d.addBoth(self.cleanup)
590 def _check_mkdir(self, name_u):
591 return self._check_file(name_u + u"/", "", directory=True)
593 def _check_file(self, name_u, data, temporary=False, directory=False):
594 precondition(not (temporary and directory), temporary=temporary, directory=directory)
596 print "%r._check_file(%r, %r, temporary=%r, directory=%r)" % (self, name_u, data, temporary, directory)
597 previously_uploaded = self._get_count('uploader.objects_succeeded')
598 previously_disappeared = self._get_count('uploader.objects_disappeared')
600 d = self.magicfolder.uploader.set_hook('processed')
602 path_u = abspath_expanduser_unicode(name_u, base=self.local_dir)
603 path = to_filepath(path_u)
607 event_mask = self.inotify.IN_CREATE | self.inotify.IN_ISDIR
609 # We don't use FilePath.setContent() here because it creates a temporary file that
610 # is renamed into place, which causes events that the test is not expecting.
611 f = open(path_u, "wb")
613 if temporary and sys.platform != "win32":
618 if temporary and sys.platform == "win32":
620 self.notify(path, self.inotify.IN_DELETE, flush=False)
621 event_mask = self.inotify.IN_CLOSE_WRITE
623 self.notify(path, event_mask)
624 encoded_name_u = magicpath.path2magic(name_u)
626 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_failed'), 0))
628 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_disappeared'),
629 previously_disappeared + 1))
632 print "here %r %r" % (n, res)
635 d.addCallback(lambda ign: self.upload_dirnode.list())
636 d.addBoth(_here, 1.5)
637 d.addCallback(lambda ign: self.upload_dirnode.get(encoded_name_u))
639 d.addCallback(download_to_data)
641 d.addCallback(lambda actual_data: self.failUnlessReallyEqual(actual_data, data))
643 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_succeeded'),
644 previously_uploaded + 1))
646 d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('uploader.objects_queued'), 0))
649 def _check_version_in_dmd(self, magicfolder, relpath_u, expected_version):
650 encoded_name_u = magicpath.path2magic(relpath_u)
651 d = magicfolder.downloader._get_collective_latest_file(encoded_name_u)
652 def check_latest(result):
653 if result[0] is not None:
654 node, metadata = result
655 d.addCallback(lambda ign: self.failUnlessEqual(metadata['version'], expected_version))
656 d.addCallback(check_latest)
659 def _check_version_in_local_db(self, magicfolder, relpath_u, expected_version):
660 version = magicfolder._db.get_local_file_version(relpath_u)
661 #print "_check_version_in_local_db: %r has version %s" % (relpath_u, version)
662 self.failUnlessEqual(version, expected_version)
664 def _check_file_gone(self, magicfolder, relpath_u):
665 path = os.path.join(magicfolder.uploader._local_path_u, relpath_u)
666 self.assertTrue(not os.path.exists(path))
668 def _check_uploader_count(self, name, expected, magic=None):
669 self.failUnlessReallyEqual(self._get_count('uploader.'+name, client=(magic or self.alice_magicfolder)._client),
672 def _check_downloader_count(self, name, expected, magic=None):
673 self.failUnlessReallyEqual(self._get_count('downloader.'+name, client=(magic or self.bob_magicfolder)._client),
676 def test_alice_bob(self):
677 alice_clock = task.Clock()
678 bob_clock = task.Clock()
679 d = self.setup_alice_and_bob(alice_clock, bob_clock)
681 def _wait_for_Alice(ign, downloaded_d):
682 print "Now waiting for Alice to download\n"
683 alice_clock.advance(0)
686 def _wait_for_Bob(ign, downloaded_d):
687 print "Now waiting for Bob to download\n"
691 def _wait_for(ign, something_to_do, alice=True):
693 downloaded_d = self.bob_magicfolder.downloader.set_hook('processed')
694 uploaded_d = self.alice_magicfolder.uploader.set_hook('processed')
696 downloaded_d = self.alice_magicfolder.downloader.set_hook('processed')
697 uploaded_d = self.bob_magicfolder.uploader.set_hook('processed')
700 print "Waiting for Alice to upload\n"
701 alice_clock.advance(0)
702 uploaded_d.addCallback(_wait_for_Bob, downloaded_d)
704 print "Waiting for Bob to upload\n"
706 uploaded_d.addCallback(_wait_for_Alice, downloaded_d)
709 def Alice_to_write_a_file():
710 print "Alice writes a file\n"
711 self.file_path = abspath_expanduser_unicode(u"file1", base=self.alice_magicfolder.uploader._local_path_u)
712 fileutil.write(self.file_path, "meow, meow meow. meow? meow meow! meow.")
713 self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
714 d.addCallback(_wait_for, Alice_to_write_a_file)
716 d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file1", 0))
717 d.addCallback(lambda ign: self._check_version_in_local_db(self.alice_magicfolder, u"file1", 0))
718 d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0))
719 d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 1))
720 d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 1))
721 d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0))
722 d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0))
723 d.addCallback(lambda ign: self._check_uploader_count('objects_conflicted', 0))
724 d.addCallback(lambda ign: self._check_uploader_count('objects_conflicted', 0, magic=self.bob_magicfolder))
726 d.addCallback(lambda ign: self._check_version_in_local_db(self.bob_magicfolder, u"file1", 0))
727 d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0))
728 d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 1))
729 d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 0, magic=self.bob_magicfolder))
731 def Alice_to_delete_file():
732 print "Alice deletes the file!\n"
733 os.unlink(self.file_path)
734 self.notify(to_filepath(self.file_path), self.inotify.IN_DELETE, magic=self.alice_magicfolder)
735 d.addCallback(_wait_for, Alice_to_delete_file)
737 def notify_bob_moved(ign):
738 d0 = self.bob_magicfolder.uploader.set_hook('processed')
739 p = abspath_expanduser_unicode(u"file1", base=self.bob_magicfolder.uploader._local_path_u)
740 self.notify(to_filepath(p), self.inotify.IN_MOVED_FROM, magic=self.bob_magicfolder, flush=False)
741 self.notify(to_filepath(p + u'.backup'), self.inotify.IN_MOVED_TO, magic=self.bob_magicfolder)
744 d.addCallback(notify_bob_moved)
746 d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file1", 1))
747 d.addCallback(lambda ign: self._check_version_in_local_db(self.alice_magicfolder, u"file1", 1))
748 d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0))
749 d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 2))
750 d.addCallback(lambda ign: self._check_uploader_count('objects_not_uploaded', 1, magic=self.bob_magicfolder))
751 d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 1, magic=self.bob_magicfolder))
753 d.addCallback(lambda ign: self._check_version_in_local_db(self.bob_magicfolder, u"file1", 1))
754 d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file1", 1))
755 d.addCallback(lambda ign: self._check_file_gone(self.bob_magicfolder, u"file1"))
756 d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0))
757 d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 2))
759 def Alice_to_rewrite_file():
760 print "Alice rewrites file\n"
761 self.file_path = abspath_expanduser_unicode(u"file1", base=self.alice_magicfolder.uploader._local_path_u)
762 fileutil.write(self.file_path, "Alice suddenly sees the white rabbit running into the forest.")
763 self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE, magic=self.alice_magicfolder)
764 d.addCallback(_wait_for, Alice_to_rewrite_file)
766 d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file1", 2))
767 d.addCallback(lambda ign: self._check_version_in_local_db(self.alice_magicfolder, u"file1", 2))
768 d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0))
769 d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 3))
770 d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 3))
771 d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0))
772 d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0))
773 d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
774 d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0))
776 d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file1", 2))
777 d.addCallback(lambda ign: self._check_version_in_local_db(self.bob_magicfolder, u"file1", 2))
778 d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0))
779 d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 3))
781 path_u = u"/tmp/magic_folder_test"
782 encoded_path_u = magicpath.path2magic(u"/tmp/magic_folder_test")
784 def Alice_tries_to_p0wn_Bob(ign):
785 print "Alice tries to p0wn Bob\n"
786 self.objects_excluded = self._get_count('downloader.objects_excluded', client=self.bob_magicfolder._client)
787 processed_d = self.bob_magicfolder.downloader.set_hook('processed')
789 # upload a file that would provoke the security bug from #2506
790 uploadable = Data("", self.alice_magicfolder._client.convergence)
791 alice_dmd = self.alice_magicfolder.uploader._upload_dirnode
793 d2 = alice_dmd.add_file(encoded_path_u, uploadable, metadata={"version": 0}, overwrite=True)
794 d2.addCallback(lambda ign: self.failUnless(alice_dmd.has_child(encoded_path_u)))
795 d2.addCallback(_wait_for_Bob, processed_d)
797 d.addCallback(Alice_tries_to_p0wn_Bob)
799 d.addCallback(lambda ign: self.failIf(os.path.exists(path_u)))
800 d.addCallback(lambda ign: self._check_version_in_local_db(self.bob_magicfolder, encoded_path_u, None))
801 d.addCallback(lambda ign: self._check_downloader_count('objects_excluded', self.objects_excluded+1))
802 d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 3))
803 d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
804 d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0))
806 def Bob_to_rewrite_file():
807 print "Bob rewrites file\n"
808 self.file_path = abspath_expanduser_unicode(u"file1", base=self.bob_magicfolder.uploader._local_path_u)
809 print "---- bob's file is %r" % (self.file_path,)
810 fileutil.write(self.file_path, "No white rabbit to be found.")
811 self.magicfolder = self.bob_magicfolder
812 self.notify(to_filepath(self.file_path), self.inotify.IN_CLOSE_WRITE)
813 d.addCallback(lambda ign: _wait_for(None, Bob_to_rewrite_file, alice=False))
815 d.addCallback(lambda ign: self._check_version_in_dmd(self.bob_magicfolder, u"file1", 3))
816 d.addCallback(lambda ign: self._check_version_in_local_db(self.bob_magicfolder, u"file1", 3))
817 d.addCallback(lambda ign: self._check_uploader_count('objects_failed', 0, magic=self.bob_magicfolder))
818 d.addCallback(lambda ign: self._check_uploader_count('objects_succeeded', 2, magic=self.bob_magicfolder))
819 d.addCallback(lambda ign: self._check_uploader_count('files_uploaded', 1, magic=self.bob_magicfolder))
820 d.addCallback(lambda ign: self._check_uploader_count('objects_queued', 0, magic=self.bob_magicfolder))
821 d.addCallback(lambda ign: self._check_uploader_count('directories_created', 0, magic=self.bob_magicfolder))
822 d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0))
824 d.addCallback(lambda ign: self._check_version_in_dmd(self.alice_magicfolder, u"file1", 3))
825 d.addCallback(lambda ign: self._check_version_in_local_db(self.alice_magicfolder, u"file1", 3))
826 d.addCallback(lambda ign: self._check_downloader_count('objects_failed', 0, magic=self.alice_magicfolder))
827 d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 1, magic=self.alice_magicfolder))
828 d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 0, magic=self.alice_magicfolder))
830 def Alice_conflicts_with_Bob():
831 print "Alice conflicts with Bob\n"
832 downloaded_d = self.bob_magicfolder.downloader.set_hook('processed')
833 uploadable = Data("do not follow the white rabbit", self.alice_magicfolder._client.convergence)
834 alice_dmd = self.alice_magicfolder.uploader._upload_dirnode
835 d2 = alice_dmd.add_file(u"file1", uploadable,
836 metadata={"version": 5,
837 "last_downloaded_uri" : "URI:LIT:" },
839 print "Waiting for Alice to upload\n"
840 d2.addCallback(lambda ign: bob_clock.advance(6))
841 d2.addCallback(lambda ign: downloaded_d)
842 d2.addCallback(lambda ign: self.failUnless(alice_dmd.has_child(encoded_path_u)))
845 d.addCallback(lambda ign: Alice_conflicts_with_Bob())
846 # XXX fix the code so that it doesn't increment objects_excluded each turn
847 #d.addCallback(lambda ign: self._check_downloader_count('objects_excluded', 1))
848 d.addCallback(lambda ign: self._check_downloader_count('objects_downloaded', 4))
849 d.addCallback(lambda ign: self._check_downloader_count('objects_conflicted', 1))
851 def _cleanup(ign, magicfolder, clock):
852 if magicfolder is not None:
853 d2 = magicfolder.finish()
857 def cleanup_Alice_and_Bob(result):
858 print "cleanup alice bob test\n"
859 d = defer.succeed(None)
860 d.addCallback(_cleanup, self.alice_magicfolder, alice_clock)
861 d.addCallback(_cleanup, self.bob_magicfolder, bob_clock)
862 d.addCallback(lambda ign: result)
864 d.addBoth(cleanup_Alice_and_Bob)
868 class MockTest(MagicFolderTestMixin, unittest.TestCase):
869 """This can run on any platform, and even if twisted.internet.inotify can't be imported."""
872 MagicFolderTestMixin.setUp(self)
873 self.inotify = fake_inotify
874 self.patch(magic_folder, 'get_inotify_module', lambda: self.inotify)
876 def notify(self, path, mask, magic=None, flush=True):
878 magic = self.magicfolder
879 magic.uploader._notifier.event(path, mask)
880 # no flush for the mock test.
882 def test_errors(self):
885 errors_dir = abspath_expanduser_unicode(u"errors_dir", base=self.basedir)
887 not_a_dir = abspath_expanduser_unicode(u"NOT_A_DIR", base=self.basedir)
888 fileutil.write(not_a_dir, "")
889 magicfolderdb = abspath_expanduser_unicode(u"magicfolderdb", base=self.basedir)
890 doesnotexist = abspath_expanduser_unicode(u"doesnotexist", base=self.basedir)
892 client = self.g.clients[0]
893 d = client.create_dirnode()
894 def _check_errors(n):
895 self.failUnless(IDirectoryNode.providedBy(n))
896 upload_dircap = n.get_uri()
897 readonly_dircap = n.get_readonly_uri()
899 self.shouldFail(AssertionError, 'nonexistent local.directory', 'there is no directory',
900 MagicFolder, client, upload_dircap, '', doesnotexist, magicfolderdb)
901 self.shouldFail(AssertionError, 'non-directory local.directory', 'is not a directory',
902 MagicFolder, client, upload_dircap, '', not_a_dir, magicfolderdb)
903 self.shouldFail(AssertionError, 'bad upload.dircap', 'does not refer to a directory',
904 MagicFolder, client, 'bad', '', errors_dir, magicfolderdb)
905 self.shouldFail(AssertionError, 'non-directory upload.dircap', 'does not refer to a directory',
906 MagicFolder, client, 'URI:LIT:foo', '', errors_dir, magicfolderdb)
907 self.shouldFail(AssertionError, 'readonly upload.dircap', 'is not a writecap to a directory',
908 MagicFolder, client, readonly_dircap, '', errors_dir, magicfolderdb,)
909 self.shouldFail(AssertionError, 'collective dircap',
910 "The URI in '%s' is not a readonly cap to a directory." % os.path.join('private', 'collective_dircap'),
911 MagicFolder, client, upload_dircap, upload_dircap, errors_dir, magicfolderdb)
913 def _not_implemented():
914 raise NotImplementedError("blah")
915 self.patch(magic_folder, 'get_inotify_module', _not_implemented)
916 self.shouldFail(NotImplementedError, 'unsupported', 'blah',
917 MagicFolder, client, upload_dircap, '', errors_dir, magicfolderdb)
918 d.addCallback(_check_errors)
921 def test_write_downloaded_file(self):
922 workdir = u"cli/MagicFolder/write-downloaded-file"
923 local_file = fileutil.abspath_expanduser_unicode(os.path.join(workdir, "foobar"))
925 class TestWriteFileMixin(WriteFileMixin):
929 writefile = TestWriteFileMixin()
931 # create a file with name "foobar" with content "foo"
932 # write downloaded file content "bar" into "foobar" with is_conflict = False
933 fileutil.make_dirs(workdir)
934 fileutil.write(local_file, "foo")
936 # if is_conflict is False, then the .conflict file shouldn't exist.
937 writefile._write_downloaded_file(local_file, "bar", False, None)
938 conflicted_path = local_file + u".conflict"
939 self.failIf(os.path.exists(conflicted_path))
941 # At this point, the backup file should exist with content "foo"
942 backup_path = local_file + u".backup"
943 self.failUnless(os.path.exists(backup_path))
944 self.failUnlessEqual(fileutil.read(backup_path), "foo")
946 # .tmp file shouldn't exist
947 self.failIf(os.path.exists(local_file + u".tmp"))
949 # .. and the original file should have the new content
950 self.failUnlessEqual(fileutil.read(local_file), "bar")
952 # now a test for conflicted case
953 writefile._write_downloaded_file(local_file, "bar", True, None)
954 self.failUnless(os.path.exists(conflicted_path))
956 # .tmp file shouldn't exist
957 self.failIf(os.path.exists(local_file + u".tmp"))
960 class RealTest(MagicFolderTestMixin, unittest.TestCase):
961 """This is skipped unless both Twisted and the platform support inotify."""
964 MagicFolderTestMixin.setUp(self)
965 self.inotify = magic_folder.get_inotify_module()
967 def notify(self, path, mask, magic=None, flush=True):
968 # Writing to the filesystem causes the notification.
969 # However, flushing filesystem buffers may be necessary on Windows.
971 fileutil.flush_volume(path.path)
974 magic_folder.get_inotify_module()
975 except NotImplementedError:
976 RealTest.skip = "Magic Folder support can only be tested for-real on an OS that supports inotify or equivalent."