Merge branch '2438.magic-folder-stable.9' of https://github.com/tahoe-lafs/tahoe...

[tahoe-lafs/tahoe-lafs.git] / src / allmydata / frontends / magic_folder.py

diff --git a/src/allmydata/frontends/magic_folder.py b/src/allmydata/frontends/magic_folder.py
index c0c4f051f47dbc0a1db3968514bc71662facaedf..bd226156d23b81c576b988c32d1babe63c3d3512 100644 (file)
--- a/src/allmydata/frontends/magic_folder.py
+++ b/src/allmydata/frontends/magic_folder.py
@@ -42,10 +42,21 @@ def get_inotify_module():
         raise
 
 
+def is_new_file(pathinfo, db_entry):
+    if db_entry is None:
+        return True
+
+    if not pathinfo.exists and db_entry.size is None:
+        return False
+
+    return ((pathinfo.size, pathinfo.ctime, pathinfo.mtime) !=
+            (db_entry.size, db_entry.ctime, db_entry.mtime))
+
+
 class MagicFolder(service.MultiService):
     name = 'magic-folder'
 
-    def __init__(self, client, upload_dircap, collective_dircap, local_path_u, dbfile,
+    def __init__(self, client, upload_dircap, collective_dircap, local_path_u, dbfile, umask,
                  pending_delay=1.0, clock=None):
         precondition_abspath(local_path_u)
 
@@ -66,7 +77,7 @@ class MagicFolder(service.MultiService):
 
         self.uploader = Uploader(client, local_path_u, db, upload_dirnode, pending_delay, clock, immediate)
         self.downloader = Downloader(client, local_path_u, db, collective_dirnode,
-                                     upload_dirnode.get_readonly_uri(), clock, self.uploader.is_pending)
+                                     upload_dirnode.get_readonly_uri(), clock, self.uploader.is_pending, umask)
 
     def startService(self):
         # TODO: why is this being called more than once?
@@ -336,7 +347,7 @@ class Uploader(QueueMixin):
 
                 last_downloaded_timestamp = now  # is this correct?
 
-                if self._db.is_new_file(pathinfo, relpath_u):
+                if is_new_file(pathinfo, db_entry):
                     new_version = db_entry.version + 1
                 else:
                     self._log("Not uploading %r" % (relpath_u,))
@@ -389,7 +400,7 @@ class Uploader(QueueMixin):
 
                 if db_entry is None:
                     new_version = 0
-                elif self._db.is_new_file(pathinfo, relpath_u):
+                elif is_new_file(pathinfo, db_entry):
                     new_version = db_entry.version + 1
                 else:
                     self._log("Not uploading %r" % (relpath_u,))
@@ -473,10 +484,18 @@ class WriteFileMixin(object):
 
         # ensure parent directory exists
         head, tail = os.path.split(abspath_u)
-        mode = 0777 # XXX
-        fileutil.make_dirs(head, mode)
 
-        fileutil.write(replacement_path_u, file_contents)
+        old_mask = os.umask(self._umask)
+        try:
+            fileutil.make_dirs(head, (~ self._umask) & 0777)
+            fileutil.write(replacement_path_u, file_contents)
+        finally:
+            os.umask(old_mask)
+
+        # FUDGE_SECONDS is used to determine if another process
+        # has written to the same file concurrently. This is described
+        # in the Earth Dragon section of our design document:
+        # docs/proposed/magic-folder/remote-to-local-sync.rst
         os.utime(replacement_path_u, (now, now - self.FUDGE_SECONDS))
         if is_conflict:
             print "0x00 ------------ <><> is conflict; calling _rename_conflicted_file... %r %r" % (abspath_u, replacement_path_u)
@@ -514,7 +533,7 @@ class Downloader(QueueMixin, WriteFileMixin):
     REMOTE_SCAN_INTERVAL = 3  # facilitates tests
 
     def __init__(self, client, local_path_u, db, collective_dirnode,
-                 upload_readonly_dircap, clock, is_upload_pending):
+                 upload_readonly_dircap, clock, is_upload_pending, umask):
         QueueMixin.__init__(self, client, local_path_u, db, 'downloader', clock)
 
         if not IDirectoryNode.providedBy(collective_dirnode):
@@ -527,8 +546,7 @@ class Downloader(QueueMixin, WriteFileMixin):
         self._collective_dirnode = collective_dirnode
         self._upload_readonly_dircap = upload_readonly_dircap
         self._is_upload_pending = is_upload_pending
-
-        self._turn_delay = self.REMOTE_SCAN_INTERVAL
+        self._umask = umask
 
     def start_scanning(self):
         self._log("start_scanning")
@@ -598,6 +616,10 @@ class Downloader(QueueMixin, WriteFileMixin):
             node = None
             for success, result in deferredList:
                 if success:
+                    if 'version' not in result[1]:
+                        self._log("invalid remote metadata detected")
+                        continue
+
                     if result[1]['version'] > max_version:
                         node, metadata = result
                         max_version = result[1]['version']
@@ -615,10 +637,15 @@ class Downloader(QueueMixin, WriteFileMixin):
 
                 file_node, metadata = listing_map[encoded_relpath_u]
                 local_version = self._get_local_latest(relpath_u)
-                remote_version = metadata.get('version', None)
+
+                if 'version' not in metadata:
+                    self._log("invalid remote metadata detected for %r" % (relpath_u,))
+                    continue
+
+                remote_version = metadata['version']
                 self._log("%r has local version %r, remote version %r" % (relpath_u, local_version, remote_version))
 
-                if local_version is None or remote_version is None or local_version < remote_version:
+                if local_version is None or local_version < remote_version:
                     self._log("%r added to download queue" % (relpath_u,))
                     if scan_batch.has_key(relpath_u):
                         scan_batch[relpath_u] += [(file_node, metadata)]
@@ -638,7 +665,7 @@ class Downloader(QueueMixin, WriteFileMixin):
             d2 = defer.succeed(None)
             for dir_name in dirmap:
                 (dirnode, metadata) = dirmap[dir_name]
-                if scan_self is True or dirnode.get_readonly_uri() != self._upload_readonly_dircap:
+                if scan_self or dirnode.get_readonly_uri() != self._upload_readonly_dircap:
                     d2.addCallback(lambda ign, dir_name=dir_name, dirnode=dirnode:
                                    self._scan_remote_dmd(dir_name, dirnode, scan_batch))
                     def _err(f, dir_name=dir_name):
@@ -658,7 +685,6 @@ class Downloader(QueueMixin, WriteFileMixin):
                     self._deque.append( (relpath_u, file_node, metadata) )
                 else:
                     self._log("Excluding %r" % (relpath_u,))
-                    self._count('objects_excluded')
                     self._call_hook(None, 'processed')
 
             self._log("deque after = %r" % (self._deque,))
@@ -666,7 +692,7 @@ class Downloader(QueueMixin, WriteFileMixin):
         return d
 
     def _when_queue_is_empty(self):
-        d = task.deferLater(self._clock, self._turn_delay, self._scan_remote_collective)
+        d = task.deferLater(self._clock, self.REMOTE_SCAN_INTERVAL, self._scan_remote_collective)
         d.addBoth(self._logcb, "after _scan_remote_collective 1")
         d.addCallback(lambda ign: self._turn_deque())
         return d