Add version to magic folder db schema

- also update handling file addition and deletion events
to utilize local and remote version numbers in the uploader...

^ untested so far althought he basic Alice + Bob test continues to pass

diff --git a/src/allmydata/backupdb.py b/src/allmydata/backupdb.py
index 537d4305051a726d967377d4f181a1990f9fc31d..18290213502a9974a9516bcd14b42d64a2ea08d0 100644 (file)
--- a/src/allmydata/backupdb.py
+++ b/src/allmydata/backupdb.py
@@ -64,6 +64,40 @@ UPDATERS = {
     2: UPDATE_v1_to_v2,
 }
 
+MAIN_v3 = """
+CREATE TABLE version
+(
+ version INTEGER  -- contains one row, set to 3
+);
+
+CREATE TABLE local_files
+(
+ path  VARCHAR(1024) PRIMARY KEY, -- index, this is an absolute UTF-8-encoded local filename
+ size  INTEGER,       -- os.stat(fn)[stat.ST_SIZE]
+ mtime NUMBER,        -- os.stat(fn)[stat.ST_MTIME]
+ ctime NUMBER,        -- os.stat(fn)[stat.ST_CTIME]
+ fileid INTEGER,
+ version INTEGER
+);
+
+CREATE TABLE caps
+(
+ fileid INTEGER PRIMARY KEY AUTOINCREMENT,
+ filecap VARCHAR(256) UNIQUE       -- URI:CHK:...
+);
+
+CREATE TABLE last_upload
+(
+ fileid INTEGER PRIMARY KEY,
+ last_uploaded TIMESTAMP,
+ last_checked TIMESTAMP
+);
+
+"""
+
+SCHEMA_v3 = MAIN_v3 + TABLE_DIRECTORY
+
+
 def get_backupdb(dbfile, stderr=sys.stderr,
                  create_version=(SCHEMA_v2, 2), just_create=False):
     # Open or create the given backupdb file. The parent directory must
@@ -71,7 +105,15 @@ def get_backupdb(dbfile, stderr=sys.stderr,
     try:
         (sqlite3, db) = get_db(dbfile, stderr, create_version, updaters=UPDATERS,
                                just_create=just_create, dbname="backupdb")
-        return BackupDB_v2(sqlite3, db)
+        if create_version[1] == 2:
+            print "ver 2!"
+            return BackupDB_v2(sqlite3, db)
+        elif create_version[1] == 3:
+            print "ver 3!"
+            return BackupDB_v3(sqlite3, db)
+        else:
+            print >>stderr, "invalid db schema version specified"
+            return None
     except DBError, e:
         print >>stderr, e
         return None
@@ -351,3 +393,27 @@ class BackupDB_v2:
                             " WHERE dircap=?",
                             (now, dircap))
         self.connection.commit()
+
+
+class BackupDB_v3(BackupDB_v2):
+    VERSION = 3 # XXX does this override the class var from parent class?
+
+    def __init__(self, sqlite_module, connection):
+        self.sqlite_module = sqlite_module
+        self.connection = connection
+        self.cursor = connection.cursor()
+
+    def get_local_file_version(self, path):
+        """I will tell you the version of a local file tracked by our magic folder db.
+        If no db entry found then I'll return None.
+        """
+        c = self.cursor
+        c.execute("SELECT version"
+                  " FROM local_files"
+                  " WHERE path=?",
+                  (path,))
+        row = self.cursor.fetchone()
+        if not row:
+            return None
+        else:
+            return row[0]

diff --git a/src/allmydata/frontends/magic_folder.py b/src/allmydata/frontends/magic_folder.py
index e587b59ec37d45b6691f546c0ccd255ac9a66968..0ecc54892aeb860661a2e1b887ce9ea9330f648e 100644 (file)
--- a/src/allmydata/frontends/magic_folder.py
+++ b/src/allmydata/frontends/magic_folder.py
@@ -237,7 +237,7 @@ class MagicFolder(service.MultiService):
                 self.warn("WARNING: cannot backup special file %s" % quote_local_unicode_path(childpath))
 
     def startService(self):
-        self._db = backupdb.get_backupdb(self._dbfile)
+        self._db = backupdb.get_backupdb(self._dbfile, create_version=(backupdb.SCHEMA_v3, 3))
         if self._db is None:
             return Failure(Exception('ERROR: Unable to load magic folder db.'))
 
@@ -307,44 +307,53 @@ class MagicFolder(service.MultiService):
     def _process(self, path):
         d = defer.succeed(None)
 
-        def _add_file(name):
+        def _add_file(name, version):
             u = FileName(path, self._convergence)
-            return self._upload_dirnode.add_file(name, u, metadata={"version":1}, overwrite=True)
+            return self._upload_dirnode.add_file(name, u, metadata={"version":version}, overwrite=True)
 
         def _add_dir(name):
             self._notifier.watch(to_filepath(path), mask=self.mask, callbacks=[self._notify], recursive=True)
             u = Data("", self._convergence)
             name += "@_"
-            d2 = self._upload_dirnode.add_file(name, u, metadata={"version":1}, overwrite=True)
+            upload_d = self._upload_dirnode.add_file(name, u, metadata={"version":1}, overwrite=True)
             def _succeeded(ign):
                 self._log("created subdirectory %r" % (path,))
                 self._stats_provider.count('magic_folder.directories_created', 1)
             def _failed(f):
                 self._log("failed to create subdirectory %r" % (path,))
                 return f
-            d2.addCallbacks(_succeeded, _failed)
-            d2.addCallback(lambda ign: self._scan(path))
-            return d2
+            upload_d.addCallbacks(_succeeded, _failed)
+            upload_d.addCallback(lambda ign: self._scan(path))
+            return upload_d
 
         def _maybe_upload(val):
             self._upload_pending.remove(path)
             relpath = os.path.relpath(path, self._local_dir)
             name = magicpath.path2magic(relpath)
 
+            def get_metadata(result):
+                try:
+                    metadata_d = self._parent.get_metadata_for(name)
+                except KeyError:
+                    return failure.Failure()
+                return metadata_d
+
+            def get_local_version(path):
+                v = self._db.get_local_file_version(path)
+                if v is None:
+                    return 1
+                else:
+                    return v
+
             if not os.path.exists(path):
                 self._log("drop-upload: notified object %r disappeared "
                           "(this is normal for temporary objects)" % (path,))
                 self._stats_provider.count('magic_folder.objects_disappeared', 1)
-
                 d2 = defer.succeed(None)
-                if not self._db.check_file_db_exists(path):
-                    pass
-                else:
-                    def get_metadata(d):
-                        return self._parent.get_metadata_for(name)
+                if self._db.check_file_db_exists(path):
                     d2.addCallback(get_metadata)
                     def set_deleted(metadata):
-                        metadata['version'] += 1
+                        metadata['version'] = get_local_version(path) + 1
                         metadata['deleted'] = True
                         emptyUploadable = Data("", self._convergence)
                         return self._parent.add_file(name, emptyUploadable, overwrite=True, metadata=metadata)
@@ -356,7 +365,8 @@ class MagicFolder(service.MultiService):
             if os.path.isdir(path):
                 return _add_dir(name)
             elif os.path.isfile(path):
-                d2 = _add_file(name)
+                version = get_local_version(path)
+                d2 = _add_file(name, version)
                 def add_db_entry(filenode):
                     filecap = filenode.get_uri()
                     s = os.stat(path)

diff --git a/src/allmydata/test/test_magic_folder.py b/src/allmydata/test/test_magic_folder.py
index 2d44f8c2408fc43f2ae407ed7c25cbb2c7fdc794..34087a2800f8c1463c9434999ef12c6b74506619 100644 (file)
--- a/src/allmydata/test/test_magic_folder.py
+++ b/src/allmydata/test/test_magic_folder.py
@@ -39,9 +39,9 @@ class MagicFolderTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, ReallyEqual
 
     def _createdb(self):
         dbfile = abspath_expanduser_unicode(u"magicfolderdb.sqlite", base=self.basedir)
-        bdb = backupdb.get_backupdb(dbfile)
+        bdb = backupdb.get_backupdb(dbfile, create_version=(backupdb.SCHEMA_v3, 3))
         self.failUnless(bdb, "unable to create backupdb from %r" % (dbfile,))
-        self.failUnlessEqual(bdb.VERSION, 2)
+        self.failUnlessEqual(bdb.VERSION, 3)
         return bdb
 
     def _made_upload_dir(self, n):