From d8ba2a3265d8d8adf98d6fb3468e5581903aa665 Mon Sep 17 00:00:00 2001 From: David Stainton Date: Thu, 2 Jul 2015 11:49:13 -0700 Subject: [PATCH] Add version to magic folder db schema - also update handling file addition and deletion events to utilize local and remote version numbers in the uploader... ^ untested so far althought he basic Alice + Bob test continues to pass --- src/allmydata/backupdb.py | 68 ++++++++++++++++++++++++- src/allmydata/frontends/magic_folder.py | 40 +++++++++------ src/allmydata/test/test_magic_folder.py | 4 +- 3 files changed, 94 insertions(+), 18 deletions(-) diff --git a/src/allmydata/backupdb.py b/src/allmydata/backupdb.py index 537d4305..18290213 100644 --- a/src/allmydata/backupdb.py +++ b/src/allmydata/backupdb.py @@ -64,6 +64,40 @@ UPDATERS = { 2: UPDATE_v1_to_v2, } +MAIN_v3 = """ +CREATE TABLE version +( + version INTEGER -- contains one row, set to 3 +); + +CREATE TABLE local_files +( + path VARCHAR(1024) PRIMARY KEY, -- index, this is an absolute UTF-8-encoded local filename + size INTEGER, -- os.stat(fn)[stat.ST_SIZE] + mtime NUMBER, -- os.stat(fn)[stat.ST_MTIME] + ctime NUMBER, -- os.stat(fn)[stat.ST_CTIME] + fileid INTEGER, + version INTEGER +); + +CREATE TABLE caps +( + fileid INTEGER PRIMARY KEY AUTOINCREMENT, + filecap VARCHAR(256) UNIQUE -- URI:CHK:... +); + +CREATE TABLE last_upload +( + fileid INTEGER PRIMARY KEY, + last_uploaded TIMESTAMP, + last_checked TIMESTAMP +); + +""" + +SCHEMA_v3 = MAIN_v3 + TABLE_DIRECTORY + + def get_backupdb(dbfile, stderr=sys.stderr, create_version=(SCHEMA_v2, 2), just_create=False): # Open or create the given backupdb file. The parent directory must @@ -71,7 +105,15 @@ def get_backupdb(dbfile, stderr=sys.stderr, try: (sqlite3, db) = get_db(dbfile, stderr, create_version, updaters=UPDATERS, just_create=just_create, dbname="backupdb") - return BackupDB_v2(sqlite3, db) + if create_version[1] == 2: + print "ver 2!" + return BackupDB_v2(sqlite3, db) + elif create_version[1] == 3: + print "ver 3!" + return BackupDB_v3(sqlite3, db) + else: + print >>stderr, "invalid db schema version specified" + return None except DBError, e: print >>stderr, e return None @@ -351,3 +393,27 @@ class BackupDB_v2: " WHERE dircap=?", (now, dircap)) self.connection.commit() + + +class BackupDB_v3(BackupDB_v2): + VERSION = 3 # XXX does this override the class var from parent class? + + def __init__(self, sqlite_module, connection): + self.sqlite_module = sqlite_module + self.connection = connection + self.cursor = connection.cursor() + + def get_local_file_version(self, path): + """I will tell you the version of a local file tracked by our magic folder db. + If no db entry found then I'll return None. + """ + c = self.cursor + c.execute("SELECT version" + " FROM local_files" + " WHERE path=?", + (path,)) + row = self.cursor.fetchone() + if not row: + return None + else: + return row[0] diff --git a/src/allmydata/frontends/magic_folder.py b/src/allmydata/frontends/magic_folder.py index e587b59e..0ecc5489 100644 --- a/src/allmydata/frontends/magic_folder.py +++ b/src/allmydata/frontends/magic_folder.py @@ -237,7 +237,7 @@ class MagicFolder(service.MultiService): self.warn("WARNING: cannot backup special file %s" % quote_local_unicode_path(childpath)) def startService(self): - self._db = backupdb.get_backupdb(self._dbfile) + self._db = backupdb.get_backupdb(self._dbfile, create_version=(backupdb.SCHEMA_v3, 3)) if self._db is None: return Failure(Exception('ERROR: Unable to load magic folder db.')) @@ -307,44 +307,53 @@ class MagicFolder(service.MultiService): def _process(self, path): d = defer.succeed(None) - def _add_file(name): + def _add_file(name, version): u = FileName(path, self._convergence) - return self._upload_dirnode.add_file(name, u, metadata={"version":1}, overwrite=True) + return self._upload_dirnode.add_file(name, u, metadata={"version":version}, overwrite=True) def _add_dir(name): self._notifier.watch(to_filepath(path), mask=self.mask, callbacks=[self._notify], recursive=True) u = Data("", self._convergence) name += "@_" - d2 = self._upload_dirnode.add_file(name, u, metadata={"version":1}, overwrite=True) + upload_d = self._upload_dirnode.add_file(name, u, metadata={"version":1}, overwrite=True) def _succeeded(ign): self._log("created subdirectory %r" % (path,)) self._stats_provider.count('magic_folder.directories_created', 1) def _failed(f): self._log("failed to create subdirectory %r" % (path,)) return f - d2.addCallbacks(_succeeded, _failed) - d2.addCallback(lambda ign: self._scan(path)) - return d2 + upload_d.addCallbacks(_succeeded, _failed) + upload_d.addCallback(lambda ign: self._scan(path)) + return upload_d def _maybe_upload(val): self._upload_pending.remove(path) relpath = os.path.relpath(path, self._local_dir) name = magicpath.path2magic(relpath) + def get_metadata(result): + try: + metadata_d = self._parent.get_metadata_for(name) + except KeyError: + return failure.Failure() + return metadata_d + + def get_local_version(path): + v = self._db.get_local_file_version(path) + if v is None: + return 1 + else: + return v + if not os.path.exists(path): self._log("drop-upload: notified object %r disappeared " "(this is normal for temporary objects)" % (path,)) self._stats_provider.count('magic_folder.objects_disappeared', 1) - d2 = defer.succeed(None) - if not self._db.check_file_db_exists(path): - pass - else: - def get_metadata(d): - return self._parent.get_metadata_for(name) + if self._db.check_file_db_exists(path): d2.addCallback(get_metadata) def set_deleted(metadata): - metadata['version'] += 1 + metadata['version'] = get_local_version(path) + 1 metadata['deleted'] = True emptyUploadable = Data("", self._convergence) return self._parent.add_file(name, emptyUploadable, overwrite=True, metadata=metadata) @@ -356,7 +365,8 @@ class MagicFolder(service.MultiService): if os.path.isdir(path): return _add_dir(name) elif os.path.isfile(path): - d2 = _add_file(name) + version = get_local_version(path) + d2 = _add_file(name, version) def add_db_entry(filenode): filecap = filenode.get_uri() s = os.stat(path) diff --git a/src/allmydata/test/test_magic_folder.py b/src/allmydata/test/test_magic_folder.py index 2d44f8c2..34087a28 100644 --- a/src/allmydata/test/test_magic_folder.py +++ b/src/allmydata/test/test_magic_folder.py @@ -39,9 +39,9 @@ class MagicFolderTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, ReallyEqual def _createdb(self): dbfile = abspath_expanduser_unicode(u"magicfolderdb.sqlite", base=self.basedir) - bdb = backupdb.get_backupdb(dbfile) + bdb = backupdb.get_backupdb(dbfile, create_version=(backupdb.SCHEMA_v3, 3)) self.failUnless(bdb, "unable to create backupdb from %r" % (dbfile,)) - self.failUnlessEqual(bdb.VERSION, 2) + self.failUnlessEqual(bdb.VERSION, 3) return bdb def _made_upload_dir(self, n): -- 2.45.2