From 31ad6e86dfde4f4a8434bcb77912dbf1a8ba7716 Mon Sep 17 00:00:00 2001 From: David Stainton Date: Wed, 7 Oct 2015 18:45:44 +0200 Subject: [PATCH] WIP --- src/allmydata/backupdb.py | 87 ++++++++++++++++++++----- src/allmydata/frontends/magic_folder.py | 12 +++- src/allmydata/test/test_magic_folder.py | 2 +- 3 files changed, 79 insertions(+), 22 deletions(-) diff --git a/src/allmydata/backupdb.py b/src/allmydata/backupdb.py index b75bd7aa..bf1a3576 100644 --- a/src/allmydata/backupdb.py +++ b/src/allmydata/backupdb.py @@ -68,7 +68,25 @@ UPDATERS = { } -SCHEMA_v3 = MAIN_SCHEMA % (3, ",\nversion INTEGER\n") + TABLE_DIRECTORY +# magic-folder db schema version 3 +MAGIC_FOLDER_SCHEMA_v3 = """ +CREATE TABLE version +( + version INTEGER -- contains one row, set to %s +); + +CREATE TABLE local_files +( + path VARCHAR(1024) PRIMARY KEY, -- index, this is an absolute UTF-8-encoded local filename + -- note that size is before mtime and ctime here, but after in function parameters + size INTEGER, -- os.stat(fn)[stat.ST_SIZE] (NULL if the file has been deleted) + mtime NUMBER, -- os.stat(fn)[stat.ST_MTIME] + ctime NUMBER, -- os.stat(fn)[stat.ST_CTIME] + version INTEGER, + last_downloaded_uri VARCHAR(256) UNIQUE, -- URI:CHK:... + last_downloaded_timestamp NUMBER +); +""" % (3,) def get_backupdb(dbfile, stderr=sys.stderr, @@ -368,9 +386,29 @@ class BackupDB: self.connection.commit() -class MagicFolderDB(BackupDB): +class MagicFolderDB(): VERSION = 3 + def __init__(self, sqlite_module, connection): + self.sqlite_module = sqlite_module + self.connection = connection + self.cursor = connection.cursor() + + def check_file_db_exists(self, path): + """I will tell you if a given file has an entry in my database or not + by returning True or False. + """ + c = self.cursor + c.execute("SELECT size,mtime,ctime" + " FROM local_files" + " WHERE path=?", + (path,)) + row = self.cursor.fetchone() + if not row: + return False + else: + return True + def get_all_relpaths(self): """ Retrieve a set of all relpaths of files that have had an entry in magic folder db @@ -380,13 +418,29 @@ class MagicFolderDB(BackupDB): rows = self.cursor.fetchall() return set([r[0] for r in rows]) + def get_last_downloaded_uri(self, relpath_u): + """ + Return the last downloaded uri recorded in the magic folder db. + If none are found then return None. + """ + c = self.cursor + c.execute("SELECT last_downloaded_uri" + " FROM local_files" + " WHERE path=?", + (relpath_u,)) + row = self.cursor.fetchone() + if not row: + return None + else: + return row[0] + def get_local_file_version(self, relpath_u): """ Return the version of a local file tracked by our magic folder db. If no db entry is found then return None. """ c = self.cursor - c.execute("SELECT version, fileid" + c.execute("SELECT version" " FROM local_files" " WHERE path=?", (relpath_u,)) @@ -397,26 +451,23 @@ class MagicFolderDB(BackupDB): return row[0] def did_upload_version(self, filecap, relpath_u, version, pathinfo): - #print "did_upload_version(%r, %r, %r, %r)" % (filecap, relpath_u, version, pathinfo) + print "did_upload_version(%r, %r, %r, %r)" % (filecap, relpath_u, version, pathinfo) now = time.time() - fileid = self.get_or_allocate_fileid_for_cap(filecap) - try: - self.cursor.execute("INSERT INTO last_upload VALUES (?,?,?)", - (fileid, now, now)) - except (self.sqlite_module.IntegrityError, self.sqlite_module.OperationalError): - self.cursor.execute("UPDATE last_upload" - " SET last_uploaded=?, last_checked=?" - " WHERE fileid=?", - (now, now, fileid)) try: + print "insert" self.cursor.execute("INSERT INTO local_files VALUES (?,?,?,?,?,?)", - (relpath_u, pathinfo.size, pathinfo.mtime, pathinfo.ctime, fileid, version)) + (relpath_u, pathinfo.size, pathinfo.mtime, pathinfo.ctime, version, filecap, pathinfo.mtime)) except (self.sqlite_module.IntegrityError, self.sqlite_module.OperationalError): - self.cursor.execute("UPDATE local_files" - " SET size=?, mtime=?, ctime=?, fileid=?, version=?" - " WHERE path=?", - (pathinfo.size, pathinfo.mtime, pathinfo.ctime, fileid, version, relpath_u)) + print "err... update" + try: + self.cursor.execute("UPDATE local_files" + " SET size=?, mtime=?, ctime=?, version=?, last_downloaded_uri=?, last_downloaded_timestamp=?" + " WHERE path=?", + (pathinfo.size, pathinfo.mtime, pathinfo.ctime, version, filecap, pathinfo.mtime, relpath_u)) + except (self.sqlite_module.IntegrityError, self.sqlite_module.OperationalError): + print "WTF BBQ OMG" self.connection.commit() + print "commited" def is_new_file(self, pathinfo, relpath_u): """ diff --git a/src/allmydata/frontends/magic_folder.py b/src/allmydata/frontends/magic_folder.py index e32459c1..16cdaad0 100644 --- a/src/allmydata/frontends/magic_folder.py +++ b/src/allmydata/frontends/magic_folder.py @@ -51,7 +51,7 @@ class MagicFolder(service.MultiService): service.MultiService.__init__(self) - db = backupdb.get_backupdb(dbfile, create_version=(backupdb.SCHEMA_v3, 3)) + db = backupdb.get_backupdb(dbfile, create_version=(backupdb.MAGIC_FOLDER_SCHEMA_v3, 3)) if db is None: return Failure(Exception('ERROR: Unable to load magic folder db.')) @@ -294,8 +294,10 @@ class Uploader(QueueMixin): d2 = defer.succeed(None) if self._db.check_file_db_exists(relpath_u): d2.addCallback(lambda ign: self._get_metadata(encoded_path_u)) + last_downloaded_uri = self._db.get_last_downloaded_uri(relpath_u) current_version = self._db.get_local_file_version(relpath_u) + 1 def set_deleted(metadata): + metadata['last_downloaded_uri'] = last_downloaded_uri metadata['version'] = current_version metadata['deleted'] = True empty_uploadable = Data("", self._client.convergence) @@ -305,6 +307,7 @@ class Uploader(QueueMixin): filecap = filenode.get_uri() self._db.did_upload_version(filecap, relpath_u, current_version, pathinfo) self._count('files_uploaded') + # FIXME consider whether it's correct to retrieve the filenode again. d2.addCallback(lambda x: self._get_filenode(encoded_path_u)) d2.addCallback(add_db_entry) @@ -330,6 +333,7 @@ class Uploader(QueueMixin): return upload_d elif pathinfo.isfile: version = self._db.get_local_file_version(relpath_u) + last_downloaded_uri = self._db.get_last_downloaded_uri(relpath_u) if version is None: version = 0 elif self._db.is_new_file(pathinfo, relpath_u): @@ -338,11 +342,13 @@ class Uploader(QueueMixin): return None uploadable = FileName(unicode_from_filepath(fp), self._client.convergence) - d2 = self._upload_dirnode.add_file(encoded_path_u, uploadable, metadata={"version":version}, overwrite=True) + metadata = { "version":version } + if last_downloaded_uri is not None: + metadata["last_downloaded_uri"] = last_downloaded_uri + d2 = self._upload_dirnode.add_file(encoded_path_u, uploadable, metadata=metadata, overwrite=True) def add_db_entry(filenode): filecap = filenode.get_uri() self._db.did_upload_version(filecap, relpath_u, version, pathinfo) - self._count('files_uploaded') d2.addCallback(add_db_entry) return d2 else: diff --git a/src/allmydata/test/test_magic_folder.py b/src/allmydata/test/test_magic_folder.py index 9608b772..4a6b60eb 100644 --- a/src/allmydata/test/test_magic_folder.py +++ b/src/allmydata/test/test_magic_folder.py @@ -39,7 +39,7 @@ class MagicFolderTestMixin(MagicFolderCLITestMixin, ShouldFailMixin, ReallyEqual def _createdb(self): dbfile = abspath_expanduser_unicode(u"magicfolderdb.sqlite", base=self.basedir) - bdb = backupdb.get_backupdb(dbfile, create_version=(backupdb.SCHEMA_v3, 3)) + bdb = backupdb.get_backupdb(dbfile, create_version=(backupdb.MAGIC_FOLDER_SCHEMA_v3, 3)) self.failUnless(bdb, "unable to create backupdb from %r" % (dbfile,)) self.failUnlessEqual(bdb.VERSION, 3) return bdb -- 2.45.2