From: Daira Hopwood Date: Mon, 15 Apr 2013 19:50:40 +0000 (+0100) Subject: Make backupdb use dbutil. X-Git-Url: https://git.rkrishnan.org/components/%22news.html/module-simplejson.decoder.html?a=commitdiff_plain;h=9160181d83bfc72b9ffbd4a4a27169c1de9f057e;p=tahoe-lafs%2Ftahoe-lafs.git Make backupdb use dbutil. Signed-off-by: Daira Hopwood --- diff --git a/src/allmydata/scripts/backupdb.py b/src/allmydata/scripts/backupdb.py index 75ee0d9c..d0c22616 100644 --- a/src/allmydata/scripts/backupdb.py +++ b/src/allmydata/scripts/backupdb.py @@ -6,6 +6,8 @@ from allmydata.util.hashutil import backupdb_dirhash from allmydata.util import base32 from allmydata.util.fileutil import abspath_expanduser_unicode from allmydata.util.encodingutil import to_str +from allmydata.util.dbutil import get_db, DBError + DAY = 24*60*60 MONTH = 30*DAY @@ -58,47 +60,22 @@ UPDATE_v1_to_v2 = TABLE_DIRECTORY + """ UPDATE version SET version=2; """ +UPDATERS = { + 2: UPDATE_v1_to_v2, +} def get_backupdb(dbfile, stderr=sys.stderr, create_version=(SCHEMA_v2, 2), just_create=False): - # open or create the given backupdb file. The parent directory must + # Open or create the given backupdb file. The parent directory must # exist. - import sqlite3 - - must_create = not os.path.exists(dbfile) - try: - db = sqlite3.connect(dbfile) - except (EnvironmentError, sqlite3.OperationalError), e: - print >>stderr, "Unable to create/open backupdb file %s: %s" % (dbfile, e) - return None - - c = db.cursor() - if must_create: - schema, version = create_version - c.executescript(schema) - c.execute("INSERT INTO version (version) VALUES (?)", (version,)) - db.commit() - try: - c.execute("SELECT version FROM version") - version = c.fetchone()[0] - except sqlite3.DatabaseError, e: - # this indicates that the file is not a compatible database format. - # Perhaps it was created with an old version, or it might be junk. - print >>stderr, "backupdb file is unusable: %s" % e + (sqlite3, db) = get_db(dbfile, stderr, create_version, updaters=UPDATERS, + just_create=just_create, dbname="backupdb") + return BackupDB_v2(sqlite3, db) + except DBError, e: + print >>stderr, e return None - if just_create: # for tests - return True - - if version == 1: - c.executescript(UPDATE_v1_to_v2) - db.commit() - version = 2 - if version == 2: - return BackupDB_v2(sqlite3, db) - print >>stderr, "Unable to handle backupdb version %s" % version - return None class FileResult: def __init__(self, bdb, filecap, should_check, @@ -127,6 +104,7 @@ class FileResult: def did_check_healthy(self, results): self.bdb.did_check_file_healthy(self.filecap, results) + class DirectoryResult: def __init__(self, bdb, dirhash, dircap, should_check): self.bdb = bdb @@ -148,6 +126,7 @@ class DirectoryResult: def did_check_healthy(self, results): self.bdb.did_check_directory_healthy(self.dircap, results) + class BackupDB_v2: VERSION = 2 NO_CHECK_BEFORE = 1*MONTH @@ -180,7 +159,7 @@ class BackupDB_v2: is not healthy, please upload the file and call r.did_upload(filecap) when you're done. - I use_timestamps=True (the default), I will compare ctime and mtime + If use_timestamps=True (the default), I will compare ctime and mtime of the local file against an entry in my database, and consider the file to be unchanged if ctime, mtime, and filesize are all the same as the earlier version. If use_timestamps=False, I will not trust the diff --git a/src/allmydata/storage/leasedb.py b/src/allmydata/storage/leasedb.py index d64233d8..dbd72f90 100644 --- a/src/allmydata/storage/leasedb.py +++ b/src/allmydata/storage/leasedb.py @@ -119,8 +119,16 @@ class LeaseDB: STARTER_LEASE_DURATION = 2*MONTH def __init__(self, dbfile): + # synchronous = OFF is necessary for leasedb to pass tests for the time being, + # since using synchronous = NORMAL causes failures that are apparently due to + # a file descriptor leak, and the default synchronous = FULL causes the tests + # to time out. For discussion see + # https://tahoe-lafs.org/pipermail/tahoe-dev/2012-December/007877.html + (self._sqlite, - self._db) = dbutil.get_db(dbfile, create_version=(LEASE_SCHEMA_V1, 1)) + self._db) = dbutil.get_db(dbfile, create_version=(LEASE_SCHEMA_V1, 1), + # journal_mode="WAL", + synchronous="OFF") self._cursor = self._db.cursor() self.debug = False self.retained_history_entries = 10 diff --git a/src/allmydata/util/dbutil.py b/src/allmydata/util/dbutil.py index fc12d4c0..f098f1c1 100644 --- a/src/allmydata/util/dbutil.py +++ b/src/allmydata/util/dbutil.py @@ -11,7 +11,8 @@ class DBError(Exception): def get_db(dbfile, stderr=sys.stderr, - create_version=(None, None), updaters={}, just_create=False): + create_version=(None, None), updaters={}, just_create=False, dbname="db", + journal_mode=None, synchronous=None): """Open or create the given db file. The parent directory must exist. create_version=(SCHEMA, VERNUM), and SCHEMA must have a 'version' table. Updaters is a {newver: commands} mapping, where e.g. updaters[2] is used @@ -22,7 +23,7 @@ def get_db(dbfile, stderr=sys.stderr, try: db = sqlite3.connect(dbfile) except (EnvironmentError, sqlite3.OperationalError), e: - raise DBError("Unable to create/open db file %s: %s" % (dbfile, e)) + raise DBError("Unable to create/open %s file %s: %s" % (dbname, dbfile, e)) schema, target_version = create_version c = db.cursor() @@ -31,13 +32,11 @@ def get_db(dbfile, stderr=sys.stderr, # The default is unspecified according to . c.execute("PRAGMA foreign_keys = ON;") - # This is necessary to pass tests for the time being, since using - # synchronous = NORMAL causes failures that are apparently due to a - # file descriptor leak, and the default synchronous = FULL causes the - # tests to time out. For discussion see - # https://tahoe-lafs.org/pipermail/tahoe-dev/2012-December/007877.html - #c.execute("PRAGMA journal_mode = WAL;") - c.execute("PRAGMA synchronous = OFF;") + if journal_mode is not None: + c.execute("PRAGMA journal_mode = %s;" % (journal_mode,)) + + if synchronous is not None: + c.execute("PRAGMA synchronous = %s;" % (synchronous,)) if must_create: c.executescript(schema) @@ -50,17 +49,17 @@ def get_db(dbfile, stderr=sys.stderr, except sqlite3.DatabaseError, e: # this indicates that the file is not a compatible database format. # Perhaps it was created with an old version, or it might be junk. - raise DBError("db file is unusable: %s" % e) + raise DBError("%s file is unusable: %s" % (dbname, e)) if just_create: # for tests return (sqlite3, db) - while version < target_version: + while version < target_version and version+1 in updaters: c.executescript(updaters[version+1]) db.commit() version = version+1 if version != target_version: - raise DBError("Unable to handle db version %s" % version) + raise DBError("Unable to handle %s version %s" % (dbname, version)) return (sqlite3, db)