From 8c18ac38a9b71d43f7c2798e21d14f789c0210b9 Mon Sep 17 00:00:00 2001 From: Zooko O'Whielacronx Date: Thu, 4 Jun 2009 10:31:31 -0700 Subject: [PATCH] backup: remove the --no-backupdb command, the handling of "can't import sqlite", and the related tests, and change an error message to more correctly indicate failure to load the database from disk rather than failure to import sqlite module Fixes #728. --- docs/backupdb.txt | 16 ++++++++++------ src/allmydata/scripts/backupdb.py | 25 ++++--------------------- src/allmydata/scripts/cli.py | 1 - src/allmydata/scripts/tahoe_backup.py | 18 +++++++----------- src/allmydata/test/test_cli.py | 18 +----------------- 5 files changed, 22 insertions(+), 56 deletions(-) diff --git a/docs/backupdb.txt b/docs/backupdb.txt index c6cea645..23daaeda 100644 --- a/docs/backupdb.txt +++ b/docs/backupdb.txt @@ -5,9 +5,9 @@ To speed up backup operations, Tahoe maintains a small database known as the uploaded recently. This database lives in ~/.tahoe/private/backupdb.sqlite, and is a SQLite -single-file database. It is used by the "tahoe backup" command (unless the ---no-backupdb option is included). In the future, it will also be used by -"tahoe mirror", and by "tahoe cp" when the --use-backupdb option is included. +single-file database. It is used by the "tahoe backup" command. In the future, +it will also be used by "tahoe mirror", and by "tahoe cp" when the +--use-backupdb option is included. The purpose of this database is specifically to manage the file-to-cap translation (the "upload" step). It does not address directory updates. A @@ -23,9 +23,13 @@ subsequent backup operation may use more effort (network bandwidth, CPU cycles, and disk IO) than it would have without the backupdb. The database uses sqlite3, which is included as part of the standard python -library with python2.5 and later. For python2.4, please install the -"pysqlite2" package (which, despite the name, actually provides sqlite3 -rather than sqlite2). +library with python2.5 and later. For python2.4, Tahoe will try to install the +"pysqlite" package at build-time, but this will succeed only if sqlite3 with +development headers is already installed. On Debian and Debian derivatives +you can install the "python-pysqlite2" package (which, despite the name, +actually provides sqlite3 rather than sqlite2), but on old distributions such +as Debian etch (4.0 "oldstable") or Ubuntu Edgy (6.10) the "python-pysqlite2" +package won't work, but the "sqlite3-dev" package will. == Schema == diff --git a/src/allmydata/scripts/backupdb.py b/src/allmydata/scripts/backupdb.py index 8ab41d15..add7c02c 100644 --- a/src/allmydata/scripts/backupdb.py +++ b/src/allmydata/scripts/backupdb.py @@ -47,27 +47,10 @@ def get_backupdb(dbfile, stderr=sys.stderr): import sqlite3 sqlite = sqlite3 # pyflakes whines about 'import sqlite3 as sqlite' .. except ImportError: - try: - from pysqlite2 import dbapi2 - sqlite = dbapi2 # .. when this clause does it too - except ImportError: - print >>stderr, """\ -The backup command uses a SQLite database to avoid duplicate uploads, but -I was unable to import a python sqlite library. You have two options: - - 1: Install a python sqlite library. python2.5 and beyond have one built-in. - If you are using python2.4, you can install the 'pysqlite' package, - perhaps with 'apt-get install python-pysqlite2', or 'easy_install - pysqlite', or by installing the 'pysqlite' package from - http://pypi.python.org . Make sure you get the version with support for - SQLite 3. - - 2: Run me with the --no-backupdb option to disable use of the database. This - will be somewhat slower, since I will be unable to avoid re-uploading - files that were uploaded in the past, but the basic functionality will be - unimpaired. -""" - return None + from pysqlite2 import dbapi2 + sqlite = dbapi2 # .. when this clause does it too + # This import should never fail, because setuptools requires that the + # "pysqlite" distribution is present at start time (if on Python < 2.5). must_create = not os.path.exists(dbfile) try: diff --git a/src/allmydata/scripts/cli.py b/src/allmydata/scripts/cli.py index de8d6249..e65196c6 100644 --- a/src/allmydata/scripts/cli.py +++ b/src/allmydata/scripts/cli.py @@ -208,7 +208,6 @@ class BackupConfigurationError(Exception): class BackupOptions(VDriveOptions): optFlags = [ ("verbose", "v", "Be noisy about what is happening."), - ("no-backupdb", None, "Do not use the SQLite-based backup-database (always upload all files)."), ("ignore-timestamps", None, "Do not use backupdb timestamps to decide if a local file is unchanged."), ] diff --git a/src/allmydata/scripts/tahoe_backup.py b/src/allmydata/scripts/tahoe_backup.py index d87196b7..c8d5ea35 100644 --- a/src/allmydata/scripts/tahoe_backup.py +++ b/src/allmydata/scripts/tahoe_backup.py @@ -157,17 +157,13 @@ class BackerUpper: start_timestamp = datetime.datetime.now() self.backupdb = None - use_backupdb = not options["no-backupdb"] - if use_backupdb: - bdbfile = os.path.join(options["node-directory"], - "private", "backupdb.sqlite") - bdbfile = os.path.abspath(bdbfile) - self.backupdb = backupdb.get_backupdb(bdbfile, stderr) - if not self.backupdb: - # get_backupdb() has already delivered a lengthy speech about - # where to find pysqlite and how to add --no-backupdb - print >>stderr, "ERROR: Unable to import sqlite." - return 1 + bdbfile = os.path.join(options["node-directory"], + "private", "backupdb.sqlite") + bdbfile = os.path.abspath(bdbfile) + self.backupdb = backupdb.get_backupdb(bdbfile, stderr) + if not self.backupdb: + print >>stderr, "ERROR: Unable to load backup db." + return 1 rootcap, path = get_alias(options.aliases, options.to_dir, DEFAULT_ALIAS) to_url = nodeurl + "uri/%s/" % urllib.quote(rootcap) diff --git a/src/allmydata/test/test_cli.py b/src/allmydata/test/test_cli.py index dcab060f..12502f85 100644 --- a/src/allmydata/test/test_cli.py +++ b/src/allmydata/test/test_cli.py @@ -871,10 +871,8 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): self.writeto("parent/subdir/bar.txt", "bar\n" * 1000) self.writeto("parent/blah.txt", "blah") - def do_backup(use_backupdb=True, verbose=False): + def do_backup(verbose=False): cmd = ["backup"] - if not have_bdb or not use_backupdb: - cmd.append("--no-backupdb") if verbose: cmd.append("--verbose") cmd.append(source) @@ -1055,20 +1053,6 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): self.failUnlessEqual(out, "foo") d.addCallback(_check8) - d.addCallback(self.stall, 1.1) - d.addCallback(lambda res: do_backup(use_backupdb=False)) - def _check9((rc, out, err)): - # --no-backupdb means re-upload everything. We still get to - # re-use the directories, since nothing changed. - self.failUnlessEqual(err, "") - self.failUnlessEqual(rc, 0) - fu, fr, dc, dr = self.count_output(out) - self.failUnlessEqual(fu, 5) - self.failUnlessEqual(fr, 0) - self.failUnlessEqual(dc, 0) - self.failUnlessEqual(dr, 5) - d.addCallback(_check9) - return d # on our old dapper buildslave, this test takes a long time (usually -- 2.45.2