src/allmydata/test/test_backupdb.py

   1
   2 import os.path, time
   3 from StringIO import StringIO
   4 from twisted.trial import unittest
   5
   6 from allmydata.util import fileutil
   7 from allmydata.util.encodingutil import listdir_unicode, get_filesystem_encoding, unicode_platform
   8 from allmydata.util.assertutil import precondition
   9 from allmydata.scripts import backupdb
  10
  11 class BackupDB(unittest.TestCase):
  12     def create(self, dbfile):
  13         stderr = StringIO()
  14         bdb = backupdb.get_backupdb(dbfile, stderr=stderr)
  15         self.failUnless(bdb, "unable to create backupdb from %r" % (dbfile,))
  16         return bdb
  17
  18     def skip_if_cannot_represent_filename(self, u):
  19         precondition(isinstance(u, unicode))
  20
  21         enc = get_filesystem_encoding()
  22         if not unicode_platform():
  23             try:
  24                 u.encode(enc)
  25             except UnicodeEncodeError:
  26                 raise unittest.SkipTest("A non-ASCII filename could not be encoded on this platform.")
  27
  28     def test_basic(self):
  29         self.basedir = basedir = os.path.join("backupdb", "create")
  30         fileutil.make_dirs(basedir)
  31         dbfile = os.path.join(basedir, "dbfile")
  32         bdb = self.create(dbfile)
  33         self.failUnlessEqual(bdb.VERSION, 2)
  34
  35     def test_upgrade_v1_v2(self):
  36         self.basedir = basedir = os.path.join("backupdb", "upgrade_v1_v2")
  37         fileutil.make_dirs(basedir)
  38         dbfile = os.path.join(basedir, "dbfile")
  39         stderr = StringIO()
  40         created = backupdb.get_backupdb(dbfile, stderr=stderr,
  41                                         create_version=(backupdb.SCHEMA_v1, 1),
  42                                         just_create=True)
  43         self.failUnless(created, "unable to create v1 backupdb")
  44         # now we should have a v1 database on disk
  45         bdb = self.create(dbfile)
  46         self.failUnlessEqual(bdb.VERSION, 2)
  47
  48     def test_fail(self):
  49         self.basedir = basedir = os.path.join("backupdb", "fail")
  50         fileutil.make_dirs(basedir)
  51
  52         # put a non-DB file in the way
  53         not_a_db = ("I do not look like a sqlite database\n" +
  54                     "I'M NOT" * 1000) # OS-X sqlite-2.3.2 takes some convincing
  55         self.writeto("not-a-database", not_a_db)
  56         stderr_f = StringIO()
  57         bdb = backupdb.get_backupdb(os.path.join(basedir, "not-a-database"),
  58                                     stderr_f)
  59         self.failUnlessEqual(bdb, None)
  60         stderr = stderr_f.getvalue()
  61         self.failUnlessIn("backupdb file is unusable", stderr)
  62         self.failUnlessIn("file is encrypted or is not a database", stderr)
  63
  64         # put a directory in the way, to exercise a different error path
  65         where = os.path.join(basedir, "roadblock-dir")
  66         fileutil.make_dirs(where)
  67         stderr_f = StringIO()
  68         bdb = backupdb.get_backupdb(where, stderr_f)
  69         self.failUnlessEqual(bdb, None)
  70         stderr = stderr_f.getvalue()
  71         self.failUnlessIn("Unable to create/open backupdb file %s" % (where,), stderr)
  72         self.failUnlessIn("unable to open database file", stderr)
  73
  74
  75     def writeto(self, filename, data):
  76         fn = os.path.join(self.basedir, unicode(filename))
  77         parentdir = os.path.dirname(fn)
  78         fileutil.make_dirs(parentdir)
  79         fileutil.write(fn, data)
  80         return fn
  81
  82     def test_check(self):
  83         self.basedir = basedir = os.path.join("backupdb", "check")
  84         fileutil.make_dirs(basedir)
  85         dbfile = os.path.join(basedir, "dbfile")
  86         bdb = self.create(dbfile)
  87
  88         foo_fn = self.writeto("foo.txt", "foo.txt")
  89         blah_fn = self.writeto("bar/blah.txt", "blah.txt")
  90
  91         r = bdb.check_file(foo_fn)
  92         self.failUnlessEqual(r.was_uploaded(), False)
  93         r.did_upload("foo-cap")
  94
  95         r = bdb.check_file(blah_fn)
  96         self.failUnlessEqual(r.was_uploaded(), False)
  97         r.did_upload("blah-cap")
  98
  99         r = bdb.check_file(foo_fn)
 100         self.failUnlessEqual(r.was_uploaded(), "foo-cap")
 101         self.failUnlessEqual(type(r.was_uploaded()), str)
 102         self.failUnlessEqual(r.should_check(), False)
 103
 104         time.sleep(1.0) # make sure the timestamp changes
 105         self.writeto("foo.txt", "NEW")
 106
 107         r = bdb.check_file(foo_fn)
 108         self.failUnlessEqual(r.was_uploaded(), False)
 109         r.did_upload("new-cap")
 110
 111         r = bdb.check_file(foo_fn)
 112         self.failUnlessEqual(r.was_uploaded(), "new-cap")
 113         self.failUnlessEqual(r.should_check(), False)
 114         # if we spontaneously decide to upload it anyways, nothing should
 115         # break
 116         r.did_upload("new-cap")
 117
 118         r = bdb.check_file(foo_fn, use_timestamps=False)
 119         self.failUnlessEqual(r.was_uploaded(), False)
 120         r.did_upload("new-cap")
 121
 122         r = bdb.check_file(foo_fn)
 123         self.failUnlessEqual(r.was_uploaded(), "new-cap")
 124         self.failUnlessEqual(r.should_check(), False)
 125
 126         bdb.NO_CHECK_BEFORE = 0
 127         bdb.ALWAYS_CHECK_AFTER = 0.1
 128
 129         r = bdb.check_file(blah_fn)
 130         self.failUnlessEqual(r.was_uploaded(), "blah-cap")
 131         self.failUnlessEqual(r.should_check(), True)
 132         r.did_check_healthy("results") # we know they're ignored for now
 133
 134         bdb.NO_CHECK_BEFORE = 200
 135         bdb.ALWAYS_CHECK_AFTER = 400
 136
 137         r = bdb.check_file(blah_fn)
 138         self.failUnlessEqual(r.was_uploaded(), "blah-cap")
 139         self.failUnlessEqual(r.should_check(), False)
 140
 141         os.unlink(os.path.join(basedir, "foo.txt"))
 142         fileutil.make_dirs(os.path.join(basedir, "foo.txt")) # file becomes dir
 143         r = bdb.check_file(foo_fn)
 144         self.failUnlessEqual(r.was_uploaded(), False)
 145
 146     def test_wrong_version(self):
 147         self.basedir = basedir = os.path.join("backupdb", "wrong_version")
 148         fileutil.make_dirs(basedir)
 149
 150         where = os.path.join(basedir, "tooold.db")
 151         bdb = self.create(where)
 152         # reach into the DB and make it old
 153         bdb.cursor.execute("UPDATE version SET version=0")
 154         bdb.connection.commit()
 155
 156         # now the next time we open the database, it should be an unusable
 157         # version
 158         stderr_f = StringIO()
 159         bdb = backupdb.get_backupdb(where, stderr_f)
 160         self.failUnlessEqual(bdb, None)
 161         stderr = stderr_f.getvalue()
 162         self.failUnlessEqual(stderr.strip(),
 163                              "Unable to handle backupdb version 0")
 164
 165     def test_directory(self):
 166         self.basedir = basedir = os.path.join("backupdb", "directory")
 167         fileutil.make_dirs(basedir)
 168         dbfile = os.path.join(basedir, "dbfile")
 169         bdb = self.create(dbfile)
 170
 171         contents = {u"file1": "URI:CHK:blah1",
 172                     u"file2": "URI:CHK:blah2",
 173                     u"dir1": "URI:DIR2-CHK:baz2"}
 174         r = bdb.check_directory(contents)
 175         self.failUnless(isinstance(r, backupdb.DirectoryResult))
 176         self.failIf(r.was_created())
 177         dircap = "URI:DIR2-CHK:foo1"
 178         r.did_create(dircap)
 179
 180         r = bdb.check_directory(contents)
 181         self.failUnless(r.was_created())
 182         self.failUnlessEqual(r.was_created(), dircap)
 183         self.failUnlessEqual(r.should_check(), False)
 184
 185         # if we spontaneously decide to upload it anyways, nothing should
 186         # break
 187         r.did_create(dircap)
 188         r = bdb.check_directory(contents)
 189         self.failUnless(r.was_created())
 190         self.failUnlessEqual(r.was_created(), dircap)
 191         self.failUnlessEqual(type(r.was_created()), str)
 192         self.failUnlessEqual(r.should_check(), False)
 193
 194         bdb.NO_CHECK_BEFORE = 0
 195         bdb.ALWAYS_CHECK_AFTER = 0.1
 196         time.sleep(1.0)
 197
 198         r = bdb.check_directory(contents)
 199         self.failUnless(r.was_created())
 200         self.failUnlessEqual(r.was_created(), dircap)
 201         self.failUnlessEqual(r.should_check(), True)
 202         r.did_check_healthy("results")
 203
 204         bdb.NO_CHECK_BEFORE = 200
 205         bdb.ALWAYS_CHECK_AFTER = 400
 206
 207         r = bdb.check_directory(contents)
 208         self.failUnless(r.was_created())
 209         self.failUnlessEqual(r.was_created(), dircap)
 210         self.failUnlessEqual(r.should_check(), False)
 211
 212
 213         contents2 = {u"file1": "URI:CHK:blah1",
 214                      u"dir1": "URI:DIR2-CHK:baz2"}
 215         r = bdb.check_directory(contents2)
 216         self.failIf(r.was_created())
 217
 218         contents3 = {u"file1": "URI:CHK:blah1",
 219                      u"file2": "URI:CHK:blah3",
 220                      u"dir1": "URI:DIR2-CHK:baz2"}
 221         r = bdb.check_directory(contents3)
 222         self.failIf(r.was_created())
 223
 224     def test_unicode(self):
 225         self.skip_if_cannot_represent_filename(u"f\u00f6\u00f6.txt")
 226         self.skip_if_cannot_represent_filename(u"b\u00e5r.txt")
 227
 228         self.basedir = basedir = os.path.join("backupdb", "unicode")
 229         fileutil.make_dirs(basedir)
 230         dbfile = os.path.join(basedir, "dbfile")
 231         bdb = self.create(dbfile)
 232
 233         self.writeto(u"f\u00f6\u00f6.txt", "foo.txt")
 234         files = [fn for fn in listdir_unicode(unicode(basedir)) if fn.endswith(".txt")]
 235         self.failUnlessEqual(len(files), 1)
 236         foo_fn = os.path.join(basedir, files[0])
 237         #print foo_fn, type(foo_fn)
 238
 239         r = bdb.check_file(foo_fn)
 240         self.failUnlessEqual(r.was_uploaded(), False)
 241         r.did_upload("foo-cap")
 242
 243         r = bdb.check_file(foo_fn)
 244         self.failUnlessEqual(r.was_uploaded(), "foo-cap")
 245         self.failUnlessEqual(r.should_check(), False)
 246
 247         bar_fn = self.writeto(u"b\u00e5r.txt", "bar.txt")
 248         #print bar_fn, type(bar_fn)
 249
 250         r = bdb.check_file(bar_fn)
 251         self.failUnlessEqual(r.was_uploaded(), False)
 252         r.did_upload("bar-cap")
 253
 254         r = bdb.check_file(bar_fn)
 255         self.failUnlessEqual(r.was_uploaded(), "bar-cap")
 256         self.failUnlessEqual(r.should_check(), False)
 257