From: David Stainton Date: Tue, 14 Apr 2015 04:33:26 +0000 (+0000) Subject: Add very basic unit tests for dir scan and db X-Git-Url: https://git.rkrishnan.org/frontends/reliability?a=commitdiff_plain;h=e8b67cf4c53183de3919d93b321493931cb7f5d8;p=tahoe-lafs%2Ftahoe-lafs.git Add very basic unit tests for dir scan and db --- diff --git a/src/allmydata/frontends/drop_upload.py b/src/allmydata/frontends/drop_upload.py index 8375e0a7..1190347c 100644 --- a/src/allmydata/frontends/drop_upload.py +++ b/src/allmydata/frontends/drop_upload.py @@ -1,5 +1,6 @@ import sys +import os from twisted.internet import defer from twisted.python.failure import Failure @@ -14,6 +15,10 @@ from allmydata.util.fileutil import abspath_expanduser_unicode from allmydata.immutable.upload import FileName from allmydata.scripts import backupdb, tahoe_backup +from allmydata.util.encodingutil import listdir_unicode, quote_output, \ + quote_local_unicode_path, to_str, FilenameEncodingError, unicode_to_url + + class DropUploader(service.MultiService): name = 'drop-upload' @@ -35,6 +40,7 @@ class DropUploader(service.MultiService): self._stats_provider = client.stats_provider self._convergence = client.convergence self._local_path = FilePath(local_dir) + self._local_dir = unicode(local_dir, 'UTF-8') self._dbfile = dbfile if inotify is None: @@ -69,11 +75,10 @@ class DropUploader(service.MultiService): assert self._db != None use_timestamps = True r = self._db.check_file(childpath, use_timestamps) - return !r.was_uploaded() + return not r.was_uploaded() def _scan(self, localpath): quoted_path = quote_local_unicode_path(localpath) - try: children = listdir_unicode(localpath) except EnvironmentError: @@ -89,25 +94,29 @@ class DropUploader(service.MultiService): # recurse on the child directory self._scan(childpath) elif os.path.isfile(childpath) and not os.path.islink(childpath): - try: - must_upload = self._check_db_file(childpath) - if must_upload: - self._add_to_dequeue(childpath) + must_upload = self._check_db_file(childpath) + if must_upload: + self._add_to_dequeue(childpath) + else: + if os.path.islink(childpath): + self.warn("WARNING: cannot backup symlink %s" % quote_local_unicode_path(childpath)) + else: + self.warn("WARNING: cannot backup special file %s" % quote_local_unicode_path(childpath)) def startService(self): - self._db = backupdb.get_backupdb(self._dbfile, stderr) - if not self.backupdb: - # XXX or raise an exception? + self._db = backupdb.get_backupdb(self._dbfile) + if self._db is None: return Failure(Exception('ERROR: Unable to load magic folder db.')) - self._scan(self._local_path) - service.MultiService.startService(self) d = self._notifier.startReading() + + self._scan(self._local_dir) + self._stats_provider.count('drop_upload.dirs_monitored', 1) return d - def _add_to_dequeue(self, path, events_mask): + def _add_to_dequeue(self, path): # XXX stub function. fix me later. print "adding file to upload queue %s" % (path,) diff --git a/src/allmydata/test/test_drop_upload.py b/src/allmydata/test/test_drop_upload.py index 30ff8811..c88abcca 100644 --- a/src/allmydata/test/test_drop_upload.py +++ b/src/allmydata/test/test_drop_upload.py @@ -7,7 +7,7 @@ from twisted.internet import defer from allmydata.interfaces import IDirectoryNode, NoSuchChildError -from allmydata.util import fake_inotify +from allmydata.util import fake_inotify, fileutil from allmydata.util.encodingutil import get_filesystem_encoding from allmydata.util.consumer import download_to_data from allmydata.test.no_network import GridTestMixin @@ -15,6 +15,7 @@ from allmydata.test.common_util import ReallyEqualMixin, NonASCIIPathMixin from allmydata.test.common import ShouldFailMixin from allmydata.frontends.drop_upload import DropUploader +from allmydata.scripts import backupdb class DropUploadTestMixin(GridTestMixin, ShouldFailMixin, ReallyEqualMixin, NonASCIIPathMixin): @@ -41,7 +42,7 @@ class DropUploadTestMixin(GridTestMixin, ShouldFailMixin, ReallyEqualMixin, NonA self.upload_dirnode = n self.upload_dircap = n.get_uri() self.uploader = DropUploader(self.client, self.upload_dircap, self.local_dir.encode('utf-8'), - inotify=self.inotify) + "magicfolderdb.sqlite", inotify=self.inotify) return self.uploader.startService() d.addCallback(_made_upload_dir) @@ -166,6 +167,45 @@ class MockTest(DropUploadTestMixin, unittest.TestCase): class RealTest(DropUploadTestMixin, unittest.TestCase): """This is skipped unless both Twisted and the platform support inotify.""" + def create(self, dbfile): + bdb = backupdb.get_backupdb(dbfile) + self.failUnless(bdb, "unable to create backupdb from %r" % (dbfile,)) + return bdb + + def test_db_basic(self): + self.basedir = basedir = os.path.join("dropupload", "basic") + fileutil.make_dirs(basedir) + dbfile = os.path.join(basedir, "dbfile") + bdb = self.create(dbfile) + + def test_uploader_startService(self): + self.uploader = None + self.inotify = None # use the appropriate inotify for the platform + self.basedir = "drop_upload.RealTest.test_uploader_startService" + self.set_up_grid() + self.client = self.g.clients[0] + + d = self.client.create_dirnode() + def _made_upload_dir(n): + self.failUnless(IDirectoryNode.providedBy(n)) + self.upload_dirnode = n + self.upload_dircap = n.get_uri() + self.uploader = DropUploader(self.client, self.upload_dircap, self.basedir.encode('utf-8'), + "magicfolderdb.sqlite", inotify=self.inotify) + self.uploader.startService() + self.failUnlessEqual(self.uploader._db.VERSION, 2) + d.addCallback(_made_upload_dir) + + # Prevent unclean reactor errors. + def _cleanup(res): + d = defer.succeed(None) + if self.uploader is not None: + d.addCallback(lambda ign: self.uploader.finish(for_tests=True)) + d.addCallback(lambda ign: res) + return d + d.addBoth(_cleanup) + return d + def test_drop_upload(self): # We should always have runtime.platform.supportsINotify, because we're using # Twisted >= 10.1.