From: francois Date: Wed, 20 Jan 2010 09:42:49 +0000 (-0800) Subject: tahoe_backup.py: display warnings on errors instead of stopping the whole backup... X-Git-Url: https://git.rkrishnan.org/pf/content/en/-?a=commitdiff_plain;h=b03406af9d2162786dd5794f9bf784a3e7a0018a;p=tahoe-lafs%2Ftahoe-lafs.git tahoe_backup.py: display warnings on errors instead of stopping the whole backup. Fix #729. This patch displays a warning to the user in two cases: 1. When special files like symlinks, fifos, devices, etc. are found in the local source. 2. If files or directories are not readables by the user running the 'tahoe backup' command. In verbose mode, the number of skipped files and directories is printed at the end of the backup. Exit status returned by 'tahoe backup': - 0 everything went fine - 1 the backup failed - 2 files were skipped during the backup --- diff --git a/src/allmydata/scripts/tahoe_backup.py b/src/allmydata/scripts/tahoe_backup.py index 122018c9..ee8cb569 100644 --- a/src/allmydata/scripts/tahoe_backup.py +++ b/src/allmydata/scripts/tahoe_backup.py @@ -65,9 +65,11 @@ class BackerUpper: self.files_uploaded = 0 self.files_reused = 0 self.files_checked = 0 + self.files_skipped = 0 self.directories_created = 0 self.directories_reused = 0 self.directories_checked = 0 + self.directories_skipped = 0 def run(self): options = self.options @@ -123,16 +125,25 @@ class BackerUpper: if self.verbosity >= 1: print >>stdout, (" %d files uploaded (%d reused), " - "%d directories created (%d reused)" + "%d files skipped, " + "%d directories created (%d reused), " + "%d directories skipped" % (self.files_uploaded, self.files_reused, + self.files_skipped, self.directories_created, - self.directories_reused)) + self.directories_reused, + self.directories_skipped)) if self.verbosity >= 2: print >>stdout, (" %d files checked, %d directories checked" % (self.files_checked, self.directories_checked)) print >>stdout, " backup done, elapsed time: %s" % elapsed_time + + # The command exits with code 2 if files or directories were skipped + if self.files_skipped or self.directories_skipped: + return 2 + # done! return 0 @@ -140,13 +151,24 @@ class BackerUpper: if self.verbosity >= 2: print >>self.options.stdout, msg + def warn(self, msg): + print >>self.options.stderr, msg + def process(self, localpath): # returns newdircap self.verboseprint("processing %s" % localpath) create_contents = {} # childname -> (type, rocap, metadata) compare_contents = {} # childname -> rocap - for child in self.options.filter_listdir(os.listdir(localpath)): + + try: + children = os.listdir(localpath) + except EnvironmentError: + self.directories_skipped += 1 + self.warn("WARNING: permission denied on directory %s" % localpath) + children = [] + + for child in self.options.filter_listdir(children): childpath = os.path.join(localpath, child) child = unicode(child) if os.path.isdir(childpath): @@ -157,12 +179,17 @@ class BackerUpper: create_contents[child] = ("dirnode", childcap, metadata) compare_contents[child] = childcap elif os.path.isfile(childpath): - childcap, metadata = self.upload(childpath) - assert isinstance(childcap, str) - create_contents[child] = ("filenode", childcap, metadata) - compare_contents[child] = childcap + try: + childcap, metadata = self.upload(childpath) + assert isinstance(childcap, str) + create_contents[child] = ("filenode", childcap, metadata) + compare_contents[child] = childcap + except EnvironmentError: + self.files_skipped += 1 + self.warn("WARNING: permission denied on file %s" % childpath) else: - raise BackupProcessingError("Cannot backup child %r" % childpath) + self.files_skipped += 1 + self.warn("WARNING: cannot backup special file %s" % childpath) must_create, r = self.check_backupdb_directory(compare_contents) if must_create: @@ -245,6 +272,7 @@ class BackerUpper: r.did_check_healthy(cr) return False, r + # This function will raise an IOError exception when called on an unreadable file def upload(self, childpath): #self.verboseprint("uploading %s.." % childpath) metadata = get_local_metadata(childpath) diff --git a/src/allmydata/test/test_cli.py b/src/allmydata/test/test_cli.py index 30037910..dc45e2fe 100644 --- a/src/allmydata/test/test_cli.py +++ b/src/allmydata/test/test_cli.py @@ -1072,7 +1072,10 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): f.close() def count_output(self, out): - mo = re.search(r"(\d)+ files uploaded \((\d+) reused\), (\d+) directories created \((\d+) reused\)", out) + mo = re.search(r"(\d)+ files uploaded \((\d+) reused\), " + "(\d)+ files skipped, " + "(\d+) directories created \((\d+) reused\), " + "(\d+) directories skipped", out) return [int(s) for s in mo.groups()] def count_output2(self, out): @@ -1117,13 +1120,15 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): def _check0((rc, out, err)): self.failUnlessEqual(err, "") self.failUnlessEqual(rc, 0) - fu, fr, dc, dr = self.count_output(out) + fu, fr, fs, dc, dr, ds = self.count_output(out) # foo.txt, bar.txt, blah.txt self.failUnlessEqual(fu, 3) self.failUnlessEqual(fr, 0) + self.failUnlessEqual(fs, 0) # empty, home, home/parent, home/parent/subdir self.failUnlessEqual(dc, 4) self.failUnlessEqual(dr, 0) + self.failUnlessEqual(ds, 0) d.addCallback(_check0) d.addCallback(lambda res: self.do_cli("ls", "--uri", "tahoe:backups")) @@ -1172,13 +1177,15 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): self.failUnlessEqual(err, "") self.failUnlessEqual(rc, 0) if have_bdb: - fu, fr, dc, dr = self.count_output(out) + fu, fr, fs, dc, dr, ds = self.count_output(out) # foo.txt, bar.txt, blah.txt self.failUnlessEqual(fu, 0) self.failUnlessEqual(fr, 3) + self.failUnlessEqual(fs, 0) # empty, home, home/parent, home/parent/subdir self.failUnlessEqual(dc, 0) self.failUnlessEqual(dr, 4) + self.failUnlessEqual(ds, 0) d.addCallback(_check4a) if have_bdb: @@ -1203,14 +1210,16 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): # re-use all of them too. self.failUnlessEqual(err, "") self.failUnlessEqual(rc, 0) - fu, fr, dc, dr = self.count_output(out) + fu, fr, fs, dc, dr, ds = self.count_output(out) fchecked, dchecked = self.count_output2(out) self.failUnlessEqual(fchecked, 3) self.failUnlessEqual(fu, 0) self.failUnlessEqual(fr, 3) + self.failUnlessEqual(fs, 0) self.failUnlessEqual(dchecked, 4) self.failUnlessEqual(dc, 0) self.failUnlessEqual(dr, 4) + self.failUnlessEqual(ds, 0) d.addCallback(_check4b) d.addCallback(lambda res: self.do_cli("ls", "tahoe:backups/Archives")) @@ -1247,14 +1256,16 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): self.failUnlessEqual(err, "") self.failUnlessEqual(rc, 0) if have_bdb: - fu, fr, dc, dr = self.count_output(out) + fu, fr, fs, dc, dr, ds = self.count_output(out) # new foo.txt, surprise file, subfile, empty self.failUnlessEqual(fu, 4) # old bar.txt self.failUnlessEqual(fr, 1) + self.failUnlessEqual(fs, 0) # home, parent, subdir, blah.txt, surprisedir self.failUnlessEqual(dc, 5) self.failUnlessEqual(dr, 0) + self.failUnlessEqual(ds, 0) d.addCallback(_check5a) d.addCallback(lambda res: self.do_cli("ls", "tahoe:backups/Archives")) def _check6((rc, out, err)): @@ -1356,6 +1367,107 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): _check_filtering(filtered, root_listdir, ('lib.a', '_darcs', 'subdir'), ('nice_doc.lyx',)) + def test_ignore_symlinks(self): + if not hasattr(os, 'symlink'): + raise unittest.SkipTest("There is no symlink on this platform.") + + self.basedir = os.path.dirname(self.mktemp()) + self.set_up_grid() + + source = os.path.join(self.basedir, "home") + self.writeto("foo.txt", "foo") + os.symlink(os.path.join(source, "foo.txt"), os.path.join(source, "foo2.txt")) + + d = self.do_cli("create-alias", "tahoe") + d.addCallback(lambda res: self.do_cli("backup", "--verbose", source, "tahoe:test")) + + def _check((rc, out, err)): + self.failUnlessEqual(rc, 2) + self.failUnlessEqual(err, "WARNING: cannot backup special file %s\n" % os.path.join(source, "foo2.txt")) + + fu, fr, fs, dc, dr, ds = self.count_output(out) + # foo.txt + self.failUnlessEqual(fu, 1) + self.failUnlessEqual(fr, 0) + # foo2.txt + self.failUnlessEqual(fs, 1) + # home + self.failUnlessEqual(dc, 1) + self.failUnlessEqual(dr, 0) + self.failUnlessEqual(ds, 0) + + d.addCallback(_check) + return d + + def test_ignore_unreadable_file(self): + self.basedir = os.path.dirname(self.mktemp()) + self.set_up_grid() + + source = os.path.join(self.basedir, "home") + self.writeto("foo.txt", "foo") + os.chmod(os.path.join(source, "foo.txt"), 0000) + + d = self.do_cli("create-alias", "tahoe") + d.addCallback(lambda res: self.do_cli("backup", source, "tahoe:test")) + + def _check((rc, out, err)): + self.failUnlessEqual(rc, 2) + self.failUnlessEqual(err, "WARNING: permission denied on file %s\n" % os.path.join(source, "foo.txt")) + + fu, fr, fs, dc, dr, ds = self.count_output(out) + self.failUnlessEqual(fu, 0) + self.failUnlessEqual(fr, 0) + # foo.txt + self.failUnlessEqual(fs, 1) + # home + self.failUnlessEqual(dc, 1) + self.failUnlessEqual(dr, 0) + self.failUnlessEqual(ds, 0) + d.addCallback(_check) + + # This is necessary for the temp files to be correctly removed + def _cleanup(self): + os.chmod(os.path.join(source, "foo.txt"), 0644) + d.addCallback(_cleanup) + d.addErrback(_cleanup) + + return d + + def test_ignore_unreadable_directory(self): + self.basedir = os.path.dirname(self.mktemp()) + self.set_up_grid() + + source = os.path.join(self.basedir, "home") + os.mkdir(source) + os.mkdir(os.path.join(source, "test")) + os.chmod(os.path.join(source, "test"), 0000) + + d = self.do_cli("create-alias", "tahoe") + d.addCallback(lambda res: self.do_cli("backup", source, "tahoe:test")) + + def _check((rc, out, err)): + self.failUnlessEqual(rc, 2) + self.failUnlessEqual(err, "WARNING: permission denied on directory %s\n" % os.path.join(source, "test")) + + fu, fr, fs, dc, dr, ds = self.count_output(out) + self.failUnlessEqual(fu, 0) + self.failUnlessEqual(fr, 0) + self.failUnlessEqual(fs, 0) + # home, test + self.failUnlessEqual(dc, 2) + self.failUnlessEqual(dr, 0) + # test + self.failUnlessEqual(ds, 1) + d.addCallback(_check) + + # This is necessary for the temp files to be correctly removed + def _cleanup(self): + os.chmod(os.path.join(source, "test"), 0655) + d.addCallback(_cleanup) + d.addErrback(_cleanup) + return d + + class Check(GridTestMixin, CLITestMixin, unittest.TestCase): def test_check(self):