tahoe_backup.py: display warnings on errors instead of stopping the whole backup...

This patch displays a warning to the user in two cases:

  1. When special files like symlinks, fifos, devices, etc. are found in the
     local source.

  2. If files or directories are not readables by the user running the 'tahoe
     backup' command.

In verbose mode, the number of skipped files and directories is printed at the
end of the backup.

Exit status returned by 'tahoe backup':

  - 0 everything went fine
  - 1 the backup failed
  - 2 files were skipped during the backup

diff --git a/src/allmydata/scripts/tahoe_backup.py b/src/allmydata/scripts/tahoe_backup.py
index 122018c982c404b69dd24235149311f16bf9ccbf..ee8cb56944d8c3acca685a073601d08270fe9695 100644 (file)
--- a/src/allmydata/scripts/tahoe_backup.py
+++ b/src/allmydata/scripts/tahoe_backup.py
@@ -65,9 +65,11 @@ class BackerUpper:
         self.files_uploaded = 0
         self.files_reused = 0
         self.files_checked = 0
+        self.files_skipped = 0
         self.directories_created = 0
         self.directories_reused = 0
         self.directories_checked = 0
+        self.directories_skipped = 0
 
     def run(self):
         options = self.options
@@ -123,16 +125,25 @@ class BackerUpper:
 
         if self.verbosity >= 1:
             print >>stdout, (" %d files uploaded (%d reused), "
-                             "%d directories created (%d reused)"
+                             "%d files skipped, "
+                             "%d directories created (%d reused), "
+                             "%d directories skipped"
                              % (self.files_uploaded,
                                 self.files_reused,
+                                self.files_skipped,
                                 self.directories_created,
-                                self.directories_reused))
+                                self.directories_reused,
+                                self.directories_skipped))
             if self.verbosity >= 2:
                 print >>stdout, (" %d files checked, %d directories checked"
                                  % (self.files_checked,
                                     self.directories_checked))
             print >>stdout, " backup done, elapsed time: %s" % elapsed_time
+
+        # The command exits with code 2 if files or directories were skipped
+        if self.files_skipped or self.directories_skipped:
+            return 2
+
         # done!
         return 0
 
@@ -140,13 +151,24 @@ class BackerUpper:
         if self.verbosity >= 2:
             print >>self.options.stdout, msg
 
+    def warn(self, msg):
+        print >>self.options.stderr, msg
+
     def process(self, localpath):
         # returns newdircap
 
         self.verboseprint("processing %s" % localpath)
         create_contents = {} # childname -> (type, rocap, metadata)
         compare_contents = {} # childname -> rocap
-        for child in self.options.filter_listdir(os.listdir(localpath)):
+
+        try:
+            children = os.listdir(localpath)
+        except EnvironmentError:
+            self.directories_skipped += 1
+            self.warn("WARNING: permission denied on directory %s" % localpath)
+            children = []
+
+        for child in self.options.filter_listdir(children):
             childpath = os.path.join(localpath, child)
             child = unicode(child)
             if os.path.isdir(childpath):
@@ -157,12 +179,17 @@ class BackerUpper:
                 create_contents[child] = ("dirnode", childcap, metadata)
                 compare_contents[child] = childcap
             elif os.path.isfile(childpath):
-                childcap, metadata = self.upload(childpath)
-                assert isinstance(childcap, str)
-                create_contents[child] = ("filenode", childcap, metadata)
-                compare_contents[child] = childcap
+                try:
+                    childcap, metadata = self.upload(childpath)
+                    assert isinstance(childcap, str)
+                    create_contents[child] = ("filenode", childcap, metadata)
+                    compare_contents[child] = childcap
+                except EnvironmentError:
+                    self.files_skipped += 1
+                    self.warn("WARNING: permission denied on file %s" % childpath)
             else:
-                raise BackupProcessingError("Cannot backup child %r" % childpath)
+                self.files_skipped += 1
+                self.warn("WARNING: cannot backup special file %s" % childpath)
 
         must_create, r = self.check_backupdb_directory(compare_contents)
         if must_create:
@@ -245,6 +272,7 @@ class BackerUpper:
         r.did_check_healthy(cr)
         return False, r
 
+    # This function will raise an IOError exception when called on an unreadable file
     def upload(self, childpath):
         #self.verboseprint("uploading %s.." % childpath)
         metadata = get_local_metadata(childpath)

diff --git a/src/allmydata/test/test_cli.py b/src/allmydata/test/test_cli.py
index 30037910f51bc2a7427cfcb28310d833b19c1c40..dc45e2fe7ab8638c25f2569ff1397b7901deac1b 100644 (file)
--- a/src/allmydata/test/test_cli.py
+++ b/src/allmydata/test/test_cli.py
@@ -1072,7 +1072,10 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase):
         f.close()
 
     def count_output(self, out):
-        mo = re.search(r"(\d)+ files uploaded \((\d+) reused\), (\d+) directories created \((\d+) reused\)", out)
+        mo = re.search(r"(\d)+ files uploaded \((\d+) reused\), "
+                        "(\d)+ files skipped, "
+                        "(\d+) directories created \((\d+) reused\), "
+                        "(\d+) directories skipped", out)
         return [int(s) for s in mo.groups()]
 
     def count_output2(self, out):
@@ -1117,13 +1120,15 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase):
         def _check0((rc, out, err)):
             self.failUnlessEqual(err, "")
             self.failUnlessEqual(rc, 0)
-            fu, fr, dc, dr = self.count_output(out)
+            fu, fr, fs, dc, dr, ds = self.count_output(out)
             # foo.txt, bar.txt, blah.txt
             self.failUnlessEqual(fu, 3)
             self.failUnlessEqual(fr, 0)
+            self.failUnlessEqual(fs, 0)
             # empty, home, home/parent, home/parent/subdir
             self.failUnlessEqual(dc, 4)
             self.failUnlessEqual(dr, 0)
+            self.failUnlessEqual(ds, 0)
         d.addCallback(_check0)
 
         d.addCallback(lambda res: self.do_cli("ls", "--uri", "tahoe:backups"))
@@ -1172,13 +1177,15 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase):
             self.failUnlessEqual(err, "")
             self.failUnlessEqual(rc, 0)
             if have_bdb:
-                fu, fr, dc, dr = self.count_output(out)
+                fu, fr, fs, dc, dr, ds = self.count_output(out)
                 # foo.txt, bar.txt, blah.txt
                 self.failUnlessEqual(fu, 0)
                 self.failUnlessEqual(fr, 3)
+                self.failUnlessEqual(fs, 0)
                 # empty, home, home/parent, home/parent/subdir
                 self.failUnlessEqual(dc, 0)
                 self.failUnlessEqual(dr, 4)
+                self.failUnlessEqual(ds, 0)
         d.addCallback(_check4a)
 
         if have_bdb:
@@ -1203,14 +1210,16 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase):
                 # re-use all of them too.
                 self.failUnlessEqual(err, "")
                 self.failUnlessEqual(rc, 0)
-                fu, fr, dc, dr = self.count_output(out)
+                fu, fr, fs, dc, dr, ds = self.count_output(out)
                 fchecked, dchecked = self.count_output2(out)
                 self.failUnlessEqual(fchecked, 3)
                 self.failUnlessEqual(fu, 0)
                 self.failUnlessEqual(fr, 3)
+                self.failUnlessEqual(fs, 0)
                 self.failUnlessEqual(dchecked, 4)
                 self.failUnlessEqual(dc, 0)
                 self.failUnlessEqual(dr, 4)
+                self.failUnlessEqual(ds, 0)
             d.addCallback(_check4b)
 
         d.addCallback(lambda res: self.do_cli("ls", "tahoe:backups/Archives"))
@@ -1247,14 +1256,16 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase):
             self.failUnlessEqual(err, "")
             self.failUnlessEqual(rc, 0)
             if have_bdb:
-                fu, fr, dc, dr = self.count_output(out)
+                fu, fr, fs, dc, dr, ds = self.count_output(out)
                 # new foo.txt, surprise file, subfile, empty
                 self.failUnlessEqual(fu, 4)
                 # old bar.txt
                 self.failUnlessEqual(fr, 1)
+                self.failUnlessEqual(fs, 0)
                 # home, parent, subdir, blah.txt, surprisedir
                 self.failUnlessEqual(dc, 5)
                 self.failUnlessEqual(dr, 0)
+                self.failUnlessEqual(ds, 0)
         d.addCallback(_check5a)
         d.addCallback(lambda res: self.do_cli("ls", "tahoe:backups/Archives"))
         def _check6((rc, out, err)):
@@ -1356,6 +1367,107 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase):
         _check_filtering(filtered, root_listdir, ('lib.a', '_darcs', 'subdir'),
                          ('nice_doc.lyx',))
 
+    def test_ignore_symlinks(self):
+        if not hasattr(os, 'symlink'):
+            raise unittest.SkipTest("There is no symlink on this platform.")
+
+        self.basedir = os.path.dirname(self.mktemp())
+        self.set_up_grid()
+
+        source = os.path.join(self.basedir, "home")
+        self.writeto("foo.txt", "foo")
+        os.symlink(os.path.join(source, "foo.txt"), os.path.join(source, "foo2.txt"))
+
+        d = self.do_cli("create-alias", "tahoe")
+        d.addCallback(lambda res: self.do_cli("backup", "--verbose", source, "tahoe:test"))
+
+        def _check((rc, out, err)):
+            self.failUnlessEqual(rc, 2)
+            self.failUnlessEqual(err, "WARNING: cannot backup special file %s\n" % os.path.join(source, "foo2.txt"))
+
+            fu, fr, fs, dc, dr, ds = self.count_output(out)
+            # foo.txt
+            self.failUnlessEqual(fu, 1)
+            self.failUnlessEqual(fr, 0)
+            # foo2.txt
+            self.failUnlessEqual(fs, 1)
+            # home
+            self.failUnlessEqual(dc, 1)
+            self.failUnlessEqual(dr, 0)
+            self.failUnlessEqual(ds, 0)
+
+        d.addCallback(_check)
+        return d
+
+    def test_ignore_unreadable_file(self):
+        self.basedir = os.path.dirname(self.mktemp())
+        self.set_up_grid()
+
+        source = os.path.join(self.basedir, "home")
+        self.writeto("foo.txt", "foo")
+        os.chmod(os.path.join(source, "foo.txt"), 0000)
+
+        d = self.do_cli("create-alias", "tahoe")
+        d.addCallback(lambda res: self.do_cli("backup", source, "tahoe:test"))
+
+        def _check((rc, out, err)):
+            self.failUnlessEqual(rc, 2)
+            self.failUnlessEqual(err, "WARNING: permission denied on file %s\n" % os.path.join(source, "foo.txt"))
+
+            fu, fr, fs, dc, dr, ds = self.count_output(out)
+            self.failUnlessEqual(fu, 0)
+            self.failUnlessEqual(fr, 0)
+            # foo.txt
+            self.failUnlessEqual(fs, 1)
+            # home
+            self.failUnlessEqual(dc, 1)
+            self.failUnlessEqual(dr, 0)
+            self.failUnlessEqual(ds, 0)
+        d.addCallback(_check)
+
+        # This is necessary for the temp files to be correctly removed
+        def _cleanup(self):
+            os.chmod(os.path.join(source, "foo.txt"), 0644)
+        d.addCallback(_cleanup)
+        d.addErrback(_cleanup)
+
+        return d
+
+    def test_ignore_unreadable_directory(self):
+        self.basedir = os.path.dirname(self.mktemp())
+        self.set_up_grid()
+
+        source = os.path.join(self.basedir, "home")
+        os.mkdir(source)
+        os.mkdir(os.path.join(source, "test"))
+        os.chmod(os.path.join(source, "test"), 0000)
+
+        d = self.do_cli("create-alias", "tahoe")
+        d.addCallback(lambda res: self.do_cli("backup", source, "tahoe:test"))
+
+        def _check((rc, out, err)):
+            self.failUnlessEqual(rc, 2)
+            self.failUnlessEqual(err, "WARNING: permission denied on directory %s\n" % os.path.join(source, "test"))
+
+            fu, fr, fs, dc, dr, ds = self.count_output(out)
+            self.failUnlessEqual(fu, 0)
+            self.failUnlessEqual(fr, 0)
+            self.failUnlessEqual(fs, 0)
+            # home, test
+            self.failUnlessEqual(dc, 2)
+            self.failUnlessEqual(dr, 0)
+            # test
+            self.failUnlessEqual(ds, 1)
+        d.addCallback(_check)
+
+        # This is necessary for the temp files to be correctly removed
+        def _cleanup(self):
+            os.chmod(os.path.join(source, "test"), 0655)
+        d.addCallback(_cleanup)
+        d.addErrback(_cleanup)
+        return d
+
+
 class Check(GridTestMixin, CLITestMixin, unittest.TestCase):
 
     def test_check(self):