Changeset b03406a in trunk

Timestamp:

2010-01-20T09:42:49Z (15 years ago)

Author:

francois <francois@…>

Branches:

master

Children:

00b895c

Parents:

66646d9d

Message:

tahoe_backup.py: display warnings on errors instead of stopping the whole backup. Fix #729.

This patch displays a warning to the user in two cases:

1. When special files like symlinks, fifos, devices, etc. are found in the local source.

2. If files or directories are not readables by the user running the 'tahoe backup' command.

In verbose mode, the number of skipped files and directories is printed at the
end of the backup.

Exit status returned by 'tahoe backup':

• 0 everything went fine
• 1 the backup failed
• 2 files were skipped during the backup

Location:

src/allmydata

Files:

• scripts/tahoe_backup.py (modified) (7 diffs)
• test/test_cli.py (modified) (6 diffs)

src/allmydata/scripts/tahoe_backup.py ¶

@@ -66 +66 @@
         self.files_reused = 0
         self.files_checked = 0
+        self.files_skipped = 0
         self.directories_created = 0
         self.directories_reused = 0
         self.directories_checked = 0
+        self.directories_skipped = 0
 
     def run(self):
@@ -124 +126 @@
         if self.verbosity >= 1:
             print >>stdout, (" %d files uploaded (%d reused), "
-                             "%d directories created (%d reused)"
+                             "%d files skipped, "
+                             "%d directories created (%d reused), "
+                             "%d directories skipped"
                              % (self.files_uploaded,
                                 self.files_reused,
+                                self.files_skipped,
                                 self.directories_created,
-                                self.directories_reused))
+                                self.directories_reused,
+                                self.directories_skipped))
             if self.verbosity >= 2:
                 print >>stdout, (" %d files checked, %d directories checked"
@@ -134 +140 @@
                                     self.directories_checked))
             print >>stdout, " backup done, elapsed time: %s" % elapsed_time
+
+        # The command exits with code 2 if files or directories were skipped
+        if self.files_skipped or self.directories_skipped:
+            return 2
+
         # done!
         return 0
@@ -141 +152 @@
             print >>self.options.stdout, msg
 
+    def warn(self, msg):
+        print >>self.options.stderr, msg
+
     def process(self, localpath):
         # returns newdircap
@@ -147 +161 @@
         create_contents = {} # childname -> (type, rocap, metadata)
         compare_contents = {} # childname -> rocap
-        for child in self.options.filter_listdir(os.listdir(localpath)):
+
+        try:
+            children = os.listdir(localpath)
+        except EnvironmentError:
+            self.directories_skipped += 1
+            self.warn("WARNING: permission denied on directory %s" % localpath)
+            children = []
+
+        for child in self.options.filter_listdir(children):
             childpath = os.path.join(localpath, child)
             child = unicode(child)
@@ -158 +180 @@
                 compare_contents[child] = childcap
             elif os.path.isfile(childpath):
-                childcap, metadata = self.upload(childpath)
-                assert isinstance(childcap, str)
-                create_contents[child] = ("filenode", childcap, metadata)
-                compare_contents[child] = childcap
+                try:
+                    childcap, metadata = self.upload(childpath)
+                    assert isinstance(childcap, str)
+                    create_contents[child] = ("filenode", childcap, metadata)
+                    compare_contents[child] = childcap
+                except EnvironmentError:
+                    self.files_skipped += 1
+                    self.warn("WARNING: permission denied on file %s" % childpath)
             else:
-                raise BackupProcessingError("Cannot backup child %r" % childpath)
+                self.files_skipped += 1
+                self.warn("WARNING: cannot backup special file %s" % childpath)
 
         must_create, r = self.check_backupdb_directory(compare_contents)
@@ -246 +273 @@
         return False, r
 
+    # This function will raise an IOError exception when called on an unreadable file
     def upload(self, childpath):
         #self.verboseprint("uploading %s.." % childpath)

src/allmydata/test/test_cli.py ¶

@@ -1073 +1073 @@
 
     def count_output(self, out):
-        mo = re.search(r"(\d)+ files uploaded \((\d+) reused\), (\d+) directories created \((\d+) reused\)", out)
+        mo = re.search(r"(\d)+ files uploaded \((\d+) reused\), "
+                        "(\d)+ files skipped, "
+                        "(\d+) directories created \((\d+) reused\), "
+                        "(\d+) directories skipped", out)
         return [int(s) for s in mo.groups()]
 
@@ -1118 +1121 @@
             self.failUnlessEqual(err, "")
             self.failUnlessEqual(rc, 0)
-            fu, fr, dc, dr = self.count_output(out)
+            fu, fr, fs, dc, dr, ds = self.count_output(out)
             # foo.txt, bar.txt, blah.txt
             self.failUnlessEqual(fu, 3)
             self.failUnlessEqual(fr, 0)
+            self.failUnlessEqual(fs, 0)
             # empty, home, home/parent, home/parent/subdir
             self.failUnlessEqual(dc, 4)
             self.failUnlessEqual(dr, 0)
+            self.failUnlessEqual(ds, 0)
         d.addCallback(_check0)
 
@@ -1173 +1178 @@
             self.failUnlessEqual(rc, 0)
             if have_bdb:
-                fu, fr, dc, dr = self.count_output(out)
+                fu, fr, fs, dc, dr, ds = self.count_output(out)
                 # foo.txt, bar.txt, blah.txt
                 self.failUnlessEqual(fu, 0)
                 self.failUnlessEqual(fr, 3)
+                self.failUnlessEqual(fs, 0)
                 # empty, home, home/parent, home/parent/subdir
                 self.failUnlessEqual(dc, 0)
                 self.failUnlessEqual(dr, 4)
+                self.failUnlessEqual(ds, 0)
         d.addCallback(_check4a)
 
@@ -1204 +1211 @@
                 self.failUnlessEqual(err, "")
                 self.failUnlessEqual(rc, 0)
-                fu, fr, dc, dr = self.count_output(out)
+                fu, fr, fs, dc, dr, ds = self.count_output(out)
                 fchecked, dchecked = self.count_output2(out)
                 self.failUnlessEqual(fchecked, 3)
                 self.failUnlessEqual(fu, 0)
                 self.failUnlessEqual(fr, 3)
+                self.failUnlessEqual(fs, 0)
                 self.failUnlessEqual(dchecked, 4)
                 self.failUnlessEqual(dc, 0)
                 self.failUnlessEqual(dr, 4)
+                self.failUnlessEqual(ds, 0)
             d.addCallback(_check4b)
 
@@ -1248 +1257 @@
             self.failUnlessEqual(rc, 0)
             if have_bdb:
-                fu, fr, dc, dr = self.count_output(out)
+                fu, fr, fs, dc, dr, ds = self.count_output(out)
                 # new foo.txt, surprise file, subfile, empty
                 self.failUnlessEqual(fu, 4)
                 # old bar.txt
                 self.failUnlessEqual(fr, 1)
+                self.failUnlessEqual(fs, 0)
                 # home, parent, subdir, blah.txt, surprisedir
                 self.failUnlessEqual(dc, 5)
                 self.failUnlessEqual(dr, 0)
+                self.failUnlessEqual(ds, 0)
         d.addCallback(_check5a)
         d.addCallback(lambda res: self.do_cli("ls", "tahoe:backups/Archives"))
@@ -1357 +1368 @@
                          ('nice_doc.lyx',))
 
+    def test_ignore_symlinks(self):
+        if not hasattr(os, 'symlink'):
+            raise unittest.SkipTest("There is no symlink on this platform.")
+
+        self.basedir = os.path.dirname(self.mktemp())
+        self.set_up_grid()
+
+        source = os.path.join(self.basedir, "home")
+        self.writeto("foo.txt", "foo")
+        os.symlink(os.path.join(source, "foo.txt"), os.path.join(source, "foo2.txt"))
+
+        d = self.do_cli("create-alias", "tahoe")
+        d.addCallback(lambda res: self.do_cli("backup", "--verbose", source, "tahoe:test"))
+
+        def _check((rc, out, err)):
+            self.failUnlessEqual(rc, 2)
+            self.failUnlessEqual(err, "WARNING: cannot backup special file %s\n" % os.path.join(source, "foo2.txt"))
+
+            fu, fr, fs, dc, dr, ds = self.count_output(out)
+            # foo.txt
+            self.failUnlessEqual(fu, 1)
+            self.failUnlessEqual(fr, 0)
+            # foo2.txt
+            self.failUnlessEqual(fs, 1)
+            # home
+            self.failUnlessEqual(dc, 1)
+            self.failUnlessEqual(dr, 0)
+            self.failUnlessEqual(ds, 0)
+
+        d.addCallback(_check)
+        return d
+
+    def test_ignore_unreadable_file(self):
+        self.basedir = os.path.dirname(self.mktemp())
+        self.set_up_grid()
+
+        source = os.path.join(self.basedir, "home")
+        self.writeto("foo.txt", "foo")
+        os.chmod(os.path.join(source, "foo.txt"), 0000)
+
+        d = self.do_cli("create-alias", "tahoe")
+        d.addCallback(lambda res: self.do_cli("backup", source, "tahoe:test"))
+
+        def _check((rc, out, err)):
+            self.failUnlessEqual(rc, 2)
+            self.failUnlessEqual(err, "WARNING: permission denied on file %s\n" % os.path.join(source, "foo.txt"))
+
+            fu, fr, fs, dc, dr, ds = self.count_output(out)
+            self.failUnlessEqual(fu, 0)
+            self.failUnlessEqual(fr, 0)
+            # foo.txt
+            self.failUnlessEqual(fs, 1)
+            # home
+            self.failUnlessEqual(dc, 1)
+            self.failUnlessEqual(dr, 0)
+            self.failUnlessEqual(ds, 0)
+        d.addCallback(_check)
+
+        # This is necessary for the temp files to be correctly removed
+        def _cleanup(self):
+            os.chmod(os.path.join(source, "foo.txt"), 0644)
+        d.addCallback(_cleanup)
+        d.addErrback(_cleanup)
+
+        return d
+
+    def test_ignore_unreadable_directory(self):
+        self.basedir = os.path.dirname(self.mktemp())
+        self.set_up_grid()
+
+        source = os.path.join(self.basedir, "home")
+        os.mkdir(source)
+        os.mkdir(os.path.join(source, "test"))
+        os.chmod(os.path.join(source, "test"), 0000)
+
+        d = self.do_cli("create-alias", "tahoe")
+        d.addCallback(lambda res: self.do_cli("backup", source, "tahoe:test"))
+
+        def _check((rc, out, err)):
+            self.failUnlessEqual(rc, 2)
+            self.failUnlessEqual(err, "WARNING: permission denied on directory %s\n" % os.path.join(source, "test"))
+
+            fu, fr, fs, dc, dr, ds = self.count_output(out)
+            self.failUnlessEqual(fu, 0)
+            self.failUnlessEqual(fr, 0)
+            self.failUnlessEqual(fs, 0)
+            # home, test
+            self.failUnlessEqual(dc, 2)
+            self.failUnlessEqual(dr, 0)
+            # test
+            self.failUnlessEqual(ds, 1)
+        d.addCallback(_check)
+
+        # This is necessary for the temp files to be correctly removed
+        def _cleanup(self):
+            os.chmod(os.path.join(source, "test"), 0655)
+        d.addCallback(_cleanup)
+        d.addErrback(_cleanup)
+        return d
+
+
 class Check(GridTestMixin, CLITestMixin, unittest.TestCase):