From: Brian Warner Date: Wed, 25 Feb 2009 06:44:15 +0000 (-0700) Subject: CLI: modify 'tahoe manifest' and 'tahoe deep-check' to report ERROR: properly. For... X-Git-Url: https://git.rkrishnan.org/Site/Content/Exhibitors/something?a=commitdiff_plain;h=a3c1fe35d9eda0dfb557504be08ad67be6f36c6c;p=tahoe-lafs%2Ftahoe-lafs.git CLI: modify 'tahoe manifest' and 'tahoe deep-check' to report ERROR: properly. For #590. --- diff --git a/src/allmydata/scripts/tahoe_check.py b/src/allmydata/scripts/tahoe_check.py index f702b0c5..5c1a2387 100644 --- a/src/allmydata/scripts/tahoe_check.py +++ b/src/allmydata/scripts/tahoe_check.py @@ -89,16 +89,28 @@ class FakeTransport: class DeepCheckOutput(LineOnlyReceiver): delimiter = "\n" - def __init__(self, options): + def __init__(self, streamer, options): + self.streamer = streamer self.transport = FakeTransport() self.verbose = bool(options["verbose"]) self.stdout = options.stdout + self.stderr = options.stderr self.num_objects = 0 self.files_healthy = 0 self.files_unhealthy = 0 + self.in_error = False def lineReceived(self, line): + if self.in_error: + print >>self.stderr, line + return + if line.startswith("ERROR:"): + self.in_error = True + self.streamer.rc = 1 + print >>self.stderr, line + return + d = simplejson.loads(line) stdout = self.stdout if d["type"] not in ("file", "directory"): @@ -131,17 +143,21 @@ class DeepCheckOutput(LineOnlyReceiver): (serverid, storage_index, sharenum) def done(self): + if self.in_error: + return stdout = self.stdout print >>stdout, "done: %d objects checked, %d healthy, %d unhealthy" \ % (self.num_objects, self.files_healthy, self.files_unhealthy) class DeepCheckAndRepairOutput(LineOnlyReceiver): delimiter = "\n" - def __init__(self, options): + def __init__(self, streamer, options): + self.streamer = streamer self.transport = FakeTransport() self.verbose = bool(options["verbose"]) self.stdout = options.stdout + self.stderr = options.stderr self.num_objects = 0 self.pre_repair_files_healthy = 0 self.pre_repair_files_unhealthy = 0 @@ -149,8 +165,18 @@ class DeepCheckAndRepairOutput(LineOnlyReceiver): self.repairs_successful = 0 self.post_repair_files_healthy = 0 self.post_repair_files_unhealthy = 0 + self.in_error = False def lineReceived(self, line): + if self.in_error: + print >>self.stderr, line + return + if line.startswith("ERROR:"): + self.in_error = True + self.streamer.rc = 1 + print >>self.stderr, line + return + d = simplejson.loads(line) stdout = self.stdout if d["type"] not in ("file", "directory"): @@ -211,6 +237,8 @@ class DeepCheckAndRepairOutput(LineOnlyReceiver): print >>stdout, " repair failed" def done(self): + if self.in_error: + return stdout = self.stdout print >>stdout, "done: %d objects checked" % self.num_objects print >>stdout, " pre-repair: %d healthy, %d unhealthy" \ @@ -229,6 +257,7 @@ class DeepCheckStreamer(LineOnlyReceiver): def run(self, options): stdout = options.stdout stderr = options.stderr + self.rc = 0 self.options = options nodeurl = options['node-url'] if not nodeurl.endswith("/"): @@ -247,9 +276,9 @@ class DeepCheckStreamer(LineOnlyReceiver): url += "&verify=true" if options["repair"]: url += "&repair=true" - output = DeepCheckAndRepairOutput(options) + output = DeepCheckAndRepairOutput(self, options) else: - output = DeepCheckOutput(options) + output = DeepCheckOutput(self, options) if options["add-lease"]: url += "&add-lease=true" resp = do_http("POST", url) @@ -268,7 +297,7 @@ class DeepCheckStreamer(LineOnlyReceiver): output.dataReceived(chunk) if not self.options["raw"]: output.done() - return 0 + return self.rc def deepcheck(options): return DeepCheckStreamer().run(options) diff --git a/src/allmydata/scripts/tahoe_manifest.py b/src/allmydata/scripts/tahoe_manifest.py index 0f20f9a9..a43b5b97 100644 --- a/src/allmydata/scripts/tahoe_manifest.py +++ b/src/allmydata/scripts/tahoe_manifest.py @@ -16,6 +16,7 @@ class ManifestStreamer(LineOnlyReceiver): self.transport = FakeTransport() def run(self, options): + self.rc = 0 stdout = options.stdout stderr = options.stderr self.options = options @@ -38,6 +39,7 @@ class ManifestStreamer(LineOnlyReceiver): return 1 #print "RESP", dir(resp) # use Twisted to split this into lines + self.in_error = False while True: chunk = resp.read(100) if not chunk: @@ -46,11 +48,21 @@ class ManifestStreamer(LineOnlyReceiver): stdout.write(chunk) else: self.dataReceived(chunk) - return 0 + return self.rc def lineReceived(self, line): - d = simplejson.loads(line) stdout = self.options.stdout + stderr = self.options.stderr + if self.in_error: + print >>stderr, line + return + if line.startswith("ERROR:"): + self.in_error = True + self.rc = 1 + print >>stderr, line + return + + d = simplejson.loads(line) if d["type"] in ("file", "directory"): if self.options["storage-index"]: si = d["storage-index"] diff --git a/src/allmydata/test/test_cli.py b/src/allmydata/test/test_cli.py index 5330a7b7..5dc529a5 100644 --- a/src/allmydata/test/test_cli.py +++ b/src/allmydata/test/test_cli.py @@ -1198,6 +1198,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(_stash_root_and_create_file) def _stash_uri(fn, which): self.uris[which] = fn.get_uri() + return fn d.addCallback(_stash_uri, "good") d.addCallback(lambda ign: self.rootnode.add_file(u"small", @@ -1217,6 +1218,11 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): in lines, out) d.addCallback(_check1) + # root + # root/good + # root/small + # root/mutable + d.addCallback(lambda ign: self.do_cli("deep-check", "--verbose", self.rooturi)) def _check2((rc, out, err)): @@ -1248,6 +1254,11 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): debug.corrupt_share(cso) d.addCallback(_clobber_shares) + # root + # root/good [9 shares] + # root/small + # root/mutable [1 corrupt share] + d.addCallback(lambda ign: self.do_cli("deep-check", "--verbose", self.rooturi)) def _check3((rc, out, err)): @@ -1315,5 +1326,55 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnless(" post-repair: 4 healthy, 0 unhealthy" in lines,out) d.addCallback(_check6) + # now add a subdir, and a file below that, then make the subdir + # unrecoverable + + d.addCallback(lambda ign: + self.rootnode.create_empty_directory(u"subdir")) + d.addCallback(_stash_uri, "subdir") + d.addCallback(lambda fn: + fn.add_file(u"subfile", upload.Data(DATA+"2", ""))) + d.addCallback(lambda ign: + self.delete_shares_numbered(self.uris["subdir"], + range(10))) + + # root + # root/good + # root/small + # root/mutable + # root/subdir [unrecoverable: 0 shares] + # root/subfile + + d.addCallback(lambda ign: self.do_cli("manifest", self.rooturi)) + def _manifest_failed((rc, out, err)): + self.failIfEqual(rc, 0) + self.failUnlessIn("ERROR: UnrecoverableFileError", err) + # the fatal directory should still show up, as the last line + self.failUnlessIn(" subdir\n", out) + d.addCallback(_manifest_failed) + + d.addCallback(lambda ign: self.do_cli("deep-check", self.rooturi)) + def _deep_check_failed((rc, out, err)): + self.failIfEqual(rc, 0) + self.failUnlessIn("ERROR: UnrecoverableFileError", err) + # we want to make sure that the error indication is the last + # thing that gets emitted + self.failIf("done:" in out, out) + d.addCallback(_deep_check_failed) + + # this test is disabled until the deep-repair response to an + # unrepairable directory is fixed. The failure-to-repair should not + # throw an exception, but the failure-to-traverse that follows + # should throw UnrecoverableFileError. + + #d.addCallback(lambda ign: + # self.do_cli("deep-check", "--repair", self.rooturi)) + #def _deep_check_repair_failed((rc, out, err)): + # self.failIfEqual(rc, 0) + # print err + # self.failUnlessIn("ERROR: UnrecoverableFileError", err) + # self.failIf("done:" in out, out) + #d.addCallback(_deep_check_repair_failed) + return d