From: Brian Warner Date: Wed, 13 Aug 2008 00:05:01 +0000 (-0700) Subject: CLI: add 'tahoe debug corrupt-share', and use it for deep-verify tests, and fix non... X-Git-Url: https://git.rkrishnan.org/%5B/%5D%20/uri/?a=commitdiff_plain;h=014c9b59690fd54b9303be8289c6f1f5cbdc7b97;p=tahoe-lafs%2Ftahoe-lafs.git CLI: add 'tahoe debug corrupt-share', and use it for deep-verify tests, and fix non-deep web checker API to pass verify=true into node --- diff --git a/docs/CLI.txt b/docs/CLI.txt index d2e6ede6..04a1a8b8 100644 --- a/docs/CLI.txt +++ b/docs/CLI.txt @@ -356,3 +356,7 @@ data for this file. Tahoe packages and modules are available on sys.path (e.g. by using 'import allmydata'). This is most useful from a source tree: it simply sets the PYTHONPATH correctly and runs the 'python' executable. + +"tahoe debug corrupt-share SHAREFILE" will flip a bit in the given sharefile. +This can be used to test the client-side verification/repair code. Obviously +this command should not be used during normal operation. diff --git a/src/allmydata/scripts/debug.py b/src/allmydata/scripts/debug.py index 4ca32aaa..243e49d8 100644 --- a/src/allmydata/scripts/debug.py +++ b/src/allmydata/scripts/debug.py @@ -619,6 +619,83 @@ def catalog_shares(options): describe_share(abs_sharefile, si_s, shnum_s, now, out) return 0 +class CorruptShareOptions(usage.Options): + def getSynopsis(self): + return "Usage: tahoe debug corrupt-share SHARE_FILENAME" + + optParameters = [ + ["offset", "o", "block-random", "Which bit to flip."], + ] + + def getUsage(self, width=None): + t = usage.Options.getUsage(self, width) + t += """ +Corrupt the given share by flipping a bit. This will cause a +verifying/downloading client to log an integrity-check failure incident, and +downloads will proceed with a different share. + +The --offset parameter controls which bit should be flipped. The default is +to flip a single random bit of the block data. + + tahoe debug corrupt-share testgrid/node-3/storage/shares/4v/4vozh77tsrw7mdhnj7qvp5ky74/0 + +Obviously, this command should not be used in normal operation. +""" + return t + def parseArgs(self, filename): + self['filename'] = filename + +def corrupt_share(options): + import random + from allmydata import storage + from allmydata.mutable.layout import unpack_header + out = options.stdout + fn = options['filename'] + assert options["offset"] == "block-random", "other offsets not implemented" + # first, what kind of share is it? + + def flip_bit(start, end): + offset = random.randrange(start, end) + bit = random.randrange(0, 8) + print >>out, "[%d..%d): %d.b%d" % (start, end, offset, bit) + f = open(fn, "rb+") + f.seek(offset) + d = f.read(1) + d = chr(ord(d) ^ 0x01) + f.seek(offset) + f.write(d) + f.close() + + f = open(fn, "rb") + prefix = f.read(32) + f.close() + if prefix == storage.MutableShareFile.MAGIC: + # mutable + m = storage.MutableShareFile(fn) + f = open(fn, "rb") + f.seek(m.DATA_OFFSET) + data = f.read(2000) + # make sure this slot contains an SMDF share + assert data[0] == "\x00", "non-SDMF mutable shares not supported" + f.close() + + (version, ig_seqnum, ig_roothash, ig_IV, ig_k, ig_N, ig_segsize, + ig_datalen, offsets) = unpack_header(data) + + assert version == 0, "we only handle v0 SDMF files" + start = m.DATA_OFFSET + offsets["share_data"] + end = m.DATA_OFFSET + offsets["enc_privkey"] + flip_bit(start, end) + else: + # otherwise assume it's immutable + f = storage.ShareFile(fn) + bp = storage.ReadBucketProxy(None) + offsets = bp._parse_offsets(f.read_share_data(0, 0x24)) + start = f._data_offset + offsets["data"] + end = f._data_offset + offsets["plaintext_hash_tree"] + flip_bit(start, end) + + class ReplOptions(usage.Options): pass @@ -635,6 +712,7 @@ class DebugCommand(usage.Options): ["dump-cap", None, DumpCapOptions, "Unpack a read-cap or write-cap"], ["find-shares", None, FindSharesOptions, "Locate sharefiles in node dirs"], ["catalog-shares", None, CatalogSharesOptions, "Describe shares in node dirs"], + ["corrupt-share", None, CorruptShareOptions, "Corrupt a share"], ["repl", None, ReplOptions, "Open a python interpreter"], ] def postOptions(self): @@ -650,6 +728,7 @@ Subcommands: tahoe debug dump-cap Unpack a read-cap or write-cap tahoe debug find-shares Locate sharefiles in node directories tahoe debug catalog-shares Describe all shares in node dirs + tahoe debug corrupt-share Corrupt a share by flipping a bit. Please run e.g. 'tahoe debug dump-share --help' for more details on each subcommand. @@ -661,6 +740,7 @@ subDispatch = { "dump-cap": dump_cap, "find-shares": find_shares, "catalog-shares": catalog_shares, + "corrupt-share": corrupt_share, "repl": repl, } diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index af64ec97..aa5846d3 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -1,5 +1,5 @@ from base64 import b32encode -import os, random, struct, sys, time, re, simplejson +import os, random, struct, sys, time, re, simplejson, urllib from cStringIO import StringIO from twisted.trial import unittest from twisted.internet import defer @@ -1851,3 +1851,71 @@ class ImmutableChecker(ShareManglingMixin, unittest.TestCase): d.addCallback(_check2) return d test_check_with_verify.todo = "We haven't implemented a verifier this thorough yet." + +class MutableChecker(SystemTestMixin, unittest.TestCase): + + def _run_cli(self, argv): + stdout, stderr = StringIO(), StringIO() + runner.runner(argv, run_by_human=False, stdout=stdout, stderr=stderr) + return stdout.getvalue() + + def test_good(self): + self.basedir = self.mktemp() + d = self.set_up_nodes() + CONTENTS = "a little bit of data" + d.addCallback(lambda res: self.clients[0].create_mutable_file(CONTENTS)) + def _created(node): + self.node = node + si = self.node.get_storage_index() + d.addCallback(_created) + # now make sure the webapi verifier sees no problems + def _do_check(res): + url = (self.webish_url + + "uri/%s" % urllib.quote(self.node.get_uri()) + + "?t=check&verify=true") + return getPage(url, method="POST") + d.addCallback(_do_check) + def _got_results(out): + self.failUnless("
Healthy!" in out, out)
+            self.failIf("Not Healthy!" in out, out)
+            self.failIf("Unhealthy" in out, out)
+            self.failIf("Corrupt Shares" in out, out)
+        d.addCallback(_got_results)
+        return d
+
+    def test_corrupt(self):
+        self.basedir = self.mktemp()
+        d = self.set_up_nodes()
+        CONTENTS = "a little bit of data"
+        d.addCallback(lambda res: self.clients[0].create_mutable_file(CONTENTS))
+        def _created(node):
+            self.node = node
+            si = self.node.get_storage_index()
+            out = self._run_cli(["debug", "find-shares", base32.b2a(si),
+                                self.clients[1].basedir])
+            files = out.split("\n")
+            # corrupt one of them, using the CLI debug command
+            f = files[0]
+            shnum = os.path.basename(f)
+            nodeid = self.clients[1].nodeid
+            nodeid_prefix = idlib.shortnodeid_b2a(nodeid)
+            self.corrupt_shareid = "%s-sh%s" % (nodeid_prefix, shnum)
+            out = self._run_cli(["debug", "corrupt-share", files[0]])
+        d.addCallback(_created)
+        # now make sure the webapi verifier notices it
+        def _do_check(res):
+            url = (self.webish_url +
+                   "uri/%s" % urllib.quote(self.node.get_uri()) +
+                   "?t=check&verify=true")
+            return getPage(url, method="POST")
+        d.addCallback(_do_check)
+        def _got_results(out):
+            self.failUnless("Not Healthy!" in out, out)
+            self.failUnless("Unhealthy: best recoverable version has only 9 shares (encoding is 3-of-10)" in out, out)
+            shid_re = (r"Corrupt Shares:\s+%s: block hash tree failure" %
+                       self.corrupt_shareid)
+            self.failUnless(re.search(shid_re, out), out)
+
+        d.addCallback(_got_results)
+        return d
+
diff --git a/src/allmydata/web/directory.py b/src/allmydata/web/directory.py
index 6adcd072..3b62b4a0 100644
--- a/src/allmydata/web/directory.py
+++ b/src/allmydata/web/directory.py
@@ -339,8 +339,7 @@ class DirectoryNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin):
     def _POST_deep_check(self, req):
         # check this directory and everything reachable from it
         verify = boolean_of_arg(get_arg(req, "verify", "false"))
-        #repair = boolean_of_arg(get_arg(req, "repair", "false"))
-        repair = False # make sure it works first
+        repair = boolean_of_arg(get_arg(req, "repair", "false"))
         d = self.node.deep_check(verify, repair)
         d.addCallback(lambda res: DeepCheckResults(res))
         return d
diff --git a/src/allmydata/web/filenode.py b/src/allmydata/web/filenode.py
index 7996e757..185ab1dd 100644
--- a/src/allmydata/web/filenode.py
+++ b/src/allmydata/web/filenode.py
@@ -255,7 +255,9 @@ class FileNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin):
         return d
 
     def _POST_check(self, req):
-        d = self.node.check()
+        verify = boolean_of_arg(get_arg(req, "verify", "false"))
+        repair = boolean_of_arg(get_arg(req, "repair", "false"))
+        d = self.node.check(verify, repair)
         d.addCallback(lambda res: CheckerResults(res))
         return d