From 014c9b59690fd54b9303be8289c6f1f5cbdc7b97 Mon Sep 17 00:00:00 2001 From: Brian Warner <warner@allmydata.com> Date: Tue, 12 Aug 2008 17:05:01 -0700 Subject: [PATCH] CLI: add 'tahoe debug corrupt-share', and use it for deep-verify tests, and fix non-deep web checker API to pass verify=true into node --- docs/CLI.txt | 4 ++ src/allmydata/scripts/debug.py | 80 +++++++++++++++++++++++++++++++ src/allmydata/test/test_system.py | 70 ++++++++++++++++++++++++++- src/allmydata/web/directory.py | 3 +- src/allmydata/web/filenode.py | 4 +- 5 files changed, 157 insertions(+), 4 deletions(-) diff --git a/docs/CLI.txt b/docs/CLI.txt index d2e6ede6..04a1a8b8 100644 --- a/docs/CLI.txt +++ b/docs/CLI.txt @@ -356,3 +356,7 @@ data for this file. Tahoe packages and modules are available on sys.path (e.g. by using 'import allmydata'). This is most useful from a source tree: it simply sets the PYTHONPATH correctly and runs the 'python' executable. + +"tahoe debug corrupt-share SHAREFILE" will flip a bit in the given sharefile. +This can be used to test the client-side verification/repair code. Obviously +this command should not be used during normal operation. diff --git a/src/allmydata/scripts/debug.py b/src/allmydata/scripts/debug.py index 4ca32aaa..243e49d8 100644 --- a/src/allmydata/scripts/debug.py +++ b/src/allmydata/scripts/debug.py @@ -619,6 +619,83 @@ def catalog_shares(options): describe_share(abs_sharefile, si_s, shnum_s, now, out) return 0 +class CorruptShareOptions(usage.Options): + def getSynopsis(self): + return "Usage: tahoe debug corrupt-share SHARE_FILENAME" + + optParameters = [ + ["offset", "o", "block-random", "Which bit to flip."], + ] + + def getUsage(self, width=None): + t = usage.Options.getUsage(self, width) + t += """ +Corrupt the given share by flipping a bit. This will cause a +verifying/downloading client to log an integrity-check failure incident, and +downloads will proceed with a different share. + +The --offset parameter controls which bit should be flipped. The default is +to flip a single random bit of the block data. + + tahoe debug corrupt-share testgrid/node-3/storage/shares/4v/4vozh77tsrw7mdhnj7qvp5ky74/0 + +Obviously, this command should not be used in normal operation. +""" + return t + def parseArgs(self, filename): + self['filename'] = filename + +def corrupt_share(options): + import random + from allmydata import storage + from allmydata.mutable.layout import unpack_header + out = options.stdout + fn = options['filename'] + assert options["offset"] == "block-random", "other offsets not implemented" + # first, what kind of share is it? + + def flip_bit(start, end): + offset = random.randrange(start, end) + bit = random.randrange(0, 8) + print >>out, "[%d..%d): %d.b%d" % (start, end, offset, bit) + f = open(fn, "rb+") + f.seek(offset) + d = f.read(1) + d = chr(ord(d) ^ 0x01) + f.seek(offset) + f.write(d) + f.close() + + f = open(fn, "rb") + prefix = f.read(32) + f.close() + if prefix == storage.MutableShareFile.MAGIC: + # mutable + m = storage.MutableShareFile(fn) + f = open(fn, "rb") + f.seek(m.DATA_OFFSET) + data = f.read(2000) + # make sure this slot contains an SMDF share + assert data[0] == "\x00", "non-SDMF mutable shares not supported" + f.close() + + (version, ig_seqnum, ig_roothash, ig_IV, ig_k, ig_N, ig_segsize, + ig_datalen, offsets) = unpack_header(data) + + assert version == 0, "we only handle v0 SDMF files" + start = m.DATA_OFFSET + offsets["share_data"] + end = m.DATA_OFFSET + offsets["enc_privkey"] + flip_bit(start, end) + else: + # otherwise assume it's immutable + f = storage.ShareFile(fn) + bp = storage.ReadBucketProxy(None) + offsets = bp._parse_offsets(f.read_share_data(0, 0x24)) + start = f._data_offset + offsets["data"] + end = f._data_offset + offsets["plaintext_hash_tree"] + flip_bit(start, end) + + class ReplOptions(usage.Options): pass @@ -635,6 +712,7 @@ class DebugCommand(usage.Options): ["dump-cap", None, DumpCapOptions, "Unpack a read-cap or write-cap"], ["find-shares", None, FindSharesOptions, "Locate sharefiles in node dirs"], ["catalog-shares", None, CatalogSharesOptions, "Describe shares in node dirs"], + ["corrupt-share", None, CorruptShareOptions, "Corrupt a share"], ["repl", None, ReplOptions, "Open a python interpreter"], ] def postOptions(self): @@ -650,6 +728,7 @@ Subcommands: tahoe debug dump-cap Unpack a read-cap or write-cap tahoe debug find-shares Locate sharefiles in node directories tahoe debug catalog-shares Describe all shares in node dirs + tahoe debug corrupt-share Corrupt a share by flipping a bit. Please run e.g. 'tahoe debug dump-share --help' for more details on each subcommand. @@ -661,6 +740,7 @@ subDispatch = { "dump-cap": dump_cap, "find-shares": find_shares, "catalog-shares": catalog_shares, + "corrupt-share": corrupt_share, "repl": repl, } diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index af64ec97..aa5846d3 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -1,5 +1,5 @@ from base64 import b32encode -import os, random, struct, sys, time, re, simplejson +import os, random, struct, sys, time, re, simplejson, urllib from cStringIO import StringIO from twisted.trial import unittest from twisted.internet import defer @@ -1851,3 +1851,71 @@ class ImmutableChecker(ShareManglingMixin, unittest.TestCase): d.addCallback(_check2) return d test_check_with_verify.todo = "We haven't implemented a verifier this thorough yet." + +class MutableChecker(SystemTestMixin, unittest.TestCase): + + def _run_cli(self, argv): + stdout, stderr = StringIO(), StringIO() + runner.runner(argv, run_by_human=False, stdout=stdout, stderr=stderr) + return stdout.getvalue() + + def test_good(self): + self.basedir = self.mktemp() + d = self.set_up_nodes() + CONTENTS = "a little bit of data" + d.addCallback(lambda res: self.clients[0].create_mutable_file(CONTENTS)) + def _created(node): + self.node = node + si = self.node.get_storage_index() + d.addCallback(_created) + # now make sure the webapi verifier sees no problems + def _do_check(res): + url = (self.webish_url + + "uri/%s" % urllib.quote(self.node.get_uri()) + + "?t=check&verify=true") + return getPage(url, method="POST") + d.addCallback(_do_check) + def _got_results(out): + self.failUnless("<pre>Healthy!" in out, out) + self.failIf("Not Healthy!" in out, out) + self.failIf("Unhealthy" in out, out) + self.failIf("Corrupt Shares" in out, out) + d.addCallback(_got_results) + return d + + def test_corrupt(self): + self.basedir = self.mktemp() + d = self.set_up_nodes() + CONTENTS = "a little bit of data" + d.addCallback(lambda res: self.clients[0].create_mutable_file(CONTENTS)) + def _created(node): + self.node = node + si = self.node.get_storage_index() + out = self._run_cli(["debug", "find-shares", base32.b2a(si), + self.clients[1].basedir]) + files = out.split("\n") + # corrupt one of them, using the CLI debug command + f = files[0] + shnum = os.path.basename(f) + nodeid = self.clients[1].nodeid + nodeid_prefix = idlib.shortnodeid_b2a(nodeid) + self.corrupt_shareid = "%s-sh%s" % (nodeid_prefix, shnum) + out = self._run_cli(["debug", "corrupt-share", files[0]]) + d.addCallback(_created) + # now make sure the webapi verifier notices it + def _do_check(res): + url = (self.webish_url + + "uri/%s" % urllib.quote(self.node.get_uri()) + + "?t=check&verify=true") + return getPage(url, method="POST") + d.addCallback(_do_check) + def _got_results(out): + self.failUnless("Not Healthy!" in out, out) + self.failUnless("Unhealthy: best recoverable version has only 9 shares (encoding is 3-of-10)" in out, out) + shid_re = (r"Corrupt Shares:\s+%s: block hash tree failure" % + self.corrupt_shareid) + self.failUnless(re.search(shid_re, out), out) + + d.addCallback(_got_results) + return d + diff --git a/src/allmydata/web/directory.py b/src/allmydata/web/directory.py index 6adcd072..3b62b4a0 100644 --- a/src/allmydata/web/directory.py +++ b/src/allmydata/web/directory.py @@ -339,8 +339,7 @@ class DirectoryNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin): def _POST_deep_check(self, req): # check this directory and everything reachable from it verify = boolean_of_arg(get_arg(req, "verify", "false")) - #repair = boolean_of_arg(get_arg(req, "repair", "false")) - repair = False # make sure it works first + repair = boolean_of_arg(get_arg(req, "repair", "false")) d = self.node.deep_check(verify, repair) d.addCallback(lambda res: DeepCheckResults(res)) return d diff --git a/src/allmydata/web/filenode.py b/src/allmydata/web/filenode.py index 7996e757..185ab1dd 100644 --- a/src/allmydata/web/filenode.py +++ b/src/allmydata/web/filenode.py @@ -255,7 +255,9 @@ class FileNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin): return d def _POST_check(self, req): - d = self.node.check() + verify = boolean_of_arg(get_arg(req, "verify", "false")) + repair = boolean_of_arg(get_arg(req, "repair", "false")) + d = self.node.check(verify, repair) d.addCallback(lambda res: CheckerResults(res)) return d -- 2.45.2