From: Zooko O'Whielacronx Date: Thu, 25 Sep 2008 17:16:53 +0000 (-0700) Subject: repairer: add basic test of repairer, move tests of immutable checker/repairer from... X-Git-Url: https://git.rkrishnan.org/components/com_hotproperty/%22doc.html/%3C?a=commitdiff_plain;h=1e8d37cc2dd98ba51b73ddffac1c3cfaf346e8ff;p=tahoe-lafs%2Ftahoe-lafs.git repairer: add basic test of repairer, move tests of immutable checker/repairer from test_system to test_immutable_checker, remove obsolete test helper code from test_filenode Hm... "Checker" ought to be renamed to "CheckerRepairer" or "Repairer" at some point... --- diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index c68ae11a..f2da66f8 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -1477,13 +1477,8 @@ class ICheckable(Interface): """Like check(), but if the file/directory is not healthy, attempt to repair the damage. - This returns a Deferred which fires with a tuple of (pre, post), each - is either None or an ICheckerResults instance. For non-distributed - files (i.e. a LIT file) both are None. Otherwise, 'pre' is an - ICheckerResults representing the state of the object before any - repair attempt is made. If the file was unhealthy and repair was - attempted, 'post' will be another ICheckerResults instance with the - state of the object after repair.""" + This returns a Deferred which fires with an instance of + ICheckAndRepairResults.""" class IDeepCheckable(Interface): def deep_check(verify=False): diff --git a/src/allmydata/test/test_filenode.py b/src/allmydata/test/test_filenode.py index f7dfd8bf..16879d41 100644 --- a/src/allmydata/test/test_filenode.py +++ b/src/allmydata/test/test_filenode.py @@ -111,31 +111,7 @@ class Node(unittest.TestCase): v = n.get_verifier() self.failUnless(isinstance(v, uri.SSKVerifierURI)) -class Checker(unittest.TestCase): - def test_chk_filenode(self): - u = uri.CHKFileURI(key="\x00"*16, - uri_extension_hash="\x00"*32, - needed_shares=3, - total_shares=10, - size=1000) - c = None - fn1 = filenode.FileNode(u, c) - - fn1.checker_class = FakeImmutableChecker - fn1.verifier_class = FakeImmutableVerifier - - d = fn1.check() - def _check_checker_results(cr): - self.failUnless(cr.is_healthy()) - d.addCallback(_check_checker_results) - - d.addCallback(lambda res: fn1.check(verify=True)) - d.addCallback(_check_checker_results) - - # TODO: check-and-repair - - return d - +class LiteralChecker(unittest.TestCase): def test_literal_filenode(self): DATA = "I am a short file." u = uri.LiteralFileURI(data=DATA) @@ -150,35 +126,3 @@ class Checker(unittest.TestCase): d.addCallback(_check_checker_results) return d - -class FakeMutableChecker: - def __init__(self, node): - self.r = CheckerResults(node.get_storage_index()) - self.r.set_healthy(True) - - def check(self, verify): - return defer.succeed(self.r) - -class FakeMutableCheckAndRepairer: - def __init__(self, node): - cr = CheckerResults(node.get_storage_index()) - cr.set_healthy(True) - self.r = CheckAndRepairResults(node.get_storage_index()) - self.r.pre_repair_results = self.r.post_repair_results = cr - - def check(self, verify): - return defer.succeed(self.r) - -class FakeImmutableChecker: - def __init__(self, client, storage_index, needed_shares, total_shares): - self.r = CheckerResults(storage_index) - self.r.set_healthy(True) - - def start(self): - return defer.succeed(self.r) - -def FakeImmutableVerifier(client, - storage_index, needed_shares, total_shares, size, - ueb_hash): - return FakeImmutableChecker(client, - storage_index, needed_shares, total_shares) diff --git a/src/allmydata/test/test_immutable_checker.py b/src/allmydata/test/test_immutable_checker.py new file mode 100644 index 00000000..a561cf29 --- /dev/null +++ b/src/allmydata/test/test_immutable_checker.py @@ -0,0 +1,226 @@ +from allmydata.immutable import upload +from allmydata.test.common import SystemTestMixin, ShareManglingMixin +from allmydata.util import testutil +from twisted.internet import defer +from twisted.trial import unittest +import random, struct + +class Test(ShareManglingMixin, unittest.TestCase): + def setUp(self): + # Set self.basedir to a temp dir which has the name of the current test method in its + # name. + self.basedir = self.mktemp() + TEST_DATA="\x02"*(upload.Uploader.URI_LIT_SIZE_THRESHOLD+1) + + d = defer.maybeDeferred(SystemTestMixin.setUp, self) + d.addCallback(lambda x: self.set_up_nodes()) + + def _upload_a_file(ignored): + d2 = self.clients[0].upload(upload.Data(TEST_DATA, convergence="")) + d2.addCallback(lambda u: self.clients[0].create_node_from_uri(u.uri)) + return d2 + d.addCallback(_upload_a_file) + + def _stash_it(filenode): + self.filenode = filenode + d.addCallback(_stash_it) + return d + + def _delete_a_share(self, unused=None): + """ Exactly one bit of exactly one share on disk will be flipped (randomly selected from + among the bits of the 'share data' -- the verifiable bits).""" + + shares = self.find_shares() + ks = shares.keys() + k = random.choice(ks) + del shares[k] + self.replace_shares(shares) + + return unused + + def _corrupt_a_share(self, unused=None): + """ Delete one share. """ + + shares = self.find_shares() + ks = shares.keys() + k = random.choice(ks) + data = shares[k] + + (version, size, num_leases) = struct.unpack(">LLL", data[:0xc]) + sharedata = data[0xc:0xc+size] + + corruptedsharedata = testutil.flip_one_bit(sharedata) + corrupteddata = data[:0xc]+corruptedsharedata+data[0xc+size:] + shares[k] = corrupteddata + + self.replace_shares(shares) + + return unused + + def test_test_code(self): + # The following process of stashing the shares, running + # replace_shares, and asserting that the new set of shares equals the + # old is more to test this test code than to test the Tahoe code... + d = defer.succeed(None) + d.addCallback(self.find_shares) + stash = [None] + def _stash_it(res): + stash[0] = res + return res + + d.addCallback(_stash_it) + d.addCallback(self.replace_shares) + + def _compare(res): + oldshares = stash[0] + self.failUnless(isinstance(oldshares, dict), oldshares) + self.failUnlessEqual(oldshares, res) + + d.addCallback(self.find_shares) + d.addCallback(_compare) + + d.addCallback(lambda ignore: self.replace_shares({})) + d.addCallback(self.find_shares) + d.addCallback(lambda x: self.failUnlessEqual(x, {})) + + return d + + def _count_reads(self): + sum_of_read_counts = 0 + for client in self.clients: + counters = client.stats_provider.get_stats()['counters'] + sum_of_read_counts += counters.get('storage_server.read', 0) + return sum_of_read_counts + + def _count_allocates(self): + sum_of_allocate_counts = 0 + for client in self.clients: + counters = client.stats_provider.get_stats()['counters'] + sum_of_allocate_counts += counters.get('storage_server.allocate', 0) + return sum_of_allocate_counts + + def test_check_without_verify(self): + """ Check says the file is healthy when none of the shares have been + touched. It says that the file is unhealthy when all of them have + been removed. It says that the file is healthy if one bit of one share + has been flipped.""" + d = defer.succeed(self.filenode) + def _check1(filenode): + before_check_reads = self._count_reads() + + d2 = filenode.check(verify=False) + def _after_check(checkresults): + after_check_reads = self._count_reads() + self.failIf(after_check_reads - before_check_reads > 0, after_check_reads - before_check_reads) + self.failUnless(checkresults.is_healthy()) + + d2.addCallback(_after_check) + return d2 + d.addCallback(_check1) + + d.addCallback(self._corrupt_a_share) + def _check2(ignored): + before_check_reads = self._count_reads() + d2 = self.filenode.check(verify=False) + + def _after_check(checkresults): + after_check_reads = self._count_reads() + self.failIf(after_check_reads - before_check_reads > 0, after_check_reads - before_check_reads) + + d2.addCallback(_after_check) + return d2 + d.addCallback(_check2) + return d + + d.addCallback(lambda ignore: self.replace_shares({})) + def _check3(ignored): + before_check_reads = self._count_reads() + d2 = self.filenode.check(verify=False) + + def _after_check(checkresults): + after_check_reads = self._count_reads() + self.failIf(after_check_reads - before_check_reads > 0, after_check_reads - before_check_reads) + self.failIf(checkresults.is_healthy()) + + d2.addCallback(_after_check) + return d2 + d.addCallback(_check3) + + return d + + def test_check_with_verify(self): + """ Check says the file is healthy when none of the shares have been touched. It says + that the file is unhealthy if one bit of one share has been flipped.""" + # N == 10. 2 is the "efficiency leeway" -- we'll allow you to pass this test even if + # you trigger twice as many disk reads and blocks sends as would be optimal. + DELTA_READS = 10 * 2 + d = defer.succeed(self.filenode) + def _check1(filenode): + before_check_reads = self._count_reads() + + d2 = filenode.check(verify=True) + def _after_check(checkresults): + after_check_reads = self._count_reads() + # print "delta was ", after_check_reads - before_check_reads + self.failIf(after_check_reads - before_check_reads > DELTA_READS) + self.failUnless(checkresults.is_healthy()) + + d2.addCallback(_after_check) + return d2 + d.addCallback(_check1) + + d.addCallback(self._corrupt_a_share) + def _check2(ignored): + before_check_reads = self._count_reads() + d2 = self.filenode.check(verify=True) + + def _after_check(checkresults): + after_check_reads = self._count_reads() + # print "delta was ", after_check_reads - before_check_reads + self.failIf(after_check_reads - before_check_reads > DELTA_READS) + self.failIf(checkresults.is_healthy()) + + d2.addCallback(_after_check) + return d2 + d.addCallback(_check2) + return d + test_check_with_verify.todo = "We haven't implemented a verifier this thorough yet." + + def test_repair(self): + """ Repair replaces a share that got deleted. """ + # N == 10. 2 is the "efficiency leeway" -- we'll allow you to pass this test even if + # you trigger twice as many disk reads and blocks sends as would be optimal. + DELTA_READS = 10 * 2 + # We'll allow you to pass this test only if you repair the missing share using only a + # single allocate. + DELTA_ALLOCATES = 1 + + d = defer.succeed(self.filenode) + d.addCallback(self._delete_a_share) + + def _repair(filenode): + before_repair_reads = self._count_reads() + before_repair_allocates = self._count_allocates() + + d2 = filenode.check_and_repair(verify=False) + def _after_repair(checkandrepairresults): + prerepairres = checkandrepairresults.get_pre_repair_results() + postrepairres = checkandrepairresults.get_post_repair_results() + after_repair_reads = self._count_reads() + after_repair_allocates = self._count_allocates() + + # print "delta was ", after_repair_reads - before_repair_reads, after_repair_allocates - before_repair_allocates + self.failIf(after_repair_reads - before_repair_reads > DELTA_READS) + self.failIf(after_repair_allocates - before_repair_allocates > DELTA_ALLOCATES) + self.failIf(prerepairres.is_healthy()) + self.failUnless(postrepairres.is_healthy()) + + # Now we inspect the filesystem to make sure that it is really there. + shares = self.find_shares() + self.failIf(len(shares) < 10) + + d2.addCallback(_after_repair) + return d2 + d.addCallback(_repair) + return d + test_repair.todo = "We haven't implemented a checker yet." diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index bb29acb5..530b2052 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -1695,165 +1695,6 @@ class SystemTest(SystemTestMixin, unittest.TestCase): return d -class ImmutableChecker(ShareManglingMixin, unittest.TestCase): - def setUp(self): - # Set self.basedir to a temp dir which has the name of the current test method in its - # name. - self.basedir = self.mktemp() - TEST_DATA="\x02"*(upload.Uploader.URI_LIT_SIZE_THRESHOLD+1) - - d = defer.maybeDeferred(SystemTestMixin.setUp, self) - d.addCallback(lambda x: self.set_up_nodes()) - - def _upload_a_file(ignored): - d2 = self.clients[0].upload(upload.Data(TEST_DATA, convergence="")) - d2.addCallback(lambda u: self.clients[0].create_node_from_uri(u.uri)) - return d2 - d.addCallback(_upload_a_file) - - def _stash_it(filenode): - self.filenode = filenode - d.addCallback(_stash_it) - return d - - def _corrupt_a_share(self, unused=None): - """ Exactly one bit of exactly one share on disk will be flipped (randomly selected from - among the bits of the 'share data' -- the verifiable bits).""" - - shares = self.find_shares() - ks = shares.keys() - k = random.choice(ks) - data = shares[k] - - (version, size, num_leases) = struct.unpack(">LLL", data[:0xc]) - sharedata = data[0xc:0xc+size] - - corruptedsharedata = testutil.flip_one_bit(sharedata) - corrupteddata = data[:0xc]+corruptedsharedata+data[0xc+size:] - shares[k] = corrupteddata - - self.replace_shares(shares) - - def test_test_code(self): - # The following process of stashing the shares, running - # replace_shares, and asserting that the new set of shares equals the - # old is more to test this test code than to test the Tahoe code... - d = defer.succeed(None) - d.addCallback(self.find_shares) - stash = [None] - def _stash_it(res): - stash[0] = res - return res - - d.addCallback(_stash_it) - d.addCallback(self.replace_shares) - - def _compare(res): - oldshares = stash[0] - self.failUnless(isinstance(oldshares, dict), oldshares) - self.failUnlessEqual(oldshares, res) - - d.addCallback(self.find_shares) - d.addCallback(_compare) - - d.addCallback(lambda ignore: self.replace_shares({})) - d.addCallback(self.find_shares) - d.addCallback(lambda x: self.failUnlessEqual(x, {})) - - return d - - def _count_reads(self): - sum_of_read_counts = 0 - for client in self.clients: - counters = client.stats_provider.get_stats()['counters'] - sum_of_read_counts += counters.get('storage_server.read', 0) - return sum_of_read_counts - - def test_check_without_verify(self): - """ Check says the file is healthy when none of the shares have been - touched. It says that the file is unhealthy when all of them have - been removed. It says that the file is healthy if one bit of one share - has been flipped.""" - d = defer.succeed(self.filenode) - def _check1(filenode): - before_check_reads = self._count_reads() - - d2 = filenode.check(verify=False) - def _after_check(checkresults): - after_check_reads = self._count_reads() - self.failIf(after_check_reads - before_check_reads > 0, after_check_reads - before_check_reads) - self.failUnless(checkresults.is_healthy()) - - d2.addCallback(_after_check) - return d2 - d.addCallback(_check1) - - d.addCallback(self._corrupt_a_share) - def _check2(ignored): - before_check_reads = self._count_reads() - d2 = self.filenode.check(verify=False) - - def _after_check(checkresults): - after_check_reads = self._count_reads() - self.failIf(after_check_reads - before_check_reads > 0, after_check_reads - before_check_reads) - - d2.addCallback(_after_check) - return d2 - d.addCallback(_check2) - return d - - d.addCallback(lambda ignore: self.replace_shares({})) - def _check3(ignored): - before_check_reads = self._count_reads() - d2 = self.filenode.check(verify=False) - - def _after_check(checkresults): - after_check_reads = self._count_reads() - self.failIf(after_check_reads - before_check_reads > 0, after_check_reads - before_check_reads) - self.failIf(checkresults.is_healthy()) - - d2.addCallback(_after_check) - return d2 - d.addCallback(_check3) - - return d - - def test_check_with_verify(self): - """ Check says the file is healthy when none of the shares have been touched. It says - that the file is unhealthy if one bit of one share has been flipped.""" - DELTA_READS = 10 * 2 # N == 10 - d = defer.succeed(self.filenode) - def _check1(filenode): - before_check_reads = self._count_reads() - - d2 = filenode.check(verify=True) - def _after_check(checkresults): - after_check_reads = self._count_reads() - # print "delta was ", after_check_reads - before_check_reads - self.failIf(after_check_reads - before_check_reads > DELTA_READS) - self.failUnless(checkresults.is_healthy()) - - d2.addCallback(_after_check) - return d2 - d.addCallback(_check1) - - d.addCallback(self._corrupt_a_share) - def _check2(ignored): - before_check_reads = self._count_reads() - d2 = self.filenode.check(verify=True) - - def _after_check(checkresults): - after_check_reads = self._count_reads() - # print "delta was ", after_check_reads - before_check_reads - self.failIf(after_check_reads - before_check_reads > DELTA_READS) - self.failIf(checkresults.is_healthy()) - - d2.addCallback(_after_check) - return d2 - d.addCallback(_check2) - return d - test_check_with_verify.todo = "We haven't implemented a verifier this thorough yet." - class MutableChecker(SystemTestMixin, unittest.TestCase): def _run_cli(self, argv):