From d106e411af6c6bdc824953782c313616162655bd Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Mon, 11 Aug 2008 21:03:26 -0700 Subject: [PATCH] checker: add information to results, add some deep-check tests, fix a bug in which unhealthy files were not counted --- src/allmydata/immutable/checker.py | 10 ++++++-- src/allmydata/interfaces.py | 7 +++++- src/allmydata/mutable/checker.py | 2 ++ src/allmydata/test/common.py | 27 ++++++++++++++++++---- src/allmydata/test/test_dirnode.py | 19 +++++++++++++++ src/allmydata/web/checker_results.py | 18 +++++++++++++++ src/allmydata/web/deep-check-results.xhtml | 20 +++++++++++++++- 7 files changed, 94 insertions(+), 9 deletions(-) diff --git a/src/allmydata/immutable/checker.py b/src/allmydata/immutable/checker.py index 3328daba..c84f7fe9 100644 --- a/src/allmydata/immutable/checker.py +++ b/src/allmydata/immutable/checker.py @@ -28,6 +28,9 @@ class Results: def is_healthy(self): return self.healthy + def get_storage_index(self): + return self.storage_index + def get_storage_index_string(self): return self.storage_index_s @@ -59,6 +62,7 @@ class DeepCheckResults: self.repairs_attempted = 0 self.repairs_successful = 0 self.problems = [] + self.all_results = {} self.server_problems = {} def get_root_storage_index_string(self): @@ -66,10 +70,11 @@ class DeepCheckResults: def add_check(self, r): self.objects_checked += 1 - if r.is_healthy: + if r.is_healthy(): self.objects_healthy += 1 else: self.problems.append(r) + self.all_results[r.get_storage_index()] = r def add_repair(self, is_successful): self.repairs_attempted += 1 @@ -88,7 +93,8 @@ class DeepCheckResults: return self.server_problems def get_problems(self): return self.problems - + def get_all_results(self): + return self.all_results class SimpleCHKFileChecker: """Return a list of (needed, total, found, sharemap), where sharemap maps diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index d5941658..80de5ef6 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -1478,8 +1478,10 @@ class ICheckerResults(Interface): """Return a bool, True if the file is fully healthy, False if it is damaged in any way.""" + def get_storage_index(): + """Return a string with the (binary) storage index.""" def get_storage_index_string(): - """Return a string with the abbreviated storage index.""" + """Return a string with the (printable) abbreviated storage index.""" def get_mutability_string(): """Return a string with 'mutable' or 'immutable'.""" @@ -1524,6 +1526,9 @@ class IDeepCheckResults(Interface): def get_problems(): """Return a list of ICheckerResults, one for each object that was not fully healthy.""" + def get_all_results(): + """Return a dict mapping storage_index (a binary string) to an + ICheckerResults instance, one for each object that was checked.""" class IRepairable(Interface): def repair(checker_results): diff --git a/src/allmydata/mutable/checker.py b/src/allmydata/mutable/checker.py index 691521ef..b25859be 100644 --- a/src/allmydata/mutable/checker.py +++ b/src/allmydata/mutable/checker.py @@ -212,6 +212,8 @@ class Results: def is_healthy(self): return self.healthy + def get_storage_index(self): + return self.storage_index def get_storage_index_string(self): return self.storage_index_s diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index be2234cd..fc3a32a9 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -12,6 +12,8 @@ from allmydata.interfaces import IURI, IMutableFileNode, IFileNode, \ FileTooLargeError, ICheckable from allmydata.immutable import checker from allmydata.immutable.encode import NotEnoughSharesError +from allmydata.mutable.checker import Results as MutableCheckerResults +from allmydata.mutable.common import CorruptShareError from allmydata.util import log, testutil, fileutil from allmydata.stats import PickleStatsGatherer from allmydata.key_generator import KeyGeneratorService @@ -29,10 +31,12 @@ class FakeCHKFileNode: dictionary.""" implements(IFileNode) all_contents = {} + bad_shares = {} def __init__(self, u, client): self.client = client self.my_uri = u.to_string() + self.storage_index = u.storage_index def get_uri(self): return self.my_uri @@ -42,8 +46,13 @@ class FakeCHKFileNode: return IURI(self.my_uri).get_verifier() def check(self, verify=False, repair=False): r = checker.Results(None) - r.healthy = True - r.problems = [] + is_bad = self.bad_shares.get(self.storage_index, None) + if is_bad: + r.healthy = False + r.problems = failure.Failure(CorruptShareError(is_bad)) + else: + r.healthy = True + r.problems = [] return defer.succeed(r) def is_mutable(self): return False @@ -90,6 +99,7 @@ class FakeMutableFileNode: implements(IMutableFileNode, ICheckable) MUTABLE_SIZELIMIT = 10000 all_contents = {} + bad_shares = {} def __init__(self, client): self.client = client @@ -125,9 +135,16 @@ class FakeMutableFileNode: return self.storage_index def check(self, verify=False, repair=False): - r = checker.Results(None) - r.healthy = True - r.problems = [] + r = MutableCheckerResults(self.storage_index) + is_bad = self.bad_shares.get(self.storage_index, None) + if is_bad: + r.healthy = False + r.problems = failure.Failure(CorruptShareError("peerid", + 0, # shnum + is_bad)) + else: + r.healthy = True + r.problems = [] return defer.succeed(r) def deep_check(self, verify=False, repair=False): diff --git a/src/allmydata/test/test_dirnode.py b/src/allmydata/test/test_dirnode.py index 84d51b0e..399f3989 100644 --- a/src/allmydata/test/test_dirnode.py +++ b/src/allmydata/test/test_dirnode.py @@ -159,6 +159,25 @@ class Dirnode(unittest.TestCase, testutil.ShouldFailMixin, testutil.StallMixin): d.addCallback(_check_results) return d + def _mark_file_bad(self, rootnode): + si = IURI(rootnode.get_uri())._filenode_uri.storage_index + rootnode._node.bad_shares[si] = "unhealthy" + return rootnode + + def test_deepcheck_problems(self): + d = self._test_deepcheck_create() + d.addCallback(lambda rootnode: self._mark_file_bad(rootnode)) + d.addCallback(lambda rootnode: rootnode.deep_check()) + def _check_results(r): + self.failUnlessEqual(r.count_objects_checked(), 3) + self.failUnlessEqual(r.count_objects_healthy(), 2) + self.failUnlessEqual(r.count_repairs_attempted(), 0) + self.failUnlessEqual(r.count_repairs_successful(), 0) + self.failUnlessEqual(len(r.get_server_problems()), 0) + self.failUnlessEqual(len(r.get_problems()), 1) + d.addCallback(_check_results) + return d + def test_readonly(self): fileuri = make_chk_file_uri(1234) filenode = self.client.create_node_from_uri(fileuri) diff --git a/src/allmydata/web/checker_results.py b/src/allmydata/web/checker_results.py index 5a4fff5f..50ea475b 100644 --- a/src/allmydata/web/checker_results.py +++ b/src/allmydata/web/checker_results.py @@ -2,6 +2,7 @@ from nevow import rend, inevow, tags as T from allmydata.web.common import getxmlfile, get_arg from allmydata.interfaces import ICheckerResults, IDeepCheckResults +from allmydata.util import base32 class CheckerResults(rend.Page): docFactory = getxmlfile("checker-results.xhtml") @@ -48,6 +49,23 @@ class DeepCheckResults(rend.Page): def data_problems(self, ctx, data): for cr in self.r.get_problems(): yield cr + def render_problem(self, ctx, data): + cr = data + text = cr.get_storage_index_string() + text += ": " + text += cr.status_report + return ctx.tag[text] + + def data_all_objects(self, ctx, data): + r = self.r.get_all_results() + for storage_index in sorted(r.keys()): + yield r[storage_index] + + def render_object(self, ctx, data): + r = data + ctx.fillSlots("storage_index", r.get_storage_index_string()) + ctx.fillSlots("healthy", str(r.is_healthy())) + return ctx.tag def render_return(self, ctx, data): req = inevow.IRequest(ctx) diff --git a/src/allmydata/web/deep-check-results.xhtml b/src/allmydata/web/deep-check-results.xhtml index d89da6c6..2f819808 100644 --- a/src/allmydata/web/deep-check-results.xhtml +++ b/src/allmydata/web/deep-check-results.xhtml @@ -18,7 +18,7 @@

Problems:

@@ -28,6 +28,24 @@
  • Repairs Successful:
    • +

    Objects Checked

    +
    + + + + + + + + + + + + +
    Storage IndexHealthy?
    no objects?
    +
    + +
    -- 2.45.2