src/allmydata/test/test_deepcheck.py

   1
   2 import os, simplejson, urllib
   3 from cStringIO import StringIO
   4 from twisted.trial import unittest
   5 from twisted.internet import defer
   6 from twisted.internet import threads # CLI tests use deferToThread
   7 from allmydata.immutable import upload
   8 from allmydata.util import idlib
   9 from allmydata.util import base32
  10 from allmydata.scripts import runner
  11 from allmydata.interfaces import ICheckResults, ICheckAndRepairResults, \
  12      IDeepCheckResults, IDeepCheckAndRepairResults
  13 from allmydata.monitor import Monitor, OperationCancelledError
  14 from twisted.web.client import getPage
  15
  16 from allmydata.test.common import SystemTestMixin, ErrorMixin
  17
  18 class MutableChecker(SystemTestMixin, unittest.TestCase, ErrorMixin):
  19
  20     def _run_cli(self, argv):
  21         stdout, stderr = StringIO(), StringIO()
  22         # this can only do synchronous operations
  23         assert argv[0] == "debug"
  24         runner.runner(argv, run_by_human=False, stdout=stdout, stderr=stderr)
  25         return stdout.getvalue()
  26
  27     def test_good(self):
  28         self.basedir = self.mktemp()
  29         d = self.set_up_nodes()
  30         CONTENTS = "a little bit of data"
  31         d.addCallback(lambda res: self.clients[0].create_mutable_file(CONTENTS))
  32         def _created(node):
  33             self.node = node
  34             si = self.node.get_storage_index()
  35         d.addCallback(_created)
  36         # now make sure the webapi verifier sees no problems
  37         def _do_check(res):
  38             url = (self.webish_url +
  39                    "uri/%s" % urllib.quote(self.node.get_uri()) +
  40                    "?t=check&verify=true")
  41             return getPage(url, method="POST")
  42         d.addCallback(_do_check)
  43         def _got_results(out):
  44             self.failUnless("<span>Healthy : Healthy</span>" in out, out)
  45             self.failUnless("Recoverable Versions: 10*seq1-" in out, out)
  46             self.failIf("Not Healthy!" in out, out)
  47             self.failIf("Unhealthy" in out, out)
  48             self.failIf("Corrupt Shares" in out, out)
  49         d.addCallback(_got_results)
  50         d.addErrback(self.explain_web_error)
  51         return d
  52
  53     def test_corrupt(self):
  54         self.basedir = self.mktemp()
  55         d = self.set_up_nodes()
  56         CONTENTS = "a little bit of data"
  57         d.addCallback(lambda res: self.clients[0].create_mutable_file(CONTENTS))
  58         def _created(node):
  59             self.node = node
  60             si = self.node.get_storage_index()
  61             out = self._run_cli(["debug", "find-shares", base32.b2a(si),
  62                                 self.clients[1].basedir])
  63             files = out.split("\n")
  64             # corrupt one of them, using the CLI debug command
  65             f = files[0]
  66             shnum = os.path.basename(f)
  67             nodeid = self.clients[1].nodeid
  68             nodeid_prefix = idlib.shortnodeid_b2a(nodeid)
  69             self.corrupt_shareid = "%s-sh%s" % (nodeid_prefix, shnum)
  70             out = self._run_cli(["debug", "corrupt-share", files[0]])
  71         d.addCallback(_created)
  72         # now make sure the webapi verifier notices it
  73         def _do_check(res):
  74             url = (self.webish_url +
  75                    "uri/%s" % urllib.quote(self.node.get_uri()) +
  76                    "?t=check&verify=true")
  77             return getPage(url, method="POST")
  78         d.addCallback(_do_check)
  79         def _got_results(out):
  80             self.failUnless("Not Healthy!" in out, out)
  81             self.failUnless("Unhealthy: best version has only 9 shares (encoding is 3-of-10)" in out, out)
  82             self.failUnless("Corrupt Shares:" in out, out)
  83         d.addCallback(_got_results)
  84
  85         # now make sure the webapi repairer can fix it
  86         def _do_repair(res):
  87             url = (self.webish_url +
  88                    "uri/%s" % urllib.quote(self.node.get_uri()) +
  89                    "?t=check&verify=true&repair=true")
  90             return getPage(url, method="POST")
  91         d.addCallback(_do_repair)
  92         def _got_repair_results(out):
  93             self.failUnless("<div>Repair successful</div>" in out, out)
  94         d.addCallback(_got_repair_results)
  95         d.addCallback(_do_check)
  96         def _got_postrepair_results(out):
  97             self.failIf("Not Healthy!" in out, out)
  98             self.failUnless("Recoverable Versions: 10*seq" in out, out)
  99         d.addCallback(_got_postrepair_results)
 100         d.addErrback(self.explain_web_error)
 101
 102         return d
 103
 104     def test_delete_share(self):
 105         self.basedir = self.mktemp()
 106         d = self.set_up_nodes()
 107         CONTENTS = "a little bit of data"
 108         d.addCallback(lambda res: self.clients[0].create_mutable_file(CONTENTS))
 109         def _created(node):
 110             self.node = node
 111             si = self.node.get_storage_index()
 112             out = self._run_cli(["debug", "find-shares", base32.b2a(si),
 113                                 self.clients[1].basedir])
 114             files = out.split("\n")
 115             # corrupt one of them, using the CLI debug command
 116             f = files[0]
 117             shnum = os.path.basename(f)
 118             nodeid = self.clients[1].nodeid
 119             nodeid_prefix = idlib.shortnodeid_b2a(nodeid)
 120             self.corrupt_shareid = "%s-sh%s" % (nodeid_prefix, shnum)
 121             os.unlink(files[0])
 122         d.addCallback(_created)
 123         # now make sure the webapi checker notices it
 124         def _do_check(res):
 125             url = (self.webish_url +
 126                    "uri/%s" % urllib.quote(self.node.get_uri()) +
 127                    "?t=check&verify=false")
 128             return getPage(url, method="POST")
 129         d.addCallback(_do_check)
 130         def _got_results(out):
 131             self.failUnless("Not Healthy!" in out, out)
 132             self.failUnless("Unhealthy: best version has only 9 shares (encoding is 3-of-10)" in out, out)
 133             self.failIf("Corrupt Shares" in out, out)
 134         d.addCallback(_got_results)
 135
 136         # now make sure the webapi repairer can fix it
 137         def _do_repair(res):
 138             url = (self.webish_url +
 139                    "uri/%s" % urllib.quote(self.node.get_uri()) +
 140                    "?t=check&verify=false&repair=true")
 141             return getPage(url, method="POST")
 142         d.addCallback(_do_repair)
 143         def _got_repair_results(out):
 144             self.failUnless("Repair successful" in out)
 145         d.addCallback(_got_repair_results)
 146         d.addCallback(_do_check)
 147         def _got_postrepair_results(out):
 148             self.failIf("Not Healthy!" in out, out)
 149             self.failUnless("Recoverable Versions: 10*seq" in out)
 150         d.addCallback(_got_postrepair_results)
 151         d.addErrback(self.explain_web_error)
 152
 153         return d
 154
 155
 156 class DeepCheckBase(SystemTestMixin, ErrorMixin):
 157
 158     def web_json(self, n, **kwargs):
 159         kwargs["output"] = "json"
 160         d = self.web(n, "POST", **kwargs)
 161         d.addCallback(self.decode_json)
 162         return d
 163
 164     def decode_json(self, (s,url)):
 165         try:
 166             data = simplejson.loads(s)
 167         except ValueError:
 168             self.fail("%s: not JSON: '%s'" % (url, s))
 169         return data
 170
 171     def parse_streamed_json(self, s):
 172         for unit in s.split("\n"):
 173             if not unit:
 174                 # stream should end with a newline, so split returns ""
 175                 continue
 176             yield simplejson.loads(unit)
 177
 178     def web(self, n, method="GET", **kwargs):
 179         # returns (data, url)
 180         url = (self.webish_url + "uri/%s" % urllib.quote(n.get_uri())
 181                + "?" + "&".join(["%s=%s" % (k,v) for (k,v) in kwargs.items()]))
 182         d = getPage(url, method=method)
 183         d.addCallback(lambda data: (data,url))
 184         return d
 185
 186     def wait_for_operation(self, ignored, ophandle):
 187         url = self.webish_url + "operations/" + ophandle
 188         url += "?t=status&output=JSON"
 189         d = getPage(url)
 190         def _got(res):
 191             try:
 192                 data = simplejson.loads(res)
 193             except ValueError:
 194                 self.fail("%s: not JSON: '%s'" % (url, res))
 195             if not data["finished"]:
 196                 d = self.stall(delay=1.0)
 197                 d.addCallback(self.wait_for_operation, ophandle)
 198                 return d
 199             return data
 200         d.addCallback(_got)
 201         return d
 202
 203     def get_operation_results(self, ignored, ophandle, output=None):
 204         url = self.webish_url + "operations/" + ophandle
 205         url += "?t=status"
 206         if output:
 207             url += "&output=" + output
 208         d = getPage(url)
 209         def _got(res):
 210             if output and output.lower() == "json":
 211                 try:
 212                     return simplejson.loads(res)
 213                 except ValueError:
 214                     self.fail("%s: not JSON: '%s'" % (url, res))
 215             return res
 216         d.addCallback(_got)
 217         return d
 218
 219     def slow_web(self, n, output=None, **kwargs):
 220         # use ophandle=
 221         handle = base32.b2a(os.urandom(4))
 222         d = self.web(n, "POST", ophandle=handle, **kwargs)
 223         d.addCallback(self.wait_for_operation, handle)
 224         d.addCallback(self.get_operation_results, handle, output=output)
 225         return d
 226
 227
 228 class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
 229     # construct a small directory tree (with one dir, one immutable file, one
 230     # mutable file, one LIT file, and a loop), and then check/examine it in
 231     # various ways.
 232
 233     def set_up_tree(self, ignored):
 234         # 2.9s
 235
 236         # root
 237         #   mutable
 238         #   large
 239         #   small
 240         #   small2
 241         #   loop -> root
 242         c0 = self.clients[0]
 243         d = c0.create_empty_dirnode()
 244         def _created_root(n):
 245             self.root = n
 246             self.root_uri = n.get_uri()
 247         d.addCallback(_created_root)
 248         d.addCallback(lambda ign: c0.create_mutable_file("mutable file contents"))
 249         d.addCallback(lambda n: self.root.set_node(u"mutable", n))
 250         def _created_mutable(n):
 251             self.mutable = n
 252             self.mutable_uri = n.get_uri()
 253         d.addCallback(_created_mutable)
 254
 255         large = upload.Data("Lots of data\n" * 1000, None)
 256         d.addCallback(lambda ign: self.root.add_file(u"large", large))
 257         def _created_large(n):
 258             self.large = n
 259             self.large_uri = n.get_uri()
 260         d.addCallback(_created_large)
 261
 262         small = upload.Data("Small enough for a LIT", None)
 263         d.addCallback(lambda ign: self.root.add_file(u"small", small))
 264         def _created_small(n):
 265             self.small = n
 266             self.small_uri = n.get_uri()
 267         d.addCallback(_created_small)
 268
 269         small2 = upload.Data("Small enough for a LIT too", None)
 270         d.addCallback(lambda ign: self.root.add_file(u"small2", small2))
 271         def _created_small2(n):
 272             self.small2 = n
 273             self.small2_uri = n.get_uri()
 274         d.addCallback(_created_small2)
 275
 276         d.addCallback(lambda ign: self.root.set_node(u"loop", self.root))
 277         return d
 278
 279     def check_is_healthy(self, cr, n, where, incomplete=False):
 280         self.failUnless(ICheckResults.providedBy(cr), where)
 281         self.failUnless(cr.is_healthy(), where)
 282         self.failUnlessEqual(cr.get_storage_index(), n.get_storage_index(),
 283                              where)
 284         self.failUnlessEqual(cr.get_storage_index_string(),
 285                              base32.b2a(n.get_storage_index()), where)
 286         needs_rebalancing = bool( len(self.clients) < 10 )
 287         if not incomplete:
 288             self.failUnlessEqual(cr.needs_rebalancing(), needs_rebalancing, str((where, cr, cr.get_data())))
 289         d = cr.get_data()
 290         self.failUnlessEqual(d["count-shares-good"], 10, where)
 291         self.failUnlessEqual(d["count-shares-needed"], 3, where)
 292         self.failUnlessEqual(d["count-shares-expected"], 10, where)
 293         if not incomplete:
 294             self.failUnlessEqual(d["count-good-share-hosts"], len(self.clients), where)
 295         self.failUnlessEqual(d["count-corrupt-shares"], 0, where)
 296         self.failUnlessEqual(d["list-corrupt-shares"], [], where)
 297         if not incomplete:
 298             self.failUnlessEqual(sorted(d["servers-responding"]),
 299                                  sorted([c.nodeid for c in self.clients]),
 300                                  where)
 301             self.failUnless("sharemap" in d, str((where, d)))
 302             all_serverids = set()
 303             for (shareid, serverids) in d["sharemap"].items():
 304                 all_serverids.update(serverids)
 305             self.failUnlessEqual(sorted(all_serverids),
 306                                  sorted([c.nodeid for c in self.clients]),
 307                                  where)
 308
 309         self.failUnlessEqual(d["count-wrong-shares"], 0, where)
 310         self.failUnlessEqual(d["count-recoverable-versions"], 1, where)
 311         self.failUnlessEqual(d["count-unrecoverable-versions"], 0, where)
 312
 313
 314     def check_and_repair_is_healthy(self, cr, n, where, incomplete=False):
 315         self.failUnless(ICheckAndRepairResults.providedBy(cr), (where, cr))
 316         self.failUnless(cr.get_pre_repair_results().is_healthy(), where)
 317         self.check_is_healthy(cr.get_pre_repair_results(), n, where, incomplete)
 318         self.failUnless(cr.get_post_repair_results().is_healthy(), where)
 319         self.check_is_healthy(cr.get_post_repair_results(), n, where, incomplete)
 320         self.failIf(cr.get_repair_attempted(), where)
 321
 322     def deep_check_is_healthy(self, cr, num_healthy, where):
 323         self.failUnless(IDeepCheckResults.providedBy(cr))
 324         self.failUnlessEqual(cr.get_counters()["count-objects-healthy"],
 325                              num_healthy, where)
 326
 327     def deep_check_and_repair_is_healthy(self, cr, num_healthy, where):
 328         self.failUnless(IDeepCheckAndRepairResults.providedBy(cr), where)
 329         c = cr.get_counters()
 330         self.failUnlessEqual(c["count-objects-healthy-pre-repair"],
 331                              num_healthy, where)
 332         self.failUnlessEqual(c["count-objects-healthy-post-repair"],
 333                              num_healthy, where)
 334         self.failUnlessEqual(c["count-repairs-attempted"], 0, where)
 335
 336     def test_good(self):
 337         self.basedir = self.mktemp()
 338         d = self.set_up_nodes()
 339         d.addCallback(self.set_up_tree)
 340         d.addCallback(self.do_stats)
 341         d.addCallback(self.do_web_stream_manifest)
 342         d.addCallback(self.do_web_stream_check)
 343         d.addCallback(self.do_test_check_good)
 344         d.addCallback(self.do_test_web_good)
 345         d.addCallback(self.do_test_cli_good)
 346         d.addErrback(self.explain_web_error)
 347         d.addErrback(self.explain_error)
 348         return d
 349
 350     def do_stats(self, ignored):
 351         d = defer.succeed(None)
 352         d.addCallback(lambda ign: self.root.start_deep_stats().when_done())
 353         d.addCallback(self.check_stats_good)
 354         return d
 355
 356     def check_stats_good(self, s):
 357         self.failUnlessEqual(s["count-directories"], 1)
 358         self.failUnlessEqual(s["count-files"], 4)
 359         self.failUnlessEqual(s["count-immutable-files"], 1)
 360         self.failUnlessEqual(s["count-literal-files"], 2)
 361         self.failUnlessEqual(s["count-mutable-files"], 1)
 362         # don't check directories: their size will vary
 363         # s["largest-directory"]
 364         # s["size-directories"]
 365         self.failUnlessEqual(s["largest-directory-children"], 5)
 366         self.failUnlessEqual(s["largest-immutable-file"], 13000)
 367         # to re-use this function for both the local
 368         # dirnode.start_deep_stats() and the webapi t=start-deep-stats, we
 369         # coerce the result into a list of tuples. dirnode.start_deep_stats()
 370         # returns a list of tuples, but JSON only knows about lists., so
 371         # t=start-deep-stats returns a list of lists.
 372         histogram = [tuple(stuff) for stuff in s["size-files-histogram"]]
 373         self.failUnlessEqual(histogram, [(11, 31, 2),
 374                                          (10001, 31622, 1),
 375                                          ])
 376         self.failUnlessEqual(s["size-immutable-files"], 13000)
 377         self.failUnlessEqual(s["size-literal-files"], 48)
 378
 379     def do_web_stream_manifest(self, ignored):
 380         d = self.web(self.root, method="POST", t="stream-manifest")
 381         d.addCallback(lambda (output,url):
 382                       self._check_streamed_manifest(output))
 383         return d
 384
 385     def _check_streamed_manifest(self, output):
 386         units = list(self.parse_streamed_json(output))
 387         files = [u for u in units if u["type"] in ("file", "directory")]
 388         assert units[-1]["type"] == "stats"
 389         stats = units[-1]["stats"]
 390         self.failUnlessEqual(len(files), 5)
 391         # [root,mutable,large] are distributed, [small,small2] are not
 392         self.failUnlessEqual(len([f for f in files
 393                                   if f["verifycap"] is not None]), 3)
 394         self.failUnlessEqual(len([f for f in files
 395                                   if f["verifycap"] is None]), 2)
 396         self.failUnlessEqual(len([f for f in files
 397                                   if f["repaircap"] is not None]), 3)
 398         self.failUnlessEqual(len([f for f in files
 399                                   if f["repaircap"] is None]), 2)
 400         self.failUnlessEqual(len([f for f in files
 401                                   if f["storage-index"] is not None]), 3)
 402         self.failUnlessEqual(len([f for f in files
 403                                   if f["storage-index"] is None]), 2)
 404         # make sure that a mutable file has filecap==repaircap!=verifycap
 405         mutable = [f for f in files
 406                    if f["cap"] is not None
 407                    and f["cap"].startswith("URI:SSK:")][0]
 408         self.failUnlessEqual(mutable["cap"], self.mutable_uri)
 409         self.failIfEqual(mutable["cap"], mutable["verifycap"])
 410         self.failUnlessEqual(mutable["cap"], mutable["repaircap"])
 411         # for immutable file, verifycap==repaircap!=filecap
 412         large = [f for f in files
 413                    if f["cap"] is not None
 414                    and f["cap"].startswith("URI:CHK:")][0]
 415         self.failUnlessEqual(large["cap"], self.large_uri)
 416         self.failIfEqual(large["cap"], large["verifycap"])
 417         self.failUnlessEqual(large["verifycap"], large["repaircap"])
 418         self.check_stats_good(stats)
 419
 420     def do_web_stream_check(self, ignored):
 421         return
 422         d = self.web(self.root, t="stream-deep-check")
 423         def _check(res):
 424             units = list(self.parse_streamed_json(res))
 425             files = [u for u in units if u["type"] in ("file", "directory")]
 426             assert units[-1]["type"] == "stats"
 427             stats = units[-1]["stats"]
 428             # ...
 429         d.addCallback(_check)
 430         return d
 431
 432     def do_test_check_good(self, ignored):
 433         d = defer.succeed(None)
 434         # check the individual items
 435         d.addCallback(lambda ign: self.root.check(Monitor()))
 436         d.addCallback(self.check_is_healthy, self.root, "root")
 437         d.addCallback(lambda ign: self.mutable.check(Monitor()))
 438         d.addCallback(self.check_is_healthy, self.mutable, "mutable")
 439         d.addCallback(lambda ign: self.large.check(Monitor()))
 440         d.addCallback(self.check_is_healthy, self.large, "large")
 441         d.addCallback(lambda ign: self.small.check(Monitor()))
 442         d.addCallback(self.failUnlessEqual, None, "small")
 443         d.addCallback(lambda ign: self.small2.check(Monitor()))
 444         d.addCallback(self.failUnlessEqual, None, "small2")
 445
 446         # and again with verify=True
 447         d.addCallback(lambda ign: self.root.check(Monitor(), verify=True))
 448         d.addCallback(self.check_is_healthy, self.root, "root")
 449         d.addCallback(lambda ign: self.mutable.check(Monitor(), verify=True))
 450         d.addCallback(self.check_is_healthy, self.mutable, "mutable")
 451         d.addCallback(lambda ign: self.large.check(Monitor(), verify=True))
 452         d.addCallback(self.check_is_healthy, self.large, "large", incomplete=True)
 453         d.addCallback(lambda ign: self.small.check(Monitor(), verify=True))
 454         d.addCallback(self.failUnlessEqual, None, "small")
 455         d.addCallback(lambda ign: self.small2.check(Monitor(), verify=True))
 456         d.addCallback(self.failUnlessEqual, None, "small2")
 457
 458         # and check_and_repair(), which should be a nop
 459         d.addCallback(lambda ign: self.root.check_and_repair(Monitor()))
 460         d.addCallback(self.check_and_repair_is_healthy, self.root, "root")
 461         d.addCallback(lambda ign: self.mutable.check_and_repair(Monitor()))
 462         d.addCallback(self.check_and_repair_is_healthy, self.mutable, "mutable")
 463         #TODO d.addCallback(lambda ign: self.large.check_and_repair(Monitor()))
 464         #TODO d.addCallback(self.check_and_repair_is_healthy, self.large, "large")
 465         #TODO d.addCallback(lambda ign: self.small.check_and_repair(Monitor()))
 466         #TODO d.addCallback(self.failUnlessEqual, None, "small")
 467         #TODO d.addCallback(lambda ign: self.small2.check_and_repair(Monitor()))
 468         #TODO d.addCallback(self.failUnlessEqual, None, "small2")
 469
 470         # check_and_repair(verify=True)
 471         d.addCallback(lambda ign: self.root.check_and_repair(Monitor(), verify=True))
 472         d.addCallback(self.check_and_repair_is_healthy, self.root, "root")
 473         d.addCallback(lambda ign: self.mutable.check_and_repair(Monitor(), verify=True))
 474         d.addCallback(self.check_and_repair_is_healthy, self.mutable, "mutable")
 475         #TODO d.addCallback(lambda ign: self.large.check_and_repair(Monitor(), verify=True))
 476         #TODO d.addCallback(self.check_and_repair_is_healthy, self.large, "large",
 477         #TODO               incomplete=True)
 478         #TODO d.addCallback(lambda ign: self.small.check_and_repair(Monitor(), verify=True))
 479         #TODO d.addCallback(self.failUnlessEqual, None, "small")
 480         #TODO d.addCallback(lambda ign: self.small2.check_and_repair(Monitor(), verify=True))
 481         #TODO d.addCallback(self.failUnlessEqual, None, "small2")
 482
 483
 484         # now deep-check the root, with various verify= and repair= options
 485         d.addCallback(lambda ign:
 486                       self.root.start_deep_check().when_done())
 487         d.addCallback(self.deep_check_is_healthy, 3, "root")
 488         d.addCallback(lambda ign:
 489                       self.root.start_deep_check(verify=True).when_done())
 490         d.addCallback(self.deep_check_is_healthy, 3, "root")
 491         d.addCallback(lambda ign:
 492                       self.root.start_deep_check_and_repair().when_done())
 493         d.addCallback(self.deep_check_and_repair_is_healthy, 3, "root")
 494         d.addCallback(lambda ign:
 495                       self.root.start_deep_check_and_repair(verify=True).when_done())
 496         d.addCallback(self.deep_check_and_repair_is_healthy, 3, "root")
 497
 498         # and finally, start a deep-check, but then cancel it.
 499         d.addCallback(lambda ign: self.root.start_deep_check())
 500         def _checking(monitor):
 501             monitor.cancel()
 502             d = monitor.when_done()
 503             # this should fire as soon as the next dirnode.list finishes.
 504             # TODO: add a counter to measure how many list() calls are made,
 505             # assert that no more than one gets to run before the cancel()
 506             # takes effect.
 507             def _finished_normally(res):
 508                 self.fail("this was supposed to fail, not finish normally")
 509             def _cancelled(f):
 510                 f.trap(OperationCancelledError)
 511             d.addCallbacks(_finished_normally, _cancelled)
 512             return d
 513         d.addCallback(_checking)
 514
 515         return d
 516
 517     def json_check_is_healthy(self, data, n, where, incomplete=False):
 518
 519         self.failUnlessEqual(data["storage-index"],
 520                              base32.b2a(n.get_storage_index()), where)
 521         self.failUnless("summary" in data, (where, data))
 522         self.failUnlessEqual(data["summary"].lower(), "healthy",
 523                              "%s: '%s'" % (where, data["summary"]))
 524         r = data["results"]
 525         self.failUnlessEqual(r["healthy"], True, where)
 526         needs_rebalancing = bool( len(self.clients) < 10 )
 527         if not incomplete:
 528             self.failUnlessEqual(r["needs-rebalancing"], needs_rebalancing, where)
 529         self.failUnlessEqual(r["count-shares-good"], 10, where)
 530         self.failUnlessEqual(r["count-shares-needed"], 3, where)
 531         self.failUnlessEqual(r["count-shares-expected"], 10, where)
 532         if not incomplete:
 533             self.failUnlessEqual(r["count-good-share-hosts"], len(self.clients), where)
 534         self.failUnlessEqual(r["count-corrupt-shares"], 0, where)
 535         self.failUnlessEqual(r["list-corrupt-shares"], [], where)
 536         if not incomplete:
 537             self.failUnlessEqual(sorted(r["servers-responding"]),
 538                                  sorted([idlib.nodeid_b2a(c.nodeid)
 539                                          for c in self.clients]), where)
 540             self.failUnless("sharemap" in r, where)
 541             all_serverids = set()
 542             for (shareid, serverids_s) in r["sharemap"].items():
 543                 all_serverids.update(serverids_s)
 544             self.failUnlessEqual(sorted(all_serverids),
 545                                  sorted([idlib.nodeid_b2a(c.nodeid)
 546                                          for c in self.clients]), where)
 547         self.failUnlessEqual(r["count-wrong-shares"], 0, where)
 548         self.failUnlessEqual(r["count-recoverable-versions"], 1, where)
 549         self.failUnlessEqual(r["count-unrecoverable-versions"], 0, where)
 550
 551     def json_check_and_repair_is_healthy(self, data, n, where, incomplete=False):
 552         self.failUnlessEqual(data["storage-index"],
 553                              base32.b2a(n.get_storage_index()), where)
 554         self.failUnlessEqual(data["repair-attempted"], False, where)
 555         self.json_check_is_healthy(data["pre-repair-results"],
 556                                    n, where, incomplete)
 557         self.json_check_is_healthy(data["post-repair-results"],
 558                                    n, where, incomplete)
 559
 560     def json_full_deepcheck_is_healthy(self, data, n, where):
 561         self.failUnlessEqual(data["root-storage-index"],
 562                              base32.b2a(n.get_storage_index()), where)
 563         self.failUnlessEqual(data["count-objects-checked"], 3, where)
 564         self.failUnlessEqual(data["count-objects-healthy"], 3, where)
 565         self.failUnlessEqual(data["count-objects-unhealthy"], 0, where)
 566         self.failUnlessEqual(data["count-corrupt-shares"], 0, where)
 567         self.failUnlessEqual(data["list-corrupt-shares"], [], where)
 568         self.failUnlessEqual(data["list-unhealthy-files"], [], where)
 569         self.json_check_stats_good(data["stats"], where)
 570
 571     def json_full_deepcheck_and_repair_is_healthy(self, data, n, where):
 572         self.failUnlessEqual(data["root-storage-index"],
 573                              base32.b2a(n.get_storage_index()), where)
 574         self.failUnlessEqual(data["count-objects-checked"], 3, where)
 575
 576         self.failUnlessEqual(data["count-objects-healthy-pre-repair"], 3, where)
 577         self.failUnlessEqual(data["count-objects-unhealthy-pre-repair"], 0, where)
 578         self.failUnlessEqual(data["count-corrupt-shares-pre-repair"], 0, where)
 579
 580         self.failUnlessEqual(data["count-objects-healthy-post-repair"], 3, where)
 581         self.failUnlessEqual(data["count-objects-unhealthy-post-repair"], 0, where)
 582         self.failUnlessEqual(data["count-corrupt-shares-post-repair"], 0, where)
 583
 584         self.failUnlessEqual(data["list-corrupt-shares"], [], where)
 585         self.failUnlessEqual(data["list-remaining-corrupt-shares"], [], where)
 586         self.failUnlessEqual(data["list-unhealthy-files"], [], where)
 587
 588         self.failUnlessEqual(data["count-repairs-attempted"], 0, where)
 589         self.failUnlessEqual(data["count-repairs-successful"], 0, where)
 590         self.failUnlessEqual(data["count-repairs-unsuccessful"], 0, where)
 591
 592
 593     def json_check_lit(self, data, n, where):
 594         self.failUnlessEqual(data["storage-index"], "", where)
 595         self.failUnlessEqual(data["results"]["healthy"], True, where)
 596
 597     def json_check_stats_good(self, data, where):
 598         self.check_stats_good(data)
 599
 600     def do_test_web_good(self, ignored):
 601         d = defer.succeed(None)
 602
 603         # stats
 604         d.addCallback(lambda ign:
 605                       self.slow_web(self.root,
 606                                     t="start-deep-stats", output="json"))
 607         d.addCallback(self.json_check_stats_good, "deep-stats")
 608
 609         # check, no verify
 610         d.addCallback(lambda ign: self.web_json(self.root, t="check"))
 611         d.addCallback(self.json_check_is_healthy, self.root, "root")
 612         d.addCallback(lambda ign: self.web_json(self.mutable, t="check"))
 613         d.addCallback(self.json_check_is_healthy, self.mutable, "mutable")
 614         d.addCallback(lambda ign: self.web_json(self.large, t="check"))
 615         d.addCallback(self.json_check_is_healthy, self.large, "large")
 616         d.addCallback(lambda ign: self.web_json(self.small, t="check"))
 617         d.addCallback(self.json_check_lit, self.small, "small")
 618         d.addCallback(lambda ign: self.web_json(self.small2, t="check"))
 619         d.addCallback(self.json_check_lit, self.small2, "small2")
 620
 621         # check and verify
 622         d.addCallback(lambda ign:
 623                       self.web_json(self.root, t="check", verify="true"))
 624         d.addCallback(self.json_check_is_healthy, self.root, "root+v")
 625         d.addCallback(lambda ign:
 626                       self.web_json(self.mutable, t="check", verify="true"))
 627         d.addCallback(self.json_check_is_healthy, self.mutable, "mutable+v")
 628         d.addCallback(lambda ign:
 629                       self.web_json(self.large, t="check", verify="true"))
 630         d.addCallback(self.json_check_is_healthy, self.large, "large+v",
 631                       incomplete=True)
 632         d.addCallback(lambda ign:
 633                       self.web_json(self.small, t="check", verify="true"))
 634         d.addCallback(self.json_check_lit, self.small, "small+v")
 635         d.addCallback(lambda ign:
 636                       self.web_json(self.small2, t="check", verify="true"))
 637         d.addCallback(self.json_check_lit, self.small2, "small2+v")
 638
 639         # check and repair, no verify
 640         d.addCallback(lambda ign:
 641                       self.web_json(self.root, t="check", repair="true"))
 642         d.addCallback(self.json_check_and_repair_is_healthy, self.root, "root+r")
 643         d.addCallback(lambda ign:
 644                       self.web_json(self.mutable, t="check", repair="true"))
 645         d.addCallback(self.json_check_and_repair_is_healthy, self.mutable, "mutable+r")
 646         d.addCallback(lambda ign:
 647                       self.web_json(self.large, t="check", repair="true"))
 648         d.addCallback(self.json_check_and_repair_is_healthy, self.large, "large+r")
 649         d.addCallback(lambda ign:
 650                       self.web_json(self.small, t="check", repair="true"))
 651         d.addCallback(self.json_check_lit, self.small, "small+r")
 652         d.addCallback(lambda ign:
 653                       self.web_json(self.small2, t="check", repair="true"))
 654         d.addCallback(self.json_check_lit, self.small2, "small2+r")
 655
 656         # check+verify+repair
 657         d.addCallback(lambda ign:
 658                       self.web_json(self.root, t="check", repair="true", verify="true"))
 659         d.addCallback(self.json_check_and_repair_is_healthy, self.root, "root+vr")
 660         d.addCallback(lambda ign:
 661                       self.web_json(self.mutable, t="check", repair="true", verify="true"))
 662         d.addCallback(self.json_check_and_repair_is_healthy, self.mutable, "mutable+vr")
 663         d.addCallback(lambda ign:
 664                       self.web_json(self.large, t="check", repair="true", verify="true"))
 665         d.addCallback(self.json_check_and_repair_is_healthy, self.large, "large+vr", incomplete=True)
 666         d.addCallback(lambda ign:
 667                       self.web_json(self.small, t="check", repair="true", verify="true"))
 668         d.addCallback(self.json_check_lit, self.small, "small+vr")
 669         d.addCallback(lambda ign:
 670                       self.web_json(self.small2, t="check", repair="true", verify="true"))
 671         d.addCallback(self.json_check_lit, self.small2, "small2+vr")
 672
 673         # now run a deep-check, with various verify= and repair= flags
 674         d.addCallback(lambda ign:
 675                       self.slow_web(self.root, t="start-deep-check", output="json"))
 676         d.addCallback(self.json_full_deepcheck_is_healthy, self.root, "root+d")
 677         d.addCallback(lambda ign:
 678                       self.slow_web(self.root, t="start-deep-check", verify="true",
 679                                     output="json"))
 680         d.addCallback(self.json_full_deepcheck_is_healthy, self.root, "root+dv")
 681         d.addCallback(lambda ign:
 682                       self.slow_web(self.root, t="start-deep-check", repair="true",
 683                                     output="json"))
 684         d.addCallback(self.json_full_deepcheck_and_repair_is_healthy, self.root, "root+dr")
 685         d.addCallback(lambda ign:
 686                       self.slow_web(self.root, t="start-deep-check", verify="true", repair="true", output="json"))
 687         d.addCallback(self.json_full_deepcheck_and_repair_is_healthy, self.root, "root+dvr")
 688
 689         # now look at t=info
 690         d.addCallback(lambda ign: self.web(self.root, t="info"))
 691         # TODO: examine the output
 692         d.addCallback(lambda ign: self.web(self.mutable, t="info"))
 693         d.addCallback(lambda ign: self.web(self.large, t="info"))
 694         d.addCallback(lambda ign: self.web(self.small, t="info"))
 695         d.addCallback(lambda ign: self.web(self.small2, t="info"))
 696
 697         return d
 698
 699     def _run_cli(self, argv, stdin=""):
 700         #print "CLI:", argv
 701         stdout, stderr = StringIO(), StringIO()
 702         d = threads.deferToThread(runner.runner, argv, run_by_human=False,
 703                                   stdin=StringIO(stdin),
 704                                   stdout=stdout, stderr=stderr)
 705         def _done(res):
 706             return stdout.getvalue(), stderr.getvalue()
 707         d.addCallback(_done)
 708         return d
 709
 710     def do_test_cli_good(self, ignored):
 711         d = defer.succeed(None)
 712         d.addCallback(lambda ign: self.do_cli_manifest_stream1())
 713         d.addCallback(lambda ign: self.do_cli_manifest_stream2())
 714         d.addCallback(lambda ign: self.do_cli_manifest_stream3())
 715         d.addCallback(lambda ign: self.do_cli_manifest_stream4())
 716         d.addCallback(lambda ign: self.do_cli_manifest_stream5())
 717         d.addCallback(lambda ign: self.do_cli_stats1())
 718         d.addCallback(lambda ign: self.do_cli_stats2())
 719         return d
 720
 721     def _check_manifest_storage_index(self, out):
 722         lines = [l for l in out.split("\n") if l]
 723         self.failUnlessEqual(len(lines), 3)
 724         self.failUnless(base32.b2a(self.root.get_storage_index()) in lines)
 725         self.failUnless(base32.b2a(self.mutable.get_storage_index()) in lines)
 726         self.failUnless(base32.b2a(self.large.get_storage_index()) in lines)
 727
 728     def do_cli_manifest_stream1(self):
 729         basedir = self.getdir("client0")
 730         d = self._run_cli(["manifest",
 731                            "--node-directory", basedir,
 732                            self.root_uri])
 733         def _check((out,err)):
 734             self.failUnlessEqual(err, "")
 735             lines = [l for l in out.split("\n") if l]
 736             self.failUnlessEqual(len(lines), 5)
 737             caps = {}
 738             for l in lines:
 739                 try:
 740                     cap, path = l.split(None, 1)
 741                 except ValueError:
 742                     cap = l.strip()
 743                     path = ""
 744                 caps[cap] = path
 745             self.failUnless(self.root.get_uri() in caps)
 746             self.failUnlessEqual(caps[self.root.get_uri()], "")
 747             self.failUnlessEqual(caps[self.mutable.get_uri()], "mutable")
 748             self.failUnlessEqual(caps[self.large.get_uri()], "large")
 749             self.failUnlessEqual(caps[self.small.get_uri()], "small")
 750             self.failUnlessEqual(caps[self.small2.get_uri()], "small2")
 751         d.addCallback(_check)
 752         return d
 753
 754     def do_cli_manifest_stream2(self):
 755         basedir = self.getdir("client0")
 756         d = self._run_cli(["manifest",
 757                            "--node-directory", basedir,
 758                            "--raw",
 759                            self.root_uri])
 760         def _check((out,err)):
 761             self.failUnlessEqual(err, "")
 762             # this should be the same as the POST t=stream-manifest output
 763             self._check_streamed_manifest(out)
 764         d.addCallback(_check)
 765         return d
 766
 767     def do_cli_manifest_stream3(self):
 768         basedir = self.getdir("client0")
 769         d = self._run_cli(["manifest",
 770                            "--node-directory", basedir,
 771                            "--storage-index",
 772                            self.root_uri])
 773         def _check((out,err)):
 774             self.failUnlessEqual(err, "")
 775             self._check_manifest_storage_index(out)
 776         d.addCallback(_check)
 777         return d
 778
 779     def do_cli_manifest_stream4(self):
 780         basedir = self.getdir("client0")
 781         d = self._run_cli(["manifest",
 782                            "--node-directory", basedir,
 783                            "--verify-cap",
 784                            self.root_uri])
 785         def _check((out,err)):
 786             self.failUnlessEqual(err, "")
 787             lines = [l for l in out.split("\n") if l]
 788             self.failUnlessEqual(len(lines), 3)
 789             self.failUnless(self.root.get_verify_cap().to_string() in lines)
 790             self.failUnless(self.mutable.get_verify_cap().to_string() in lines)
 791             self.failUnless(self.large.get_verify_cap().to_string() in lines)
 792         d.addCallback(_check)
 793         return d
 794
 795     def do_cli_manifest_stream5(self):
 796         basedir = self.getdir("client0")
 797         d = self._run_cli(["manifest",
 798                            "--node-directory", basedir,
 799                            "--repair-cap",
 800                            self.root_uri])
 801         def _check((out,err)):
 802             self.failUnlessEqual(err, "")
 803             lines = [l for l in out.split("\n") if l]
 804             self.failUnlessEqual(len(lines), 3)
 805             self.failUnless(self.root.get_repair_cap().to_string() in lines)
 806             self.failUnless(self.mutable.get_repair_cap().to_string() in lines)
 807             self.failUnless(self.large.get_repair_cap().to_string() in lines)
 808         d.addCallback(_check)
 809         return d
 810
 811     def do_cli_stats1(self):
 812         basedir = self.getdir("client0")
 813         d = self._run_cli(["stats",
 814                            "--node-directory", basedir,
 815                            self.root_uri])
 816         def _check3((out,err)):
 817             lines = [l.strip() for l in out.split("\n") if l]
 818             self.failUnless("count-immutable-files: 1" in lines)
 819             self.failUnless("count-mutable-files: 1" in lines)
 820             self.failUnless("count-literal-files: 2" in lines)
 821             self.failUnless("count-files: 4" in lines)
 822             self.failUnless("count-directories: 1" in lines)
 823             self.failUnless("size-immutable-files: 13000    (13.00 kB, 12.70 kiB)" in lines, lines)
 824             self.failUnless("size-literal-files: 48" in lines)
 825             self.failUnless("   11-31    : 2    (31 B, 31 B)".strip() in lines)
 826             self.failUnless("10001-31622 : 1    (31.62 kB, 30.88 kiB)".strip() in lines)
 827         d.addCallback(_check3)
 828         return d
 829
 830     def do_cli_stats2(self):
 831         basedir = self.getdir("client0")
 832         d = self._run_cli(["stats",
 833                            "--node-directory", basedir,
 834                            "--raw",
 835                            self.root_uri])
 836         def _check4((out,err)):
 837             data = simplejson.loads(out)
 838             self.failUnlessEqual(data["count-immutable-files"], 1)
 839             self.failUnlessEqual(data["count-immutable-files"], 1)
 840             self.failUnlessEqual(data["count-mutable-files"], 1)
 841             self.failUnlessEqual(data["count-literal-files"], 2)
 842             self.failUnlessEqual(data["count-files"], 4)
 843             self.failUnlessEqual(data["count-directories"], 1)
 844             self.failUnlessEqual(data["size-immutable-files"], 13000)
 845             self.failUnlessEqual(data["size-literal-files"], 48)
 846             self.failUnless([11,31,2] in data["size-files-histogram"])
 847             self.failUnless([10001,31622,1] in data["size-files-histogram"])
 848         d.addCallback(_check4)
 849         return d
 850
 851
 852 class DeepCheckWebBad(DeepCheckBase, unittest.TestCase):
 853
 854     def test_bad(self):
 855         self.basedir = self.mktemp()
 856         d = self.set_up_nodes()
 857         d.addCallback(self.set_up_damaged_tree)
 858         d.addCallback(self.do_check)
 859         d.addCallback(self.do_deepcheck)
 860         d.addCallback(self.do_test_web_bad)
 861         d.addErrback(self.explain_web_error)
 862         d.addErrback(self.explain_error)
 863         return d
 864
 865
 866
 867     def set_up_damaged_tree(self, ignored):
 868         # 6.4s
 869
 870         # root
 871         #   mutable-good
 872         #   mutable-missing-shares
 873         #   mutable-corrupt-shares
 874         #   mutable-unrecoverable
 875         #   large-good
 876         #   large-missing-shares
 877         #   large-corrupt-shares
 878         #   large-unrecoverable
 879
 880         self.nodes = {}
 881
 882         c0 = self.clients[0]
 883         d = c0.create_empty_dirnode()
 884         def _created_root(n):
 885             self.root = n
 886             self.root_uri = n.get_uri()
 887         d.addCallback(_created_root)
 888         d.addCallback(self.create_mangled, "mutable-good")
 889         d.addCallback(self.create_mangled, "mutable-missing-shares")
 890         d.addCallback(self.create_mangled, "mutable-corrupt-shares")
 891         d.addCallback(self.create_mangled, "mutable-unrecoverable")
 892         d.addCallback(self.create_mangled, "large-good")
 893         d.addCallback(self.create_mangled, "large-missing-shares")
 894         d.addCallback(self.create_mangled, "large-corrupt-shares")
 895         d.addCallback(self.create_mangled, "large-unrecoverable")
 896
 897         return d
 898
 899
 900     def create_mangled(self, ignored, name):
 901         nodetype, mangletype = name.split("-", 1)
 902         if nodetype == "mutable":
 903             d = self.clients[0].create_mutable_file("mutable file contents")
 904             d.addCallback(lambda n: self.root.set_node(unicode(name), n))
 905         elif nodetype == "large":
 906             large = upload.Data("Lots of data\n" * 1000 + name + "\n", None)
 907             d = self.root.add_file(unicode(name), large)
 908         elif nodetype == "small":
 909             small = upload.Data("Small enough for a LIT", None)
 910             d = self.root.add_file(unicode(name), small)
 911
 912         def _stash_node(node):
 913             self.nodes[name] = node
 914             return node
 915         d.addCallback(_stash_node)
 916
 917         if mangletype == "good":
 918             pass
 919         elif mangletype == "missing-shares":
 920             d.addCallback(self._delete_some_shares)
 921         elif mangletype == "corrupt-shares":
 922             d.addCallback(self._corrupt_some_shares)
 923         else:
 924             assert mangletype == "unrecoverable"
 925             d.addCallback(self._delete_most_shares)
 926
 927         return d
 928
 929     def _run_cli(self, argv):
 930         stdout, stderr = StringIO(), StringIO()
 931         # this can only do synchronous operations
 932         assert argv[0] == "debug"
 933         runner.runner(argv, run_by_human=False, stdout=stdout, stderr=stderr)
 934         return stdout.getvalue()
 935
 936     def _find_shares(self, node):
 937         si = node.get_storage_index()
 938         out = self._run_cli(["debug", "find-shares", base32.b2a(si)] +
 939                             [c.basedir for c in self.clients])
 940         files = out.split("\n")
 941         return [f for f in files if f]
 942
 943     def _delete_some_shares(self, node):
 944         shares = self._find_shares(node)
 945         os.unlink(shares[0])
 946         os.unlink(shares[1])
 947
 948     def _corrupt_some_shares(self, node):
 949         shares = self._find_shares(node)
 950         self._run_cli(["debug", "corrupt-share", shares[0]])
 951         self._run_cli(["debug", "corrupt-share", shares[1]])
 952
 953     def _delete_most_shares(self, node):
 954         shares = self._find_shares(node)
 955         for share in shares[1:]:
 956             os.unlink(share)
 957
 958
 959     def check_is_healthy(self, cr, where):
 960         try:
 961             self.failUnless(ICheckResults.providedBy(cr), (cr, type(cr), where))
 962             self.failUnless(cr.is_healthy(), (cr.get_report(), cr.is_healthy(), cr.get_summary(), where))
 963             self.failUnless(cr.is_recoverable(), where)
 964             d = cr.get_data()
 965             self.failUnlessEqual(d["count-recoverable-versions"], 1, where)
 966             self.failUnlessEqual(d["count-unrecoverable-versions"], 0, where)
 967             return cr
 968         except Exception, le:
 969             le.args = tuple(le.args + (where,))
 970             raise
 971
 972     def check_is_missing_shares(self, cr, where):
 973         self.failUnless(ICheckResults.providedBy(cr), where)
 974         self.failIf(cr.is_healthy(), where)
 975         self.failUnless(cr.is_recoverable(), where)
 976         d = cr.get_data()
 977         self.failUnlessEqual(d["count-recoverable-versions"], 1, where)
 978         self.failUnlessEqual(d["count-unrecoverable-versions"], 0, where)
 979         return cr
 980
 981     def check_has_corrupt_shares(self, cr, where):
 982         # by "corrupt-shares" we mean the file is still recoverable
 983         self.failUnless(ICheckResults.providedBy(cr), where)
 984         d = cr.get_data()
 985         self.failIf(cr.is_healthy(), (where, cr))
 986         self.failUnless(cr.is_recoverable(), where)
 987         d = cr.get_data()
 988         self.failUnless(d["count-shares-good"] < 10, where)
 989         self.failUnless(d["count-corrupt-shares"], where)
 990         self.failUnless(d["list-corrupt-shares"], where)
 991         return cr
 992
 993     def check_is_unrecoverable(self, cr, where):
 994         self.failUnless(ICheckResults.providedBy(cr), where)
 995         d = cr.get_data()
 996         self.failIf(cr.is_healthy(), where)
 997         self.failIf(cr.is_recoverable(), where)
 998         self.failUnless(d["count-shares-good"] < d["count-shares-needed"], (d["count-shares-good"], d["count-shares-needed"], where))
 999         self.failUnlessEqual(d["count-recoverable-versions"], 0, where)
1000         self.failUnlessEqual(d["count-unrecoverable-versions"], 1, where)
1001         return cr
1002
1003     def do_check(self, ignored):
1004         d = defer.succeed(None)
1005
1006         # check the individual items, without verification. This will not
1007         # detect corrupt shares.
1008         def _check(which, checker):
1009             d = self.nodes[which].check(Monitor())
1010             d.addCallback(checker, which + "--check")
1011             return d
1012
1013         d.addCallback(lambda ign: _check("mutable-good", self.check_is_healthy))
1014         d.addCallback(lambda ign: _check("mutable-missing-shares",
1015                                          self.check_is_missing_shares))
1016         d.addCallback(lambda ign: _check("mutable-corrupt-shares",
1017                                          self.check_is_healthy))
1018         d.addCallback(lambda ign: _check("mutable-unrecoverable",
1019                                          self.check_is_unrecoverable))
1020         d.addCallback(lambda ign: _check("large-good", self.check_is_healthy))
1021         d.addCallback(lambda ign: _check("large-missing-shares",
1022                                          self.check_is_missing_shares))
1023         d.addCallback(lambda ign: _check("large-corrupt-shares",
1024                                          self.check_is_healthy))
1025         d.addCallback(lambda ign: _check("large-unrecoverable",
1026                                          self.check_is_unrecoverable))
1027
1028         # and again with verify=True, which *does* detect corrupt shares.
1029         def _checkv(which, checker):
1030             d = self.nodes[which].check(Monitor(), verify=True)
1031             d.addCallback(checker, which + "--check-and-verify")
1032             return d
1033
1034         d.addCallback(lambda ign: _checkv("mutable-good", self.check_is_healthy))
1035         d.addCallback(lambda ign: _checkv("mutable-missing-shares",
1036                                          self.check_is_missing_shares))
1037         d.addCallback(lambda ign: _checkv("mutable-corrupt-shares",
1038                                          self.check_has_corrupt_shares))
1039         d.addCallback(lambda ign: _checkv("mutable-unrecoverable",
1040                                          self.check_is_unrecoverable))
1041         d.addCallback(lambda ign: _checkv("large-good", self.check_is_healthy))
1042         d.addCallback(lambda ign: _checkv("large-missing-shares", self.check_is_missing_shares))
1043         d.addCallback(lambda ign: _checkv("large-corrupt-shares", self.check_has_corrupt_shares))
1044         d.addCallback(lambda ign: _checkv("large-unrecoverable",
1045                                          self.check_is_unrecoverable))
1046
1047         return d
1048
1049     def do_deepcheck(self, ignored):
1050         d = defer.succeed(None)
1051
1052         # now deep-check the root, with various verify= and repair= options
1053         d.addCallback(lambda ign:
1054                       self.root.start_deep_check().when_done())
1055         def _check1(cr):
1056             self.failUnless(IDeepCheckResults.providedBy(cr))
1057             c = cr.get_counters()
1058             self.failUnlessEqual(c["count-objects-checked"], 9)
1059             self.failUnlessEqual(c["count-objects-healthy"], 5)
1060             self.failUnlessEqual(c["count-objects-unhealthy"], 4)
1061             self.failUnlessEqual(c["count-objects-unrecoverable"], 2)
1062         d.addCallback(_check1)
1063
1064         d.addCallback(lambda ign:
1065                       self.root.start_deep_check(verify=True).when_done())
1066         def _check2(cr):
1067             self.failUnless(IDeepCheckResults.providedBy(cr))
1068             c = cr.get_counters()
1069             self.failUnlessEqual(c["count-objects-checked"], 9)
1070             self.failUnlessEqual(c["count-objects-healthy"], 3)
1071             self.failUnlessEqual(c["count-objects-unhealthy"], 6)
1072             self.failUnlessEqual(c["count-objects-healthy"], 3) # root, mutable good, large good
1073             self.failUnlessEqual(c["count-objects-unrecoverable"], 2) # mutable unrecoverable, large unrecoverable
1074         d.addCallback(_check2)
1075
1076         return d
1077
1078     def json_is_healthy(self, data, where):
1079         r = data["results"]
1080         self.failUnless(r["healthy"], where)
1081         self.failUnless(r["recoverable"], where)
1082         self.failUnlessEqual(r["count-recoverable-versions"], 1, where)
1083         self.failUnlessEqual(r["count-unrecoverable-versions"], 0, where)
1084
1085     def json_is_missing_shares(self, data, where):
1086         r = data["results"]
1087         self.failIf(r["healthy"], where)
1088         self.failUnless(r["recoverable"], where)
1089         self.failUnlessEqual(r["count-recoverable-versions"], 1, where)
1090         self.failUnlessEqual(r["count-unrecoverable-versions"], 0, where)
1091
1092     def json_has_corrupt_shares(self, data, where):
1093         # by "corrupt-shares" we mean the file is still recoverable
1094         r = data["results"]
1095         self.failIf(r["healthy"], where)
1096         self.failUnless(r["recoverable"], where)
1097         self.failUnless(r["count-shares-good"] < 10, where)
1098         self.failUnless(r["count-corrupt-shares"], where)
1099         self.failUnless(r["list-corrupt-shares"], where)
1100
1101     def json_is_unrecoverable(self, data, where):
1102         r = data["results"]
1103         self.failIf(r["healthy"], where)
1104         self.failIf(r["recoverable"], where)
1105         self.failUnless(r["count-shares-good"] < r["count-shares-needed"],
1106                         where)
1107         self.failUnlessEqual(r["count-recoverable-versions"], 0, where)
1108         self.failUnlessEqual(r["count-unrecoverable-versions"], 1, where)
1109
1110     def do_test_web_bad(self, ignored):
1111         d = defer.succeed(None)
1112
1113         # check, no verify
1114         def _check(which, checker):
1115             d = self.web_json(self.nodes[which], t="check")
1116             d.addCallback(checker, which + "--webcheck")
1117             return d
1118
1119         d.addCallback(lambda ign: _check("mutable-good",
1120                                          self.json_is_healthy))
1121         d.addCallback(lambda ign: _check("mutable-missing-shares",
1122                                          self.json_is_missing_shares))
1123         d.addCallback(lambda ign: _check("mutable-corrupt-shares",
1124                                          self.json_is_healthy))
1125         d.addCallback(lambda ign: _check("mutable-unrecoverable",
1126                                          self.json_is_unrecoverable))
1127         d.addCallback(lambda ign: _check("large-good",
1128                                          self.json_is_healthy))
1129         d.addCallback(lambda ign: _check("large-missing-shares",
1130                                          self.json_is_missing_shares))
1131         d.addCallback(lambda ign: _check("large-corrupt-shares",
1132                                          self.json_is_healthy))
1133         d.addCallback(lambda ign: _check("large-unrecoverable",
1134                                          self.json_is_unrecoverable))
1135
1136         # check and verify
1137         def _checkv(which, checker):
1138             d = self.web_json(self.nodes[which], t="check", verify="true")
1139             d.addCallback(checker, which + "--webcheck-and-verify")
1140             return d
1141
1142         d.addCallback(lambda ign: _checkv("mutable-good",
1143                                           self.json_is_healthy))
1144         d.addCallback(lambda ign: _checkv("mutable-missing-shares",
1145                                          self.json_is_missing_shares))
1146         d.addCallback(lambda ign: _checkv("mutable-corrupt-shares",
1147                                          self.json_has_corrupt_shares))
1148         d.addCallback(lambda ign: _checkv("mutable-unrecoverable",
1149                                          self.json_is_unrecoverable))
1150         d.addCallback(lambda ign: _checkv("large-good",
1151                                           self.json_is_healthy))
1152         d.addCallback(lambda ign: _checkv("large-missing-shares", self.json_is_missing_shares))
1153         d.addCallback(lambda ign: _checkv("large-corrupt-shares", self.json_has_corrupt_shares))
1154         d.addCallback(lambda ign: _checkv("large-unrecoverable",
1155                                          self.json_is_unrecoverable))
1156
1157         return d