src/allmydata/test/test_deepcheck.py

   1
   2 import os, simplejson, urllib
   3 from cStringIO import StringIO
   4 from twisted.trial import unittest
   5 from twisted.internet import defer
   6 from twisted.internet import threads # CLI tests use deferToThread
   7 from allmydata.immutable import upload
   8 from allmydata.mutable.common import UnrecoverableFileError
   9 from allmydata.util import idlib
  10 from allmydata.util import base32
  11 from allmydata.scripts import runner
  12 from allmydata.interfaces import ICheckResults, ICheckAndRepairResults, \
  13      IDeepCheckResults, IDeepCheckAndRepairResults
  14 from allmydata.monitor import Monitor, OperationCancelledError
  15 from allmydata.uri import LiteralFileURI
  16 from twisted.web.client import getPage
  17
  18 from allmydata.test.common import ErrorMixin, _corrupt_mutable_share_data, \
  19      ShouldFailMixin
  20 from allmydata.test.common_util import StallMixin
  21 from allmydata.test.no_network import GridTestMixin
  22
  23 timeout = 2400 # One of these took 1046.091s on Zandr's ARM box.
  24
  25 class MutableChecker(GridTestMixin, unittest.TestCase, ErrorMixin):
  26     def _run_cli(self, argv):
  27         stdout, stderr = StringIO(), StringIO()
  28         # this can only do synchronous operations
  29         assert argv[0] == "debug"
  30         runner.runner(argv, run_by_human=False, stdout=stdout, stderr=stderr)
  31         return stdout.getvalue()
  32
  33     def test_good(self):
  34         self.basedir = "deepcheck/MutableChecker/good"
  35         self.set_up_grid()
  36         CONTENTS = "a little bit of data"
  37         d = self.g.clients[0].create_mutable_file(CONTENTS)
  38         def _created(node):
  39             self.node = node
  40             self.fileurl = "uri/" + urllib.quote(node.get_uri())
  41             si = self.node.get_storage_index()
  42         d.addCallback(_created)
  43         # now make sure the webapi verifier sees no problems
  44         d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=true",
  45                                            method="POST"))
  46         def _got_results(out):
  47             self.failUnless("<span>Healthy : Healthy</span>" in out, out)
  48             self.failUnless("Recoverable Versions: 10*seq1-" in out, out)
  49             self.failIf("Not Healthy!" in out, out)
  50             self.failIf("Unhealthy" in out, out)
  51             self.failIf("Corrupt Shares" in out, out)
  52         d.addCallback(_got_results)
  53         d.addErrback(self.explain_web_error)
  54         return d
  55
  56     def test_corrupt(self):
  57         self.basedir = "deepcheck/MutableChecker/corrupt"
  58         self.set_up_grid()
  59         CONTENTS = "a little bit of data"
  60         d = self.g.clients[0].create_mutable_file(CONTENTS)
  61         def _stash_and_corrupt(node):
  62             self.node = node
  63             self.fileurl = "uri/" + urllib.quote(node.get_uri())
  64             self.corrupt_shares_numbered(node.get_uri(), [0],
  65                                          _corrupt_mutable_share_data)
  66         d.addCallback(_stash_and_corrupt)
  67         # now make sure the webapi verifier notices it
  68         d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=true",
  69                                            method="POST"))
  70         def _got_results(out):
  71             self.failUnless("Not Healthy!" in out, out)
  72             self.failUnless("Unhealthy: best version has only 9 shares (encoding is 3-of-10)" in out, out)
  73             self.failUnless("Corrupt Shares:" in out, out)
  74         d.addCallback(_got_results)
  75
  76         # now make sure the webapi repairer can fix it
  77         d.addCallback(lambda ign:
  78                       self.GET(self.fileurl+"?t=check&verify=true&repair=true",
  79                                method="POST"))
  80         def _got_repair_results(out):
  81             self.failUnless("<div>Repair successful</div>" in out, out)
  82         d.addCallback(_got_repair_results)
  83         d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=true",
  84                                            method="POST"))
  85         def _got_postrepair_results(out):
  86             self.failIf("Not Healthy!" in out, out)
  87             self.failUnless("Recoverable Versions: 10*seq" in out, out)
  88         d.addCallback(_got_postrepair_results)
  89         d.addErrback(self.explain_web_error)
  90
  91         return d
  92
  93     def test_delete_share(self):
  94         self.basedir = "deepcheck/MutableChecker/delete_share"
  95         self.set_up_grid()
  96         CONTENTS = "a little bit of data"
  97         d = self.g.clients[0].create_mutable_file(CONTENTS)
  98         def _stash_and_delete(node):
  99             self.node = node
 100             self.fileurl = "uri/" + urllib.quote(node.get_uri())
 101             self.delete_shares_numbered(node.get_uri(), [0])
 102         d.addCallback(_stash_and_delete)
 103         # now make sure the webapi checker notices it
 104         d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=false",
 105                                            method="POST"))
 106         def _got_results(out):
 107             self.failUnless("Not Healthy!" in out, out)
 108             self.failUnless("Unhealthy: best version has only 9 shares (encoding is 3-of-10)" in out, out)
 109             self.failIf("Corrupt Shares" in out, out)
 110         d.addCallback(_got_results)
 111
 112         # now make sure the webapi repairer can fix it
 113         d.addCallback(lambda ign:
 114                       self.GET(self.fileurl+"?t=check&verify=false&repair=true",
 115                                method="POST"))
 116         def _got_repair_results(out):
 117             self.failUnless("Repair successful" in out)
 118         d.addCallback(_got_repair_results)
 119         d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=false",
 120                                            method="POST"))
 121         def _got_postrepair_results(out):
 122             self.failIf("Not Healthy!" in out, out)
 123             self.failUnless("Recoverable Versions: 10*seq" in out)
 124         d.addCallback(_got_postrepair_results)
 125         d.addErrback(self.explain_web_error)
 126
 127         return d
 128
 129
 130 class DeepCheckBase(GridTestMixin, ErrorMixin, StallMixin, ShouldFailMixin):
 131
 132     def web_json(self, n, **kwargs):
 133         kwargs["output"] = "json"
 134         d = self.web(n, "POST", **kwargs)
 135         d.addCallback(self.decode_json)
 136         return d
 137
 138     def decode_json(self, (s,url)):
 139         try:
 140             data = simplejson.loads(s)
 141         except ValueError:
 142             self.fail("%s: not JSON: '%s'" % (url, s))
 143         return data
 144
 145     def parse_streamed_json(self, s):
 146         for unit in s.split("\n"):
 147             if not unit:
 148                 # stream should end with a newline, so split returns ""
 149                 continue
 150             yield simplejson.loads(unit)
 151
 152     def web(self, n, method="GET", **kwargs):
 153         # returns (data, url)
 154         url = (self.client_baseurls[0] + "uri/%s" % urllib.quote(n.get_uri())
 155                + "?" + "&".join(["%s=%s" % (k,v) for (k,v) in kwargs.items()]))
 156         d = getPage(url, method=method)
 157         d.addCallback(lambda data: (data,url))
 158         return d
 159
 160     def wait_for_operation(self, ignored, ophandle):
 161         url = self.client_baseurls[0] + "operations/" + ophandle
 162         url += "?t=status&output=JSON"
 163         d = getPage(url)
 164         def _got(res):
 165             try:
 166                 data = simplejson.loads(res)
 167             except ValueError:
 168                 self.fail("%s: not JSON: '%s'" % (url, res))
 169             if not data["finished"]:
 170                 d = self.stall(delay=1.0)
 171                 d.addCallback(self.wait_for_operation, ophandle)
 172                 return d
 173             return data
 174         d.addCallback(_got)
 175         return d
 176
 177     def get_operation_results(self, ignored, ophandle, output=None):
 178         url = self.client_baseurls[0] + "operations/" + ophandle
 179         url += "?t=status"
 180         if output:
 181             url += "&output=" + output
 182         d = getPage(url)
 183         def _got(res):
 184             if output and output.lower() == "json":
 185                 try:
 186                     return simplejson.loads(res)
 187                 except ValueError:
 188                     self.fail("%s: not JSON: '%s'" % (url, res))
 189             return res
 190         d.addCallback(_got)
 191         return d
 192
 193     def slow_web(self, n, output=None, **kwargs):
 194         # use ophandle=
 195         handle = base32.b2a(os.urandom(4))
 196         d = self.web(n, "POST", ophandle=handle, **kwargs)
 197         d.addCallback(self.wait_for_operation, handle)
 198         d.addCallback(self.get_operation_results, handle, output=output)
 199         return d
 200
 201
 202 class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
 203     # construct a small directory tree (with one dir, one immutable file, one
 204     # mutable file, one LIT file, and a loop), and then check/examine it in
 205     # various ways.
 206
 207     def set_up_tree(self):
 208         # 2.9s
 209
 210         # root
 211         #   mutable
 212         #   large
 213         #   small
 214         #   small2
 215         #   loop -> root
 216         c0 = self.g.clients[0]
 217         d = c0.create_dirnode()
 218         def _created_root(n):
 219             self.root = n
 220             self.root_uri = n.get_uri()
 221         d.addCallback(_created_root)
 222         d.addCallback(lambda ign: c0.create_mutable_file("mutable file contents"))
 223         d.addCallback(lambda n: self.root.set_node(u"mutable", n))
 224         def _created_mutable(n):
 225             self.mutable = n
 226             self.mutable_uri = n.get_uri()
 227         d.addCallback(_created_mutable)
 228
 229         large = upload.Data("Lots of data\n" * 1000, None)
 230         d.addCallback(lambda ign: self.root.add_file(u"large", large))
 231         def _created_large(n):
 232             self.large = n
 233             self.large_uri = n.get_uri()
 234         d.addCallback(_created_large)
 235
 236         small = upload.Data("Small enough for a LIT", None)
 237         d.addCallback(lambda ign: self.root.add_file(u"small", small))
 238         def _created_small(n):
 239             self.small = n
 240             self.small_uri = n.get_uri()
 241         d.addCallback(_created_small)
 242
 243         small2 = upload.Data("Small enough for a LIT too", None)
 244         d.addCallback(lambda ign: self.root.add_file(u"small2", small2))
 245         def _created_small2(n):
 246             self.small2 = n
 247             self.small2_uri = n.get_uri()
 248         d.addCallback(_created_small2)
 249
 250         d.addCallback(lambda ign: self.root.set_node(u"loop", self.root))
 251         return d
 252
 253     def check_is_healthy(self, cr, n, where, incomplete=False):
 254         self.failUnless(ICheckResults.providedBy(cr), where)
 255         self.failUnless(cr.is_healthy(), where)
 256         self.failUnlessEqual(cr.get_storage_index(), n.get_storage_index(),
 257                              where)
 258         self.failUnlessEqual(cr.get_storage_index_string(),
 259                              base32.b2a(n.get_storage_index()), where)
 260         num_servers = len(self.g.all_servers)
 261         needs_rebalancing = bool( num_servers < 10 )
 262         if not incomplete:
 263             self.failUnlessEqual(cr.needs_rebalancing(), needs_rebalancing,
 264                                  str((where, cr, cr.get_data())))
 265         d = cr.get_data()
 266         self.failUnlessEqual(d["count-shares-good"], 10, where)
 267         self.failUnlessEqual(d["count-shares-needed"], 3, where)
 268         self.failUnlessEqual(d["count-shares-expected"], 10, where)
 269         if not incomplete:
 270             self.failUnlessEqual(d["count-good-share-hosts"], num_servers,
 271                                  where)
 272         self.failUnlessEqual(d["count-corrupt-shares"], 0, where)
 273         self.failUnlessEqual(d["list-corrupt-shares"], [], where)
 274         if not incomplete:
 275             self.failUnlessEqual(sorted(d["servers-responding"]),
 276                                  sorted(self.g.servers_by_id.keys()),
 277                                  where)
 278             self.failUnless("sharemap" in d, str((where, d)))
 279             all_serverids = set()
 280             for (shareid, serverids) in d["sharemap"].items():
 281                 all_serverids.update(serverids)
 282             self.failUnlessEqual(sorted(all_serverids),
 283                                  sorted(self.g.servers_by_id.keys()),
 284                                  where)
 285
 286         self.failUnlessEqual(d["count-wrong-shares"], 0, where)
 287         self.failUnlessEqual(d["count-recoverable-versions"], 1, where)
 288         self.failUnlessEqual(d["count-unrecoverable-versions"], 0, where)
 289
 290
 291     def check_and_repair_is_healthy(self, cr, n, where, incomplete=False):
 292         self.failUnless(ICheckAndRepairResults.providedBy(cr), (where, cr))
 293         self.failUnless(cr.get_pre_repair_results().is_healthy(), where)
 294         self.check_is_healthy(cr.get_pre_repair_results(), n, where, incomplete)
 295         self.failUnless(cr.get_post_repair_results().is_healthy(), where)
 296         self.check_is_healthy(cr.get_post_repair_results(), n, where, incomplete)
 297         self.failIf(cr.get_repair_attempted(), where)
 298
 299     def deep_check_is_healthy(self, cr, num_healthy, where):
 300         self.failUnless(IDeepCheckResults.providedBy(cr))
 301         self.failUnlessEqual(cr.get_counters()["count-objects-healthy"],
 302                              num_healthy, where)
 303
 304     def deep_check_and_repair_is_healthy(self, cr, num_healthy, where):
 305         self.failUnless(IDeepCheckAndRepairResults.providedBy(cr), where)
 306         c = cr.get_counters()
 307         self.failUnlessEqual(c["count-objects-healthy-pre-repair"],
 308                              num_healthy, where)
 309         self.failUnlessEqual(c["count-objects-healthy-post-repair"],
 310                              num_healthy, where)
 311         self.failUnlessEqual(c["count-repairs-attempted"], 0, where)
 312
 313     def test_good(self):
 314         self.basedir = "deepcheck/DeepCheckWebGood/good"
 315         self.set_up_grid()
 316         d = self.set_up_tree()
 317         d.addCallback(self.do_stats)
 318         d.addCallback(self.do_web_stream_manifest)
 319         d.addCallback(self.do_web_stream_check)
 320         d.addCallback(self.do_test_check_good)
 321         d.addCallback(self.do_test_web_good)
 322         d.addCallback(self.do_test_cli_good)
 323         d.addErrback(self.explain_web_error)
 324         d.addErrback(self.explain_error)
 325         return d
 326
 327     def do_stats(self, ignored):
 328         d = defer.succeed(None)
 329         d.addCallback(lambda ign: self.root.start_deep_stats().when_done())
 330         d.addCallback(self.check_stats_good)
 331         return d
 332
 333     def check_stats_good(self, s):
 334         self.failUnlessEqual(s["count-directories"], 1)
 335         self.failUnlessEqual(s["count-files"], 4)
 336         self.failUnlessEqual(s["count-immutable-files"], 1)
 337         self.failUnlessEqual(s["count-literal-files"], 2)
 338         self.failUnlessEqual(s["count-mutable-files"], 1)
 339         # don't check directories: their size will vary
 340         # s["largest-directory"]
 341         # s["size-directories"]
 342         self.failUnlessEqual(s["largest-directory-children"], 5)
 343         self.failUnlessEqual(s["largest-immutable-file"], 13000)
 344         # to re-use this function for both the local
 345         # dirnode.start_deep_stats() and the webapi t=start-deep-stats, we
 346         # coerce the result into a list of tuples. dirnode.start_deep_stats()
 347         # returns a list of tuples, but JSON only knows about lists., so
 348         # t=start-deep-stats returns a list of lists.
 349         histogram = [tuple(stuff) for stuff in s["size-files-histogram"]]
 350         self.failUnlessEqual(histogram, [(11, 31, 2),
 351                                          (10001, 31622, 1),
 352                                          ])
 353         self.failUnlessEqual(s["size-immutable-files"], 13000)
 354         self.failUnlessEqual(s["size-literal-files"], 48)
 355
 356     def do_web_stream_manifest(self, ignored):
 357         d = self.web(self.root, method="POST", t="stream-manifest")
 358         d.addCallback(lambda (output,url):
 359                       self._check_streamed_manifest(output))
 360         return d
 361
 362     def _check_streamed_manifest(self, output):
 363         units = list(self.parse_streamed_json(output))
 364         files = [u for u in units if u["type"] in ("file", "directory")]
 365         assert units[-1]["type"] == "stats"
 366         stats = units[-1]["stats"]
 367         self.failUnlessEqual(len(files), 5)
 368         # [root,mutable,large] are distributed, [small,small2] are not
 369         self.failUnlessEqual(len([f for f in files
 370                                   if f["verifycap"] is not None]), 3)
 371         self.failUnlessEqual(len([f for f in files
 372                                   if f["verifycap"] is None]), 2)
 373         self.failUnlessEqual(len([f for f in files
 374                                   if f["repaircap"] is not None]), 3)
 375         self.failUnlessEqual(len([f for f in files
 376                                   if f["repaircap"] is None]), 2)
 377         self.failUnlessEqual(len([f for f in files
 378                                   if f["storage-index"] is not None]), 3)
 379         self.failUnlessEqual(len([f for f in files
 380                                   if f["storage-index"] is None]), 2)
 381         # make sure that a mutable file has filecap==repaircap!=verifycap
 382         mutable = [f for f in files
 383                    if f["cap"] is not None
 384                    and f["cap"].startswith("URI:SSK:")][0]
 385         self.failUnlessEqual(mutable["cap"], self.mutable_uri)
 386         self.failIfEqual(mutable["cap"], mutable["verifycap"])
 387         self.failUnlessEqual(mutable["cap"], mutable["repaircap"])
 388         # for immutable file, verifycap==repaircap!=filecap
 389         large = [f for f in files
 390                    if f["cap"] is not None
 391                    and f["cap"].startswith("URI:CHK:")][0]
 392         self.failUnlessEqual(large["cap"], self.large_uri)
 393         self.failIfEqual(large["cap"], large["verifycap"])
 394         self.failUnlessEqual(large["verifycap"], large["repaircap"])
 395         self.check_stats_good(stats)
 396
 397     def do_web_stream_check(self, ignored):
 398         return
 399         d = self.web(self.root, t="stream-deep-check")
 400         def _check(res):
 401             units = list(self.parse_streamed_json(res))
 402             files = [u for u in units if u["type"] in ("file", "directory")]
 403             assert units[-1]["type"] == "stats"
 404             stats = units[-1]["stats"]
 405             # ...
 406         d.addCallback(_check)
 407         return d
 408
 409     def do_test_check_good(self, ignored):
 410         d = defer.succeed(None)
 411         # check the individual items
 412         d.addCallback(lambda ign: self.root.check(Monitor()))
 413         d.addCallback(self.check_is_healthy, self.root, "root")
 414         d.addCallback(lambda ign: self.mutable.check(Monitor()))
 415         d.addCallback(self.check_is_healthy, self.mutable, "mutable")
 416         d.addCallback(lambda ign: self.large.check(Monitor()))
 417         d.addCallback(self.check_is_healthy, self.large, "large")
 418         d.addCallback(lambda ign: self.small.check(Monitor()))
 419         d.addCallback(self.failUnlessEqual, None, "small")
 420         d.addCallback(lambda ign: self.small2.check(Monitor()))
 421         d.addCallback(self.failUnlessEqual, None, "small2")
 422
 423         # and again with verify=True
 424         d.addCallback(lambda ign: self.root.check(Monitor(), verify=True))
 425         d.addCallback(self.check_is_healthy, self.root, "root")
 426         d.addCallback(lambda ign: self.mutable.check(Monitor(), verify=True))
 427         d.addCallback(self.check_is_healthy, self.mutable, "mutable")
 428         d.addCallback(lambda ign: self.large.check(Monitor(), verify=True))
 429         d.addCallback(self.check_is_healthy, self.large, "large", incomplete=True)
 430         d.addCallback(lambda ign: self.small.check(Monitor(), verify=True))
 431         d.addCallback(self.failUnlessEqual, None, "small")
 432         d.addCallback(lambda ign: self.small2.check(Monitor(), verify=True))
 433         d.addCallback(self.failUnlessEqual, None, "small2")
 434
 435         # and check_and_repair(), which should be a nop
 436         d.addCallback(lambda ign: self.root.check_and_repair(Monitor()))
 437         d.addCallback(self.check_and_repair_is_healthy, self.root, "root")
 438         d.addCallback(lambda ign: self.mutable.check_and_repair(Monitor()))
 439         d.addCallback(self.check_and_repair_is_healthy, self.mutable, "mutable")
 440         #TODO d.addCallback(lambda ign: self.large.check_and_repair(Monitor()))
 441         #TODO d.addCallback(self.check_and_repair_is_healthy, self.large, "large")
 442         #TODO d.addCallback(lambda ign: self.small.check_and_repair(Monitor()))
 443         #TODO d.addCallback(self.failUnlessEqual, None, "small")
 444         #TODO d.addCallback(lambda ign: self.small2.check_and_repair(Monitor()))
 445         #TODO d.addCallback(self.failUnlessEqual, None, "small2")
 446
 447         # check_and_repair(verify=True)
 448         d.addCallback(lambda ign: self.root.check_and_repair(Monitor(), verify=True))
 449         d.addCallback(self.check_and_repair_is_healthy, self.root, "root")
 450         d.addCallback(lambda ign: self.mutable.check_and_repair(Monitor(), verify=True))
 451         d.addCallback(self.check_and_repair_is_healthy, self.mutable, "mutable")
 452         #TODO d.addCallback(lambda ign: self.large.check_and_repair(Monitor(), verify=True))
 453         #TODO d.addCallback(self.check_and_repair_is_healthy, self.large, "large",
 454         #TODO               incomplete=True)
 455         #TODO d.addCallback(lambda ign: self.small.check_and_repair(Monitor(), verify=True))
 456         #TODO d.addCallback(self.failUnlessEqual, None, "small")
 457         #TODO d.addCallback(lambda ign: self.small2.check_and_repair(Monitor(), verify=True))
 458         #TODO d.addCallback(self.failUnlessEqual, None, "small2")
 459
 460
 461         # now deep-check the root, with various verify= and repair= options
 462         d.addCallback(lambda ign:
 463                       self.root.start_deep_check().when_done())
 464         d.addCallback(self.deep_check_is_healthy, 3, "root")
 465         d.addCallback(lambda ign:
 466                       self.root.start_deep_check(verify=True).when_done())
 467         d.addCallback(self.deep_check_is_healthy, 3, "root")
 468         d.addCallback(lambda ign:
 469                       self.root.start_deep_check_and_repair().when_done())
 470         d.addCallback(self.deep_check_and_repair_is_healthy, 3, "root")
 471         d.addCallback(lambda ign:
 472                       self.root.start_deep_check_and_repair(verify=True).when_done())
 473         d.addCallback(self.deep_check_and_repair_is_healthy, 3, "root")
 474
 475         # and finally, start a deep-check, but then cancel it.
 476         d.addCallback(lambda ign: self.root.start_deep_check())
 477         def _checking(monitor):
 478             monitor.cancel()
 479             d = monitor.when_done()
 480             # this should fire as soon as the next dirnode.list finishes.
 481             # TODO: add a counter to measure how many list() calls are made,
 482             # assert that no more than one gets to run before the cancel()
 483             # takes effect.
 484             def _finished_normally(res):
 485                 self.fail("this was supposed to fail, not finish normally")
 486             def _cancelled(f):
 487                 f.trap(OperationCancelledError)
 488             d.addCallbacks(_finished_normally, _cancelled)
 489             return d
 490         d.addCallback(_checking)
 491
 492         return d
 493
 494     def json_check_is_healthy(self, data, n, where, incomplete=False):
 495
 496         self.failUnlessEqual(data["storage-index"],
 497                              base32.b2a(n.get_storage_index()), where)
 498         self.failUnless("summary" in data, (where, data))
 499         self.failUnlessEqual(data["summary"].lower(), "healthy",
 500                              "%s: '%s'" % (where, data["summary"]))
 501         r = data["results"]
 502         self.failUnlessEqual(r["healthy"], True, where)
 503         num_servers = len(self.g.all_servers)
 504         needs_rebalancing = bool( num_servers < 10 )
 505         if not incomplete:
 506             self.failUnlessEqual(r["needs-rebalancing"], needs_rebalancing,
 507                                  where)
 508         self.failUnlessEqual(r["count-shares-good"], 10, where)
 509         self.failUnlessEqual(r["count-shares-needed"], 3, where)
 510         self.failUnlessEqual(r["count-shares-expected"], 10, where)
 511         if not incomplete:
 512             self.failUnlessEqual(r["count-good-share-hosts"], num_servers,
 513                                  where)
 514         self.failUnlessEqual(r["count-corrupt-shares"], 0, where)
 515         self.failUnlessEqual(r["list-corrupt-shares"], [], where)
 516         if not incomplete:
 517             self.failUnlessEqual(sorted(r["servers-responding"]),
 518                                  sorted([idlib.nodeid_b2a(sid)
 519                                          for sid in self.g.servers_by_id]),
 520                                  where)
 521             self.failUnless("sharemap" in r, where)
 522             all_serverids = set()
 523             for (shareid, serverids_s) in r["sharemap"].items():
 524                 all_serverids.update(serverids_s)
 525             self.failUnlessEqual(sorted(all_serverids),
 526                                  sorted([idlib.nodeid_b2a(sid)
 527                                          for sid in self.g.servers_by_id]),
 528                                  where)
 529         self.failUnlessEqual(r["count-wrong-shares"], 0, where)
 530         self.failUnlessEqual(r["count-recoverable-versions"], 1, where)
 531         self.failUnlessEqual(r["count-unrecoverable-versions"], 0, where)
 532
 533     def json_check_and_repair_is_healthy(self, data, n, where, incomplete=False):
 534         self.failUnlessEqual(data["storage-index"],
 535                              base32.b2a(n.get_storage_index()), where)
 536         self.failUnlessEqual(data["repair-attempted"], False, where)
 537         self.json_check_is_healthy(data["pre-repair-results"],
 538                                    n, where, incomplete)
 539         self.json_check_is_healthy(data["post-repair-results"],
 540                                    n, where, incomplete)
 541
 542     def json_full_deepcheck_is_healthy(self, data, n, where):
 543         self.failUnlessEqual(data["root-storage-index"],
 544                              base32.b2a(n.get_storage_index()), where)
 545         self.failUnlessEqual(data["count-objects-checked"], 3, where)
 546         self.failUnlessEqual(data["count-objects-healthy"], 3, where)
 547         self.failUnlessEqual(data["count-objects-unhealthy"], 0, where)
 548         self.failUnlessEqual(data["count-corrupt-shares"], 0, where)
 549         self.failUnlessEqual(data["list-corrupt-shares"], [], where)
 550         self.failUnlessEqual(data["list-unhealthy-files"], [], where)
 551         self.json_check_stats_good(data["stats"], where)
 552
 553     def json_full_deepcheck_and_repair_is_healthy(self, data, n, where):
 554         self.failUnlessEqual(data["root-storage-index"],
 555                              base32.b2a(n.get_storage_index()), where)
 556         self.failUnlessEqual(data["count-objects-checked"], 3, where)
 557
 558         self.failUnlessEqual(data["count-objects-healthy-pre-repair"], 3, where)
 559         self.failUnlessEqual(data["count-objects-unhealthy-pre-repair"], 0, where)
 560         self.failUnlessEqual(data["count-corrupt-shares-pre-repair"], 0, where)
 561
 562         self.failUnlessEqual(data["count-objects-healthy-post-repair"], 3, where)
 563         self.failUnlessEqual(data["count-objects-unhealthy-post-repair"], 0, where)
 564         self.failUnlessEqual(data["count-corrupt-shares-post-repair"], 0, where)
 565
 566         self.failUnlessEqual(data["list-corrupt-shares"], [], where)
 567         self.failUnlessEqual(data["list-remaining-corrupt-shares"], [], where)
 568         self.failUnlessEqual(data["list-unhealthy-files"], [], where)
 569
 570         self.failUnlessEqual(data["count-repairs-attempted"], 0, where)
 571         self.failUnlessEqual(data["count-repairs-successful"], 0, where)
 572         self.failUnlessEqual(data["count-repairs-unsuccessful"], 0, where)
 573
 574
 575     def json_check_lit(self, data, n, where):
 576         self.failUnlessEqual(data["storage-index"], "", where)
 577         self.failUnlessEqual(data["results"]["healthy"], True, where)
 578
 579     def json_check_stats_good(self, data, where):
 580         self.check_stats_good(data)
 581
 582     def do_test_web_good(self, ignored):
 583         d = defer.succeed(None)
 584
 585         # stats
 586         d.addCallback(lambda ign:
 587                       self.slow_web(self.root,
 588                                     t="start-deep-stats", output="json"))
 589         d.addCallback(self.json_check_stats_good, "deep-stats")
 590
 591         # check, no verify
 592         d.addCallback(lambda ign: self.web_json(self.root, t="check"))
 593         d.addCallback(self.json_check_is_healthy, self.root, "root")
 594         d.addCallback(lambda ign: self.web_json(self.mutable, t="check"))
 595         d.addCallback(self.json_check_is_healthy, self.mutable, "mutable")
 596         d.addCallback(lambda ign: self.web_json(self.large, t="check"))
 597         d.addCallback(self.json_check_is_healthy, self.large, "large")
 598         d.addCallback(lambda ign: self.web_json(self.small, t="check"))
 599         d.addCallback(self.json_check_lit, self.small, "small")
 600         d.addCallback(lambda ign: self.web_json(self.small2, t="check"))
 601         d.addCallback(self.json_check_lit, self.small2, "small2")
 602
 603         # check and verify
 604         d.addCallback(lambda ign:
 605                       self.web_json(self.root, t="check", verify="true"))
 606         d.addCallback(self.json_check_is_healthy, self.root, "root+v")
 607         d.addCallback(lambda ign:
 608                       self.web_json(self.mutable, t="check", verify="true"))
 609         d.addCallback(self.json_check_is_healthy, self.mutable, "mutable+v")
 610         d.addCallback(lambda ign:
 611                       self.web_json(self.large, t="check", verify="true"))
 612         d.addCallback(self.json_check_is_healthy, self.large, "large+v",
 613                       incomplete=True)
 614         d.addCallback(lambda ign:
 615                       self.web_json(self.small, t="check", verify="true"))
 616         d.addCallback(self.json_check_lit, self.small, "small+v")
 617         d.addCallback(lambda ign:
 618                       self.web_json(self.small2, t="check", verify="true"))
 619         d.addCallback(self.json_check_lit, self.small2, "small2+v")
 620
 621         # check and repair, no verify
 622         d.addCallback(lambda ign:
 623                       self.web_json(self.root, t="check", repair="true"))
 624         d.addCallback(self.json_check_and_repair_is_healthy, self.root, "root+r")
 625         d.addCallback(lambda ign:
 626                       self.web_json(self.mutable, t="check", repair="true"))
 627         d.addCallback(self.json_check_and_repair_is_healthy, self.mutable, "mutable+r")
 628         d.addCallback(lambda ign:
 629                       self.web_json(self.large, t="check", repair="true"))
 630         d.addCallback(self.json_check_and_repair_is_healthy, self.large, "large+r")
 631         d.addCallback(lambda ign:
 632                       self.web_json(self.small, t="check", repair="true"))
 633         d.addCallback(self.json_check_lit, self.small, "small+r")
 634         d.addCallback(lambda ign:
 635                       self.web_json(self.small2, t="check", repair="true"))
 636         d.addCallback(self.json_check_lit, self.small2, "small2+r")
 637
 638         # check+verify+repair
 639         d.addCallback(lambda ign:
 640                       self.web_json(self.root, t="check", repair="true", verify="true"))
 641         d.addCallback(self.json_check_and_repair_is_healthy, self.root, "root+vr")
 642         d.addCallback(lambda ign:
 643                       self.web_json(self.mutable, t="check", repair="true", verify="true"))
 644         d.addCallback(self.json_check_and_repair_is_healthy, self.mutable, "mutable+vr")
 645         d.addCallback(lambda ign:
 646                       self.web_json(self.large, t="check", repair="true", verify="true"))
 647         d.addCallback(self.json_check_and_repair_is_healthy, self.large, "large+vr", incomplete=True)
 648         d.addCallback(lambda ign:
 649                       self.web_json(self.small, t="check", repair="true", verify="true"))
 650         d.addCallback(self.json_check_lit, self.small, "small+vr")
 651         d.addCallback(lambda ign:
 652                       self.web_json(self.small2, t="check", repair="true", verify="true"))
 653         d.addCallback(self.json_check_lit, self.small2, "small2+vr")
 654
 655         # now run a deep-check, with various verify= and repair= flags
 656         d.addCallback(lambda ign:
 657                       self.slow_web(self.root, t="start-deep-check", output="json"))
 658         d.addCallback(self.json_full_deepcheck_is_healthy, self.root, "root+d")
 659         d.addCallback(lambda ign:
 660                       self.slow_web(self.root, t="start-deep-check", verify="true",
 661                                     output="json"))
 662         d.addCallback(self.json_full_deepcheck_is_healthy, self.root, "root+dv")
 663         d.addCallback(lambda ign:
 664                       self.slow_web(self.root, t="start-deep-check", repair="true",
 665                                     output="json"))
 666         d.addCallback(self.json_full_deepcheck_and_repair_is_healthy, self.root, "root+dr")
 667         d.addCallback(lambda ign:
 668                       self.slow_web(self.root, t="start-deep-check", verify="true", repair="true", output="json"))
 669         d.addCallback(self.json_full_deepcheck_and_repair_is_healthy, self.root, "root+dvr")
 670
 671         # now look at t=info
 672         d.addCallback(lambda ign: self.web(self.root, t="info"))
 673         # TODO: examine the output
 674         d.addCallback(lambda ign: self.web(self.mutable, t="info"))
 675         d.addCallback(lambda ign: self.web(self.large, t="info"))
 676         d.addCallback(lambda ign: self.web(self.small, t="info"))
 677         d.addCallback(lambda ign: self.web(self.small2, t="info"))
 678
 679         return d
 680
 681     def _run_cli(self, argv, stdin=""):
 682         #print "CLI:", argv
 683         stdout, stderr = StringIO(), StringIO()
 684         d = threads.deferToThread(runner.runner, argv, run_by_human=False,
 685                                   stdin=StringIO(stdin),
 686                                   stdout=stdout, stderr=stderr)
 687         def _done(res):
 688             return stdout.getvalue(), stderr.getvalue()
 689         d.addCallback(_done)
 690         return d
 691
 692     def do_test_cli_good(self, ignored):
 693         d = defer.succeed(None)
 694         d.addCallback(lambda ign: self.do_cli_manifest_stream1())
 695         d.addCallback(lambda ign: self.do_cli_manifest_stream2())
 696         d.addCallback(lambda ign: self.do_cli_manifest_stream3())
 697         d.addCallback(lambda ign: self.do_cli_manifest_stream4())
 698         d.addCallback(lambda ign: self.do_cli_manifest_stream5())
 699         d.addCallback(lambda ign: self.do_cli_stats1())
 700         d.addCallback(lambda ign: self.do_cli_stats2())
 701         return d
 702
 703     def _check_manifest_storage_index(self, out):
 704         lines = [l for l in out.split("\n") if l]
 705         self.failUnlessEqual(len(lines), 3)
 706         self.failUnless(base32.b2a(self.root.get_storage_index()) in lines)
 707         self.failUnless(base32.b2a(self.mutable.get_storage_index()) in lines)
 708         self.failUnless(base32.b2a(self.large.get_storage_index()) in lines)
 709
 710     def do_cli_manifest_stream1(self):
 711         basedir = self.get_clientdir(0)
 712         d = self._run_cli(["manifest",
 713                            "--node-directory", basedir,
 714                            self.root_uri])
 715         def _check((out,err)):
 716             self.failUnlessEqual(err, "")
 717             lines = [l for l in out.split("\n") if l]
 718             self.failUnlessEqual(len(lines), 5)
 719             caps = {}
 720             for l in lines:
 721                 try:
 722                     cap, path = l.split(None, 1)
 723                 except ValueError:
 724                     cap = l.strip()
 725                     path = ""
 726                 caps[cap] = path
 727             self.failUnless(self.root.get_uri() in caps)
 728             self.failUnlessEqual(caps[self.root.get_uri()], "")
 729             self.failUnlessEqual(caps[self.mutable.get_uri()], "mutable")
 730             self.failUnlessEqual(caps[self.large.get_uri()], "large")
 731             self.failUnlessEqual(caps[self.small.get_uri()], "small")
 732             self.failUnlessEqual(caps[self.small2.get_uri()], "small2")
 733         d.addCallback(_check)
 734         return d
 735
 736     def do_cli_manifest_stream2(self):
 737         basedir = self.get_clientdir(0)
 738         d = self._run_cli(["manifest",
 739                            "--node-directory", basedir,
 740                            "--raw",
 741                            self.root_uri])
 742         def _check((out,err)):
 743             self.failUnlessEqual(err, "")
 744             # this should be the same as the POST t=stream-manifest output
 745             self._check_streamed_manifest(out)
 746         d.addCallback(_check)
 747         return d
 748
 749     def do_cli_manifest_stream3(self):
 750         basedir = self.get_clientdir(0)
 751         d = self._run_cli(["manifest",
 752                            "--node-directory", basedir,
 753                            "--storage-index",
 754                            self.root_uri])
 755         def _check((out,err)):
 756             self.failUnlessEqual(err, "")
 757             self._check_manifest_storage_index(out)
 758         d.addCallback(_check)
 759         return d
 760
 761     def do_cli_manifest_stream4(self):
 762         basedir = self.get_clientdir(0)
 763         d = self._run_cli(["manifest",
 764                            "--node-directory", basedir,
 765                            "--verify-cap",
 766                            self.root_uri])
 767         def _check((out,err)):
 768             self.failUnlessEqual(err, "")
 769             lines = [l for l in out.split("\n") if l]
 770             self.failUnlessEqual(len(lines), 3)
 771             self.failUnless(self.root.get_verify_cap().to_string() in lines)
 772             self.failUnless(self.mutable.get_verify_cap().to_string() in lines)
 773             self.failUnless(self.large.get_verify_cap().to_string() in lines)
 774         d.addCallback(_check)
 775         return d
 776
 777     def do_cli_manifest_stream5(self):
 778         basedir = self.get_clientdir(0)
 779         d = self._run_cli(["manifest",
 780                            "--node-directory", basedir,
 781                            "--repair-cap",
 782                            self.root_uri])
 783         def _check((out,err)):
 784             self.failUnlessEqual(err, "")
 785             lines = [l for l in out.split("\n") if l]
 786             self.failUnlessEqual(len(lines), 3)
 787             self.failUnless(self.root.get_repair_cap().to_string() in lines)
 788             self.failUnless(self.mutable.get_repair_cap().to_string() in lines)
 789             self.failUnless(self.large.get_repair_cap().to_string() in lines)
 790         d.addCallback(_check)
 791         return d
 792
 793     def do_cli_stats1(self):
 794         basedir = self.get_clientdir(0)
 795         d = self._run_cli(["stats",
 796                            "--node-directory", basedir,
 797                            self.root_uri])
 798         def _check3((out,err)):
 799             lines = [l.strip() for l in out.split("\n") if l]
 800             self.failUnless("count-immutable-files: 1" in lines)
 801             self.failUnless("count-mutable-files: 1" in lines)
 802             self.failUnless("count-literal-files: 2" in lines)
 803             self.failUnless("count-files: 4" in lines)
 804             self.failUnless("count-directories: 1" in lines)
 805             self.failUnless("size-immutable-files: 13000    (13.00 kB, 12.70 kiB)" in lines, lines)
 806             self.failUnless("size-literal-files: 48" in lines)
 807             self.failUnless("   11-31    : 2    (31 B, 31 B)".strip() in lines)
 808             self.failUnless("10001-31622 : 1    (31.62 kB, 30.88 kiB)".strip() in lines)
 809         d.addCallback(_check3)
 810         return d
 811
 812     def do_cli_stats2(self):
 813         basedir = self.get_clientdir(0)
 814         d = self._run_cli(["stats",
 815                            "--node-directory", basedir,
 816                            "--raw",
 817                            self.root_uri])
 818         def _check4((out,err)):
 819             data = simplejson.loads(out)
 820             self.failUnlessEqual(data["count-immutable-files"], 1)
 821             self.failUnlessEqual(data["count-immutable-files"], 1)
 822             self.failUnlessEqual(data["count-mutable-files"], 1)
 823             self.failUnlessEqual(data["count-literal-files"], 2)
 824             self.failUnlessEqual(data["count-files"], 4)
 825             self.failUnlessEqual(data["count-directories"], 1)
 826             self.failUnlessEqual(data["size-immutable-files"], 13000)
 827             self.failUnlessEqual(data["size-literal-files"], 48)
 828             self.failUnless([11,31,2] in data["size-files-histogram"])
 829             self.failUnless([10001,31622,1] in data["size-files-histogram"])
 830         d.addCallback(_check4)
 831         return d
 832
 833
 834 class DeepCheckWebBad(DeepCheckBase, unittest.TestCase):
 835     def test_bad(self):
 836         self.basedir = "deepcheck/DeepCheckWebBad/bad"
 837         self.set_up_grid()
 838         d = self.set_up_damaged_tree()
 839         d.addCallback(self.do_check)
 840         d.addCallback(self.do_deepcheck)
 841         d.addCallback(self.do_deepcheck_broken)
 842         d.addCallback(self.do_test_web_bad)
 843         d.addErrback(self.explain_web_error)
 844         d.addErrback(self.explain_error)
 845         return d
 846
 847
 848
 849     def set_up_damaged_tree(self):
 850         # 6.4s
 851
 852         # root
 853         #   mutable-good
 854         #   mutable-missing-shares
 855         #   mutable-corrupt-shares
 856         #   mutable-unrecoverable
 857         #   large-good
 858         #   large-missing-shares
 859         #   large-corrupt-shares
 860         #   large-unrecoverable
 861         # broken
 862         #   large1-good
 863         #   subdir-good
 864         #     large2-good
 865         #   subdir-unrecoverable
 866         #     large3-good
 867
 868         self.nodes = {}
 869
 870         c0 = self.g.clients[0]
 871         d = c0.create_dirnode()
 872         def _created_root(n):
 873             self.root = n
 874             self.root_uri = n.get_uri()
 875         d.addCallback(_created_root)
 876         d.addCallback(self.create_mangled, "mutable-good")
 877         d.addCallback(self.create_mangled, "mutable-missing-shares")
 878         d.addCallback(self.create_mangled, "mutable-corrupt-shares")
 879         d.addCallback(self.create_mangled, "mutable-unrecoverable")
 880         d.addCallback(self.create_mangled, "large-good")
 881         d.addCallback(self.create_mangled, "large-missing-shares")
 882         d.addCallback(self.create_mangled, "large-corrupt-shares")
 883         d.addCallback(self.create_mangled, "large-unrecoverable")
 884         d.addCallback(lambda ignored: c0.create_dirnode())
 885         d.addCallback(self._stash_node, "broken")
 886         large1 = upload.Data("Lots of data\n" * 1000 + "large1" + "\n", None)
 887         d.addCallback(lambda ignored:
 888                       self.nodes["broken"].add_file(u"large1", large1))
 889         d.addCallback(lambda ignored:
 890                       self.nodes["broken"].create_subdirectory(u"subdir-good"))
 891         large2 = upload.Data("Lots of data\n" * 1000 + "large2" + "\n", None)
 892         d.addCallback(lambda subdir: subdir.add_file(u"large2-good", large2))
 893         d.addCallback(lambda ignored:
 894                       self.nodes["broken"].create_subdirectory(u"subdir-unrecoverable"))
 895         d.addCallback(self._stash_node, "subdir-unrecoverable")
 896         large3 = upload.Data("Lots of data\n" * 1000 + "large3" + "\n", None)
 897         d.addCallback(lambda subdir: subdir.add_file(u"large3-good", large3))
 898         d.addCallback(lambda ignored:
 899                       self._delete_most_shares(self.nodes["broken"]))
 900         return d
 901
 902     def _stash_node(self, node, name):
 903         self.nodes[name] = node
 904         return node
 905
 906     def create_mangled(self, ignored, name):
 907         nodetype, mangletype = name.split("-", 1)
 908         if nodetype == "mutable":
 909             d = self.g.clients[0].create_mutable_file("mutable file contents")
 910             d.addCallback(lambda n: self.root.set_node(unicode(name), n))
 911         elif nodetype == "large":
 912             large = upload.Data("Lots of data\n" * 1000 + name + "\n", None)
 913             d = self.root.add_file(unicode(name), large)
 914         elif nodetype == "small":
 915             small = upload.Data("Small enough for a LIT", None)
 916             d = self.root.add_file(unicode(name), small)
 917
 918         d.addCallback(self._stash_node, name)
 919
 920         if mangletype == "good":
 921             pass
 922         elif mangletype == "missing-shares":
 923             d.addCallback(self._delete_some_shares)
 924         elif mangletype == "corrupt-shares":
 925             d.addCallback(self._corrupt_some_shares)
 926         else:
 927             assert mangletype == "unrecoverable"
 928             d.addCallback(self._delete_most_shares)
 929
 930         return d
 931
 932     def _run_cli(self, argv):
 933         stdout, stderr = StringIO(), StringIO()
 934         # this can only do synchronous operations
 935         assert argv[0] == "debug"
 936         runner.runner(argv, run_by_human=False, stdout=stdout, stderr=stderr)
 937         return stdout.getvalue()
 938
 939     def _delete_some_shares(self, node):
 940         self.delete_shares_numbered(node.get_uri(), [0,1])
 941
 942     def _corrupt_some_shares(self, node):
 943         for (shnum, serverid, sharefile) in self.find_shares(node.get_uri()):
 944             if shnum in (0,1):
 945                 self._run_cli(["debug", "corrupt-share", sharefile])
 946
 947     def _delete_most_shares(self, node):
 948         self.delete_shares_numbered(node.get_uri(), range(1,10))
 949
 950
 951     def check_is_healthy(self, cr, where):
 952         try:
 953             self.failUnless(ICheckResults.providedBy(cr), (cr, type(cr), where))
 954             self.failUnless(cr.is_healthy(), (cr.get_report(), cr.is_healthy(), cr.get_summary(), where))
 955             self.failUnless(cr.is_recoverable(), where)
 956             d = cr.get_data()
 957             self.failUnlessEqual(d["count-recoverable-versions"], 1, where)
 958             self.failUnlessEqual(d["count-unrecoverable-versions"], 0, where)
 959             return cr
 960         except Exception, le:
 961             le.args = tuple(le.args + (where,))
 962             raise
 963
 964     def check_is_missing_shares(self, cr, where):
 965         self.failUnless(ICheckResults.providedBy(cr), where)
 966         self.failIf(cr.is_healthy(), where)
 967         self.failUnless(cr.is_recoverable(), where)
 968         d = cr.get_data()
 969         self.failUnlessEqual(d["count-recoverable-versions"], 1, where)
 970         self.failUnlessEqual(d["count-unrecoverable-versions"], 0, where)
 971         return cr
 972
 973     def check_has_corrupt_shares(self, cr, where):
 974         # by "corrupt-shares" we mean the file is still recoverable
 975         self.failUnless(ICheckResults.providedBy(cr), where)
 976         d = cr.get_data()
 977         self.failIf(cr.is_healthy(), (where, cr))
 978         self.failUnless(cr.is_recoverable(), where)
 979         d = cr.get_data()
 980         self.failUnless(d["count-shares-good"] < 10, where)
 981         self.failUnless(d["count-corrupt-shares"], where)
 982         self.failUnless(d["list-corrupt-shares"], where)
 983         return cr
 984
 985     def check_is_unrecoverable(self, cr, where):
 986         self.failUnless(ICheckResults.providedBy(cr), where)
 987         d = cr.get_data()
 988         self.failIf(cr.is_healthy(), where)
 989         self.failIf(cr.is_recoverable(), where)
 990         self.failUnless(d["count-shares-good"] < d["count-shares-needed"], (d["count-shares-good"], d["count-shares-needed"], where))
 991         self.failUnlessEqual(d["count-recoverable-versions"], 0, where)
 992         self.failUnlessEqual(d["count-unrecoverable-versions"], 1, where)
 993         return cr
 994
 995     def do_check(self, ignored):
 996         d = defer.succeed(None)
 997
 998         # check the individual items, without verification. This will not
 999         # detect corrupt shares.
1000         def _check(which, checker):
1001             d = self.nodes[which].check(Monitor())
1002             d.addCallback(checker, which + "--check")
1003             return d
1004
1005         d.addCallback(lambda ign: _check("mutable-good", self.check_is_healthy))
1006         d.addCallback(lambda ign: _check("mutable-missing-shares",
1007                                          self.check_is_missing_shares))
1008         d.addCallback(lambda ign: _check("mutable-corrupt-shares",
1009                                          self.check_is_healthy))
1010         d.addCallback(lambda ign: _check("mutable-unrecoverable",
1011                                          self.check_is_unrecoverable))
1012         d.addCallback(lambda ign: _check("large-good", self.check_is_healthy))
1013         d.addCallback(lambda ign: _check("large-missing-shares",
1014                                          self.check_is_missing_shares))
1015         d.addCallback(lambda ign: _check("large-corrupt-shares",
1016                                          self.check_is_healthy))
1017         d.addCallback(lambda ign: _check("large-unrecoverable",
1018                                          self.check_is_unrecoverable))
1019
1020         # and again with verify=True, which *does* detect corrupt shares.
1021         def _checkv(which, checker):
1022             d = self.nodes[which].check(Monitor(), verify=True)
1023             d.addCallback(checker, which + "--check-and-verify")
1024             return d
1025
1026         d.addCallback(lambda ign: _checkv("mutable-good", self.check_is_healthy))
1027         d.addCallback(lambda ign: _checkv("mutable-missing-shares",
1028                                          self.check_is_missing_shares))
1029         d.addCallback(lambda ign: _checkv("mutable-corrupt-shares",
1030                                          self.check_has_corrupt_shares))
1031         d.addCallback(lambda ign: _checkv("mutable-unrecoverable",
1032                                          self.check_is_unrecoverable))
1033         d.addCallback(lambda ign: _checkv("large-good", self.check_is_healthy))
1034         d.addCallback(lambda ign: _checkv("large-missing-shares", self.check_is_missing_shares))
1035         d.addCallback(lambda ign: _checkv("large-corrupt-shares", self.check_has_corrupt_shares))
1036         d.addCallback(lambda ign: _checkv("large-unrecoverable",
1037                                          self.check_is_unrecoverable))
1038
1039         return d
1040
1041     def do_deepcheck(self, ignored):
1042         d = defer.succeed(None)
1043
1044         # now deep-check the root, with various verify= and repair= options
1045         d.addCallback(lambda ign:
1046                       self.root.start_deep_check().when_done())
1047         def _check1(cr):
1048             self.failUnless(IDeepCheckResults.providedBy(cr))
1049             c = cr.get_counters()
1050             self.failUnlessEqual(c["count-objects-checked"], 9)
1051             self.failUnlessEqual(c["count-objects-healthy"], 5)
1052             self.failUnlessEqual(c["count-objects-unhealthy"], 4)
1053             self.failUnlessEqual(c["count-objects-unrecoverable"], 2)
1054         d.addCallback(_check1)
1055
1056         d.addCallback(lambda ign:
1057                       self.root.start_deep_check(verify=True).when_done())
1058         def _check2(cr):
1059             self.failUnless(IDeepCheckResults.providedBy(cr))
1060             c = cr.get_counters()
1061             self.failUnlessEqual(c["count-objects-checked"], 9)
1062             self.failUnlessEqual(c["count-objects-healthy"], 3)
1063             self.failUnlessEqual(c["count-objects-unhealthy"], 6)
1064             self.failUnlessEqual(c["count-objects-healthy"], 3) # root, mutable good, large good
1065             self.failUnlessEqual(c["count-objects-unrecoverable"], 2) # mutable unrecoverable, large unrecoverable
1066         d.addCallback(_check2)
1067
1068         return d
1069
1070     def do_deepcheck_broken(self, ignored):
1071         # deep-check on the broken directory should fail, because of the
1072         # untraversable subdir
1073         def _do_deep_check():
1074             return self.nodes["broken"].start_deep_check().when_done()
1075         d = self.shouldFail(UnrecoverableFileError, "do_deep_check",
1076                             "no recoverable versions",
1077                             _do_deep_check)
1078         return d
1079
1080     def json_is_healthy(self, data, where):
1081         r = data["results"]
1082         self.failUnless(r["healthy"], where)
1083         self.failUnless(r["recoverable"], where)
1084         self.failUnlessEqual(r["count-recoverable-versions"], 1, where)
1085         self.failUnlessEqual(r["count-unrecoverable-versions"], 0, where)
1086
1087     def json_is_missing_shares(self, data, where):
1088         r = data["results"]
1089         self.failIf(r["healthy"], where)
1090         self.failUnless(r["recoverable"], where)
1091         self.failUnlessEqual(r["count-recoverable-versions"], 1, where)
1092         self.failUnlessEqual(r["count-unrecoverable-versions"], 0, where)
1093
1094     def json_has_corrupt_shares(self, data, where):
1095         # by "corrupt-shares" we mean the file is still recoverable
1096         r = data["results"]
1097         self.failIf(r["healthy"], where)
1098         self.failUnless(r["recoverable"], where)
1099         self.failUnless(r["count-shares-good"] < 10, where)
1100         self.failUnless(r["count-corrupt-shares"], where)
1101         self.failUnless(r["list-corrupt-shares"], where)
1102
1103     def json_is_unrecoverable(self, data, where):
1104         r = data["results"]
1105         self.failIf(r["healthy"], where)
1106         self.failIf(r["recoverable"], where)
1107         self.failUnless(r["count-shares-good"] < r["count-shares-needed"],
1108                         where)
1109         self.failUnlessEqual(r["count-recoverable-versions"], 0, where)
1110         self.failUnlessEqual(r["count-unrecoverable-versions"], 1, where)
1111
1112     def do_test_web_bad(self, ignored):
1113         d = defer.succeed(None)
1114
1115         # check, no verify
1116         def _check(which, checker):
1117             d = self.web_json(self.nodes[which], t="check")
1118             d.addCallback(checker, which + "--webcheck")
1119             return d
1120
1121         d.addCallback(lambda ign: _check("mutable-good",
1122                                          self.json_is_healthy))
1123         d.addCallback(lambda ign: _check("mutable-missing-shares",
1124                                          self.json_is_missing_shares))
1125         d.addCallback(lambda ign: _check("mutable-corrupt-shares",
1126                                          self.json_is_healthy))
1127         d.addCallback(lambda ign: _check("mutable-unrecoverable",
1128                                          self.json_is_unrecoverable))
1129         d.addCallback(lambda ign: _check("large-good",
1130                                          self.json_is_healthy))
1131         d.addCallback(lambda ign: _check("large-missing-shares",
1132                                          self.json_is_missing_shares))
1133         d.addCallback(lambda ign: _check("large-corrupt-shares",
1134                                          self.json_is_healthy))
1135         d.addCallback(lambda ign: _check("large-unrecoverable",
1136                                          self.json_is_unrecoverable))
1137
1138         # check and verify
1139         def _checkv(which, checker):
1140             d = self.web_json(self.nodes[which], t="check", verify="true")
1141             d.addCallback(checker, which + "--webcheck-and-verify")
1142             return d
1143
1144         d.addCallback(lambda ign: _checkv("mutable-good",
1145                                           self.json_is_healthy))
1146         d.addCallback(lambda ign: _checkv("mutable-missing-shares",
1147                                          self.json_is_missing_shares))
1148         d.addCallback(lambda ign: _checkv("mutable-corrupt-shares",
1149                                          self.json_has_corrupt_shares))
1150         d.addCallback(lambda ign: _checkv("mutable-unrecoverable",
1151                                          self.json_is_unrecoverable))
1152         d.addCallback(lambda ign: _checkv("large-good",
1153                                           self.json_is_healthy))
1154         d.addCallback(lambda ign: _checkv("large-missing-shares", self.json_is_missing_shares))
1155         d.addCallback(lambda ign: _checkv("large-corrupt-shares", self.json_has_corrupt_shares))
1156         d.addCallback(lambda ign: _checkv("large-unrecoverable",
1157                                          self.json_is_unrecoverable))
1158
1159         return d
1160
1161 class Large(DeepCheckBase, unittest.TestCase):
1162     def test_lots_of_lits(self):
1163         self.basedir = "deepcheck/Large/lots_of_lits"
1164         self.set_up_grid()
1165         # create the following directory structure:
1166         #  root/
1167         #   subdir/
1168         #    000-large (CHK)
1169         #    001-small (LIT)
1170         #    002-small
1171         #    ...
1172         #    399-small
1173         # then do a deepcheck and make sure it doesn't cause a
1174         # Deferred-tail-recursion stack overflow
1175
1176         COUNT = 400
1177         c0 = self.g.clients[0]
1178         d = c0.create_dirnode()
1179         self.stash = {}
1180         def _created_root(n):
1181             self.root = n
1182             return n
1183         d.addCallback(_created_root)
1184         d.addCallback(lambda root: root.create_subdirectory(u"subdir"))
1185         def _add_children(subdir_node):
1186             self.subdir_node = subdir_node
1187             kids = {}
1188             for i in range(1, COUNT):
1189                 litcap = LiteralFileURI("%03d-data" % i).to_string()
1190                 kids[u"%03d-small" % i] = (litcap, litcap)
1191             return subdir_node.set_children(kids)
1192         d.addCallback(_add_children)
1193         up = upload.Data("large enough for CHK" * 100, "")
1194         d.addCallback(lambda ign: self.subdir_node.add_file(u"0000-large", up))
1195
1196         def _start_deepcheck(ignored):
1197             return self.web(self.root, method="POST", t="stream-deep-check")
1198         d.addCallback(_start_deepcheck)
1199         def _check( (output, url) ):
1200             units = list(self.parse_streamed_json(output))
1201             self.failUnlessEqual(len(units), 2+COUNT+1)
1202         d.addCallback(_check)
1203
1204         return d