src/allmydata/test/test_deepcheck.py

   1
   2 import os, simplejson, urllib
   3 from cStringIO import StringIO
   4 from twisted.trial import unittest
   5 from twisted.internet import defer
   6 from twisted.internet import threads # CLI tests use deferToThread
   7 from allmydata.immutable import upload
   8 from allmydata.mutable.common import UnrecoverableFileError
   9 from allmydata.mutable.publish import MutableData
  10 from allmydata.util import idlib
  11 from allmydata.util import base32
  12 from allmydata.scripts import runner
  13 from allmydata.interfaces import ICheckResults, ICheckAndRepairResults, \
  14      IDeepCheckResults, IDeepCheckAndRepairResults
  15 from allmydata.monitor import Monitor, OperationCancelledError
  16 from allmydata.uri import LiteralFileURI
  17 from twisted.web.client import getPage
  18
  19 from allmydata.test.common import ErrorMixin, _corrupt_mutable_share_data, \
  20      ShouldFailMixin
  21 from allmydata.test.common_util import StallMixin
  22 from allmydata.test.no_network import GridTestMixin
  23
  24 timeout = 2400 # One of these took 1046.091s on Zandr's ARM box.
  25
  26 class MutableChecker(GridTestMixin, unittest.TestCase, ErrorMixin):
  27     def _run_cli(self, argv):
  28         stdout, stderr = StringIO(), StringIO()
  29         # this can only do synchronous operations
  30         assert argv[0] == "debug"
  31         runner.runner(argv, run_by_human=False, stdout=stdout, stderr=stderr)
  32         return stdout.getvalue()
  33
  34     def test_good(self):
  35         self.basedir = "deepcheck/MutableChecker/good"
  36         self.set_up_grid()
  37         CONTENTS = "a little bit of data"
  38         CONTENTS_uploadable = MutableData(CONTENTS)
  39         d = self.g.clients[0].create_mutable_file(CONTENTS_uploadable)
  40         def _created(node):
  41             self.node = node
  42             self.fileurl = "uri/" + urllib.quote(node.get_uri())
  43         d.addCallback(_created)
  44         # now make sure the webapi verifier sees no problems
  45         d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=true",
  46                                            method="POST"))
  47         def _got_results(out):
  48             self.failUnless("<span>Healthy : Healthy</span>" in out, out)
  49             self.failUnless("Recoverable Versions: 10*seq1-" in out, out)
  50             self.failIf("Not Healthy!" in out, out)
  51             self.failIf("Unhealthy" in out, out)
  52             self.failIf("Corrupt Shares" in out, out)
  53         d.addCallback(_got_results)
  54         d.addErrback(self.explain_web_error)
  55         return d
  56
  57     def test_corrupt(self):
  58         self.basedir = "deepcheck/MutableChecker/corrupt"
  59         self.set_up_grid()
  60         CONTENTS = "a little bit of data"
  61         CONTENTS_uploadable = MutableData(CONTENTS)
  62         d = self.g.clients[0].create_mutable_file(CONTENTS_uploadable)
  63         def _stash_and_corrupt(node):
  64             self.node = node
  65             self.fileurl = "uri/" + urllib.quote(node.get_uri())
  66             self.corrupt_shares_numbered(node.get_uri(), [0],
  67                                          _corrupt_mutable_share_data)
  68         d.addCallback(_stash_and_corrupt)
  69         # now make sure the webapi verifier notices it
  70         d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=true",
  71                                            method="POST"))
  72         def _got_results(out):
  73             self.failUnless("Not Healthy!" in out, out)
  74             self.failUnless("Unhealthy: best version has only 9 shares (encoding is 3-of-10)" in out, out)
  75             self.failUnless("Corrupt Shares:" in out, out)
  76         d.addCallback(_got_results)
  77
  78         # now make sure the webapi repairer can fix it
  79         d.addCallback(lambda ign:
  80                       self.GET(self.fileurl+"?t=check&verify=true&repair=true",
  81                                method="POST"))
  82         def _got_repair_results(out):
  83             self.failUnless("<div>Repair successful</div>" in out, out)
  84         d.addCallback(_got_repair_results)
  85         d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=true",
  86                                            method="POST"))
  87         def _got_postrepair_results(out):
  88             self.failIf("Not Healthy!" in out, out)
  89             self.failUnless("Recoverable Versions: 10*seq" in out, out)
  90         d.addCallback(_got_postrepair_results)
  91         d.addErrback(self.explain_web_error)
  92
  93         return d
  94
  95     def test_delete_share(self):
  96         self.basedir = "deepcheck/MutableChecker/delete_share"
  97         self.set_up_grid()
  98         CONTENTS = "a little bit of data"
  99         CONTENTS_uploadable = MutableData(CONTENTS)
 100         d = self.g.clients[0].create_mutable_file(CONTENTS_uploadable)
 101         def _stash_and_delete(node):
 102             self.node = node
 103             self.fileurl = "uri/" + urllib.quote(node.get_uri())
 104             self.delete_shares_numbered(node.get_uri(), [0])
 105         d.addCallback(_stash_and_delete)
 106         # now make sure the webapi checker notices it
 107         d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=false",
 108                                            method="POST"))
 109         def _got_results(out):
 110             self.failUnless("Not Healthy!" in out, out)
 111             self.failUnless("Unhealthy: best version has only 9 shares (encoding is 3-of-10)" in out, out)
 112             self.failIf("Corrupt Shares" in out, out)
 113         d.addCallback(_got_results)
 114
 115         # now make sure the webapi repairer can fix it
 116         d.addCallback(lambda ign:
 117                       self.GET(self.fileurl+"?t=check&verify=false&repair=true",
 118                                method="POST"))
 119         def _got_repair_results(out):
 120             self.failUnless("Repair successful" in out)
 121         d.addCallback(_got_repair_results)
 122         d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=false",
 123                                            method="POST"))
 124         def _got_postrepair_results(out):
 125             self.failIf("Not Healthy!" in out, out)
 126             self.failUnless("Recoverable Versions: 10*seq" in out)
 127         d.addCallback(_got_postrepair_results)
 128         d.addErrback(self.explain_web_error)
 129
 130         return d
 131
 132
 133 class DeepCheckBase(GridTestMixin, ErrorMixin, StallMixin, ShouldFailMixin):
 134
 135     def web_json(self, n, **kwargs):
 136         kwargs["output"] = "json"
 137         d = self.web(n, "POST", **kwargs)
 138         d.addCallback(self.decode_json)
 139         return d
 140
 141     def decode_json(self, (s,url)):
 142         try:
 143             data = simplejson.loads(s)
 144         except ValueError:
 145             self.fail("%s: not JSON: '%s'" % (url, s))
 146         return data
 147
 148     def parse_streamed_json(self, s):
 149         for unit in s.split("\n"):
 150             if not unit:
 151                 # stream should end with a newline, so split returns ""
 152                 continue
 153             try:
 154                 yield simplejson.loads(unit)
 155             except ValueError, le:
 156                 le.args = tuple(le.args + (unit,))
 157                 raise
 158
 159     def web(self, n, method="GET", **kwargs):
 160         # returns (data, url)
 161         url = (self.client_baseurls[0] + "uri/%s" % urllib.quote(n.get_uri())
 162                + "?" + "&".join(["%s=%s" % (k,v) for (k,v) in kwargs.items()]))
 163         d = getPage(url, method=method)
 164         d.addCallback(lambda data: (data,url))
 165         return d
 166
 167     def wait_for_operation(self, ignored, ophandle):
 168         url = self.client_baseurls[0] + "operations/" + ophandle
 169         url += "?t=status&output=JSON"
 170         d = getPage(url)
 171         def _got(res):
 172             try:
 173                 data = simplejson.loads(res)
 174             except ValueError:
 175                 self.fail("%s: not JSON: '%s'" % (url, res))
 176             if not data["finished"]:
 177                 d = self.stall(delay=1.0)
 178                 d.addCallback(self.wait_for_operation, ophandle)
 179                 return d
 180             return data
 181         d.addCallback(_got)
 182         return d
 183
 184     def get_operation_results(self, ignored, ophandle, output=None):
 185         url = self.client_baseurls[0] + "operations/" + ophandle
 186         url += "?t=status"
 187         if output:
 188             url += "&output=" + output
 189         d = getPage(url)
 190         def _got(res):
 191             if output and output.lower() == "json":
 192                 try:
 193                     return simplejson.loads(res)
 194                 except ValueError:
 195                     self.fail("%s: not JSON: '%s'" % (url, res))
 196             return res
 197         d.addCallback(_got)
 198         return d
 199
 200     def slow_web(self, n, output=None, **kwargs):
 201         # use ophandle=
 202         handle = base32.b2a(os.urandom(4))
 203         d = self.web(n, "POST", ophandle=handle, **kwargs)
 204         d.addCallback(self.wait_for_operation, handle)
 205         d.addCallback(self.get_operation_results, handle, output=output)
 206         return d
 207
 208
 209 class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
 210     # construct a small directory tree (with one dir, one immutable file, one
 211     # mutable file, two LIT files, one DIR2:LIT empty dir, one DIR2:LIT tiny
 212     # dir, and a loop), and then check/examine it in various ways.
 213
 214     def set_up_tree(self):
 215         # 2.9s
 216
 217         c0 = self.g.clients[0]
 218         d = c0.create_dirnode()
 219         def _created_root(n):
 220             self.root = n
 221             self.root_uri = n.get_uri()
 222         d.addCallback(_created_root)
 223         d.addCallback(lambda ign:
 224             c0.create_mutable_file(MutableData("mutable file contents")))
 225         d.addCallback(lambda n: self.root.set_node(u"mutable", n))
 226         def _created_mutable(n):
 227             self.mutable = n
 228             self.mutable_uri = n.get_uri()
 229         d.addCallback(_created_mutable)
 230
 231         large = upload.Data("Lots of data\n" * 1000, None)
 232         d.addCallback(lambda ign: self.root.add_file(u"large", large))
 233         def _created_large(n):
 234             self.large = n
 235             self.large_uri = n.get_uri()
 236         d.addCallback(_created_large)
 237
 238         small = upload.Data("Small enough for a LIT", None)
 239         d.addCallback(lambda ign: self.root.add_file(u"small", small))
 240         def _created_small(n):
 241             self.small = n
 242             self.small_uri = n.get_uri()
 243         d.addCallback(_created_small)
 244
 245         small2 = upload.Data("Small enough for a LIT too", None)
 246         d.addCallback(lambda ign: self.root.add_file(u"small2", small2))
 247         def _created_small2(n):
 248             self.small2 = n
 249             self.small2_uri = n.get_uri()
 250         d.addCallback(_created_small2)
 251
 252         empty_litdir_uri = "URI:DIR2-LIT:"
 253         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
 254
 255         d.addCallback(lambda ign: self.root._create_and_validate_node(None, empty_litdir_uri, name=u"test_deepcheck empty_lit_dir"))
 256         def _created_empty_lit_dir(n):
 257             self.empty_lit_dir = n
 258             self.empty_lit_dir_uri = n.get_uri()
 259             self.root.set_node(u"empty_lit_dir", n)
 260         d.addCallback(_created_empty_lit_dir)
 261
 262         d.addCallback(lambda ign: self.root._create_and_validate_node(None, tiny_litdir_uri, name=u"test_deepcheck tiny_lit_dir"))
 263         def _created_tiny_lit_dir(n):
 264             self.tiny_lit_dir = n
 265             self.tiny_lit_dir_uri = n.get_uri()
 266             self.root.set_node(u"tiny_lit_dir", n)
 267         d.addCallback(_created_tiny_lit_dir)
 268
 269         d.addCallback(lambda ign: self.root.set_node(u"loop", self.root))
 270         return d
 271
 272     def check_is_healthy(self, cr, n, where, incomplete=False):
 273         self.failUnless(ICheckResults.providedBy(cr), where)
 274         self.failUnless(cr.is_healthy(), where)
 275         self.failUnlessEqual(cr.get_storage_index(), n.get_storage_index(),
 276                              where)
 277         self.failUnlessEqual(cr.get_storage_index_string(),
 278                              base32.b2a(n.get_storage_index()), where)
 279         num_servers = len(self.g.all_servers)
 280         needs_rebalancing = bool( num_servers < 10 )
 281         if not incomplete:
 282             self.failUnlessEqual(cr.needs_rebalancing(), needs_rebalancing,
 283                                  str((where, cr, cr.get_data())))
 284         d = cr.get_data()
 285         self.failUnlessEqual(d["count-shares-good"], 10, where)
 286         self.failUnlessEqual(d["count-shares-needed"], 3, where)
 287         self.failUnlessEqual(d["count-shares-expected"], 10, where)
 288         if not incomplete:
 289             self.failUnlessEqual(d["count-good-share-hosts"], num_servers,
 290                                  where)
 291         self.failUnlessEqual(d["count-corrupt-shares"], 0, where)
 292         self.failUnlessEqual(d["list-corrupt-shares"], [], where)
 293         if not incomplete:
 294             self.failUnlessEqual(sorted(d["servers-responding"]),
 295                                  sorted(self.g.get_all_serverids()),
 296                                  where)
 297             self.failUnless("sharemap" in d, str((where, d)))
 298             all_serverids = set()
 299             for (shareid, serverids) in d["sharemap"].items():
 300                 all_serverids.update(serverids)
 301             self.failUnlessEqual(sorted(all_serverids),
 302                                  sorted(self.g.get_all_serverids()),
 303                                  where)
 304
 305         self.failUnlessEqual(d["count-wrong-shares"], 0, where)
 306         self.failUnlessEqual(d["count-recoverable-versions"], 1, where)
 307         self.failUnlessEqual(d["count-unrecoverable-versions"], 0, where)
 308
 309
 310     def check_and_repair_is_healthy(self, cr, n, where, incomplete=False):
 311         self.failUnless(ICheckAndRepairResults.providedBy(cr), (where, cr))
 312         self.failUnless(cr.get_pre_repair_results().is_healthy(), where)
 313         self.check_is_healthy(cr.get_pre_repair_results(), n, where, incomplete)
 314         self.failUnless(cr.get_post_repair_results().is_healthy(), where)
 315         self.check_is_healthy(cr.get_post_repair_results(), n, where, incomplete)
 316         self.failIf(cr.get_repair_attempted(), where)
 317
 318     def deep_check_is_healthy(self, cr, num_healthy, where):
 319         self.failUnless(IDeepCheckResults.providedBy(cr))
 320         self.failUnlessEqual(cr.get_counters()["count-objects-healthy"],
 321                              num_healthy, where)
 322
 323     def deep_check_and_repair_is_healthy(self, cr, num_healthy, where):
 324         self.failUnless(IDeepCheckAndRepairResults.providedBy(cr), where)
 325         c = cr.get_counters()
 326         self.failUnlessEqual(c["count-objects-healthy-pre-repair"],
 327                              num_healthy, where)
 328         self.failUnlessEqual(c["count-objects-healthy-post-repair"],
 329                              num_healthy, where)
 330         self.failUnlessEqual(c["count-repairs-attempted"], 0, where)
 331
 332     def test_good(self):
 333         self.basedir = "deepcheck/DeepCheckWebGood/good"
 334         self.set_up_grid()
 335         d = self.set_up_tree()
 336         d.addCallback(self.do_stats)
 337         d.addCallback(self.do_web_stream_manifest)
 338         d.addCallback(self.do_web_stream_check)
 339         d.addCallback(self.do_test_check_good)
 340         d.addCallback(self.do_test_web_good)
 341         d.addCallback(self.do_test_cli_good)
 342         d.addErrback(self.explain_web_error)
 343         d.addErrback(self.explain_error)
 344         return d
 345
 346     def do_stats(self, ignored):
 347         d = defer.succeed(None)
 348         d.addCallback(lambda ign: self.root.start_deep_stats().when_done())
 349         d.addCallback(self.check_stats_good)
 350         return d
 351
 352     def check_stats_good(self, s):
 353         self.failUnlessEqual(s["count-directories"], 3)
 354         self.failUnlessEqual(s["count-files"], 5)
 355         self.failUnlessEqual(s["count-immutable-files"], 1)
 356         self.failUnlessEqual(s["count-literal-files"], 3)
 357         self.failUnlessEqual(s["count-mutable-files"], 1)
 358         # don't check directories: their size will vary
 359         # s["largest-directory"]
 360         # s["size-directories"]
 361         self.failUnlessEqual(s["largest-directory-children"], 7)
 362         self.failUnlessEqual(s["largest-immutable-file"], 13000)
 363         # to re-use this function for both the local
 364         # dirnode.start_deep_stats() and the webapi t=start-deep-stats, we
 365         # coerce the result into a list of tuples. dirnode.start_deep_stats()
 366         # returns a list of tuples, but JSON only knows about lists., so
 367         # t=start-deep-stats returns a list of lists.
 368         histogram = [tuple(stuff) for stuff in s["size-files-histogram"]]
 369         self.failUnlessEqual(histogram, [(4, 10, 1), (11, 31, 2),
 370                                          (10001, 31622, 1),
 371                                          ])
 372         self.failUnlessEqual(s["size-immutable-files"], 13000)
 373         self.failUnlessEqual(s["size-literal-files"], 56)
 374
 375     def do_web_stream_manifest(self, ignored):
 376         d = self.web(self.root, method="POST", t="stream-manifest")
 377         d.addCallback(lambda (output,url):
 378                       self._check_streamed_manifest(output))
 379         return d
 380
 381     def _check_streamed_manifest(self, output):
 382         units = list(self.parse_streamed_json(output))
 383         files = [u for u in units if u["type"] in ("file", "directory")]
 384         assert units[-1]["type"] == "stats"
 385         stats = units[-1]["stats"]
 386         self.failUnlessEqual(len(files), 8)
 387         # [root,mutable,large] are distributed, [small,small2,empty_litdir,tiny_litdir] are not
 388         self.failUnlessEqual(len([f for f in files
 389                                   if f["verifycap"] != ""]), 3)
 390         self.failUnlessEqual(len([f for f in files
 391                                   if f["verifycap"] == ""]), 5)
 392         self.failUnlessEqual(len([f for f in files
 393                                   if f["repaircap"] != ""]), 3)
 394         self.failUnlessEqual(len([f for f in files
 395                                   if f["repaircap"] == ""]), 5)
 396         self.failUnlessEqual(len([f for f in files
 397                                   if f["storage-index"] != ""]), 3)
 398         self.failUnlessEqual(len([f for f in files
 399                                   if f["storage-index"] == ""]), 5)
 400         # make sure that a mutable file has filecap==repaircap!=verifycap
 401         mutable = [f for f in files
 402                    if f["cap"] is not None
 403                    and f["cap"].startswith("URI:SSK:")][0]
 404         self.failUnlessEqual(mutable["cap"], self.mutable_uri)
 405         self.failIfEqual(mutable["cap"], mutable["verifycap"])
 406         self.failUnlessEqual(mutable["cap"], mutable["repaircap"])
 407         # for immutable file, verifycap==repaircap!=filecap
 408         large = [f for f in files
 409                    if f["cap"] is not None
 410                    and f["cap"].startswith("URI:CHK:")][0]
 411         self.failUnlessEqual(large["cap"], self.large_uri)
 412         self.failIfEqual(large["cap"], large["verifycap"])
 413         self.failUnlessEqual(large["verifycap"], large["repaircap"])
 414         self.check_stats_good(stats)
 415
 416     def do_web_stream_check(self, ignored):
 417         # TODO
 418         return
 419         d = self.web(self.root, t="stream-deep-check")
 420         def _check(res):
 421             units = list(self.parse_streamed_json(res))
 422             #files = [u for u in units if u["type"] in ("file", "directory")]
 423             assert units[-1]["type"] == "stats"
 424             #stats = units[-1]["stats"]
 425             # ...
 426         d.addCallback(_check)
 427         return d
 428
 429     def do_test_check_good(self, ignored):
 430         d = defer.succeed(None)
 431         # check the individual items
 432         d.addCallback(lambda ign: self.root.check(Monitor()))
 433         d.addCallback(self.check_is_healthy, self.root, "root")
 434         d.addCallback(lambda ign: self.mutable.check(Monitor()))
 435         d.addCallback(self.check_is_healthy, self.mutable, "mutable")
 436         d.addCallback(lambda ign: self.large.check(Monitor()))
 437         d.addCallback(self.check_is_healthy, self.large, "large")
 438         d.addCallback(lambda ign: self.small.check(Monitor()))
 439         d.addCallback(self.failUnlessEqual, None, "small")
 440         d.addCallback(lambda ign: self.small2.check(Monitor()))
 441         d.addCallback(self.failUnlessEqual, None, "small2")
 442         d.addCallback(lambda ign: self.empty_lit_dir.check(Monitor()))
 443         d.addCallback(self.failUnlessEqual, None, "empty_lit_dir")
 444         d.addCallback(lambda ign: self.tiny_lit_dir.check(Monitor()))
 445         d.addCallback(self.failUnlessEqual, None, "tiny_lit_dir")
 446
 447         # and again with verify=True
 448         d.addCallback(lambda ign: self.root.check(Monitor(), verify=True))
 449         d.addCallback(self.check_is_healthy, self.root, "root")
 450         d.addCallback(lambda ign: self.mutable.check(Monitor(), verify=True))
 451         d.addCallback(self.check_is_healthy, self.mutable, "mutable")
 452         d.addCallback(lambda ign: self.large.check(Monitor(), verify=True))
 453         d.addCallback(self.check_is_healthy, self.large, "large", incomplete=True)
 454         d.addCallback(lambda ign: self.small.check(Monitor(), verify=True))
 455         d.addCallback(self.failUnlessEqual, None, "small")
 456         d.addCallback(lambda ign: self.small2.check(Monitor(), verify=True))
 457         d.addCallback(self.failUnlessEqual, None, "small2")
 458         d.addCallback(lambda ign: self.empty_lit_dir.check(Monitor(), verify=True))
 459         d.addCallback(self.failUnlessEqual, None, "empty_lit_dir")
 460         d.addCallback(lambda ign: self.tiny_lit_dir.check(Monitor(), verify=True))
 461         d.addCallback(self.failUnlessEqual, None, "tiny_lit_dir")
 462
 463         # and check_and_repair(), which should be a nop
 464         d.addCallback(lambda ign: self.root.check_and_repair(Monitor()))
 465         d.addCallback(self.check_and_repair_is_healthy, self.root, "root")
 466         d.addCallback(lambda ign: self.mutable.check_and_repair(Monitor()))
 467         d.addCallback(self.check_and_repair_is_healthy, self.mutable, "mutable")
 468         d.addCallback(lambda ign: self.large.check_and_repair(Monitor()))
 469         d.addCallback(self.check_and_repair_is_healthy, self.large, "large")
 470         d.addCallback(lambda ign: self.small.check_and_repair(Monitor()))
 471         d.addCallback(self.failUnlessEqual, None, "small")
 472         d.addCallback(lambda ign: self.small2.check_and_repair(Monitor()))
 473         d.addCallback(self.failUnlessEqual, None, "small2")
 474         d.addCallback(lambda ign: self.empty_lit_dir.check_and_repair(Monitor()))
 475         d.addCallback(self.failUnlessEqual, None, "empty_lit_dir")
 476         d.addCallback(lambda ign: self.tiny_lit_dir.check_and_repair(Monitor()))
 477
 478         # check_and_repair(verify=True)
 479         d.addCallback(lambda ign: self.root.check_and_repair(Monitor(), verify=True))
 480         d.addCallback(self.check_and_repair_is_healthy, self.root, "root")
 481         d.addCallback(lambda ign: self.mutable.check_and_repair(Monitor(), verify=True))
 482         d.addCallback(self.check_and_repair_is_healthy, self.mutable, "mutable")
 483         d.addCallback(lambda ign: self.large.check_and_repair(Monitor(), verify=True))
 484         d.addCallback(self.check_and_repair_is_healthy, self.large, "large", incomplete=True)
 485         d.addCallback(lambda ign: self.small.check_and_repair(Monitor(), verify=True))
 486         d.addCallback(self.failUnlessEqual, None, "small")
 487         d.addCallback(lambda ign: self.small2.check_and_repair(Monitor(), verify=True))
 488         d.addCallback(self.failUnlessEqual, None, "small2")
 489         d.addCallback(self.failUnlessEqual, None, "small2")
 490         d.addCallback(lambda ign: self.empty_lit_dir.check_and_repair(Monitor(), verify=True))
 491         d.addCallback(self.failUnlessEqual, None, "empty_lit_dir")
 492         d.addCallback(lambda ign: self.tiny_lit_dir.check_and_repair(Monitor(), verify=True))
 493
 494
 495         # now deep-check the root, with various verify= and repair= options
 496         d.addCallback(lambda ign:
 497                       self.root.start_deep_check().when_done())
 498         d.addCallback(self.deep_check_is_healthy, 3, "root")
 499         d.addCallback(lambda ign:
 500                       self.root.start_deep_check(verify=True).when_done())
 501         d.addCallback(self.deep_check_is_healthy, 3, "root")
 502         d.addCallback(lambda ign:
 503                       self.root.start_deep_check_and_repair().when_done())
 504         d.addCallback(self.deep_check_and_repair_is_healthy, 3, "root")
 505         d.addCallback(lambda ign:
 506                       self.root.start_deep_check_and_repair(verify=True).when_done())
 507         d.addCallback(self.deep_check_and_repair_is_healthy, 3, "root")
 508
 509         # and finally, start a deep-check, but then cancel it.
 510         d.addCallback(lambda ign: self.root.start_deep_check())
 511         def _checking(monitor):
 512             monitor.cancel()
 513             d = monitor.when_done()
 514             # this should fire as soon as the next dirnode.list finishes.
 515             # TODO: add a counter to measure how many list() calls are made,
 516             # assert that no more than one gets to run before the cancel()
 517             # takes effect.
 518             def _finished_normally(res):
 519                 self.fail("this was supposed to fail, not finish normally")
 520             def _cancelled(f):
 521                 f.trap(OperationCancelledError)
 522             d.addCallbacks(_finished_normally, _cancelled)
 523             return d
 524         d.addCallback(_checking)
 525
 526         return d
 527
 528     def json_check_is_healthy(self, data, n, where, incomplete=False):
 529
 530         self.failUnlessEqual(data["storage-index"],
 531                              base32.b2a(n.get_storage_index()), where)
 532         self.failUnless("summary" in data, (where, data))
 533         self.failUnlessEqual(data["summary"].lower(), "healthy",
 534                              "%s: '%s'" % (where, data["summary"]))
 535         r = data["results"]
 536         self.failUnlessEqual(r["healthy"], True, where)
 537         num_servers = len(self.g.all_servers)
 538         needs_rebalancing = bool( num_servers < 10 )
 539         if not incomplete:
 540             self.failUnlessEqual(r["needs-rebalancing"], needs_rebalancing,
 541                                  where)
 542         self.failUnlessEqual(r["count-shares-good"], 10, where)
 543         self.failUnlessEqual(r["count-shares-needed"], 3, where)
 544         self.failUnlessEqual(r["count-shares-expected"], 10, where)
 545         if not incomplete:
 546             self.failUnlessEqual(r["count-good-share-hosts"], num_servers,
 547                                  where)
 548         self.failUnlessEqual(r["count-corrupt-shares"], 0, where)
 549         self.failUnlessEqual(r["list-corrupt-shares"], [], where)
 550         if not incomplete:
 551             self.failUnlessEqual(sorted(r["servers-responding"]),
 552                                  sorted([idlib.nodeid_b2a(sid)
 553                                          for sid in self.g.get_all_serverids()]),
 554                                  where)
 555             self.failUnless("sharemap" in r, where)
 556             all_serverids = set()
 557             for (shareid, serverids_s) in r["sharemap"].items():
 558                 all_serverids.update(serverids_s)
 559             self.failUnlessEqual(sorted(all_serverids),
 560                                  sorted([idlib.nodeid_b2a(sid)
 561                                          for sid in self.g.get_all_serverids()]),
 562                                  where)
 563         self.failUnlessEqual(r["count-wrong-shares"], 0, where)
 564         self.failUnlessEqual(r["count-recoverable-versions"], 1, where)
 565         self.failUnlessEqual(r["count-unrecoverable-versions"], 0, where)
 566
 567     def json_check_and_repair_is_healthy(self, data, n, where, incomplete=False):
 568         self.failUnlessEqual(data["storage-index"],
 569                              base32.b2a(n.get_storage_index()), where)
 570         self.failUnlessEqual(data["repair-attempted"], False, where)
 571         self.json_check_is_healthy(data["pre-repair-results"],
 572                                    n, where, incomplete)
 573         self.json_check_is_healthy(data["post-repair-results"],
 574                                    n, where, incomplete)
 575
 576     def json_full_deepcheck_is_healthy(self, data, n, where):
 577         self.failUnlessEqual(data["root-storage-index"],
 578                              base32.b2a(n.get_storage_index()), where)
 579         self.failUnlessEqual(data["count-objects-checked"], 3, where)
 580         self.failUnlessEqual(data["count-objects-healthy"], 3, where)
 581         self.failUnlessEqual(data["count-objects-unhealthy"], 0, where)
 582         self.failUnlessEqual(data["count-corrupt-shares"], 0, where)
 583         self.failUnlessEqual(data["list-corrupt-shares"], [], where)
 584         self.failUnlessEqual(data["list-unhealthy-files"], [], where)
 585         self.json_check_stats_good(data["stats"], where)
 586
 587     def json_full_deepcheck_and_repair_is_healthy(self, data, n, where):
 588         self.failUnlessEqual(data["root-storage-index"],
 589                              base32.b2a(n.get_storage_index()), where)
 590         self.failUnlessEqual(data["count-objects-checked"], 3, where)
 591
 592         self.failUnlessEqual(data["count-objects-healthy-pre-repair"], 3, where)
 593         self.failUnlessEqual(data["count-objects-unhealthy-pre-repair"], 0, where)
 594         self.failUnlessEqual(data["count-corrupt-shares-pre-repair"], 0, where)
 595
 596         self.failUnlessEqual(data["count-objects-healthy-post-repair"], 3, where)
 597         self.failUnlessEqual(data["count-objects-unhealthy-post-repair"], 0, where)
 598         self.failUnlessEqual(data["count-corrupt-shares-post-repair"], 0, where)
 599
 600         self.failUnlessEqual(data["list-corrupt-shares"], [], where)
 601         self.failUnlessEqual(data["list-remaining-corrupt-shares"], [], where)
 602         self.failUnlessEqual(data["list-unhealthy-files"], [], where)
 603
 604         self.failUnlessEqual(data["count-repairs-attempted"], 0, where)
 605         self.failUnlessEqual(data["count-repairs-successful"], 0, where)
 606         self.failUnlessEqual(data["count-repairs-unsuccessful"], 0, where)
 607
 608
 609     def json_check_lit(self, data, n, where):
 610         self.failUnlessEqual(data["storage-index"], "", where)
 611         self.failUnlessEqual(data["results"]["healthy"], True, where)
 612
 613     def json_check_stats_good(self, data, where):
 614         self.check_stats_good(data)
 615
 616     def do_test_web_good(self, ignored):
 617         d = defer.succeed(None)
 618
 619         # stats
 620         d.addCallback(lambda ign:
 621                       self.slow_web(self.root,
 622                                     t="start-deep-stats", output="json"))
 623         d.addCallback(self.json_check_stats_good, "deep-stats")
 624
 625         # check, no verify
 626         d.addCallback(lambda ign: self.web_json(self.root, t="check"))
 627         d.addCallback(self.json_check_is_healthy, self.root, "root")
 628         d.addCallback(lambda ign: self.web_json(self.mutable, t="check"))
 629         d.addCallback(self.json_check_is_healthy, self.mutable, "mutable")
 630         d.addCallback(lambda ign: self.web_json(self.large, t="check"))
 631         d.addCallback(self.json_check_is_healthy, self.large, "large")
 632         d.addCallback(lambda ign: self.web_json(self.small, t="check"))
 633         d.addCallback(self.json_check_lit, self.small, "small")
 634         d.addCallback(lambda ign: self.web_json(self.small2, t="check"))
 635         d.addCallback(self.json_check_lit, self.small2, "small2")
 636         d.addCallback(lambda ign: self.web_json(self.empty_lit_dir, t="check"))
 637         d.addCallback(self.json_check_lit, self.empty_lit_dir, "empty_lit_dir")
 638         d.addCallback(lambda ign: self.web_json(self.tiny_lit_dir, t="check"))
 639         d.addCallback(self.json_check_lit, self.tiny_lit_dir, "tiny_lit_dir")
 640
 641         # check and verify
 642         d.addCallback(lambda ign:
 643                       self.web_json(self.root, t="check", verify="true"))
 644         d.addCallback(self.json_check_is_healthy, self.root, "root+v")
 645         d.addCallback(lambda ign:
 646                       self.web_json(self.mutable, t="check", verify="true"))
 647         d.addCallback(self.json_check_is_healthy, self.mutable, "mutable+v")
 648         d.addCallback(lambda ign:
 649                       self.web_json(self.large, t="check", verify="true"))
 650         d.addCallback(self.json_check_is_healthy, self.large, "large+v",
 651                       incomplete=True)
 652         d.addCallback(lambda ign:
 653                       self.web_json(self.small, t="check", verify="true"))
 654         d.addCallback(self.json_check_lit, self.small, "small+v")
 655         d.addCallback(lambda ign:
 656                       self.web_json(self.small2, t="check", verify="true"))
 657         d.addCallback(self.json_check_lit, self.small2, "small2+v")
 658         d.addCallback(lambda ign: self.web_json(self.empty_lit_dir, t="check", verify="true"))
 659         d.addCallback(self.json_check_lit, self.empty_lit_dir, "empty_lit_dir+v")
 660         d.addCallback(lambda ign: self.web_json(self.tiny_lit_dir, t="check", verify="true"))
 661         d.addCallback(self.json_check_lit, self.tiny_lit_dir, "tiny_lit_dir+v")
 662
 663         # check and repair, no verify
 664         d.addCallback(lambda ign:
 665                       self.web_json(self.root, t="check", repair="true"))
 666         d.addCallback(self.json_check_and_repair_is_healthy, self.root, "root+r")
 667         d.addCallback(lambda ign:
 668                       self.web_json(self.mutable, t="check", repair="true"))
 669         d.addCallback(self.json_check_and_repair_is_healthy, self.mutable, "mutable+r")
 670         d.addCallback(lambda ign:
 671                       self.web_json(self.large, t="check", repair="true"))
 672         d.addCallback(self.json_check_and_repair_is_healthy, self.large, "large+r")
 673         d.addCallback(lambda ign:
 674                       self.web_json(self.small, t="check", repair="true"))
 675         d.addCallback(self.json_check_lit, self.small, "small+r")
 676         d.addCallback(lambda ign:
 677                       self.web_json(self.small2, t="check", repair="true"))
 678         d.addCallback(self.json_check_lit, self.small2, "small2+r")
 679         d.addCallback(lambda ign: self.web_json(self.empty_lit_dir, t="check", repair="true"))
 680         d.addCallback(self.json_check_lit, self.empty_lit_dir, "empty_lit_dir+r")
 681         d.addCallback(lambda ign: self.web_json(self.tiny_lit_dir, t="check", repair="true"))
 682         d.addCallback(self.json_check_lit, self.tiny_lit_dir, "tiny_lit_dir+r")
 683
 684         # check+verify+repair
 685         d.addCallback(lambda ign:
 686                       self.web_json(self.root, t="check", repair="true", verify="true"))
 687         d.addCallback(self.json_check_and_repair_is_healthy, self.root, "root+vr")
 688         d.addCallback(lambda ign:
 689                       self.web_json(self.mutable, t="check", repair="true", verify="true"))
 690         d.addCallback(self.json_check_and_repair_is_healthy, self.mutable, "mutable+vr")
 691         d.addCallback(lambda ign:
 692                       self.web_json(self.large, t="check", repair="true", verify="true"))
 693         d.addCallback(self.json_check_and_repair_is_healthy, self.large, "large+vr", incomplete=True)
 694         d.addCallback(lambda ign:
 695                       self.web_json(self.small, t="check", repair="true", verify="true"))
 696         d.addCallback(self.json_check_lit, self.small, "small+vr")
 697         d.addCallback(lambda ign:
 698                       self.web_json(self.small2, t="check", repair="true", verify="true"))
 699         d.addCallback(self.json_check_lit, self.small2, "small2+vr")
 700         d.addCallback(lambda ign: self.web_json(self.empty_lit_dir, t="check", repair="true", verify=True))
 701         d.addCallback(self.json_check_lit, self.empty_lit_dir, "empty_lit_dir+vr")
 702         d.addCallback(lambda ign: self.web_json(self.tiny_lit_dir, t="check", repair="true", verify=True))
 703         d.addCallback(self.json_check_lit, self.tiny_lit_dir, "tiny_lit_dir+vr")
 704
 705         # now run a deep-check, with various verify= and repair= flags
 706         d.addCallback(lambda ign:
 707                       self.slow_web(self.root, t="start-deep-check", output="json"))
 708         d.addCallback(self.json_full_deepcheck_is_healthy, self.root, "root+d")
 709         d.addCallback(lambda ign:
 710                       self.slow_web(self.root, t="start-deep-check", verify="true",
 711                                     output="json"))
 712         d.addCallback(self.json_full_deepcheck_is_healthy, self.root, "root+dv")
 713         d.addCallback(lambda ign:
 714                       self.slow_web(self.root, t="start-deep-check", repair="true",
 715                                     output="json"))
 716         d.addCallback(self.json_full_deepcheck_and_repair_is_healthy, self.root, "root+dr")
 717         d.addCallback(lambda ign:
 718                       self.slow_web(self.root, t="start-deep-check", verify="true", repair="true", output="json"))
 719         d.addCallback(self.json_full_deepcheck_and_repair_is_healthy, self.root, "root+dvr")
 720
 721         # now look at t=info
 722         d.addCallback(lambda ign: self.web(self.root, t="info"))
 723         # TODO: examine the output
 724         d.addCallback(lambda ign: self.web(self.mutable, t="info"))
 725         d.addCallback(lambda ign: self.web(self.large, t="info"))
 726         d.addCallback(lambda ign: self.web(self.small, t="info"))
 727         d.addCallback(lambda ign: self.web(self.small2, t="info"))
 728         d.addCallback(lambda ign: self.web(self.empty_lit_dir, t="info"))
 729         d.addCallback(lambda ign: self.web(self.tiny_lit_dir, t="info"))
 730
 731         return d
 732
 733     def _run_cli(self, argv, stdin=""):
 734         #print "CLI:", argv
 735         stdout, stderr = StringIO(), StringIO()
 736         d = threads.deferToThread(runner.runner, argv, run_by_human=False,
 737                                   stdin=StringIO(stdin),
 738                                   stdout=stdout, stderr=stderr)
 739         def _done(res):
 740             return stdout.getvalue(), stderr.getvalue()
 741         d.addCallback(_done)
 742         return d
 743
 744     def do_test_cli_good(self, ignored):
 745         d = defer.succeed(None)
 746         d.addCallback(lambda ign: self.do_cli_manifest_stream1())
 747         d.addCallback(lambda ign: self.do_cli_manifest_stream2())
 748         d.addCallback(lambda ign: self.do_cli_manifest_stream3())
 749         d.addCallback(lambda ign: self.do_cli_manifest_stream4())
 750         d.addCallback(lambda ign: self.do_cli_manifest_stream5())
 751         d.addCallback(lambda ign: self.do_cli_stats1())
 752         d.addCallback(lambda ign: self.do_cli_stats2())
 753         return d
 754
 755     def _check_manifest_storage_index(self, out):
 756         lines = [l for l in out.split("\n") if l]
 757         self.failUnlessEqual(len(lines), 3)
 758         self.failUnless(base32.b2a(self.root.get_storage_index()) in lines)
 759         self.failUnless(base32.b2a(self.mutable.get_storage_index()) in lines)
 760         self.failUnless(base32.b2a(self.large.get_storage_index()) in lines)
 761
 762     def do_cli_manifest_stream1(self):
 763         basedir = self.get_clientdir(0)
 764         d = self._run_cli(["manifest",
 765                            "--node-directory", basedir,
 766                            self.root_uri])
 767         def _check((out,err)):
 768             self.failUnlessEqual(err, "")
 769             lines = [l for l in out.split("\n") if l]
 770             self.failUnlessEqual(len(lines), 8)
 771             caps = {}
 772             for l in lines:
 773                 try:
 774                     cap, path = l.split(None, 1)
 775                 except ValueError:
 776                     cap = l.strip()
 777                     path = ""
 778                 caps[cap] = path
 779             self.failUnless(self.root.get_uri() in caps)
 780             self.failUnlessEqual(caps[self.root.get_uri()], "")
 781             self.failUnlessEqual(caps[self.mutable.get_uri()], "mutable")
 782             self.failUnlessEqual(caps[self.large.get_uri()], "large")
 783             self.failUnlessEqual(caps[self.small.get_uri()], "small")
 784             self.failUnlessEqual(caps[self.small2.get_uri()], "small2")
 785             self.failUnlessEqual(caps[self.empty_lit_dir.get_uri()], "empty_lit_dir")
 786             self.failUnlessEqual(caps[self.tiny_lit_dir.get_uri()], "tiny_lit_dir")
 787         d.addCallback(_check)
 788         return d
 789
 790     def do_cli_manifest_stream2(self):
 791         basedir = self.get_clientdir(0)
 792         d = self._run_cli(["manifest",
 793                            "--node-directory", basedir,
 794                            "--raw",
 795                            self.root_uri])
 796         def _check((out,err)):
 797             self.failUnlessEqual(err, "")
 798             # this should be the same as the POST t=stream-manifest output
 799             self._check_streamed_manifest(out)
 800         d.addCallback(_check)
 801         return d
 802
 803     def do_cli_manifest_stream3(self):
 804         basedir = self.get_clientdir(0)
 805         d = self._run_cli(["manifest",
 806                            "--node-directory", basedir,
 807                            "--storage-index",
 808                            self.root_uri])
 809         def _check((out,err)):
 810             self.failUnlessEqual(err, "")
 811             self._check_manifest_storage_index(out)
 812         d.addCallback(_check)
 813         return d
 814
 815     def do_cli_manifest_stream4(self):
 816         basedir = self.get_clientdir(0)
 817         d = self._run_cli(["manifest",
 818                            "--node-directory", basedir,
 819                            "--verify-cap",
 820                            self.root_uri])
 821         def _check((out,err)):
 822             self.failUnlessEqual(err, "")
 823             lines = [l for l in out.split("\n") if l]
 824             self.failUnlessEqual(len(lines), 3)
 825             self.failUnless(self.root.get_verify_cap().to_string() in lines)
 826             self.failUnless(self.mutable.get_verify_cap().to_string() in lines)
 827             self.failUnless(self.large.get_verify_cap().to_string() in lines)
 828         d.addCallback(_check)
 829         return d
 830
 831     def do_cli_manifest_stream5(self):
 832         basedir = self.get_clientdir(0)
 833         d = self._run_cli(["manifest",
 834                            "--node-directory", basedir,
 835                            "--repair-cap",
 836                            self.root_uri])
 837         def _check((out,err)):
 838             self.failUnlessEqual(err, "")
 839             lines = [l for l in out.split("\n") if l]
 840             self.failUnlessEqual(len(lines), 3)
 841             self.failUnless(self.root.get_repair_cap().to_string() in lines)
 842             self.failUnless(self.mutable.get_repair_cap().to_string() in lines)
 843             self.failUnless(self.large.get_repair_cap().to_string() in lines)
 844         d.addCallback(_check)
 845         return d
 846
 847     def do_cli_stats1(self):
 848         basedir = self.get_clientdir(0)
 849         d = self._run_cli(["stats",
 850                            "--node-directory", basedir,
 851                            self.root_uri])
 852         def _check3((out,err)):
 853             lines = [l.strip() for l in out.split("\n") if l]
 854             self.failUnless("count-immutable-files: 1" in lines)
 855             self.failUnless("count-mutable-files: 1" in lines)
 856             self.failUnless("count-literal-files: 3" in lines)
 857             self.failUnless("count-files: 5" in lines)
 858             self.failUnless("count-directories: 3" in lines)
 859             self.failUnless("size-immutable-files: 13000    (13.00 kB, 12.70 kiB)" in lines, lines)
 860             self.failUnless("size-literal-files: 56" in lines, lines)
 861             self.failUnless("    4-10    : 1    (10 B, 10 B)".strip() in lines, lines)
 862             self.failUnless("   11-31    : 2    (31 B, 31 B)".strip() in lines, lines)
 863             self.failUnless("10001-31622 : 1    (31.62 kB, 30.88 kiB)".strip() in lines, lines)
 864         d.addCallback(_check3)
 865         return d
 866
 867     def do_cli_stats2(self):
 868         basedir = self.get_clientdir(0)
 869         d = self._run_cli(["stats",
 870                            "--node-directory", basedir,
 871                            "--raw",
 872                            self.root_uri])
 873         def _check4((out,err)):
 874             data = simplejson.loads(out)
 875             self.failUnlessEqual(data["count-immutable-files"], 1)
 876             self.failUnlessEqual(data["count-immutable-files"], 1)
 877             self.failUnlessEqual(data["count-mutable-files"], 1)
 878             self.failUnlessEqual(data["count-literal-files"], 3)
 879             self.failUnlessEqual(data["count-files"], 5)
 880             self.failUnlessEqual(data["count-directories"], 3)
 881             self.failUnlessEqual(data["size-immutable-files"], 13000)
 882             self.failUnlessEqual(data["size-literal-files"], 56)
 883             self.failUnless([4,10,1] in data["size-files-histogram"])
 884             self.failUnless([11,31,2] in data["size-files-histogram"])
 885             self.failUnless([10001,31622,1] in data["size-files-histogram"])
 886         d.addCallback(_check4)
 887         return d
 888
 889
 890 class DeepCheckWebBad(DeepCheckBase, unittest.TestCase):
 891     def test_bad(self):
 892         self.basedir = "deepcheck/DeepCheckWebBad/bad"
 893         self.set_up_grid()
 894         d = self.set_up_damaged_tree()
 895         d.addCallback(self.do_check)
 896         d.addCallback(self.do_deepcheck)
 897         d.addCallback(self.do_deepcheck_broken)
 898         d.addCallback(self.do_test_web_bad)
 899         d.addErrback(self.explain_web_error)
 900         d.addErrback(self.explain_error)
 901         return d
 902
 903
 904
 905     def set_up_damaged_tree(self):
 906         # 6.4s
 907
 908         # root
 909         #   mutable-good
 910         #   mutable-missing-shares
 911         #   mutable-corrupt-shares
 912         #   mutable-unrecoverable
 913         #   large-good
 914         #   large-missing-shares
 915         #   large-corrupt-shares
 916         #   large-unrecoverable
 917         # broken
 918         #   large1-good
 919         #   subdir-good
 920         #     large2-good
 921         #   subdir-unrecoverable
 922         #     large3-good
 923
 924         self.nodes = {}
 925
 926         c0 = self.g.clients[0]
 927         d = c0.create_dirnode()
 928         def _created_root(n):
 929             self.root = n
 930             self.root_uri = n.get_uri()
 931         d.addCallback(_created_root)
 932         d.addCallback(self.create_mangled, "mutable-good")
 933         d.addCallback(self.create_mangled, "mutable-missing-shares")
 934         d.addCallback(self.create_mangled, "mutable-corrupt-shares")
 935         d.addCallback(self.create_mangled, "mutable-unrecoverable")
 936         d.addCallback(self.create_mangled, "large-good")
 937         d.addCallback(self.create_mangled, "large-missing-shares")
 938         d.addCallback(self.create_mangled, "large-corrupt-shares")
 939         d.addCallback(self.create_mangled, "large-unrecoverable")
 940         d.addCallback(lambda ignored: c0.create_dirnode())
 941         d.addCallback(self._stash_node, "broken")
 942         large1 = upload.Data("Lots of data\n" * 1000 + "large1" + "\n", None)
 943         d.addCallback(lambda ignored:
 944                       self.nodes["broken"].add_file(u"large1", large1))
 945         d.addCallback(lambda ignored:
 946                       self.nodes["broken"].create_subdirectory(u"subdir-good"))
 947         large2 = upload.Data("Lots of data\n" * 1000 + "large2" + "\n", None)
 948         d.addCallback(lambda subdir: subdir.add_file(u"large2-good", large2))
 949         d.addCallback(lambda ignored:
 950                       self.nodes["broken"].create_subdirectory(u"subdir-unrecoverable"))
 951         d.addCallback(self._stash_node, "subdir-unrecoverable")
 952         large3 = upload.Data("Lots of data\n" * 1000 + "large3" + "\n", None)
 953         d.addCallback(lambda subdir: subdir.add_file(u"large3-good", large3))
 954         d.addCallback(lambda ignored:
 955                       self._delete_most_shares(self.nodes["broken"]))
 956         return d
 957
 958     def _stash_node(self, node, name):
 959         self.nodes[name] = node
 960         return node
 961
 962     def create_mangled(self, ignored, name):
 963         nodetype, mangletype = name.split("-", 1)
 964         if nodetype == "mutable":
 965             mutable_uploadable = MutableData("mutable file contents")
 966             d = self.g.clients[0].create_mutable_file(mutable_uploadable)
 967             d.addCallback(lambda n: self.root.set_node(unicode(name), n))
 968         elif nodetype == "large":
 969             large = upload.Data("Lots of data\n" * 1000 + name + "\n", None)
 970             d = self.root.add_file(unicode(name), large)
 971         elif nodetype == "small":
 972             small = upload.Data("Small enough for a LIT", None)
 973             d = self.root.add_file(unicode(name), small)
 974
 975         d.addCallback(self._stash_node, name)
 976
 977         if mangletype == "good":
 978             pass
 979         elif mangletype == "missing-shares":
 980             d.addCallback(self._delete_some_shares)
 981         elif mangletype == "corrupt-shares":
 982             d.addCallback(self._corrupt_some_shares)
 983         else:
 984             assert mangletype == "unrecoverable"
 985             d.addCallback(self._delete_most_shares)
 986
 987         return d
 988
 989     def _run_cli(self, argv):
 990         stdout, stderr = StringIO(), StringIO()
 991         # this can only do synchronous operations
 992         assert argv[0] == "debug"
 993         runner.runner(argv, run_by_human=False, stdout=stdout, stderr=stderr)
 994         return stdout.getvalue()
 995
 996     def _delete_some_shares(self, node):
 997         self.delete_shares_numbered(node.get_uri(), [0,1])
 998
 999     def _corrupt_some_shares(self, node):
1000         for (shnum, serverid, sharefile) in self.find_uri_shares(node.get_uri()):
1001             if shnum in (0,1):
1002                 self._run_cli(["debug", "corrupt-share", sharefile])
1003
1004     def _delete_most_shares(self, node):
1005         self.delete_shares_numbered(node.get_uri(), range(1,10))
1006
1007
1008     def check_is_healthy(self, cr, where):
1009         try:
1010             self.failUnless(ICheckResults.providedBy(cr), (cr, type(cr), where))
1011             self.failUnless(cr.is_healthy(), (cr.get_report(), cr.is_healthy(), cr.get_summary(), where))
1012             self.failUnless(cr.is_recoverable(), where)
1013             d = cr.get_data()
1014             self.failUnlessEqual(d["count-recoverable-versions"], 1, where)
1015             self.failUnlessEqual(d["count-unrecoverable-versions"], 0, where)
1016             return cr
1017         except Exception, le:
1018             le.args = tuple(le.args + (where,))
1019             raise
1020
1021     def check_is_missing_shares(self, cr, where):
1022         self.failUnless(ICheckResults.providedBy(cr), where)
1023         self.failIf(cr.is_healthy(), where)
1024         self.failUnless(cr.is_recoverable(), where)
1025         d = cr.get_data()
1026         self.failUnlessEqual(d["count-recoverable-versions"], 1, where)
1027         self.failUnlessEqual(d["count-unrecoverable-versions"], 0, where)
1028         return cr
1029
1030     def check_has_corrupt_shares(self, cr, where):
1031         # by "corrupt-shares" we mean the file is still recoverable
1032         self.failUnless(ICheckResults.providedBy(cr), where)
1033         d = cr.get_data()
1034         self.failIf(cr.is_healthy(), (where, cr))
1035         self.failUnless(cr.is_recoverable(), where)
1036         d = cr.get_data()
1037         self.failUnless(d["count-shares-good"] < 10, where)
1038         self.failUnless(d["count-corrupt-shares"], where)
1039         self.failUnless(d["list-corrupt-shares"], where)
1040         return cr
1041
1042     def check_is_unrecoverable(self, cr, where):
1043         self.failUnless(ICheckResults.providedBy(cr), where)
1044         d = cr.get_data()
1045         self.failIf(cr.is_healthy(), where)
1046         self.failIf(cr.is_recoverable(), where)
1047         self.failUnless(d["count-shares-good"] < d["count-shares-needed"], (d["count-shares-good"], d["count-shares-needed"], where))
1048         self.failUnlessEqual(d["count-recoverable-versions"], 0, where)
1049         self.failUnlessEqual(d["count-unrecoverable-versions"], 1, where)
1050         return cr
1051
1052     def do_check(self, ignored):
1053         d = defer.succeed(None)
1054
1055         # check the individual items, without verification. This will not
1056         # detect corrupt shares.
1057         def _check(which, checker):
1058             d = self.nodes[which].check(Monitor())
1059             d.addCallback(checker, which + "--check")
1060             return d
1061
1062         d.addCallback(lambda ign: _check("mutable-good", self.check_is_healthy))
1063         d.addCallback(lambda ign: _check("mutable-missing-shares",
1064                                          self.check_is_missing_shares))
1065         d.addCallback(lambda ign: _check("mutable-corrupt-shares",
1066                                          self.check_is_healthy))
1067         d.addCallback(lambda ign: _check("mutable-unrecoverable",
1068                                          self.check_is_unrecoverable))
1069         d.addCallback(lambda ign: _check("large-good", self.check_is_healthy))
1070         d.addCallback(lambda ign: _check("large-missing-shares",
1071                                          self.check_is_missing_shares))
1072         d.addCallback(lambda ign: _check("large-corrupt-shares",
1073                                          self.check_is_healthy))
1074         d.addCallback(lambda ign: _check("large-unrecoverable",
1075                                          self.check_is_unrecoverable))
1076
1077         # and again with verify=True, which *does* detect corrupt shares.
1078         def _checkv(which, checker):
1079             d = self.nodes[which].check(Monitor(), verify=True)
1080             d.addCallback(checker, which + "--check-and-verify")
1081             return d
1082
1083         d.addCallback(lambda ign: _checkv("mutable-good", self.check_is_healthy))
1084         d.addCallback(lambda ign: _checkv("mutable-missing-shares",
1085                                          self.check_is_missing_shares))
1086         d.addCallback(lambda ign: _checkv("mutable-corrupt-shares",
1087                                          self.check_has_corrupt_shares))
1088         d.addCallback(lambda ign: _checkv("mutable-unrecoverable",
1089                                          self.check_is_unrecoverable))
1090         d.addCallback(lambda ign: _checkv("large-good", self.check_is_healthy))
1091         d.addCallback(lambda ign: _checkv("large-missing-shares", self.check_is_missing_shares))
1092         d.addCallback(lambda ign: _checkv("large-corrupt-shares", self.check_has_corrupt_shares))
1093         d.addCallback(lambda ign: _checkv("large-unrecoverable",
1094                                          self.check_is_unrecoverable))
1095
1096         return d
1097
1098     def do_deepcheck(self, ignored):
1099         d = defer.succeed(None)
1100
1101         # now deep-check the root, with various verify= and repair= options
1102         d.addCallback(lambda ign:
1103                       self.root.start_deep_check().when_done())
1104         def _check1(cr):
1105             self.failUnless(IDeepCheckResults.providedBy(cr))
1106             c = cr.get_counters()
1107             self.failUnlessEqual(c["count-objects-checked"], 9)
1108             self.failUnlessEqual(c["count-objects-healthy"], 5)
1109             self.failUnlessEqual(c["count-objects-unhealthy"], 4)
1110             self.failUnlessEqual(c["count-objects-unrecoverable"], 2)
1111         d.addCallback(_check1)
1112
1113         d.addCallback(lambda ign:
1114                       self.root.start_deep_check(verify=True).when_done())
1115         def _check2(cr):
1116             self.failUnless(IDeepCheckResults.providedBy(cr))
1117             c = cr.get_counters()
1118             self.failUnlessEqual(c["count-objects-checked"], 9)
1119             self.failUnlessEqual(c["count-objects-healthy"], 3)
1120             self.failUnlessEqual(c["count-objects-unhealthy"], 6)
1121             self.failUnlessEqual(c["count-objects-healthy"], 3) # root, mutable good, large good
1122             self.failUnlessEqual(c["count-objects-unrecoverable"], 2) # mutable unrecoverable, large unrecoverable
1123         d.addCallback(_check2)
1124
1125         return d
1126
1127     def do_deepcheck_broken(self, ignored):
1128         # deep-check on the broken directory should fail, because of the
1129         # untraversable subdir
1130         def _do_deep_check():
1131             return self.nodes["broken"].start_deep_check().when_done()
1132         d = self.shouldFail(UnrecoverableFileError, "do_deep_check",
1133                             "no recoverable versions",
1134                             _do_deep_check)
1135         return d
1136
1137     def json_is_healthy(self, data, where):
1138         r = data["results"]
1139         self.failUnless(r["healthy"], where)
1140         self.failUnless(r["recoverable"], where)
1141         self.failUnlessEqual(r["count-recoverable-versions"], 1, where)
1142         self.failUnlessEqual(r["count-unrecoverable-versions"], 0, where)
1143
1144     def json_is_missing_shares(self, data, where):
1145         r = data["results"]
1146         self.failIf(r["healthy"], where)
1147         self.failUnless(r["recoverable"], where)
1148         self.failUnlessEqual(r["count-recoverable-versions"], 1, where)
1149         self.failUnlessEqual(r["count-unrecoverable-versions"], 0, where)
1150
1151     def json_has_corrupt_shares(self, data, where):
1152         # by "corrupt-shares" we mean the file is still recoverable
1153         r = data["results"]
1154         self.failIf(r["healthy"], where)
1155         self.failUnless(r["recoverable"], where)
1156         self.failUnless(r["count-shares-good"] < 10, where)
1157         self.failUnless(r["count-corrupt-shares"], where)
1158         self.failUnless(r["list-corrupt-shares"], where)
1159
1160     def json_is_unrecoverable(self, data, where):
1161         r = data["results"]
1162         self.failIf(r["healthy"], where)
1163         self.failIf(r["recoverable"], where)
1164         self.failUnless(r["count-shares-good"] < r["count-shares-needed"],
1165                         where)
1166         self.failUnlessEqual(r["count-recoverable-versions"], 0, where)
1167         self.failUnlessEqual(r["count-unrecoverable-versions"], 1, where)
1168
1169     def do_test_web_bad(self, ignored):
1170         d = defer.succeed(None)
1171
1172         # check, no verify
1173         def _check(which, checker):
1174             d = self.web_json(self.nodes[which], t="check")
1175             d.addCallback(checker, which + "--webcheck")
1176             return d
1177
1178         d.addCallback(lambda ign: _check("mutable-good",
1179                                          self.json_is_healthy))
1180         d.addCallback(lambda ign: _check("mutable-missing-shares",
1181                                          self.json_is_missing_shares))
1182         d.addCallback(lambda ign: _check("mutable-corrupt-shares",
1183                                          self.json_is_healthy))
1184         d.addCallback(lambda ign: _check("mutable-unrecoverable",
1185                                          self.json_is_unrecoverable))
1186         d.addCallback(lambda ign: _check("large-good",
1187                                          self.json_is_healthy))
1188         d.addCallback(lambda ign: _check("large-missing-shares",
1189                                          self.json_is_missing_shares))
1190         d.addCallback(lambda ign: _check("large-corrupt-shares",
1191                                          self.json_is_healthy))
1192         d.addCallback(lambda ign: _check("large-unrecoverable",
1193                                          self.json_is_unrecoverable))
1194
1195         # check and verify
1196         def _checkv(which, checker):
1197             d = self.web_json(self.nodes[which], t="check", verify="true")
1198             d.addCallback(checker, which + "--webcheck-and-verify")
1199             return d
1200
1201         d.addCallback(lambda ign: _checkv("mutable-good",
1202                                           self.json_is_healthy))
1203         d.addCallback(lambda ign: _checkv("mutable-missing-shares",
1204                                          self.json_is_missing_shares))
1205         d.addCallback(lambda ign: _checkv("mutable-corrupt-shares",
1206                                          self.json_has_corrupt_shares))
1207         d.addCallback(lambda ign: _checkv("mutable-unrecoverable",
1208                                          self.json_is_unrecoverable))
1209         d.addCallback(lambda ign: _checkv("large-good",
1210                                           self.json_is_healthy))
1211         d.addCallback(lambda ign: _checkv("large-missing-shares", self.json_is_missing_shares))
1212         d.addCallback(lambda ign: _checkv("large-corrupt-shares", self.json_has_corrupt_shares))
1213         d.addCallback(lambda ign: _checkv("large-unrecoverable",
1214                                          self.json_is_unrecoverable))
1215
1216         return d
1217
1218 class Large(DeepCheckBase, unittest.TestCase):
1219     def test_lots_of_lits(self):
1220         self.basedir = "deepcheck/Large/lots_of_lits"
1221         self.set_up_grid()
1222         # create the following directory structure:
1223         #  root/
1224         #   subdir/
1225         #    000-large (CHK)
1226         #    001-small (LIT)
1227         #    002-small
1228         #    ...
1229         #    399-small
1230         # then do a deepcheck and make sure it doesn't cause a
1231         # Deferred-tail-recursion stack overflow
1232
1233         COUNT = 400
1234         c0 = self.g.clients[0]
1235         d = c0.create_dirnode()
1236         self.stash = {}
1237         def _created_root(n):
1238             self.root = n
1239             return n
1240         d.addCallback(_created_root)
1241         d.addCallback(lambda root: root.create_subdirectory(u"subdir"))
1242         def _add_children(subdir_node):
1243             self.subdir_node = subdir_node
1244             kids = {}
1245             for i in range(1, COUNT):
1246                 litcap = LiteralFileURI("%03d-data" % i).to_string()
1247                 kids[u"%03d-small" % i] = (litcap, litcap)
1248             return subdir_node.set_children(kids)
1249         d.addCallback(_add_children)
1250         up = upload.Data("large enough for CHK" * 100, "")
1251         d.addCallback(lambda ign: self.subdir_node.add_file(u"0000-large", up))
1252
1253         def _start_deepcheck(ignored):
1254             return self.web(self.root, method="POST", t="stream-deep-check")
1255         d.addCallback(_start_deepcheck)
1256         def _check( (output, url) ):
1257             units = list(self.parse_streamed_json(output))
1258             self.failUnlessEqual(len(units), 2+COUNT+1)
1259         d.addCallback(_check)
1260
1261         return d