src/allmydata/test/test_deepcheck.py

   1
   2 import os, simplejson, urllib
   3 from cStringIO import StringIO
   4 from twisted.trial import unittest
   5 from twisted.internet import defer
   6 from twisted.internet import threads # CLI tests use deferToThread
   7 from allmydata.immutable import upload
   8 from allmydata.mutable.common import UnrecoverableFileError
   9 from allmydata.mutable.publish import MutableData
  10 from allmydata.util import idlib
  11 from allmydata.util import base32
  12 from allmydata.scripts import runner
  13 from allmydata.interfaces import ICheckResults, ICheckAndRepairResults, \
  14      IDeepCheckResults, IDeepCheckAndRepairResults
  15 from allmydata.monitor import Monitor, OperationCancelledError
  16 from allmydata.uri import LiteralFileURI
  17 from twisted.web.client import getPage
  18
  19 from allmydata.test.common import ErrorMixin, _corrupt_mutable_share_data, \
  20      ShouldFailMixin
  21 from allmydata.test.common_util import StallMixin
  22 from allmydata.test.no_network import GridTestMixin
  23
  24 timeout = 2400 # One of these took 1046.091s on Zandr's ARM box.
  25
  26 class MutableChecker(GridTestMixin, unittest.TestCase, ErrorMixin):
  27     def _run_cli(self, argv):
  28         stdout, stderr = StringIO(), StringIO()
  29         # this can only do synchronous operations
  30         assert argv[0] == "debug"
  31         runner.runner(argv, run_by_human=False, stdout=stdout, stderr=stderr)
  32         return stdout.getvalue()
  33
  34     def test_good(self):
  35         self.basedir = "deepcheck/MutableChecker/good"
  36         self.set_up_grid()
  37         CONTENTS = "a little bit of data"
  38         CONTENTS_uploadable = MutableData(CONTENTS)
  39         d = self.g.clients[0].create_mutable_file(CONTENTS_uploadable)
  40         def _created(node):
  41             self.node = node
  42             self.fileurl = "uri/" + urllib.quote(node.get_uri())
  43         d.addCallback(_created)
  44         # now make sure the webapi verifier sees no problems
  45         d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=true",
  46                                            method="POST"))
  47         def _got_results(out):
  48             self.failUnless("<span>Healthy : Healthy</span>" in out, out)
  49             self.failUnless("Recoverable Versions: 10*seq1-" in out, out)
  50             self.failIf("Not Healthy!" in out, out)
  51             self.failIf("Unhealthy" in out, out)
  52             self.failIf("Corrupt Shares" in out, out)
  53         d.addCallback(_got_results)
  54         d.addErrback(self.explain_web_error)
  55         return d
  56
  57     def test_corrupt(self):
  58         self.basedir = "deepcheck/MutableChecker/corrupt"
  59         self.set_up_grid()
  60         CONTENTS = "a little bit of data"
  61         CONTENTS_uploadable = MutableData(CONTENTS)
  62         d = self.g.clients[0].create_mutable_file(CONTENTS_uploadable)
  63         def _stash_and_corrupt(node):
  64             self.node = node
  65             self.fileurl = "uri/" + urllib.quote(node.get_uri())
  66             self.corrupt_shares_numbered(node.get_uri(), [0],
  67                                          _corrupt_mutable_share_data)
  68         d.addCallback(_stash_and_corrupt)
  69         # now make sure the webapi verifier notices it
  70         d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=true",
  71                                            method="POST"))
  72         def _got_results(out):
  73             self.failUnless("Not Healthy!" in out, out)
  74             self.failUnless("Unhealthy: best version has only 9 shares (encoding is 3-of-10)" in out, out)
  75             self.failUnless("Corrupt Shares:" in out, out)
  76         d.addCallback(_got_results)
  77
  78         # now make sure the webapi repairer can fix it
  79         d.addCallback(lambda ign:
  80                       self.GET(self.fileurl+"?t=check&verify=true&repair=true",
  81                                method="POST"))
  82         def _got_repair_results(out):
  83             self.failUnless("<div>Repair successful</div>" in out, out)
  84         d.addCallback(_got_repair_results)
  85         d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=true",
  86                                            method="POST"))
  87         def _got_postrepair_results(out):
  88             self.failIf("Not Healthy!" in out, out)
  89             self.failUnless("Recoverable Versions: 10*seq" in out, out)
  90         d.addCallback(_got_postrepair_results)
  91         d.addErrback(self.explain_web_error)
  92
  93         return d
  94
  95     def test_delete_share(self):
  96         self.basedir = "deepcheck/MutableChecker/delete_share"
  97         self.set_up_grid()
  98         CONTENTS = "a little bit of data"
  99         CONTENTS_uploadable = MutableData(CONTENTS)
 100         d = self.g.clients[0].create_mutable_file(CONTENTS_uploadable)
 101         def _stash_and_delete(node):
 102             self.node = node
 103             self.fileurl = "uri/" + urllib.quote(node.get_uri())
 104             self.delete_shares_numbered(node.get_uri(), [0])
 105         d.addCallback(_stash_and_delete)
 106         # now make sure the webapi checker notices it
 107         d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=false",
 108                                            method="POST"))
 109         def _got_results(out):
 110             self.failUnless("Not Healthy!" in out, out)
 111             self.failUnless("Unhealthy: best version has only 9 shares (encoding is 3-of-10)" in out, out)
 112             self.failIf("Corrupt Shares" in out, out)
 113         d.addCallback(_got_results)
 114
 115         # now make sure the webapi repairer can fix it
 116         d.addCallback(lambda ign:
 117                       self.GET(self.fileurl+"?t=check&verify=false&repair=true",
 118                                method="POST"))
 119         def _got_repair_results(out):
 120             self.failUnless("Repair successful" in out)
 121         d.addCallback(_got_repair_results)
 122         d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=false",
 123                                            method="POST"))
 124         def _got_postrepair_results(out):
 125             self.failIf("Not Healthy!" in out, out)
 126             self.failUnless("Recoverable Versions: 10*seq" in out)
 127         d.addCallback(_got_postrepair_results)
 128         d.addErrback(self.explain_web_error)
 129
 130         return d
 131
 132
 133 class DeepCheckBase(GridTestMixin, ErrorMixin, StallMixin, ShouldFailMixin):
 134
 135     def web_json(self, n, **kwargs):
 136         kwargs["output"] = "json"
 137         d = self.web(n, "POST", **kwargs)
 138         d.addCallback(self.decode_json)
 139         return d
 140
 141     def decode_json(self, (s,url)):
 142         try:
 143             data = simplejson.loads(s)
 144         except ValueError:
 145             self.fail("%s: not JSON: '%s'" % (url, s))
 146         return data
 147
 148     def parse_streamed_json(self, s):
 149         for unit in s.split("\n"):
 150             if not unit:
 151                 # stream should end with a newline, so split returns ""
 152                 continue
 153             try:
 154                 yield simplejson.loads(unit)
 155             except ValueError, le:
 156                 le.args = tuple(le.args + (unit,))
 157                 raise
 158
 159     def web(self, n, method="GET", **kwargs):
 160         # returns (data, url)
 161         url = (self.client_baseurls[0] + "uri/%s" % urllib.quote(n.get_uri())
 162                + "?" + "&".join(["%s=%s" % (k,v) for (k,v) in kwargs.items()]))
 163         d = getPage(url, method=method)
 164         d.addCallback(lambda data: (data,url))
 165         return d
 166
 167     def wait_for_operation(self, ignored, ophandle):
 168         url = self.client_baseurls[0] + "operations/" + ophandle
 169         url += "?t=status&output=JSON"
 170         d = getPage(url)
 171         def _got(res):
 172             try:
 173                 data = simplejson.loads(res)
 174             except ValueError:
 175                 self.fail("%s: not JSON: '%s'" % (url, res))
 176             if not data["finished"]:
 177                 d = self.stall(delay=1.0)
 178                 d.addCallback(self.wait_for_operation, ophandle)
 179                 return d
 180             return data
 181         d.addCallback(_got)
 182         return d
 183
 184     def get_operation_results(self, ignored, ophandle, output=None):
 185         url = self.client_baseurls[0] + "operations/" + ophandle
 186         url += "?t=status"
 187         if output:
 188             url += "&output=" + output
 189         d = getPage(url)
 190         def _got(res):
 191             if output and output.lower() == "json":
 192                 try:
 193                     return simplejson.loads(res)
 194                 except ValueError:
 195                     self.fail("%s: not JSON: '%s'" % (url, res))
 196             return res
 197         d.addCallback(_got)
 198         return d
 199
 200     def slow_web(self, n, output=None, **kwargs):
 201         # use ophandle=
 202         handle = base32.b2a(os.urandom(4))
 203         d = self.web(n, "POST", ophandle=handle, **kwargs)
 204         d.addCallback(self.wait_for_operation, handle)
 205         d.addCallback(self.get_operation_results, handle, output=output)
 206         return d
 207
 208
 209 class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
 210     # construct a small directory tree (with one dir, one immutable file, one
 211     # mutable file, two LIT files, one DIR2:LIT empty dir, one DIR2:LIT tiny
 212     # dir, and a loop), and then check/examine it in various ways.
 213
 214     def set_up_tree(self):
 215         # 2.9s
 216
 217         c0 = self.g.clients[0]
 218         d = c0.create_dirnode()
 219         def _created_root(n):
 220             self.root = n
 221             self.root_uri = n.get_uri()
 222         d.addCallback(_created_root)
 223         d.addCallback(lambda ign:
 224             c0.create_mutable_file(MutableData("mutable file contents")))
 225         d.addCallback(lambda n: self.root.set_node(u"mutable", n))
 226         def _created_mutable(n):
 227             self.mutable = n
 228             self.mutable_uri = n.get_uri()
 229         d.addCallback(_created_mutable)
 230
 231         large = upload.Data("Lots of data\n" * 1000, None)
 232         d.addCallback(lambda ign: self.root.add_file(u"large", large))
 233         def _created_large(n):
 234             self.large = n
 235             self.large_uri = n.get_uri()
 236         d.addCallback(_created_large)
 237
 238         small = upload.Data("Small enough for a LIT", None)
 239         d.addCallback(lambda ign: self.root.add_file(u"small", small))
 240         def _created_small(n):
 241             self.small = n
 242             self.small_uri = n.get_uri()
 243         d.addCallback(_created_small)
 244
 245         small2 = upload.Data("Small enough for a LIT too", None)
 246         d.addCallback(lambda ign: self.root.add_file(u"small2", small2))
 247         def _created_small2(n):
 248             self.small2 = n
 249             self.small2_uri = n.get_uri()
 250         d.addCallback(_created_small2)
 251
 252         empty_litdir_uri = "URI:DIR2-LIT:"
 253         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
 254
 255         d.addCallback(lambda ign: self.root._create_and_validate_node(None, empty_litdir_uri, name=u"test_deepcheck empty_lit_dir"))
 256         def _created_empty_lit_dir(n):
 257             self.empty_lit_dir = n
 258             self.empty_lit_dir_uri = n.get_uri()
 259             self.root.set_node(u"empty_lit_dir", n)
 260         d.addCallback(_created_empty_lit_dir)
 261
 262         d.addCallback(lambda ign: self.root._create_and_validate_node(None, tiny_litdir_uri, name=u"test_deepcheck tiny_lit_dir"))
 263         def _created_tiny_lit_dir(n):
 264             self.tiny_lit_dir = n
 265             self.tiny_lit_dir_uri = n.get_uri()
 266             self.root.set_node(u"tiny_lit_dir", n)
 267         d.addCallback(_created_tiny_lit_dir)
 268
 269         d.addCallback(lambda ign: self.root.set_node(u"loop", self.root))
 270         return d
 271
 272     def check_is_healthy(self, cr, n, where, incomplete=False):
 273         self.failUnless(ICheckResults.providedBy(cr), where)
 274         self.failUnless(cr.is_healthy(), where)
 275         self.failUnlessEqual(cr.get_storage_index(), n.get_storage_index(),
 276                              where)
 277         self.failUnlessEqual(cr.get_storage_index_string(),
 278                              base32.b2a(n.get_storage_index()), where)
 279         num_servers = len(self.g.all_servers)
 280         needs_rebalancing = bool( num_servers < 10 )
 281         if not incomplete:
 282             self.failUnlessEqual(cr.needs_rebalancing(), needs_rebalancing,
 283                                  str((where, cr, cr.as_dict())))
 284         self.failUnlessEqual(cr.get_share_counter_good(), 10, where)
 285         self.failUnlessEqual(cr.get_encoding_needed(), 3, where)
 286         self.failUnlessEqual(cr.get_encoding_expected(), 10, where)
 287         if not incomplete:
 288             self.failUnlessEqual(cr.get_host_counter_good_shares(),
 289                                  num_servers, where)
 290         self.failUnlessEqual(cr.get_corrupt_shares(), [], where)
 291         if not incomplete:
 292             self.failUnlessEqual(sorted([s.get_serverid()
 293                                          for s in cr.get_servers_responding()]),
 294                                  sorted(self.g.get_all_serverids()),
 295                                  where)
 296             all_serverids = set()
 297             for (shareid, servers) in cr.get_sharemap().items():
 298                 all_serverids.update([s.get_serverid() for s in servers])
 299             self.failUnlessEqual(sorted(all_serverids),
 300                                  sorted(self.g.get_all_serverids()),
 301                                  where)
 302
 303         self.failUnlessEqual(cr.get_share_counter_wrong(), 0, where)
 304         self.failUnlessEqual(cr.get_version_counter_recoverable(), 1, where)
 305         self.failUnlessEqual(cr.get_version_counter_unrecoverable(), 0, where)
 306
 307
 308     def check_and_repair_is_healthy(self, cr, n, where, incomplete=False):
 309         self.failUnless(ICheckAndRepairResults.providedBy(cr), (where, cr))
 310         self.failUnless(cr.get_pre_repair_results().is_healthy(), where)
 311         self.check_is_healthy(cr.get_pre_repair_results(), n, where, incomplete)
 312         self.failUnless(cr.get_post_repair_results().is_healthy(), where)
 313         self.check_is_healthy(cr.get_post_repair_results(), n, where, incomplete)
 314         self.failIf(cr.get_repair_attempted(), where)
 315
 316     def deep_check_is_healthy(self, cr, num_healthy, where):
 317         self.failUnless(IDeepCheckResults.providedBy(cr))
 318         self.failUnlessEqual(cr.get_counters()["count-objects-healthy"],
 319                              num_healthy, where)
 320
 321     def deep_check_and_repair_is_healthy(self, cr, num_healthy, where):
 322         self.failUnless(IDeepCheckAndRepairResults.providedBy(cr), where)
 323         c = cr.get_counters()
 324         self.failUnlessEqual(c["count-objects-healthy-pre-repair"],
 325                              num_healthy, where)
 326         self.failUnlessEqual(c["count-objects-healthy-post-repair"],
 327                              num_healthy, where)
 328         self.failUnlessEqual(c["count-repairs-attempted"], 0, where)
 329
 330     def test_good(self):
 331         self.basedir = "deepcheck/DeepCheckWebGood/good"
 332         self.set_up_grid()
 333         d = self.set_up_tree()
 334         d.addCallback(self.do_stats)
 335         d.addCallback(self.do_web_stream_manifest)
 336         d.addCallback(self.do_web_stream_check)
 337         d.addCallback(self.do_test_check_good)
 338         d.addCallback(self.do_test_web_good)
 339         d.addCallback(self.do_test_cli_good)
 340         d.addErrback(self.explain_web_error)
 341         d.addErrback(self.explain_error)
 342         return d
 343
 344     def do_stats(self, ignored):
 345         d = defer.succeed(None)
 346         d.addCallback(lambda ign: self.root.start_deep_stats().when_done())
 347         d.addCallback(self.check_stats_good)
 348         return d
 349
 350     def check_stats_good(self, s):
 351         self.failUnlessEqual(s["count-directories"], 3)
 352         self.failUnlessEqual(s["count-files"], 5)
 353         self.failUnlessEqual(s["count-immutable-files"], 1)
 354         self.failUnlessEqual(s["count-literal-files"], 3)
 355         self.failUnlessEqual(s["count-mutable-files"], 1)
 356         # don't check directories: their size will vary
 357         # s["largest-directory"]
 358         # s["size-directories"]
 359         self.failUnlessEqual(s["largest-directory-children"], 7)
 360         self.failUnlessEqual(s["largest-immutable-file"], 13000)
 361         # to re-use this function for both the local
 362         # dirnode.start_deep_stats() and the webapi t=start-deep-stats, we
 363         # coerce the result into a list of tuples. dirnode.start_deep_stats()
 364         # returns a list of tuples, but JSON only knows about lists., so
 365         # t=start-deep-stats returns a list of lists.
 366         histogram = [tuple(stuff) for stuff in s["size-files-histogram"]]
 367         self.failUnlessEqual(histogram, [(4, 10, 1), (11, 31, 2),
 368                                          (10001, 31622, 1),
 369                                          ])
 370         self.failUnlessEqual(s["size-immutable-files"], 13000)
 371         self.failUnlessEqual(s["size-literal-files"], 56)
 372
 373     def do_web_stream_manifest(self, ignored):
 374         d = self.web(self.root, method="POST", t="stream-manifest")
 375         d.addCallback(lambda (output,url):
 376                       self._check_streamed_manifest(output))
 377         return d
 378
 379     def _check_streamed_manifest(self, output):
 380         units = list(self.parse_streamed_json(output))
 381         files = [u for u in units if u["type"] in ("file", "directory")]
 382         assert units[-1]["type"] == "stats"
 383         stats = units[-1]["stats"]
 384         self.failUnlessEqual(len(files), 8)
 385         # [root,mutable,large] are distributed, [small,small2,empty_litdir,tiny_litdir] are not
 386         self.failUnlessEqual(len([f for f in files
 387                                   if f["verifycap"] != ""]), 3)
 388         self.failUnlessEqual(len([f for f in files
 389                                   if f["verifycap"] == ""]), 5)
 390         self.failUnlessEqual(len([f for f in files
 391                                   if f["repaircap"] != ""]), 3)
 392         self.failUnlessEqual(len([f for f in files
 393                                   if f["repaircap"] == ""]), 5)
 394         self.failUnlessEqual(len([f for f in files
 395                                   if f["storage-index"] != ""]), 3)
 396         self.failUnlessEqual(len([f for f in files
 397                                   if f["storage-index"] == ""]), 5)
 398         # make sure that a mutable file has filecap==repaircap!=verifycap
 399         mutable = [f for f in files
 400                    if f["cap"] is not None
 401                    and f["cap"].startswith("URI:SSK:")][0]
 402         self.failUnlessEqual(mutable["cap"], self.mutable_uri)
 403         self.failIfEqual(mutable["cap"], mutable["verifycap"])
 404         self.failUnlessEqual(mutable["cap"], mutable["repaircap"])
 405         # for immutable file, verifycap==repaircap!=filecap
 406         large = [f for f in files
 407                    if f["cap"] is not None
 408                    and f["cap"].startswith("URI:CHK:")][0]
 409         self.failUnlessEqual(large["cap"], self.large_uri)
 410         self.failIfEqual(large["cap"], large["verifycap"])
 411         self.failUnlessEqual(large["verifycap"], large["repaircap"])
 412         self.check_stats_good(stats)
 413
 414     def do_web_stream_check(self, ignored):
 415         # TODO
 416         return
 417         d = self.web(self.root, t="stream-deep-check")
 418         def _check(res):
 419             units = list(self.parse_streamed_json(res))
 420             #files = [u for u in units if u["type"] in ("file", "directory")]
 421             assert units[-1]["type"] == "stats"
 422             #stats = units[-1]["stats"]
 423             # ...
 424         d.addCallback(_check)
 425         return d
 426
 427     def do_test_check_good(self, ignored):
 428         d = defer.succeed(None)
 429         # check the individual items
 430         d.addCallback(lambda ign: self.root.check(Monitor()))
 431         d.addCallback(self.check_is_healthy, self.root, "root")
 432         d.addCallback(lambda ign: self.mutable.check(Monitor()))
 433         d.addCallback(self.check_is_healthy, self.mutable, "mutable")
 434         d.addCallback(lambda ign: self.large.check(Monitor()))
 435         d.addCallback(self.check_is_healthy, self.large, "large")
 436         d.addCallback(lambda ign: self.small.check(Monitor()))
 437         d.addCallback(self.failUnlessEqual, None, "small")
 438         d.addCallback(lambda ign: self.small2.check(Monitor()))
 439         d.addCallback(self.failUnlessEqual, None, "small2")
 440         d.addCallback(lambda ign: self.empty_lit_dir.check(Monitor()))
 441         d.addCallback(self.failUnlessEqual, None, "empty_lit_dir")
 442         d.addCallback(lambda ign: self.tiny_lit_dir.check(Monitor()))
 443         d.addCallback(self.failUnlessEqual, None, "tiny_lit_dir")
 444
 445         # and again with verify=True
 446         d.addCallback(lambda ign: self.root.check(Monitor(), verify=True))
 447         d.addCallback(self.check_is_healthy, self.root, "root")
 448         d.addCallback(lambda ign: self.mutable.check(Monitor(), verify=True))
 449         d.addCallback(self.check_is_healthy, self.mutable, "mutable")
 450         d.addCallback(lambda ign: self.large.check(Monitor(), verify=True))
 451         d.addCallback(self.check_is_healthy, self.large, "large", incomplete=True)
 452         d.addCallback(lambda ign: self.small.check(Monitor(), verify=True))
 453         d.addCallback(self.failUnlessEqual, None, "small")
 454         d.addCallback(lambda ign: self.small2.check(Monitor(), verify=True))
 455         d.addCallback(self.failUnlessEqual, None, "small2")
 456         d.addCallback(lambda ign: self.empty_lit_dir.check(Monitor(), verify=True))
 457         d.addCallback(self.failUnlessEqual, None, "empty_lit_dir")
 458         d.addCallback(lambda ign: self.tiny_lit_dir.check(Monitor(), verify=True))
 459         d.addCallback(self.failUnlessEqual, None, "tiny_lit_dir")
 460
 461         # and check_and_repair(), which should be a nop
 462         d.addCallback(lambda ign: self.root.check_and_repair(Monitor()))
 463         d.addCallback(self.check_and_repair_is_healthy, self.root, "root")
 464         d.addCallback(lambda ign: self.mutable.check_and_repair(Monitor()))
 465         d.addCallback(self.check_and_repair_is_healthy, self.mutable, "mutable")
 466         d.addCallback(lambda ign: self.large.check_and_repair(Monitor()))
 467         d.addCallback(self.check_and_repair_is_healthy, self.large, "large")
 468         d.addCallback(lambda ign: self.small.check_and_repair(Monitor()))
 469         d.addCallback(self.failUnlessEqual, None, "small")
 470         d.addCallback(lambda ign: self.small2.check_and_repair(Monitor()))
 471         d.addCallback(self.failUnlessEqual, None, "small2")
 472         d.addCallback(lambda ign: self.empty_lit_dir.check_and_repair(Monitor()))
 473         d.addCallback(self.failUnlessEqual, None, "empty_lit_dir")
 474         d.addCallback(lambda ign: self.tiny_lit_dir.check_and_repair(Monitor()))
 475
 476         # check_and_repair(verify=True)
 477         d.addCallback(lambda ign: self.root.check_and_repair(Monitor(), verify=True))
 478         d.addCallback(self.check_and_repair_is_healthy, self.root, "root")
 479         d.addCallback(lambda ign: self.mutable.check_and_repair(Monitor(), verify=True))
 480         d.addCallback(self.check_and_repair_is_healthy, self.mutable, "mutable")
 481         d.addCallback(lambda ign: self.large.check_and_repair(Monitor(), verify=True))
 482         d.addCallback(self.check_and_repair_is_healthy, self.large, "large", incomplete=True)
 483         d.addCallback(lambda ign: self.small.check_and_repair(Monitor(), verify=True))
 484         d.addCallback(self.failUnlessEqual, None, "small")
 485         d.addCallback(lambda ign: self.small2.check_and_repair(Monitor(), verify=True))
 486         d.addCallback(self.failUnlessEqual, None, "small2")
 487         d.addCallback(self.failUnlessEqual, None, "small2")
 488         d.addCallback(lambda ign: self.empty_lit_dir.check_and_repair(Monitor(), verify=True))
 489         d.addCallback(self.failUnlessEqual, None, "empty_lit_dir")
 490         d.addCallback(lambda ign: self.tiny_lit_dir.check_and_repair(Monitor(), verify=True))
 491
 492
 493         # now deep-check the root, with various verify= and repair= options
 494         d.addCallback(lambda ign:
 495                       self.root.start_deep_check().when_done())
 496         d.addCallback(self.deep_check_is_healthy, 3, "root")
 497         d.addCallback(lambda ign:
 498                       self.root.start_deep_check(verify=True).when_done())
 499         d.addCallback(self.deep_check_is_healthy, 3, "root")
 500         d.addCallback(lambda ign:
 501                       self.root.start_deep_check_and_repair().when_done())
 502         d.addCallback(self.deep_check_and_repair_is_healthy, 3, "root")
 503         d.addCallback(lambda ign:
 504                       self.root.start_deep_check_and_repair(verify=True).when_done())
 505         d.addCallback(self.deep_check_and_repair_is_healthy, 3, "root")
 506
 507         # and finally, start a deep-check, but then cancel it.
 508         d.addCallback(lambda ign: self.root.start_deep_check())
 509         def _checking(monitor):
 510             monitor.cancel()
 511             d = monitor.when_done()
 512             # this should fire as soon as the next dirnode.list finishes.
 513             # TODO: add a counter to measure how many list() calls are made,
 514             # assert that no more than one gets to run before the cancel()
 515             # takes effect.
 516             def _finished_normally(res):
 517                 self.fail("this was supposed to fail, not finish normally")
 518             def _cancelled(f):
 519                 f.trap(OperationCancelledError)
 520             d.addCallbacks(_finished_normally, _cancelled)
 521             return d
 522         d.addCallback(_checking)
 523
 524         return d
 525
 526     def json_check_is_healthy(self, data, n, where, incomplete=False):
 527
 528         self.failUnlessEqual(data["storage-index"],
 529                              base32.b2a(n.get_storage_index()), where)
 530         self.failUnless("summary" in data, (where, data))
 531         self.failUnlessEqual(data["summary"].lower(), "healthy",
 532                              "%s: '%s'" % (where, data["summary"]))
 533         r = data["results"]
 534         self.failUnlessEqual(r["healthy"], True, where)
 535         num_servers = len(self.g.all_servers)
 536         needs_rebalancing = bool( num_servers < 10 )
 537         if not incomplete:
 538             self.failUnlessEqual(r["needs-rebalancing"], needs_rebalancing,
 539                                  where)
 540         self.failUnlessEqual(r["count-shares-good"], 10, where)
 541         self.failUnlessEqual(r["count-shares-needed"], 3, where)
 542         self.failUnlessEqual(r["count-shares-expected"], 10, where)
 543         if not incomplete:
 544             self.failUnlessEqual(r["count-good-share-hosts"], num_servers,
 545                                  where)
 546         self.failUnlessEqual(r["count-corrupt-shares"], 0, where)
 547         self.failUnlessEqual(r["list-corrupt-shares"], [], where)
 548         if not incomplete:
 549             self.failUnlessEqual(sorted(r["servers-responding"]),
 550                                  sorted([idlib.nodeid_b2a(sid)
 551                                          for sid in self.g.get_all_serverids()]),
 552                                  where)
 553             self.failUnless("sharemap" in r, where)
 554             all_serverids = set()
 555             for (shareid, serverids_s) in r["sharemap"].items():
 556                 all_serverids.update(serverids_s)
 557             self.failUnlessEqual(sorted(all_serverids),
 558                                  sorted([idlib.nodeid_b2a(sid)
 559                                          for sid in self.g.get_all_serverids()]),
 560                                  where)
 561         self.failUnlessEqual(r["count-wrong-shares"], 0, where)
 562         self.failUnlessEqual(r["count-recoverable-versions"], 1, where)
 563         self.failUnlessEqual(r["count-unrecoverable-versions"], 0, where)
 564
 565     def json_check_and_repair_is_healthy(self, data, n, where, incomplete=False):
 566         self.failUnlessEqual(data["storage-index"],
 567                              base32.b2a(n.get_storage_index()), where)
 568         self.failUnlessEqual(data["repair-attempted"], False, where)
 569         self.json_check_is_healthy(data["pre-repair-results"],
 570                                    n, where, incomplete)
 571         self.json_check_is_healthy(data["post-repair-results"],
 572                                    n, where, incomplete)
 573
 574     def json_full_deepcheck_is_healthy(self, data, n, where):
 575         self.failUnlessEqual(data["root-storage-index"],
 576                              base32.b2a(n.get_storage_index()), where)
 577         self.failUnlessEqual(data["count-objects-checked"], 3, where)
 578         self.failUnlessEqual(data["count-objects-healthy"], 3, where)
 579         self.failUnlessEqual(data["count-objects-unhealthy"], 0, where)
 580         self.failUnlessEqual(data["count-corrupt-shares"], 0, where)
 581         self.failUnlessEqual(data["list-corrupt-shares"], [], where)
 582         self.failUnlessEqual(data["list-unhealthy-files"], [], where)
 583         self.json_check_stats_good(data["stats"], where)
 584
 585     def json_full_deepcheck_and_repair_is_healthy(self, data, n, where):
 586         self.failUnlessEqual(data["root-storage-index"],
 587                              base32.b2a(n.get_storage_index()), where)
 588         self.failUnlessEqual(data["count-objects-checked"], 3, where)
 589
 590         self.failUnlessEqual(data["count-objects-healthy-pre-repair"], 3, where)
 591         self.failUnlessEqual(data["count-objects-unhealthy-pre-repair"], 0, where)
 592         self.failUnlessEqual(data["count-corrupt-shares-pre-repair"], 0, where)
 593
 594         self.failUnlessEqual(data["count-objects-healthy-post-repair"], 3, where)
 595         self.failUnlessEqual(data["count-objects-unhealthy-post-repair"], 0, where)
 596         self.failUnlessEqual(data["count-corrupt-shares-post-repair"], 0, where)
 597
 598         self.failUnlessEqual(data["list-corrupt-shares"], [], where)
 599         self.failUnlessEqual(data["list-remaining-corrupt-shares"], [], where)
 600         self.failUnlessEqual(data["list-unhealthy-files"], [], where)
 601
 602         self.failUnlessEqual(data["count-repairs-attempted"], 0, where)
 603         self.failUnlessEqual(data["count-repairs-successful"], 0, where)
 604         self.failUnlessEqual(data["count-repairs-unsuccessful"], 0, where)
 605
 606
 607     def json_check_lit(self, data, n, where):
 608         self.failUnlessEqual(data["storage-index"], "", where)
 609         self.failUnlessEqual(data["results"]["healthy"], True, where)
 610
 611     def json_check_stats_good(self, data, where):
 612         self.check_stats_good(data)
 613
 614     def do_test_web_good(self, ignored):
 615         d = defer.succeed(None)
 616
 617         # stats
 618         d.addCallback(lambda ign:
 619                       self.slow_web(self.root,
 620                                     t="start-deep-stats", output="json"))
 621         d.addCallback(self.json_check_stats_good, "deep-stats")
 622
 623         # check, no verify
 624         d.addCallback(lambda ign: self.web_json(self.root, t="check"))
 625         d.addCallback(self.json_check_is_healthy, self.root, "root")
 626         d.addCallback(lambda ign: self.web_json(self.mutable, t="check"))
 627         d.addCallback(self.json_check_is_healthy, self.mutable, "mutable")
 628         d.addCallback(lambda ign: self.web_json(self.large, t="check"))
 629         d.addCallback(self.json_check_is_healthy, self.large, "large")
 630         d.addCallback(lambda ign: self.web_json(self.small, t="check"))
 631         d.addCallback(self.json_check_lit, self.small, "small")
 632         d.addCallback(lambda ign: self.web_json(self.small2, t="check"))
 633         d.addCallback(self.json_check_lit, self.small2, "small2")
 634         d.addCallback(lambda ign: self.web_json(self.empty_lit_dir, t="check"))
 635         d.addCallback(self.json_check_lit, self.empty_lit_dir, "empty_lit_dir")
 636         d.addCallback(lambda ign: self.web_json(self.tiny_lit_dir, t="check"))
 637         d.addCallback(self.json_check_lit, self.tiny_lit_dir, "tiny_lit_dir")
 638
 639         # check and verify
 640         d.addCallback(lambda ign:
 641                       self.web_json(self.root, t="check", verify="true"))
 642         d.addCallback(self.json_check_is_healthy, self.root, "root+v")
 643         d.addCallback(lambda ign:
 644                       self.web_json(self.mutable, t="check", verify="true"))
 645         d.addCallback(self.json_check_is_healthy, self.mutable, "mutable+v")
 646         d.addCallback(lambda ign:
 647                       self.web_json(self.large, t="check", verify="true"))
 648         d.addCallback(self.json_check_is_healthy, self.large, "large+v",
 649                       incomplete=True)
 650         d.addCallback(lambda ign:
 651                       self.web_json(self.small, t="check", verify="true"))
 652         d.addCallback(self.json_check_lit, self.small, "small+v")
 653         d.addCallback(lambda ign:
 654                       self.web_json(self.small2, t="check", verify="true"))
 655         d.addCallback(self.json_check_lit, self.small2, "small2+v")
 656         d.addCallback(lambda ign: self.web_json(self.empty_lit_dir, t="check", verify="true"))
 657         d.addCallback(self.json_check_lit, self.empty_lit_dir, "empty_lit_dir+v")
 658         d.addCallback(lambda ign: self.web_json(self.tiny_lit_dir, t="check", verify="true"))
 659         d.addCallback(self.json_check_lit, self.tiny_lit_dir, "tiny_lit_dir+v")
 660
 661         # check and repair, no verify
 662         d.addCallback(lambda ign:
 663                       self.web_json(self.root, t="check", repair="true"))
 664         d.addCallback(self.json_check_and_repair_is_healthy, self.root, "root+r")
 665         d.addCallback(lambda ign:
 666                       self.web_json(self.mutable, t="check", repair="true"))
 667         d.addCallback(self.json_check_and_repair_is_healthy, self.mutable, "mutable+r")
 668         d.addCallback(lambda ign:
 669                       self.web_json(self.large, t="check", repair="true"))
 670         d.addCallback(self.json_check_and_repair_is_healthy, self.large, "large+r")
 671         d.addCallback(lambda ign:
 672                       self.web_json(self.small, t="check", repair="true"))
 673         d.addCallback(self.json_check_lit, self.small, "small+r")
 674         d.addCallback(lambda ign:
 675                       self.web_json(self.small2, t="check", repair="true"))
 676         d.addCallback(self.json_check_lit, self.small2, "small2+r")
 677         d.addCallback(lambda ign: self.web_json(self.empty_lit_dir, t="check", repair="true"))
 678         d.addCallback(self.json_check_lit, self.empty_lit_dir, "empty_lit_dir+r")
 679         d.addCallback(lambda ign: self.web_json(self.tiny_lit_dir, t="check", repair="true"))
 680         d.addCallback(self.json_check_lit, self.tiny_lit_dir, "tiny_lit_dir+r")
 681
 682         # check+verify+repair
 683         d.addCallback(lambda ign:
 684                       self.web_json(self.root, t="check", repair="true", verify="true"))
 685         d.addCallback(self.json_check_and_repair_is_healthy, self.root, "root+vr")
 686         d.addCallback(lambda ign:
 687                       self.web_json(self.mutable, t="check", repair="true", verify="true"))
 688         d.addCallback(self.json_check_and_repair_is_healthy, self.mutable, "mutable+vr")
 689         d.addCallback(lambda ign:
 690                       self.web_json(self.large, t="check", repair="true", verify="true"))
 691         d.addCallback(self.json_check_and_repair_is_healthy, self.large, "large+vr", incomplete=True)
 692         d.addCallback(lambda ign:
 693                       self.web_json(self.small, t="check", repair="true", verify="true"))
 694         d.addCallback(self.json_check_lit, self.small, "small+vr")
 695         d.addCallback(lambda ign:
 696                       self.web_json(self.small2, t="check", repair="true", verify="true"))
 697         d.addCallback(self.json_check_lit, self.small2, "small2+vr")
 698         d.addCallback(lambda ign: self.web_json(self.empty_lit_dir, t="check", repair="true", verify=True))
 699         d.addCallback(self.json_check_lit, self.empty_lit_dir, "empty_lit_dir+vr")
 700         d.addCallback(lambda ign: self.web_json(self.tiny_lit_dir, t="check", repair="true", verify=True))
 701         d.addCallback(self.json_check_lit, self.tiny_lit_dir, "tiny_lit_dir+vr")
 702
 703         # now run a deep-check, with various verify= and repair= flags
 704         d.addCallback(lambda ign:
 705                       self.slow_web(self.root, t="start-deep-check", output="json"))
 706         d.addCallback(self.json_full_deepcheck_is_healthy, self.root, "root+d")
 707         d.addCallback(lambda ign:
 708                       self.slow_web(self.root, t="start-deep-check", verify="true",
 709                                     output="json"))
 710         d.addCallback(self.json_full_deepcheck_is_healthy, self.root, "root+dv")
 711         d.addCallback(lambda ign:
 712                       self.slow_web(self.root, t="start-deep-check", repair="true",
 713                                     output="json"))
 714         d.addCallback(self.json_full_deepcheck_and_repair_is_healthy, self.root, "root+dr")
 715         d.addCallback(lambda ign:
 716                       self.slow_web(self.root, t="start-deep-check", verify="true", repair="true", output="json"))
 717         d.addCallback(self.json_full_deepcheck_and_repair_is_healthy, self.root, "root+dvr")
 718
 719         # now look at t=info
 720         d.addCallback(lambda ign: self.web(self.root, t="info"))
 721         # TODO: examine the output
 722         d.addCallback(lambda ign: self.web(self.mutable, t="info"))
 723         d.addCallback(lambda ign: self.web(self.large, t="info"))
 724         d.addCallback(lambda ign: self.web(self.small, t="info"))
 725         d.addCallback(lambda ign: self.web(self.small2, t="info"))
 726         d.addCallback(lambda ign: self.web(self.empty_lit_dir, t="info"))
 727         d.addCallback(lambda ign: self.web(self.tiny_lit_dir, t="info"))
 728
 729         return d
 730
 731     def _run_cli(self, argv, stdin=""):
 732         #print "CLI:", argv
 733         stdout, stderr = StringIO(), StringIO()
 734         d = threads.deferToThread(runner.runner, argv, run_by_human=False,
 735                                   stdin=StringIO(stdin),
 736                                   stdout=stdout, stderr=stderr)
 737         def _done(res):
 738             return stdout.getvalue(), stderr.getvalue()
 739         d.addCallback(_done)
 740         return d
 741
 742     def do_test_cli_good(self, ignored):
 743         d = defer.succeed(None)
 744         d.addCallback(lambda ign: self.do_cli_manifest_stream1())
 745         d.addCallback(lambda ign: self.do_cli_manifest_stream2())
 746         d.addCallback(lambda ign: self.do_cli_manifest_stream3())
 747         d.addCallback(lambda ign: self.do_cli_manifest_stream4())
 748         d.addCallback(lambda ign: self.do_cli_manifest_stream5())
 749         d.addCallback(lambda ign: self.do_cli_stats1())
 750         d.addCallback(lambda ign: self.do_cli_stats2())
 751         return d
 752
 753     def _check_manifest_storage_index(self, out):
 754         lines = [l for l in out.split("\n") if l]
 755         self.failUnlessEqual(len(lines), 3)
 756         self.failUnless(base32.b2a(self.root.get_storage_index()) in lines)
 757         self.failUnless(base32.b2a(self.mutable.get_storage_index()) in lines)
 758         self.failUnless(base32.b2a(self.large.get_storage_index()) in lines)
 759
 760     def do_cli_manifest_stream1(self):
 761         basedir = self.get_clientdir(0)
 762         d = self._run_cli(["--node-directory", basedir,
 763                            "manifest",
 764                            self.root_uri])
 765         def _check((out,err)):
 766             self.failUnlessEqual(err, "")
 767             lines = [l for l in out.split("\n") if l]
 768             self.failUnlessEqual(len(lines), 8)
 769             caps = {}
 770             for l in lines:
 771                 try:
 772                     cap, path = l.split(None, 1)
 773                 except ValueError:
 774                     cap = l.strip()
 775                     path = ""
 776                 caps[cap] = path
 777             self.failUnless(self.root.get_uri() in caps)
 778             self.failUnlessEqual(caps[self.root.get_uri()], "")
 779             self.failUnlessEqual(caps[self.mutable.get_uri()], "mutable")
 780             self.failUnlessEqual(caps[self.large.get_uri()], "large")
 781             self.failUnlessEqual(caps[self.small.get_uri()], "small")
 782             self.failUnlessEqual(caps[self.small2.get_uri()], "small2")
 783             self.failUnlessEqual(caps[self.empty_lit_dir.get_uri()], "empty_lit_dir")
 784             self.failUnlessEqual(caps[self.tiny_lit_dir.get_uri()], "tiny_lit_dir")
 785         d.addCallback(_check)
 786         return d
 787
 788     def do_cli_manifest_stream2(self):
 789         basedir = self.get_clientdir(0)
 790         d = self._run_cli(["--node-directory", basedir,
 791                            "manifest",
 792                            "--raw",
 793                            self.root_uri])
 794         def _check((out,err)):
 795             self.failUnlessEqual(err, "")
 796             # this should be the same as the POST t=stream-manifest output
 797             self._check_streamed_manifest(out)
 798         d.addCallback(_check)
 799         return d
 800
 801     def do_cli_manifest_stream3(self):
 802         basedir = self.get_clientdir(0)
 803         d = self._run_cli(["--node-directory", basedir,
 804                            "manifest",
 805                            "--storage-index",
 806                            self.root_uri])
 807         def _check((out,err)):
 808             self.failUnlessEqual(err, "")
 809             self._check_manifest_storage_index(out)
 810         d.addCallback(_check)
 811         return d
 812
 813     def do_cli_manifest_stream4(self):
 814         basedir = self.get_clientdir(0)
 815         d = self._run_cli(["--node-directory", basedir,
 816                            "manifest",
 817                            "--verify-cap",
 818                            self.root_uri])
 819         def _check((out,err)):
 820             self.failUnlessEqual(err, "")
 821             lines = [l for l in out.split("\n") if l]
 822             self.failUnlessEqual(len(lines), 3)
 823             self.failUnless(self.root.get_verify_cap().to_string() in lines)
 824             self.failUnless(self.mutable.get_verify_cap().to_string() in lines)
 825             self.failUnless(self.large.get_verify_cap().to_string() in lines)
 826         d.addCallback(_check)
 827         return d
 828
 829     def do_cli_manifest_stream5(self):
 830         basedir = self.get_clientdir(0)
 831         d = self._run_cli(["--node-directory", basedir,
 832                            "manifest",
 833                            "--repair-cap",
 834                            self.root_uri])
 835         def _check((out,err)):
 836             self.failUnlessEqual(err, "")
 837             lines = [l for l in out.split("\n") if l]
 838             self.failUnlessEqual(len(lines), 3)
 839             self.failUnless(self.root.get_repair_cap().to_string() in lines)
 840             self.failUnless(self.mutable.get_repair_cap().to_string() in lines)
 841             self.failUnless(self.large.get_repair_cap().to_string() in lines)
 842         d.addCallback(_check)
 843         return d
 844
 845     def do_cli_stats1(self):
 846         basedir = self.get_clientdir(0)
 847         d = self._run_cli(["--node-directory", basedir,
 848                            "stats",
 849                            self.root_uri])
 850         def _check3((out,err)):
 851             lines = [l.strip() for l in out.split("\n") if l]
 852             self.failUnless("count-immutable-files: 1" in lines)
 853             self.failUnless("count-mutable-files: 1" in lines)
 854             self.failUnless("count-literal-files: 3" in lines)
 855             self.failUnless("count-files: 5" in lines)
 856             self.failUnless("count-directories: 3" in lines)
 857             self.failUnless("size-immutable-files: 13000    (13.00 kB, 12.70 kiB)" in lines, lines)
 858             self.failUnless("size-literal-files: 56" in lines, lines)
 859             self.failUnless("    4-10    : 1    (10 B, 10 B)".strip() in lines, lines)
 860             self.failUnless("   11-31    : 2    (31 B, 31 B)".strip() in lines, lines)
 861             self.failUnless("10001-31622 : 1    (31.62 kB, 30.88 kiB)".strip() in lines, lines)
 862         d.addCallback(_check3)
 863         return d
 864
 865     def do_cli_stats2(self):
 866         basedir = self.get_clientdir(0)
 867         d = self._run_cli(["--node-directory", basedir,
 868                            "stats",
 869                            "--raw",
 870                            self.root_uri])
 871         def _check4((out,err)):
 872             data = simplejson.loads(out)
 873             self.failUnlessEqual(data["count-immutable-files"], 1)
 874             self.failUnlessEqual(data["count-immutable-files"], 1)
 875             self.failUnlessEqual(data["count-mutable-files"], 1)
 876             self.failUnlessEqual(data["count-literal-files"], 3)
 877             self.failUnlessEqual(data["count-files"], 5)
 878             self.failUnlessEqual(data["count-directories"], 3)
 879             self.failUnlessEqual(data["size-immutable-files"], 13000)
 880             self.failUnlessEqual(data["size-literal-files"], 56)
 881             self.failUnless([4,10,1] in data["size-files-histogram"])
 882             self.failUnless([11,31,2] in data["size-files-histogram"])
 883             self.failUnless([10001,31622,1] in data["size-files-histogram"])
 884         d.addCallback(_check4)
 885         return d
 886
 887
 888 class DeepCheckWebBad(DeepCheckBase, unittest.TestCase):
 889     def test_bad(self):
 890         self.basedir = "deepcheck/DeepCheckWebBad/bad"
 891         self.set_up_grid()
 892         d = self.set_up_damaged_tree()
 893         d.addCallback(self.do_check)
 894         d.addCallback(self.do_deepcheck)
 895         d.addCallback(self.do_deepcheck_broken)
 896         d.addCallback(self.do_test_web_bad)
 897         d.addErrback(self.explain_web_error)
 898         d.addErrback(self.explain_error)
 899         return d
 900
 901
 902
 903     def set_up_damaged_tree(self):
 904         # 6.4s
 905
 906         # root
 907         #   mutable-good
 908         #   mutable-missing-shares
 909         #   mutable-corrupt-shares
 910         #   mutable-unrecoverable
 911         #   large-good
 912         #   large-missing-shares
 913         #   large-corrupt-shares
 914         #   large-unrecoverable
 915         # broken
 916         #   large1-good
 917         #   subdir-good
 918         #     large2-good
 919         #   subdir-unrecoverable
 920         #     large3-good
 921
 922         self.nodes = {}
 923
 924         c0 = self.g.clients[0]
 925         d = c0.create_dirnode()
 926         def _created_root(n):
 927             self.root = n
 928             self.root_uri = n.get_uri()
 929         d.addCallback(_created_root)
 930         d.addCallback(self.create_mangled, "mutable-good")
 931         d.addCallback(self.create_mangled, "mutable-missing-shares")
 932         d.addCallback(self.create_mangled, "mutable-corrupt-shares")
 933         d.addCallback(self.create_mangled, "mutable-unrecoverable")
 934         d.addCallback(self.create_mangled, "large-good")
 935         d.addCallback(self.create_mangled, "large-missing-shares")
 936         d.addCallback(self.create_mangled, "large-corrupt-shares")
 937         d.addCallback(self.create_mangled, "large-unrecoverable")
 938         d.addCallback(lambda ignored: c0.create_dirnode())
 939         d.addCallback(self._stash_node, "broken")
 940         large1 = upload.Data("Lots of data\n" * 1000 + "large1" + "\n", None)
 941         d.addCallback(lambda ignored:
 942                       self.nodes["broken"].add_file(u"large1", large1))
 943         d.addCallback(lambda ignored:
 944                       self.nodes["broken"].create_subdirectory(u"subdir-good"))
 945         large2 = upload.Data("Lots of data\n" * 1000 + "large2" + "\n", None)
 946         d.addCallback(lambda subdir: subdir.add_file(u"large2-good", large2))
 947         d.addCallback(lambda ignored:
 948                       self.nodes["broken"].create_subdirectory(u"subdir-unrecoverable"))
 949         d.addCallback(self._stash_node, "subdir-unrecoverable")
 950         large3 = upload.Data("Lots of data\n" * 1000 + "large3" + "\n", None)
 951         d.addCallback(lambda subdir: subdir.add_file(u"large3-good", large3))
 952         d.addCallback(lambda ignored:
 953                       self._delete_most_shares(self.nodes["broken"]))
 954         return d
 955
 956     def _stash_node(self, node, name):
 957         self.nodes[name] = node
 958         return node
 959
 960     def create_mangled(self, ignored, name):
 961         nodetype, mangletype = name.split("-", 1)
 962         if nodetype == "mutable":
 963             mutable_uploadable = MutableData("mutable file contents")
 964             d = self.g.clients[0].create_mutable_file(mutable_uploadable)
 965             d.addCallback(lambda n: self.root.set_node(unicode(name), n))
 966         elif nodetype == "large":
 967             large = upload.Data("Lots of data\n" * 1000 + name + "\n", None)
 968             d = self.root.add_file(unicode(name), large)
 969         elif nodetype == "small":
 970             small = upload.Data("Small enough for a LIT", None)
 971             d = self.root.add_file(unicode(name), small)
 972
 973         d.addCallback(self._stash_node, name)
 974
 975         if mangletype == "good":
 976             pass
 977         elif mangletype == "missing-shares":
 978             d.addCallback(self._delete_some_shares)
 979         elif mangletype == "corrupt-shares":
 980             d.addCallback(self._corrupt_some_shares)
 981         else:
 982             assert mangletype == "unrecoverable"
 983             d.addCallback(self._delete_most_shares)
 984
 985         return d
 986
 987     def _run_cli(self, argv):
 988         stdout, stderr = StringIO(), StringIO()
 989         # this can only do synchronous operations
 990         assert argv[0] == "debug"
 991         runner.runner(argv, run_by_human=False, stdout=stdout, stderr=stderr)
 992         return stdout.getvalue()
 993
 994     def _delete_some_shares(self, node):
 995         self.delete_shares_numbered(node.get_uri(), [0,1])
 996
 997     def _corrupt_some_shares(self, node):
 998         for (shnum, serverid, sharefile) in self.find_uri_shares(node.get_uri()):
 999             if shnum in (0,1):
1000                 self._run_cli(["debug", "corrupt-share", sharefile])
1001
1002     def _delete_most_shares(self, node):
1003         self.delete_shares_numbered(node.get_uri(), range(1,10))
1004
1005
1006     def check_is_healthy(self, cr, where):
1007         try:
1008             self.failUnless(ICheckResults.providedBy(cr), (cr, type(cr), where))
1009             self.failUnless(cr.is_healthy(), (cr.get_report(), cr.is_healthy(), cr.get_summary(), where))
1010             self.failUnless(cr.is_recoverable(), where)
1011             self.failUnlessEqual(cr.get_version_counter_recoverable(), 1, where)
1012             self.failUnlessEqual(cr.get_version_counter_unrecoverable(), 0, where)
1013             return cr
1014         except Exception, le:
1015             le.args = tuple(le.args + (where,))
1016             raise
1017
1018     def check_is_missing_shares(self, cr, where):
1019         self.failUnless(ICheckResults.providedBy(cr), where)
1020         self.failIf(cr.is_healthy(), where)
1021         self.failUnless(cr.is_recoverable(), where)
1022         self.failUnlessEqual(cr.get_version_counter_recoverable(), 1, where)
1023         self.failUnlessEqual(cr.get_version_counter_unrecoverable(), 0, where)
1024         return cr
1025
1026     def check_has_corrupt_shares(self, cr, where):
1027         # by "corrupt-shares" we mean the file is still recoverable
1028         self.failUnless(ICheckResults.providedBy(cr), where)
1029         self.failIf(cr.is_healthy(), (where, cr))
1030         self.failUnless(cr.is_recoverable(), where)
1031         self.failUnless(cr.get_share_counter_good() < 10, where)
1032         self.failUnless(cr.get_corrupt_shares(), where)
1033         return cr
1034
1035     def check_is_unrecoverable(self, cr, where):
1036         self.failUnless(ICheckResults.providedBy(cr), where)
1037         self.failIf(cr.is_healthy(), where)
1038         self.failIf(cr.is_recoverable(), where)
1039         self.failUnless(cr.get_share_counter_good() < cr.get_encoding_needed(),
1040                         (cr.get_share_counter_good(), cr.get_encoding_needed(),
1041                          where))
1042         self.failUnlessEqual(cr.get_version_counter_recoverable(), 0, where)
1043         self.failUnlessEqual(cr.get_version_counter_unrecoverable(), 1, where)
1044         return cr
1045
1046     def do_check(self, ignored):
1047         d = defer.succeed(None)
1048
1049         # check the individual items, without verification. This will not
1050         # detect corrupt shares.
1051         def _check(which, checker):
1052             d = self.nodes[which].check(Monitor())
1053             d.addCallback(checker, which + "--check")
1054             return d
1055
1056         d.addCallback(lambda ign: _check("mutable-good", self.check_is_healthy))
1057         d.addCallback(lambda ign: _check("mutable-missing-shares",
1058                                          self.check_is_missing_shares))
1059         d.addCallback(lambda ign: _check("mutable-corrupt-shares",
1060                                          self.check_is_healthy))
1061         d.addCallback(lambda ign: _check("mutable-unrecoverable",
1062                                          self.check_is_unrecoverable))
1063         d.addCallback(lambda ign: _check("large-good", self.check_is_healthy))
1064         d.addCallback(lambda ign: _check("large-missing-shares",
1065                                          self.check_is_missing_shares))
1066         d.addCallback(lambda ign: _check("large-corrupt-shares",
1067                                          self.check_is_healthy))
1068         d.addCallback(lambda ign: _check("large-unrecoverable",
1069                                          self.check_is_unrecoverable))
1070
1071         # and again with verify=True, which *does* detect corrupt shares.
1072         def _checkv(which, checker):
1073             d = self.nodes[which].check(Monitor(), verify=True)
1074             d.addCallback(checker, which + "--check-and-verify")
1075             return d
1076
1077         d.addCallback(lambda ign: _checkv("mutable-good", self.check_is_healthy))
1078         d.addCallback(lambda ign: _checkv("mutable-missing-shares",
1079                                          self.check_is_missing_shares))
1080         d.addCallback(lambda ign: _checkv("mutable-corrupt-shares",
1081                                          self.check_has_corrupt_shares))
1082         d.addCallback(lambda ign: _checkv("mutable-unrecoverable",
1083                                          self.check_is_unrecoverable))
1084         d.addCallback(lambda ign: _checkv("large-good", self.check_is_healthy))
1085         d.addCallback(lambda ign: _checkv("large-missing-shares", self.check_is_missing_shares))
1086         d.addCallback(lambda ign: _checkv("large-corrupt-shares", self.check_has_corrupt_shares))
1087         d.addCallback(lambda ign: _checkv("large-unrecoverable",
1088                                          self.check_is_unrecoverable))
1089
1090         return d
1091
1092     def do_deepcheck(self, ignored):
1093         d = defer.succeed(None)
1094
1095         # now deep-check the root, with various verify= and repair= options
1096         d.addCallback(lambda ign:
1097                       self.root.start_deep_check().when_done())
1098         def _check1(cr):
1099             self.failUnless(IDeepCheckResults.providedBy(cr))
1100             c = cr.get_counters()
1101             self.failUnlessEqual(c["count-objects-checked"], 9)
1102             self.failUnlessEqual(c["count-objects-healthy"], 5)
1103             self.failUnlessEqual(c["count-objects-unhealthy"], 4)
1104             self.failUnlessEqual(c["count-objects-unrecoverable"], 2)
1105         d.addCallback(_check1)
1106
1107         d.addCallback(lambda ign:
1108                       self.root.start_deep_check(verify=True).when_done())
1109         def _check2(cr):
1110             self.failUnless(IDeepCheckResults.providedBy(cr))
1111             c = cr.get_counters()
1112             self.failUnlessEqual(c["count-objects-checked"], 9)
1113             self.failUnlessEqual(c["count-objects-healthy"], 3)
1114             self.failUnlessEqual(c["count-objects-unhealthy"], 6)
1115             self.failUnlessEqual(c["count-objects-healthy"], 3) # root, mutable good, large good
1116             self.failUnlessEqual(c["count-objects-unrecoverable"], 2) # mutable unrecoverable, large unrecoverable
1117         d.addCallback(_check2)
1118
1119         return d
1120
1121     def do_deepcheck_broken(self, ignored):
1122         # deep-check on the broken directory should fail, because of the
1123         # untraversable subdir
1124         def _do_deep_check():
1125             return self.nodes["broken"].start_deep_check().when_done()
1126         d = self.shouldFail(UnrecoverableFileError, "do_deep_check",
1127                             "no recoverable versions",
1128                             _do_deep_check)
1129         return d
1130
1131     def json_is_healthy(self, data, where):
1132         r = data["results"]
1133         self.failUnless(r["healthy"], where)
1134         self.failUnless(r["recoverable"], where)
1135         self.failUnlessEqual(r["count-recoverable-versions"], 1, where)
1136         self.failUnlessEqual(r["count-unrecoverable-versions"], 0, where)
1137
1138     def json_is_missing_shares(self, data, where):
1139         r = data["results"]
1140         self.failIf(r["healthy"], where)
1141         self.failUnless(r["recoverable"], where)
1142         self.failUnlessEqual(r["count-recoverable-versions"], 1, where)
1143         self.failUnlessEqual(r["count-unrecoverable-versions"], 0, where)
1144
1145     def json_has_corrupt_shares(self, data, where):
1146         # by "corrupt-shares" we mean the file is still recoverable
1147         r = data["results"]
1148         self.failIf(r["healthy"], where)
1149         self.failUnless(r["recoverable"], where)
1150         self.failUnless(r["count-shares-good"] < 10, where)
1151         self.failUnless(r["count-corrupt-shares"], where)
1152         self.failUnless(r["list-corrupt-shares"], where)
1153
1154     def json_is_unrecoverable(self, data, where):
1155         r = data["results"]
1156         self.failIf(r["healthy"], where)
1157         self.failIf(r["recoverable"], where)
1158         self.failUnless(r["count-shares-good"] < r["count-shares-needed"],
1159                         where)
1160         self.failUnlessEqual(r["count-recoverable-versions"], 0, where)
1161         self.failUnlessEqual(r["count-unrecoverable-versions"], 1, where)
1162
1163     def do_test_web_bad(self, ignored):
1164         d = defer.succeed(None)
1165
1166         # check, no verify
1167         def _check(which, checker):
1168             d = self.web_json(self.nodes[which], t="check")
1169             d.addCallback(checker, which + "--webcheck")
1170             return d
1171
1172         d.addCallback(lambda ign: _check("mutable-good",
1173                                          self.json_is_healthy))
1174         d.addCallback(lambda ign: _check("mutable-missing-shares",
1175                                          self.json_is_missing_shares))
1176         d.addCallback(lambda ign: _check("mutable-corrupt-shares",
1177                                          self.json_is_healthy))
1178         d.addCallback(lambda ign: _check("mutable-unrecoverable",
1179                                          self.json_is_unrecoverable))
1180         d.addCallback(lambda ign: _check("large-good",
1181                                          self.json_is_healthy))
1182         d.addCallback(lambda ign: _check("large-missing-shares",
1183                                          self.json_is_missing_shares))
1184         d.addCallback(lambda ign: _check("large-corrupt-shares",
1185                                          self.json_is_healthy))
1186         d.addCallback(lambda ign: _check("large-unrecoverable",
1187                                          self.json_is_unrecoverable))
1188
1189         # check and verify
1190         def _checkv(which, checker):
1191             d = self.web_json(self.nodes[which], t="check", verify="true")
1192             d.addCallback(checker, which + "--webcheck-and-verify")
1193             return d
1194
1195         d.addCallback(lambda ign: _checkv("mutable-good",
1196                                           self.json_is_healthy))
1197         d.addCallback(lambda ign: _checkv("mutable-missing-shares",
1198                                          self.json_is_missing_shares))
1199         d.addCallback(lambda ign: _checkv("mutable-corrupt-shares",
1200                                          self.json_has_corrupt_shares))
1201         d.addCallback(lambda ign: _checkv("mutable-unrecoverable",
1202                                          self.json_is_unrecoverable))
1203         d.addCallback(lambda ign: _checkv("large-good",
1204                                           self.json_is_healthy))
1205         d.addCallback(lambda ign: _checkv("large-missing-shares", self.json_is_missing_shares))
1206         d.addCallback(lambda ign: _checkv("large-corrupt-shares", self.json_has_corrupt_shares))
1207         d.addCallback(lambda ign: _checkv("large-unrecoverable",
1208                                          self.json_is_unrecoverable))
1209
1210         return d
1211
1212 class Large(DeepCheckBase, unittest.TestCase):
1213     def test_lots_of_lits(self):
1214         self.basedir = "deepcheck/Large/lots_of_lits"
1215         self.set_up_grid()
1216         # create the following directory structure:
1217         #  root/
1218         #   subdir/
1219         #    000-large (CHK)
1220         #    001-small (LIT)
1221         #    002-small
1222         #    ...
1223         #    399-small
1224         # then do a deepcheck and make sure it doesn't cause a
1225         # Deferred-tail-recursion stack overflow
1226
1227         COUNT = 400
1228         c0 = self.g.clients[0]
1229         d = c0.create_dirnode()
1230         self.stash = {}
1231         def _created_root(n):
1232             self.root = n
1233             return n
1234         d.addCallback(_created_root)
1235         d.addCallback(lambda root: root.create_subdirectory(u"subdir"))
1236         def _add_children(subdir_node):
1237             self.subdir_node = subdir_node
1238             kids = {}
1239             for i in range(1, COUNT):
1240                 litcap = LiteralFileURI("%03d-data" % i).to_string()
1241                 kids[u"%03d-small" % i] = (litcap, litcap)
1242             return subdir_node.set_children(kids)
1243         d.addCallback(_add_children)
1244         up = upload.Data("large enough for CHK" * 100, "")
1245         d.addCallback(lambda ign: self.subdir_node.add_file(u"0000-large", up))
1246
1247         def _start_deepcheck(ignored):
1248             return self.web(self.root, method="POST", t="stream-deep-check")
1249         d.addCallback(_start_deepcheck)
1250         def _check( (output, url) ):
1251             units = list(self.parse_streamed_json(output))
1252             self.failUnlessEqual(len(units), 2+COUNT+1)
1253         d.addCallback(_check)
1254
1255         return d