src/allmydata/test/test_deepcheck.py

   1
   2 import os, simplejson, urllib
   3 from cStringIO import StringIO
   4 from twisted.trial import unittest
   5 from twisted.internet import defer
   6 from twisted.internet import threads # CLI tests use deferToThread
   7 from allmydata.immutable import upload
   8 from allmydata.mutable.common import UnrecoverableFileError
   9 from allmydata.mutable.publish import MutableData
  10 from allmydata.util import idlib
  11 from allmydata.util import base32
  12 from allmydata.scripts import runner
  13 from allmydata.interfaces import ICheckResults, ICheckAndRepairResults, \
  14      IDeepCheckResults, IDeepCheckAndRepairResults
  15 from allmydata.monitor import Monitor, OperationCancelledError
  16 from allmydata.uri import LiteralFileURI
  17 from twisted.web.client import getPage
  18
  19 from allmydata.test.common import ErrorMixin, _corrupt_mutable_share_data, \
  20      ShouldFailMixin
  21 from allmydata.test.common_util import StallMixin
  22 from allmydata.test.no_network import GridTestMixin
  23
  24 timeout = 2400 # One of these took 1046.091s on Zandr's ARM box.
  25
  26 class MutableChecker(GridTestMixin, unittest.TestCase, ErrorMixin):
  27     def _run_cli(self, argv):
  28         stdout, stderr = StringIO(), StringIO()
  29         # this can only do synchronous operations
  30         assert argv[0] == "debug"
  31         runner.runner(argv, run_by_human=False, stdout=stdout, stderr=stderr)
  32         return stdout.getvalue()
  33
  34     def test_good(self):
  35         self.basedir = "deepcheck/MutableChecker/good"
  36         self.set_up_grid()
  37         CONTENTS = "a little bit of data"
  38         CONTENTS_uploadable = MutableData(CONTENTS)
  39         d = self.g.clients[0].create_mutable_file(CONTENTS_uploadable)
  40         def _created(node):
  41             self.node = node
  42             self.fileurl = "uri/" + urllib.quote(node.get_uri())
  43         d.addCallback(_created)
  44         # now make sure the webapi verifier sees no problems
  45         d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=true",
  46                                            method="POST"))
  47         def _got_results(out):
  48             self.failUnless("<span>Healthy : Healthy</span>" in out, out)
  49             self.failUnless("Recoverable Versions: 10*seq1-" in out, out)
  50             self.failIf("Not Healthy!" in out, out)
  51             self.failIf("Unhealthy" in out, out)
  52             self.failIf("Corrupt Shares" in out, out)
  53         d.addCallback(_got_results)
  54         d.addErrback(self.explain_web_error)
  55         return d
  56
  57     def test_corrupt(self):
  58         self.basedir = "deepcheck/MutableChecker/corrupt"
  59         self.set_up_grid()
  60         CONTENTS = "a little bit of data"
  61         CONTENTS_uploadable = MutableData(CONTENTS)
  62         d = self.g.clients[0].create_mutable_file(CONTENTS_uploadable)
  63         def _stash_and_corrupt(node):
  64             self.node = node
  65             self.fileurl = "uri/" + urllib.quote(node.get_uri())
  66             self.corrupt_shares_numbered(node.get_uri(), [0],
  67                                          _corrupt_mutable_share_data)
  68         d.addCallback(_stash_and_corrupt)
  69         # now make sure the webapi verifier notices it
  70         d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=true",
  71                                            method="POST"))
  72         def _got_results(out):
  73             self.failUnless("Not Healthy!" in out, out)
  74             self.failUnless("Unhealthy: best version has only 9 shares (encoding is 3-of-10)" in out, out)
  75             self.failUnless("Corrupt Shares:" in out, out)
  76         d.addCallback(_got_results)
  77
  78         # now make sure the webapi repairer can fix it
  79         d.addCallback(lambda ign:
  80                       self.GET(self.fileurl+"?t=check&verify=true&repair=true",
  81                                method="POST"))
  82         def _got_repair_results(out):
  83             self.failUnless("<div>Repair successful</div>" in out, out)
  84         d.addCallback(_got_repair_results)
  85         d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=true",
  86                                            method="POST"))
  87         def _got_postrepair_results(out):
  88             self.failIf("Not Healthy!" in out, out)
  89             self.failUnless("Recoverable Versions: 10*seq" in out, out)
  90         d.addCallback(_got_postrepair_results)
  91         d.addErrback(self.explain_web_error)
  92
  93         return d
  94
  95     def test_delete_share(self):
  96         self.basedir = "deepcheck/MutableChecker/delete_share"
  97         self.set_up_grid()
  98         CONTENTS = "a little bit of data"
  99         CONTENTS_uploadable = MutableData(CONTENTS)
 100         d = self.g.clients[0].create_mutable_file(CONTENTS_uploadable)
 101         def _stash_and_delete(node):
 102             self.node = node
 103             self.fileurl = "uri/" + urllib.quote(node.get_uri())
 104             self.delete_shares_numbered(node.get_uri(), [0])
 105         d.addCallback(_stash_and_delete)
 106         # now make sure the webapi checker notices it
 107         d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=false",
 108                                            method="POST"))
 109         def _got_results(out):
 110             self.failUnless("Not Healthy!" in out, out)
 111             self.failUnless("Unhealthy: best version has only 9 shares (encoding is 3-of-10)" in out, out)
 112             self.failIf("Corrupt Shares" in out, out)
 113         d.addCallback(_got_results)
 114
 115         # now make sure the webapi repairer can fix it
 116         d.addCallback(lambda ign:
 117                       self.GET(self.fileurl+"?t=check&verify=false&repair=true",
 118                                method="POST"))
 119         def _got_repair_results(out):
 120             self.failUnless("Repair successful" in out)
 121         d.addCallback(_got_repair_results)
 122         d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=false",
 123                                            method="POST"))
 124         def _got_postrepair_results(out):
 125             self.failIf("Not Healthy!" in out, out)
 126             self.failUnless("Recoverable Versions: 10*seq" in out)
 127         d.addCallback(_got_postrepair_results)
 128         d.addErrback(self.explain_web_error)
 129
 130         return d
 131
 132
 133 class DeepCheckBase(GridTestMixin, ErrorMixin, StallMixin, ShouldFailMixin):
 134
 135     def web_json(self, n, **kwargs):
 136         kwargs["output"] = "json"
 137         d = self.web(n, "POST", **kwargs)
 138         d.addCallback(self.decode_json)
 139         return d
 140
 141     def decode_json(self, (s,url)):
 142         try:
 143             data = simplejson.loads(s)
 144         except ValueError:
 145             self.fail("%s: not JSON: '%s'" % (url, s))
 146         return data
 147
 148     def parse_streamed_json(self, s):
 149         for unit in s.split("\n"):
 150             if not unit:
 151                 # stream should end with a newline, so split returns ""
 152                 continue
 153             try:
 154                 yield simplejson.loads(unit)
 155             except ValueError, le:
 156                 le.args = tuple(le.args + (unit,))
 157                 raise
 158
 159     def web(self, n, method="GET", **kwargs):
 160         # returns (data, url)
 161         url = (self.client_baseurls[0] + "uri/%s" % urllib.quote(n.get_uri())
 162                + "?" + "&".join(["%s=%s" % (k,v) for (k,v) in kwargs.items()]))
 163         d = getPage(url, method=method)
 164         d.addCallback(lambda data: (data,url))
 165         return d
 166
 167     def wait_for_operation(self, ignored, ophandle):
 168         url = self.client_baseurls[0] + "operations/" + ophandle
 169         url += "?t=status&output=JSON"
 170         d = getPage(url)
 171         def _got(res):
 172             try:
 173                 data = simplejson.loads(res)
 174             except ValueError:
 175                 self.fail("%s: not JSON: '%s'" % (url, res))
 176             if not data["finished"]:
 177                 d = self.stall(delay=1.0)
 178                 d.addCallback(self.wait_for_operation, ophandle)
 179                 return d
 180             return data
 181         d.addCallback(_got)
 182         return d
 183
 184     def get_operation_results(self, ignored, ophandle, output=None):
 185         url = self.client_baseurls[0] + "operations/" + ophandle
 186         url += "?t=status"
 187         if output:
 188             url += "&output=" + output
 189         d = getPage(url)
 190         def _got(res):
 191             if output and output.lower() == "json":
 192                 try:
 193                     return simplejson.loads(res)
 194                 except ValueError:
 195                     self.fail("%s: not JSON: '%s'" % (url, res))
 196             return res
 197         d.addCallback(_got)
 198         return d
 199
 200     def slow_web(self, n, output=None, **kwargs):
 201         # use ophandle=
 202         handle = base32.b2a(os.urandom(4))
 203         d = self.web(n, "POST", ophandle=handle, **kwargs)
 204         d.addCallback(self.wait_for_operation, handle)
 205         d.addCallback(self.get_operation_results, handle, output=output)
 206         return d
 207
 208
 209 class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
 210     # construct a small directory tree (with one dir, one immutable file, one
 211     # mutable file, two LIT files, one DIR2:LIT empty dir, one DIR2:LIT tiny
 212     # dir, and a loop), and then check/examine it in various ways.
 213
 214     def set_up_tree(self):
 215         # 2.9s
 216
 217         c0 = self.g.clients[0]
 218         d = c0.create_dirnode()
 219         def _created_root(n):
 220             self.root = n
 221             self.root_uri = n.get_uri()
 222         d.addCallback(_created_root)
 223         d.addCallback(lambda ign:
 224             c0.create_mutable_file(MutableData("mutable file contents")))
 225         d.addCallback(lambda n: self.root.set_node(u"mutable", n))
 226         def _created_mutable(n):
 227             self.mutable = n
 228             self.mutable_uri = n.get_uri()
 229         d.addCallback(_created_mutable)
 230
 231         large = upload.Data("Lots of data\n" * 1000, None)
 232         d.addCallback(lambda ign: self.root.add_file(u"large", large))
 233         def _created_large(n):
 234             self.large = n
 235             self.large_uri = n.get_uri()
 236         d.addCallback(_created_large)
 237
 238         small = upload.Data("Small enough for a LIT", None)
 239         d.addCallback(lambda ign: self.root.add_file(u"small", small))
 240         def _created_small(n):
 241             self.small = n
 242             self.small_uri = n.get_uri()
 243         d.addCallback(_created_small)
 244
 245         small2 = upload.Data("Small enough for a LIT too", None)
 246         d.addCallback(lambda ign: self.root.add_file(u"small2", small2))
 247         def _created_small2(n):
 248             self.small2 = n
 249             self.small2_uri = n.get_uri()
 250         d.addCallback(_created_small2)
 251
 252         empty_litdir_uri = "URI:DIR2-LIT:"
 253         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
 254
 255         d.addCallback(lambda ign: self.root._create_and_validate_node(None, empty_litdir_uri, name=u"test_deepcheck empty_lit_dir"))
 256         def _created_empty_lit_dir(n):
 257             self.empty_lit_dir = n
 258             self.empty_lit_dir_uri = n.get_uri()
 259             self.root.set_node(u"empty_lit_dir", n)
 260         d.addCallback(_created_empty_lit_dir)
 261
 262         d.addCallback(lambda ign: self.root._create_and_validate_node(None, tiny_litdir_uri, name=u"test_deepcheck tiny_lit_dir"))
 263         def _created_tiny_lit_dir(n):
 264             self.tiny_lit_dir = n
 265             self.tiny_lit_dir_uri = n.get_uri()
 266             self.root.set_node(u"tiny_lit_dir", n)
 267         d.addCallback(_created_tiny_lit_dir)
 268
 269         d.addCallback(lambda ign: self.root.set_node(u"loop", self.root))
 270         return d
 271
 272     def check_is_healthy(self, cr, n, where, incomplete=False):
 273         self.failUnless(ICheckResults.providedBy(cr), where)
 274         self.failUnless(cr.is_healthy(), where)
 275         self.failUnlessEqual(cr.get_storage_index(), n.get_storage_index(),
 276                              where)
 277         self.failUnlessEqual(cr.get_storage_index_string(),
 278                              base32.b2a(n.get_storage_index()), where)
 279         num_servers = len(self.g.all_servers)
 280         needs_rebalancing = bool( num_servers < 10 )
 281         if not incomplete:
 282             self.failUnlessEqual(cr.needs_rebalancing(), needs_rebalancing,
 283                                  str((where, cr, cr.get_data())))
 284         self.failUnlessEqual(cr.get_share_counter_good(), 10, where)
 285         self.failUnlessEqual(cr.get_encoding_needed(), 3, where)
 286         self.failUnlessEqual(cr.get_encoding_expected(), 10, where)
 287         if not incomplete:
 288             self.failUnlessEqual(cr.get_host_counter_good_shares(),
 289                                  num_servers, where)
 290         self.failUnlessEqual(cr.get_corrupt_shares(), [], where)
 291         if not incomplete:
 292             self.failUnlessEqual(sorted(cr.get_servers_responding()),
 293                                  sorted(self.g.get_all_serverids()),
 294                                  where)
 295             all_serverids = set()
 296             for (shareid, serverids) in cr.get_sharemap().items():
 297                 all_serverids.update(serverids)
 298             self.failUnlessEqual(sorted(all_serverids),
 299                                  sorted(self.g.get_all_serverids()),
 300                                  where)
 301
 302         self.failUnlessEqual(cr.get_share_counter_wrong(), 0, where)
 303         self.failUnlessEqual(cr.get_version_counter_recoverable(), 1, where)
 304         self.failUnlessEqual(cr.get_version_counter_unrecoverable(), 0, where)
 305
 306
 307     def check_and_repair_is_healthy(self, cr, n, where, incomplete=False):
 308         self.failUnless(ICheckAndRepairResults.providedBy(cr), (where, cr))
 309         self.failUnless(cr.get_pre_repair_results().is_healthy(), where)
 310         self.check_is_healthy(cr.get_pre_repair_results(), n, where, incomplete)
 311         self.failUnless(cr.get_post_repair_results().is_healthy(), where)
 312         self.check_is_healthy(cr.get_post_repair_results(), n, where, incomplete)
 313         self.failIf(cr.get_repair_attempted(), where)
 314
 315     def deep_check_is_healthy(self, cr, num_healthy, where):
 316         self.failUnless(IDeepCheckResults.providedBy(cr))
 317         self.failUnlessEqual(cr.get_counters()["count-objects-healthy"],
 318                              num_healthy, where)
 319
 320     def deep_check_and_repair_is_healthy(self, cr, num_healthy, where):
 321         self.failUnless(IDeepCheckAndRepairResults.providedBy(cr), where)
 322         c = cr.get_counters()
 323         self.failUnlessEqual(c["count-objects-healthy-pre-repair"],
 324                              num_healthy, where)
 325         self.failUnlessEqual(c["count-objects-healthy-post-repair"],
 326                              num_healthy, where)
 327         self.failUnlessEqual(c["count-repairs-attempted"], 0, where)
 328
 329     def test_good(self):
 330         self.basedir = "deepcheck/DeepCheckWebGood/good"
 331         self.set_up_grid()
 332         d = self.set_up_tree()
 333         d.addCallback(self.do_stats)
 334         d.addCallback(self.do_web_stream_manifest)
 335         d.addCallback(self.do_web_stream_check)
 336         d.addCallback(self.do_test_check_good)
 337         d.addCallback(self.do_test_web_good)
 338         d.addCallback(self.do_test_cli_good)
 339         d.addErrback(self.explain_web_error)
 340         d.addErrback(self.explain_error)
 341         return d
 342
 343     def do_stats(self, ignored):
 344         d = defer.succeed(None)
 345         d.addCallback(lambda ign: self.root.start_deep_stats().when_done())
 346         d.addCallback(self.check_stats_good)
 347         return d
 348
 349     def check_stats_good(self, s):
 350         self.failUnlessEqual(s["count-directories"], 3)
 351         self.failUnlessEqual(s["count-files"], 5)
 352         self.failUnlessEqual(s["count-immutable-files"], 1)
 353         self.failUnlessEqual(s["count-literal-files"], 3)
 354         self.failUnlessEqual(s["count-mutable-files"], 1)
 355         # don't check directories: their size will vary
 356         # s["largest-directory"]
 357         # s["size-directories"]
 358         self.failUnlessEqual(s["largest-directory-children"], 7)
 359         self.failUnlessEqual(s["largest-immutable-file"], 13000)
 360         # to re-use this function for both the local
 361         # dirnode.start_deep_stats() and the webapi t=start-deep-stats, we
 362         # coerce the result into a list of tuples. dirnode.start_deep_stats()
 363         # returns a list of tuples, but JSON only knows about lists., so
 364         # t=start-deep-stats returns a list of lists.
 365         histogram = [tuple(stuff) for stuff in s["size-files-histogram"]]
 366         self.failUnlessEqual(histogram, [(4, 10, 1), (11, 31, 2),
 367                                          (10001, 31622, 1),
 368                                          ])
 369         self.failUnlessEqual(s["size-immutable-files"], 13000)
 370         self.failUnlessEqual(s["size-literal-files"], 56)
 371
 372     def do_web_stream_manifest(self, ignored):
 373         d = self.web(self.root, method="POST", t="stream-manifest")
 374         d.addCallback(lambda (output,url):
 375                       self._check_streamed_manifest(output))
 376         return d
 377
 378     def _check_streamed_manifest(self, output):
 379         units = list(self.parse_streamed_json(output))
 380         files = [u for u in units if u["type"] in ("file", "directory")]
 381         assert units[-1]["type"] == "stats"
 382         stats = units[-1]["stats"]
 383         self.failUnlessEqual(len(files), 8)
 384         # [root,mutable,large] are distributed, [small,small2,empty_litdir,tiny_litdir] are not
 385         self.failUnlessEqual(len([f for f in files
 386                                   if f["verifycap"] != ""]), 3)
 387         self.failUnlessEqual(len([f for f in files
 388                                   if f["verifycap"] == ""]), 5)
 389         self.failUnlessEqual(len([f for f in files
 390                                   if f["repaircap"] != ""]), 3)
 391         self.failUnlessEqual(len([f for f in files
 392                                   if f["repaircap"] == ""]), 5)
 393         self.failUnlessEqual(len([f for f in files
 394                                   if f["storage-index"] != ""]), 3)
 395         self.failUnlessEqual(len([f for f in files
 396                                   if f["storage-index"] == ""]), 5)
 397         # make sure that a mutable file has filecap==repaircap!=verifycap
 398         mutable = [f for f in files
 399                    if f["cap"] is not None
 400                    and f["cap"].startswith("URI:SSK:")][0]
 401         self.failUnlessEqual(mutable["cap"], self.mutable_uri)
 402         self.failIfEqual(mutable["cap"], mutable["verifycap"])
 403         self.failUnlessEqual(mutable["cap"], mutable["repaircap"])
 404         # for immutable file, verifycap==repaircap!=filecap
 405         large = [f for f in files
 406                    if f["cap"] is not None
 407                    and f["cap"].startswith("URI:CHK:")][0]
 408         self.failUnlessEqual(large["cap"], self.large_uri)
 409         self.failIfEqual(large["cap"], large["verifycap"])
 410         self.failUnlessEqual(large["verifycap"], large["repaircap"])
 411         self.check_stats_good(stats)
 412
 413     def do_web_stream_check(self, ignored):
 414         # TODO
 415         return
 416         d = self.web(self.root, t="stream-deep-check")
 417         def _check(res):
 418             units = list(self.parse_streamed_json(res))
 419             #files = [u for u in units if u["type"] in ("file", "directory")]
 420             assert units[-1]["type"] == "stats"
 421             #stats = units[-1]["stats"]
 422             # ...
 423         d.addCallback(_check)
 424         return d
 425
 426     def do_test_check_good(self, ignored):
 427         d = defer.succeed(None)
 428         # check the individual items
 429         d.addCallback(lambda ign: self.root.check(Monitor()))
 430         d.addCallback(self.check_is_healthy, self.root, "root")
 431         d.addCallback(lambda ign: self.mutable.check(Monitor()))
 432         d.addCallback(self.check_is_healthy, self.mutable, "mutable")
 433         d.addCallback(lambda ign: self.large.check(Monitor()))
 434         d.addCallback(self.check_is_healthy, self.large, "large")
 435         d.addCallback(lambda ign: self.small.check(Monitor()))
 436         d.addCallback(self.failUnlessEqual, None, "small")
 437         d.addCallback(lambda ign: self.small2.check(Monitor()))
 438         d.addCallback(self.failUnlessEqual, None, "small2")
 439         d.addCallback(lambda ign: self.empty_lit_dir.check(Monitor()))
 440         d.addCallback(self.failUnlessEqual, None, "empty_lit_dir")
 441         d.addCallback(lambda ign: self.tiny_lit_dir.check(Monitor()))
 442         d.addCallback(self.failUnlessEqual, None, "tiny_lit_dir")
 443
 444         # and again with verify=True
 445         d.addCallback(lambda ign: self.root.check(Monitor(), verify=True))
 446         d.addCallback(self.check_is_healthy, self.root, "root")
 447         d.addCallback(lambda ign: self.mutable.check(Monitor(), verify=True))
 448         d.addCallback(self.check_is_healthy, self.mutable, "mutable")
 449         d.addCallback(lambda ign: self.large.check(Monitor(), verify=True))
 450         d.addCallback(self.check_is_healthy, self.large, "large", incomplete=True)
 451         d.addCallback(lambda ign: self.small.check(Monitor(), verify=True))
 452         d.addCallback(self.failUnlessEqual, None, "small")
 453         d.addCallback(lambda ign: self.small2.check(Monitor(), verify=True))
 454         d.addCallback(self.failUnlessEqual, None, "small2")
 455         d.addCallback(lambda ign: self.empty_lit_dir.check(Monitor(), verify=True))
 456         d.addCallback(self.failUnlessEqual, None, "empty_lit_dir")
 457         d.addCallback(lambda ign: self.tiny_lit_dir.check(Monitor(), verify=True))
 458         d.addCallback(self.failUnlessEqual, None, "tiny_lit_dir")
 459
 460         # and check_and_repair(), which should be a nop
 461         d.addCallback(lambda ign: self.root.check_and_repair(Monitor()))
 462         d.addCallback(self.check_and_repair_is_healthy, self.root, "root")
 463         d.addCallback(lambda ign: self.mutable.check_and_repair(Monitor()))
 464         d.addCallback(self.check_and_repair_is_healthy, self.mutable, "mutable")
 465         d.addCallback(lambda ign: self.large.check_and_repair(Monitor()))
 466         d.addCallback(self.check_and_repair_is_healthy, self.large, "large")
 467         d.addCallback(lambda ign: self.small.check_and_repair(Monitor()))
 468         d.addCallback(self.failUnlessEqual, None, "small")
 469         d.addCallback(lambda ign: self.small2.check_and_repair(Monitor()))
 470         d.addCallback(self.failUnlessEqual, None, "small2")
 471         d.addCallback(lambda ign: self.empty_lit_dir.check_and_repair(Monitor()))
 472         d.addCallback(self.failUnlessEqual, None, "empty_lit_dir")
 473         d.addCallback(lambda ign: self.tiny_lit_dir.check_and_repair(Monitor()))
 474
 475         # check_and_repair(verify=True)
 476         d.addCallback(lambda ign: self.root.check_and_repair(Monitor(), verify=True))
 477         d.addCallback(self.check_and_repair_is_healthy, self.root, "root")
 478         d.addCallback(lambda ign: self.mutable.check_and_repair(Monitor(), verify=True))
 479         d.addCallback(self.check_and_repair_is_healthy, self.mutable, "mutable")
 480         d.addCallback(lambda ign: self.large.check_and_repair(Monitor(), verify=True))
 481         d.addCallback(self.check_and_repair_is_healthy, self.large, "large", incomplete=True)
 482         d.addCallback(lambda ign: self.small.check_and_repair(Monitor(), verify=True))
 483         d.addCallback(self.failUnlessEqual, None, "small")
 484         d.addCallback(lambda ign: self.small2.check_and_repair(Monitor(), verify=True))
 485         d.addCallback(self.failUnlessEqual, None, "small2")
 486         d.addCallback(self.failUnlessEqual, None, "small2")
 487         d.addCallback(lambda ign: self.empty_lit_dir.check_and_repair(Monitor(), verify=True))
 488         d.addCallback(self.failUnlessEqual, None, "empty_lit_dir")
 489         d.addCallback(lambda ign: self.tiny_lit_dir.check_and_repair(Monitor(), verify=True))
 490
 491
 492         # now deep-check the root, with various verify= and repair= options
 493         d.addCallback(lambda ign:
 494                       self.root.start_deep_check().when_done())
 495         d.addCallback(self.deep_check_is_healthy, 3, "root")
 496         d.addCallback(lambda ign:
 497                       self.root.start_deep_check(verify=True).when_done())
 498         d.addCallback(self.deep_check_is_healthy, 3, "root")
 499         d.addCallback(lambda ign:
 500                       self.root.start_deep_check_and_repair().when_done())
 501         d.addCallback(self.deep_check_and_repair_is_healthy, 3, "root")
 502         d.addCallback(lambda ign:
 503                       self.root.start_deep_check_and_repair(verify=True).when_done())
 504         d.addCallback(self.deep_check_and_repair_is_healthy, 3, "root")
 505
 506         # and finally, start a deep-check, but then cancel it.
 507         d.addCallback(lambda ign: self.root.start_deep_check())
 508         def _checking(monitor):
 509             monitor.cancel()
 510             d = monitor.when_done()
 511             # this should fire as soon as the next dirnode.list finishes.
 512             # TODO: add a counter to measure how many list() calls are made,
 513             # assert that no more than one gets to run before the cancel()
 514             # takes effect.
 515             def _finished_normally(res):
 516                 self.fail("this was supposed to fail, not finish normally")
 517             def _cancelled(f):
 518                 f.trap(OperationCancelledError)
 519             d.addCallbacks(_finished_normally, _cancelled)
 520             return d
 521         d.addCallback(_checking)
 522
 523         return d
 524
 525     def json_check_is_healthy(self, data, n, where, incomplete=False):
 526
 527         self.failUnlessEqual(data["storage-index"],
 528                              base32.b2a(n.get_storage_index()), where)
 529         self.failUnless("summary" in data, (where, data))
 530         self.failUnlessEqual(data["summary"].lower(), "healthy",
 531                              "%s: '%s'" % (where, data["summary"]))
 532         r = data["results"]
 533         self.failUnlessEqual(r["healthy"], True, where)
 534         num_servers = len(self.g.all_servers)
 535         needs_rebalancing = bool( num_servers < 10 )
 536         if not incomplete:
 537             self.failUnlessEqual(r["needs-rebalancing"], needs_rebalancing,
 538                                  where)
 539         self.failUnlessEqual(r["count-shares-good"], 10, where)
 540         self.failUnlessEqual(r["count-shares-needed"], 3, where)
 541         self.failUnlessEqual(r["count-shares-expected"], 10, where)
 542         if not incomplete:
 543             self.failUnlessEqual(r["count-good-share-hosts"], num_servers,
 544                                  where)
 545         self.failUnlessEqual(r["count-corrupt-shares"], 0, where)
 546         self.failUnlessEqual(r["list-corrupt-shares"], [], where)
 547         if not incomplete:
 548             self.failUnlessEqual(sorted(r["servers-responding"]),
 549                                  sorted([idlib.nodeid_b2a(sid)
 550                                          for sid in self.g.get_all_serverids()]),
 551                                  where)
 552             self.failUnless("sharemap" in r, where)
 553             all_serverids = set()
 554             for (shareid, serverids_s) in r["sharemap"].items():
 555                 all_serverids.update(serverids_s)
 556             self.failUnlessEqual(sorted(all_serverids),
 557                                  sorted([idlib.nodeid_b2a(sid)
 558                                          for sid in self.g.get_all_serverids()]),
 559                                  where)
 560         self.failUnlessEqual(r["count-wrong-shares"], 0, where)
 561         self.failUnlessEqual(r["count-recoverable-versions"], 1, where)
 562         self.failUnlessEqual(r["count-unrecoverable-versions"], 0, where)
 563
 564     def json_check_and_repair_is_healthy(self, data, n, where, incomplete=False):
 565         self.failUnlessEqual(data["storage-index"],
 566                              base32.b2a(n.get_storage_index()), where)
 567         self.failUnlessEqual(data["repair-attempted"], False, where)
 568         self.json_check_is_healthy(data["pre-repair-results"],
 569                                    n, where, incomplete)
 570         self.json_check_is_healthy(data["post-repair-results"],
 571                                    n, where, incomplete)
 572
 573     def json_full_deepcheck_is_healthy(self, data, n, where):
 574         self.failUnlessEqual(data["root-storage-index"],
 575                              base32.b2a(n.get_storage_index()), where)
 576         self.failUnlessEqual(data["count-objects-checked"], 3, where)
 577         self.failUnlessEqual(data["count-objects-healthy"], 3, where)
 578         self.failUnlessEqual(data["count-objects-unhealthy"], 0, where)
 579         self.failUnlessEqual(data["count-corrupt-shares"], 0, where)
 580         self.failUnlessEqual(data["list-corrupt-shares"], [], where)
 581         self.failUnlessEqual(data["list-unhealthy-files"], [], where)
 582         self.json_check_stats_good(data["stats"], where)
 583
 584     def json_full_deepcheck_and_repair_is_healthy(self, data, n, where):
 585         self.failUnlessEqual(data["root-storage-index"],
 586                              base32.b2a(n.get_storage_index()), where)
 587         self.failUnlessEqual(data["count-objects-checked"], 3, where)
 588
 589         self.failUnlessEqual(data["count-objects-healthy-pre-repair"], 3, where)
 590         self.failUnlessEqual(data["count-objects-unhealthy-pre-repair"], 0, where)
 591         self.failUnlessEqual(data["count-corrupt-shares-pre-repair"], 0, where)
 592
 593         self.failUnlessEqual(data["count-objects-healthy-post-repair"], 3, where)
 594         self.failUnlessEqual(data["count-objects-unhealthy-post-repair"], 0, where)
 595         self.failUnlessEqual(data["count-corrupt-shares-post-repair"], 0, where)
 596
 597         self.failUnlessEqual(data["list-corrupt-shares"], [], where)
 598         self.failUnlessEqual(data["list-remaining-corrupt-shares"], [], where)
 599         self.failUnlessEqual(data["list-unhealthy-files"], [], where)
 600
 601         self.failUnlessEqual(data["count-repairs-attempted"], 0, where)
 602         self.failUnlessEqual(data["count-repairs-successful"], 0, where)
 603         self.failUnlessEqual(data["count-repairs-unsuccessful"], 0, where)
 604
 605
 606     def json_check_lit(self, data, n, where):
 607         self.failUnlessEqual(data["storage-index"], "", where)
 608         self.failUnlessEqual(data["results"]["healthy"], True, where)
 609
 610     def json_check_stats_good(self, data, where):
 611         self.check_stats_good(data)
 612
 613     def do_test_web_good(self, ignored):
 614         d = defer.succeed(None)
 615
 616         # stats
 617         d.addCallback(lambda ign:
 618                       self.slow_web(self.root,
 619                                     t="start-deep-stats", output="json"))
 620         d.addCallback(self.json_check_stats_good, "deep-stats")
 621
 622         # check, no verify
 623         d.addCallback(lambda ign: self.web_json(self.root, t="check"))
 624         d.addCallback(self.json_check_is_healthy, self.root, "root")
 625         d.addCallback(lambda ign: self.web_json(self.mutable, t="check"))
 626         d.addCallback(self.json_check_is_healthy, self.mutable, "mutable")
 627         d.addCallback(lambda ign: self.web_json(self.large, t="check"))
 628         d.addCallback(self.json_check_is_healthy, self.large, "large")
 629         d.addCallback(lambda ign: self.web_json(self.small, t="check"))
 630         d.addCallback(self.json_check_lit, self.small, "small")
 631         d.addCallback(lambda ign: self.web_json(self.small2, t="check"))
 632         d.addCallback(self.json_check_lit, self.small2, "small2")
 633         d.addCallback(lambda ign: self.web_json(self.empty_lit_dir, t="check"))
 634         d.addCallback(self.json_check_lit, self.empty_lit_dir, "empty_lit_dir")
 635         d.addCallback(lambda ign: self.web_json(self.tiny_lit_dir, t="check"))
 636         d.addCallback(self.json_check_lit, self.tiny_lit_dir, "tiny_lit_dir")
 637
 638         # check and verify
 639         d.addCallback(lambda ign:
 640                       self.web_json(self.root, t="check", verify="true"))
 641         d.addCallback(self.json_check_is_healthy, self.root, "root+v")
 642         d.addCallback(lambda ign:
 643                       self.web_json(self.mutable, t="check", verify="true"))
 644         d.addCallback(self.json_check_is_healthy, self.mutable, "mutable+v")
 645         d.addCallback(lambda ign:
 646                       self.web_json(self.large, t="check", verify="true"))
 647         d.addCallback(self.json_check_is_healthy, self.large, "large+v",
 648                       incomplete=True)
 649         d.addCallback(lambda ign:
 650                       self.web_json(self.small, t="check", verify="true"))
 651         d.addCallback(self.json_check_lit, self.small, "small+v")
 652         d.addCallback(lambda ign:
 653                       self.web_json(self.small2, t="check", verify="true"))
 654         d.addCallback(self.json_check_lit, self.small2, "small2+v")
 655         d.addCallback(lambda ign: self.web_json(self.empty_lit_dir, t="check", verify="true"))
 656         d.addCallback(self.json_check_lit, self.empty_lit_dir, "empty_lit_dir+v")
 657         d.addCallback(lambda ign: self.web_json(self.tiny_lit_dir, t="check", verify="true"))
 658         d.addCallback(self.json_check_lit, self.tiny_lit_dir, "tiny_lit_dir+v")
 659
 660         # check and repair, no verify
 661         d.addCallback(lambda ign:
 662                       self.web_json(self.root, t="check", repair="true"))
 663         d.addCallback(self.json_check_and_repair_is_healthy, self.root, "root+r")
 664         d.addCallback(lambda ign:
 665                       self.web_json(self.mutable, t="check", repair="true"))
 666         d.addCallback(self.json_check_and_repair_is_healthy, self.mutable, "mutable+r")
 667         d.addCallback(lambda ign:
 668                       self.web_json(self.large, t="check", repair="true"))
 669         d.addCallback(self.json_check_and_repair_is_healthy, self.large, "large+r")
 670         d.addCallback(lambda ign:
 671                       self.web_json(self.small, t="check", repair="true"))
 672         d.addCallback(self.json_check_lit, self.small, "small+r")
 673         d.addCallback(lambda ign:
 674                       self.web_json(self.small2, t="check", repair="true"))
 675         d.addCallback(self.json_check_lit, self.small2, "small2+r")
 676         d.addCallback(lambda ign: self.web_json(self.empty_lit_dir, t="check", repair="true"))
 677         d.addCallback(self.json_check_lit, self.empty_lit_dir, "empty_lit_dir+r")
 678         d.addCallback(lambda ign: self.web_json(self.tiny_lit_dir, t="check", repair="true"))
 679         d.addCallback(self.json_check_lit, self.tiny_lit_dir, "tiny_lit_dir+r")
 680
 681         # check+verify+repair
 682         d.addCallback(lambda ign:
 683                       self.web_json(self.root, t="check", repair="true", verify="true"))
 684         d.addCallback(self.json_check_and_repair_is_healthy, self.root, "root+vr")
 685         d.addCallback(lambda ign:
 686                       self.web_json(self.mutable, t="check", repair="true", verify="true"))
 687         d.addCallback(self.json_check_and_repair_is_healthy, self.mutable, "mutable+vr")
 688         d.addCallback(lambda ign:
 689                       self.web_json(self.large, t="check", repair="true", verify="true"))
 690         d.addCallback(self.json_check_and_repair_is_healthy, self.large, "large+vr", incomplete=True)
 691         d.addCallback(lambda ign:
 692                       self.web_json(self.small, t="check", repair="true", verify="true"))
 693         d.addCallback(self.json_check_lit, self.small, "small+vr")
 694         d.addCallback(lambda ign:
 695                       self.web_json(self.small2, t="check", repair="true", verify="true"))
 696         d.addCallback(self.json_check_lit, self.small2, "small2+vr")
 697         d.addCallback(lambda ign: self.web_json(self.empty_lit_dir, t="check", repair="true", verify=True))
 698         d.addCallback(self.json_check_lit, self.empty_lit_dir, "empty_lit_dir+vr")
 699         d.addCallback(lambda ign: self.web_json(self.tiny_lit_dir, t="check", repair="true", verify=True))
 700         d.addCallback(self.json_check_lit, self.tiny_lit_dir, "tiny_lit_dir+vr")
 701
 702         # now run a deep-check, with various verify= and repair= flags
 703         d.addCallback(lambda ign:
 704                       self.slow_web(self.root, t="start-deep-check", output="json"))
 705         d.addCallback(self.json_full_deepcheck_is_healthy, self.root, "root+d")
 706         d.addCallback(lambda ign:
 707                       self.slow_web(self.root, t="start-deep-check", verify="true",
 708                                     output="json"))
 709         d.addCallback(self.json_full_deepcheck_is_healthy, self.root, "root+dv")
 710         d.addCallback(lambda ign:
 711                       self.slow_web(self.root, t="start-deep-check", repair="true",
 712                                     output="json"))
 713         d.addCallback(self.json_full_deepcheck_and_repair_is_healthy, self.root, "root+dr")
 714         d.addCallback(lambda ign:
 715                       self.slow_web(self.root, t="start-deep-check", verify="true", repair="true", output="json"))
 716         d.addCallback(self.json_full_deepcheck_and_repair_is_healthy, self.root, "root+dvr")
 717
 718         # now look at t=info
 719         d.addCallback(lambda ign: self.web(self.root, t="info"))
 720         # TODO: examine the output
 721         d.addCallback(lambda ign: self.web(self.mutable, t="info"))
 722         d.addCallback(lambda ign: self.web(self.large, t="info"))
 723         d.addCallback(lambda ign: self.web(self.small, t="info"))
 724         d.addCallback(lambda ign: self.web(self.small2, t="info"))
 725         d.addCallback(lambda ign: self.web(self.empty_lit_dir, t="info"))
 726         d.addCallback(lambda ign: self.web(self.tiny_lit_dir, t="info"))
 727
 728         return d
 729
 730     def _run_cli(self, argv, stdin=""):
 731         #print "CLI:", argv
 732         stdout, stderr = StringIO(), StringIO()
 733         d = threads.deferToThread(runner.runner, argv, run_by_human=False,
 734                                   stdin=StringIO(stdin),
 735                                   stdout=stdout, stderr=stderr)
 736         def _done(res):
 737             return stdout.getvalue(), stderr.getvalue()
 738         d.addCallback(_done)
 739         return d
 740
 741     def do_test_cli_good(self, ignored):
 742         d = defer.succeed(None)
 743         d.addCallback(lambda ign: self.do_cli_manifest_stream1())
 744         d.addCallback(lambda ign: self.do_cli_manifest_stream2())
 745         d.addCallback(lambda ign: self.do_cli_manifest_stream3())
 746         d.addCallback(lambda ign: self.do_cli_manifest_stream4())
 747         d.addCallback(lambda ign: self.do_cli_manifest_stream5())
 748         d.addCallback(lambda ign: self.do_cli_stats1())
 749         d.addCallback(lambda ign: self.do_cli_stats2())
 750         return d
 751
 752     def _check_manifest_storage_index(self, out):
 753         lines = [l for l in out.split("\n") if l]
 754         self.failUnlessEqual(len(lines), 3)
 755         self.failUnless(base32.b2a(self.root.get_storage_index()) in lines)
 756         self.failUnless(base32.b2a(self.mutable.get_storage_index()) in lines)
 757         self.failUnless(base32.b2a(self.large.get_storage_index()) in lines)
 758
 759     def do_cli_manifest_stream1(self):
 760         basedir = self.get_clientdir(0)
 761         d = self._run_cli(["manifest",
 762                            "--node-directory", basedir,
 763                            self.root_uri])
 764         def _check((out,err)):
 765             self.failUnlessEqual(err, "")
 766             lines = [l for l in out.split("\n") if l]
 767             self.failUnlessEqual(len(lines), 8)
 768             caps = {}
 769             for l in lines:
 770                 try:
 771                     cap, path = l.split(None, 1)
 772                 except ValueError:
 773                     cap = l.strip()
 774                     path = ""
 775                 caps[cap] = path
 776             self.failUnless(self.root.get_uri() in caps)
 777             self.failUnlessEqual(caps[self.root.get_uri()], "")
 778             self.failUnlessEqual(caps[self.mutable.get_uri()], "mutable")
 779             self.failUnlessEqual(caps[self.large.get_uri()], "large")
 780             self.failUnlessEqual(caps[self.small.get_uri()], "small")
 781             self.failUnlessEqual(caps[self.small2.get_uri()], "small2")
 782             self.failUnlessEqual(caps[self.empty_lit_dir.get_uri()], "empty_lit_dir")
 783             self.failUnlessEqual(caps[self.tiny_lit_dir.get_uri()], "tiny_lit_dir")
 784         d.addCallback(_check)
 785         return d
 786
 787     def do_cli_manifest_stream2(self):
 788         basedir = self.get_clientdir(0)
 789         d = self._run_cli(["manifest",
 790                            "--node-directory", basedir,
 791                            "--raw",
 792                            self.root_uri])
 793         def _check((out,err)):
 794             self.failUnlessEqual(err, "")
 795             # this should be the same as the POST t=stream-manifest output
 796             self._check_streamed_manifest(out)
 797         d.addCallback(_check)
 798         return d
 799
 800     def do_cli_manifest_stream3(self):
 801         basedir = self.get_clientdir(0)
 802         d = self._run_cli(["manifest",
 803                            "--node-directory", basedir,
 804                            "--storage-index",
 805                            self.root_uri])
 806         def _check((out,err)):
 807             self.failUnlessEqual(err, "")
 808             self._check_manifest_storage_index(out)
 809         d.addCallback(_check)
 810         return d
 811
 812     def do_cli_manifest_stream4(self):
 813         basedir = self.get_clientdir(0)
 814         d = self._run_cli(["manifest",
 815                            "--node-directory", basedir,
 816                            "--verify-cap",
 817                            self.root_uri])
 818         def _check((out,err)):
 819             self.failUnlessEqual(err, "")
 820             lines = [l for l in out.split("\n") if l]
 821             self.failUnlessEqual(len(lines), 3)
 822             self.failUnless(self.root.get_verify_cap().to_string() in lines)
 823             self.failUnless(self.mutable.get_verify_cap().to_string() in lines)
 824             self.failUnless(self.large.get_verify_cap().to_string() in lines)
 825         d.addCallback(_check)
 826         return d
 827
 828     def do_cli_manifest_stream5(self):
 829         basedir = self.get_clientdir(0)
 830         d = self._run_cli(["manifest",
 831                            "--node-directory", basedir,
 832                            "--repair-cap",
 833                            self.root_uri])
 834         def _check((out,err)):
 835             self.failUnlessEqual(err, "")
 836             lines = [l for l in out.split("\n") if l]
 837             self.failUnlessEqual(len(lines), 3)
 838             self.failUnless(self.root.get_repair_cap().to_string() in lines)
 839             self.failUnless(self.mutable.get_repair_cap().to_string() in lines)
 840             self.failUnless(self.large.get_repair_cap().to_string() in lines)
 841         d.addCallback(_check)
 842         return d
 843
 844     def do_cli_stats1(self):
 845         basedir = self.get_clientdir(0)
 846         d = self._run_cli(["stats",
 847                            "--node-directory", basedir,
 848                            self.root_uri])
 849         def _check3((out,err)):
 850             lines = [l.strip() for l in out.split("\n") if l]
 851             self.failUnless("count-immutable-files: 1" in lines)
 852             self.failUnless("count-mutable-files: 1" in lines)
 853             self.failUnless("count-literal-files: 3" in lines)
 854             self.failUnless("count-files: 5" in lines)
 855             self.failUnless("count-directories: 3" in lines)
 856             self.failUnless("size-immutable-files: 13000    (13.00 kB, 12.70 kiB)" in lines, lines)
 857             self.failUnless("size-literal-files: 56" in lines, lines)
 858             self.failUnless("    4-10    : 1    (10 B, 10 B)".strip() in lines, lines)
 859             self.failUnless("   11-31    : 2    (31 B, 31 B)".strip() in lines, lines)
 860             self.failUnless("10001-31622 : 1    (31.62 kB, 30.88 kiB)".strip() in lines, lines)
 861         d.addCallback(_check3)
 862         return d
 863
 864     def do_cli_stats2(self):
 865         basedir = self.get_clientdir(0)
 866         d = self._run_cli(["stats",
 867                            "--node-directory", basedir,
 868                            "--raw",
 869                            self.root_uri])
 870         def _check4((out,err)):
 871             data = simplejson.loads(out)
 872             self.failUnlessEqual(data["count-immutable-files"], 1)
 873             self.failUnlessEqual(data["count-immutable-files"], 1)
 874             self.failUnlessEqual(data["count-mutable-files"], 1)
 875             self.failUnlessEqual(data["count-literal-files"], 3)
 876             self.failUnlessEqual(data["count-files"], 5)
 877             self.failUnlessEqual(data["count-directories"], 3)
 878             self.failUnlessEqual(data["size-immutable-files"], 13000)
 879             self.failUnlessEqual(data["size-literal-files"], 56)
 880             self.failUnless([4,10,1] in data["size-files-histogram"])
 881             self.failUnless([11,31,2] in data["size-files-histogram"])
 882             self.failUnless([10001,31622,1] in data["size-files-histogram"])
 883         d.addCallback(_check4)
 884         return d
 885
 886
 887 class DeepCheckWebBad(DeepCheckBase, unittest.TestCase):
 888     def test_bad(self):
 889         self.basedir = "deepcheck/DeepCheckWebBad/bad"
 890         self.set_up_grid()
 891         d = self.set_up_damaged_tree()
 892         d.addCallback(self.do_check)
 893         d.addCallback(self.do_deepcheck)
 894         d.addCallback(self.do_deepcheck_broken)
 895         d.addCallback(self.do_test_web_bad)
 896         d.addErrback(self.explain_web_error)
 897         d.addErrback(self.explain_error)
 898         return d
 899
 900
 901
 902     def set_up_damaged_tree(self):
 903         # 6.4s
 904
 905         # root
 906         #   mutable-good
 907         #   mutable-missing-shares
 908         #   mutable-corrupt-shares
 909         #   mutable-unrecoverable
 910         #   large-good
 911         #   large-missing-shares
 912         #   large-corrupt-shares
 913         #   large-unrecoverable
 914         # broken
 915         #   large1-good
 916         #   subdir-good
 917         #     large2-good
 918         #   subdir-unrecoverable
 919         #     large3-good
 920
 921         self.nodes = {}
 922
 923         c0 = self.g.clients[0]
 924         d = c0.create_dirnode()
 925         def _created_root(n):
 926             self.root = n
 927             self.root_uri = n.get_uri()
 928         d.addCallback(_created_root)
 929         d.addCallback(self.create_mangled, "mutable-good")
 930         d.addCallback(self.create_mangled, "mutable-missing-shares")
 931         d.addCallback(self.create_mangled, "mutable-corrupt-shares")
 932         d.addCallback(self.create_mangled, "mutable-unrecoverable")
 933         d.addCallback(self.create_mangled, "large-good")
 934         d.addCallback(self.create_mangled, "large-missing-shares")
 935         d.addCallback(self.create_mangled, "large-corrupt-shares")
 936         d.addCallback(self.create_mangled, "large-unrecoverable")
 937         d.addCallback(lambda ignored: c0.create_dirnode())
 938         d.addCallback(self._stash_node, "broken")
 939         large1 = upload.Data("Lots of data\n" * 1000 + "large1" + "\n", None)
 940         d.addCallback(lambda ignored:
 941                       self.nodes["broken"].add_file(u"large1", large1))
 942         d.addCallback(lambda ignored:
 943                       self.nodes["broken"].create_subdirectory(u"subdir-good"))
 944         large2 = upload.Data("Lots of data\n" * 1000 + "large2" + "\n", None)
 945         d.addCallback(lambda subdir: subdir.add_file(u"large2-good", large2))
 946         d.addCallback(lambda ignored:
 947                       self.nodes["broken"].create_subdirectory(u"subdir-unrecoverable"))
 948         d.addCallback(self._stash_node, "subdir-unrecoverable")
 949         large3 = upload.Data("Lots of data\n" * 1000 + "large3" + "\n", None)
 950         d.addCallback(lambda subdir: subdir.add_file(u"large3-good", large3))
 951         d.addCallback(lambda ignored:
 952                       self._delete_most_shares(self.nodes["broken"]))
 953         return d
 954
 955     def _stash_node(self, node, name):
 956         self.nodes[name] = node
 957         return node
 958
 959     def create_mangled(self, ignored, name):
 960         nodetype, mangletype = name.split("-", 1)
 961         if nodetype == "mutable":
 962             mutable_uploadable = MutableData("mutable file contents")
 963             d = self.g.clients[0].create_mutable_file(mutable_uploadable)
 964             d.addCallback(lambda n: self.root.set_node(unicode(name), n))
 965         elif nodetype == "large":
 966             large = upload.Data("Lots of data\n" * 1000 + name + "\n", None)
 967             d = self.root.add_file(unicode(name), large)
 968         elif nodetype == "small":
 969             small = upload.Data("Small enough for a LIT", None)
 970             d = self.root.add_file(unicode(name), small)
 971
 972         d.addCallback(self._stash_node, name)
 973
 974         if mangletype == "good":
 975             pass
 976         elif mangletype == "missing-shares":
 977             d.addCallback(self._delete_some_shares)
 978         elif mangletype == "corrupt-shares":
 979             d.addCallback(self._corrupt_some_shares)
 980         else:
 981             assert mangletype == "unrecoverable"
 982             d.addCallback(self._delete_most_shares)
 983
 984         return d
 985
 986     def _run_cli(self, argv):
 987         stdout, stderr = StringIO(), StringIO()
 988         # this can only do synchronous operations
 989         assert argv[0] == "debug"
 990         runner.runner(argv, run_by_human=False, stdout=stdout, stderr=stderr)
 991         return stdout.getvalue()
 992
 993     def _delete_some_shares(self, node):
 994         self.delete_shares_numbered(node.get_uri(), [0,1])
 995
 996     def _corrupt_some_shares(self, node):
 997         for (shnum, serverid, sharefile) in self.find_uri_shares(node.get_uri()):
 998             if shnum in (0,1):
 999                 self._run_cli(["debug", "corrupt-share", sharefile])
1000
1001     def _delete_most_shares(self, node):
1002         self.delete_shares_numbered(node.get_uri(), range(1,10))
1003
1004
1005     def check_is_healthy(self, cr, where):
1006         try:
1007             self.failUnless(ICheckResults.providedBy(cr), (cr, type(cr), where))
1008             self.failUnless(cr.is_healthy(), (cr.get_report(), cr.is_healthy(), cr.get_summary(), where))
1009             self.failUnless(cr.is_recoverable(), where)
1010             self.failUnlessEqual(cr.get_version_counter_recoverable(), 1, where)
1011             self.failUnlessEqual(cr.get_version_counter_unrecoverable(), 0, where)
1012             return cr
1013         except Exception, le:
1014             le.args = tuple(le.args + (where,))
1015             raise
1016
1017     def check_is_missing_shares(self, cr, where):
1018         self.failUnless(ICheckResults.providedBy(cr), where)
1019         self.failIf(cr.is_healthy(), where)
1020         self.failUnless(cr.is_recoverable(), where)
1021         self.failUnlessEqual(cr.get_version_counter_recoverable(), 1, where)
1022         self.failUnlessEqual(cr.get_version_counter_unrecoverable(), 0, where)
1023         return cr
1024
1025     def check_has_corrupt_shares(self, cr, where):
1026         # by "corrupt-shares" we mean the file is still recoverable
1027         self.failUnless(ICheckResults.providedBy(cr), where)
1028         self.failIf(cr.is_healthy(), (where, cr))
1029         self.failUnless(cr.is_recoverable(), where)
1030         self.failUnless(cr.get_share_counter_good() < 10, where)
1031         self.failUnless(cr.get_corrupt_shares(), where)
1032         return cr
1033
1034     def check_is_unrecoverable(self, cr, where):
1035         self.failUnless(ICheckResults.providedBy(cr), where)
1036         self.failIf(cr.is_healthy(), where)
1037         self.failIf(cr.is_recoverable(), where)
1038         self.failUnless(cr.get_share_counter_good() < cr.get_encoding_needed(),
1039                         (cr.get_share_counter_good(), cr.get_encoding_needed(),
1040                          where))
1041         self.failUnlessEqual(cr.get_version_counter_recoverable(), 0, where)
1042         self.failUnlessEqual(cr.get_version_counter_unrecoverable(), 1, where)
1043         return cr
1044
1045     def do_check(self, ignored):
1046         d = defer.succeed(None)
1047
1048         # check the individual items, without verification. This will not
1049         # detect corrupt shares.
1050         def _check(which, checker):
1051             d = self.nodes[which].check(Monitor())
1052             d.addCallback(checker, which + "--check")
1053             return d
1054
1055         d.addCallback(lambda ign: _check("mutable-good", self.check_is_healthy))
1056         d.addCallback(lambda ign: _check("mutable-missing-shares",
1057                                          self.check_is_missing_shares))
1058         d.addCallback(lambda ign: _check("mutable-corrupt-shares",
1059                                          self.check_is_healthy))
1060         d.addCallback(lambda ign: _check("mutable-unrecoverable",
1061                                          self.check_is_unrecoverable))
1062         d.addCallback(lambda ign: _check("large-good", self.check_is_healthy))
1063         d.addCallback(lambda ign: _check("large-missing-shares",
1064                                          self.check_is_missing_shares))
1065         d.addCallback(lambda ign: _check("large-corrupt-shares",
1066                                          self.check_is_healthy))
1067         d.addCallback(lambda ign: _check("large-unrecoverable",
1068                                          self.check_is_unrecoverable))
1069
1070         # and again with verify=True, which *does* detect corrupt shares.
1071         def _checkv(which, checker):
1072             d = self.nodes[which].check(Monitor(), verify=True)
1073             d.addCallback(checker, which + "--check-and-verify")
1074             return d
1075
1076         d.addCallback(lambda ign: _checkv("mutable-good", self.check_is_healthy))
1077         d.addCallback(lambda ign: _checkv("mutable-missing-shares",
1078                                          self.check_is_missing_shares))
1079         d.addCallback(lambda ign: _checkv("mutable-corrupt-shares",
1080                                          self.check_has_corrupt_shares))
1081         d.addCallback(lambda ign: _checkv("mutable-unrecoverable",
1082                                          self.check_is_unrecoverable))
1083         d.addCallback(lambda ign: _checkv("large-good", self.check_is_healthy))
1084         d.addCallback(lambda ign: _checkv("large-missing-shares", self.check_is_missing_shares))
1085         d.addCallback(lambda ign: _checkv("large-corrupt-shares", self.check_has_corrupt_shares))
1086         d.addCallback(lambda ign: _checkv("large-unrecoverable",
1087                                          self.check_is_unrecoverable))
1088
1089         return d
1090
1091     def do_deepcheck(self, ignored):
1092         d = defer.succeed(None)
1093
1094         # now deep-check the root, with various verify= and repair= options
1095         d.addCallback(lambda ign:
1096                       self.root.start_deep_check().when_done())
1097         def _check1(cr):
1098             self.failUnless(IDeepCheckResults.providedBy(cr))
1099             c = cr.get_counters()
1100             self.failUnlessEqual(c["count-objects-checked"], 9)
1101             self.failUnlessEqual(c["count-objects-healthy"], 5)
1102             self.failUnlessEqual(c["count-objects-unhealthy"], 4)
1103             self.failUnlessEqual(c["count-objects-unrecoverable"], 2)
1104         d.addCallback(_check1)
1105
1106         d.addCallback(lambda ign:
1107                       self.root.start_deep_check(verify=True).when_done())
1108         def _check2(cr):
1109             self.failUnless(IDeepCheckResults.providedBy(cr))
1110             c = cr.get_counters()
1111             self.failUnlessEqual(c["count-objects-checked"], 9)
1112             self.failUnlessEqual(c["count-objects-healthy"], 3)
1113             self.failUnlessEqual(c["count-objects-unhealthy"], 6)
1114             self.failUnlessEqual(c["count-objects-healthy"], 3) # root, mutable good, large good
1115             self.failUnlessEqual(c["count-objects-unrecoverable"], 2) # mutable unrecoverable, large unrecoverable
1116         d.addCallback(_check2)
1117
1118         return d
1119
1120     def do_deepcheck_broken(self, ignored):
1121         # deep-check on the broken directory should fail, because of the
1122         # untraversable subdir
1123         def _do_deep_check():
1124             return self.nodes["broken"].start_deep_check().when_done()
1125         d = self.shouldFail(UnrecoverableFileError, "do_deep_check",
1126                             "no recoverable versions",
1127                             _do_deep_check)
1128         return d
1129
1130     def json_is_healthy(self, data, where):
1131         r = data["results"]
1132         self.failUnless(r["healthy"], where)
1133         self.failUnless(r["recoverable"], where)
1134         self.failUnlessEqual(r["count-recoverable-versions"], 1, where)
1135         self.failUnlessEqual(r["count-unrecoverable-versions"], 0, where)
1136
1137     def json_is_missing_shares(self, data, where):
1138         r = data["results"]
1139         self.failIf(r["healthy"], where)
1140         self.failUnless(r["recoverable"], where)
1141         self.failUnlessEqual(r["count-recoverable-versions"], 1, where)
1142         self.failUnlessEqual(r["count-unrecoverable-versions"], 0, where)
1143
1144     def json_has_corrupt_shares(self, data, where):
1145         # by "corrupt-shares" we mean the file is still recoverable
1146         r = data["results"]
1147         self.failIf(r["healthy"], where)
1148         self.failUnless(r["recoverable"], where)
1149         self.failUnless(r["count-shares-good"] < 10, where)
1150         self.failUnless(r["count-corrupt-shares"], where)
1151         self.failUnless(r["list-corrupt-shares"], where)
1152
1153     def json_is_unrecoverable(self, data, where):
1154         r = data["results"]
1155         self.failIf(r["healthy"], where)
1156         self.failIf(r["recoverable"], where)
1157         self.failUnless(r["count-shares-good"] < r["count-shares-needed"],
1158                         where)
1159         self.failUnlessEqual(r["count-recoverable-versions"], 0, where)
1160         self.failUnlessEqual(r["count-unrecoverable-versions"], 1, where)
1161
1162     def do_test_web_bad(self, ignored):
1163         d = defer.succeed(None)
1164
1165         # check, no verify
1166         def _check(which, checker):
1167             d = self.web_json(self.nodes[which], t="check")
1168             d.addCallback(checker, which + "--webcheck")
1169             return d
1170
1171         d.addCallback(lambda ign: _check("mutable-good",
1172                                          self.json_is_healthy))
1173         d.addCallback(lambda ign: _check("mutable-missing-shares",
1174                                          self.json_is_missing_shares))
1175         d.addCallback(lambda ign: _check("mutable-corrupt-shares",
1176                                          self.json_is_healthy))
1177         d.addCallback(lambda ign: _check("mutable-unrecoverable",
1178                                          self.json_is_unrecoverable))
1179         d.addCallback(lambda ign: _check("large-good",
1180                                          self.json_is_healthy))
1181         d.addCallback(lambda ign: _check("large-missing-shares",
1182                                          self.json_is_missing_shares))
1183         d.addCallback(lambda ign: _check("large-corrupt-shares",
1184                                          self.json_is_healthy))
1185         d.addCallback(lambda ign: _check("large-unrecoverable",
1186                                          self.json_is_unrecoverable))
1187
1188         # check and verify
1189         def _checkv(which, checker):
1190             d = self.web_json(self.nodes[which], t="check", verify="true")
1191             d.addCallback(checker, which + "--webcheck-and-verify")
1192             return d
1193
1194         d.addCallback(lambda ign: _checkv("mutable-good",
1195                                           self.json_is_healthy))
1196         d.addCallback(lambda ign: _checkv("mutable-missing-shares",
1197                                          self.json_is_missing_shares))
1198         d.addCallback(lambda ign: _checkv("mutable-corrupt-shares",
1199                                          self.json_has_corrupt_shares))
1200         d.addCallback(lambda ign: _checkv("mutable-unrecoverable",
1201                                          self.json_is_unrecoverable))
1202         d.addCallback(lambda ign: _checkv("large-good",
1203                                           self.json_is_healthy))
1204         d.addCallback(lambda ign: _checkv("large-missing-shares", self.json_is_missing_shares))
1205         d.addCallback(lambda ign: _checkv("large-corrupt-shares", self.json_has_corrupt_shares))
1206         d.addCallback(lambda ign: _checkv("large-unrecoverable",
1207                                          self.json_is_unrecoverable))
1208
1209         return d
1210
1211 class Large(DeepCheckBase, unittest.TestCase):
1212     def test_lots_of_lits(self):
1213         self.basedir = "deepcheck/Large/lots_of_lits"
1214         self.set_up_grid()
1215         # create the following directory structure:
1216         #  root/
1217         #   subdir/
1218         #    000-large (CHK)
1219         #    001-small (LIT)
1220         #    002-small
1221         #    ...
1222         #    399-small
1223         # then do a deepcheck and make sure it doesn't cause a
1224         # Deferred-tail-recursion stack overflow
1225
1226         COUNT = 400
1227         c0 = self.g.clients[0]
1228         d = c0.create_dirnode()
1229         self.stash = {}
1230         def _created_root(n):
1231             self.root = n
1232             return n
1233         d.addCallback(_created_root)
1234         d.addCallback(lambda root: root.create_subdirectory(u"subdir"))
1235         def _add_children(subdir_node):
1236             self.subdir_node = subdir_node
1237             kids = {}
1238             for i in range(1, COUNT):
1239                 litcap = LiteralFileURI("%03d-data" % i).to_string()
1240                 kids[u"%03d-small" % i] = (litcap, litcap)
1241             return subdir_node.set_children(kids)
1242         d.addCallback(_add_children)
1243         up = upload.Data("large enough for CHK" * 100, "")
1244         d.addCallback(lambda ign: self.subdir_node.add_file(u"0000-large", up))
1245
1246         def _start_deepcheck(ignored):
1247             return self.web(self.root, method="POST", t="stream-deep-check")
1248         d.addCallback(_start_deepcheck)
1249         def _check( (output, url) ):
1250             units = list(self.parse_streamed_json(output))
1251             self.failUnlessEqual(len(units), 2+COUNT+1)
1252         d.addCallback(_check)
1253
1254         return d