src/allmydata/test/test_deepcheck.py

   1
   2 import os, simplejson, urllib
   3 from cStringIO import StringIO
   4 from twisted.trial import unittest
   5 from twisted.internet import defer
   6 from twisted.internet import threads # CLI tests use deferToThread
   7 from allmydata.immutable import upload
   8 from allmydata.mutable.common import UnrecoverableFileError
   9 from allmydata.util import idlib
  10 from allmydata.util import base32
  11 from allmydata.scripts import runner
  12 from allmydata.interfaces import ICheckResults, ICheckAndRepairResults, \
  13      IDeepCheckResults, IDeepCheckAndRepairResults
  14 from allmydata.monitor import Monitor, OperationCancelledError
  15 from allmydata.uri import LiteralFileURI
  16 from twisted.web.client import getPage
  17
  18 from allmydata.test.common import ErrorMixin, _corrupt_mutable_share_data, \
  19      ShouldFailMixin
  20 from allmydata.test.common_util import StallMixin
  21 from allmydata.test.no_network import GridTestMixin
  22
  23 timeout = 2400 # One of these took 1046.091s on Zandr's ARM box.
  24
  25 class MutableChecker(GridTestMixin, unittest.TestCase, ErrorMixin):
  26     def _run_cli(self, argv):
  27         stdout, stderr = StringIO(), StringIO()
  28         # this can only do synchronous operations
  29         assert argv[0] == "debug"
  30         runner.runner(argv, run_by_human=False, stdout=stdout, stderr=stderr)
  31         return stdout.getvalue()
  32
  33     def test_good(self):
  34         self.basedir = "deepcheck/MutableChecker/good"
  35         self.set_up_grid()
  36         CONTENTS = "a little bit of data"
  37         d = self.g.clients[0].create_mutable_file(CONTENTS)
  38         def _created(node):
  39             self.node = node
  40             self.fileurl = "uri/" + urllib.quote(node.get_uri())
  41         d.addCallback(_created)
  42         # now make sure the webapi verifier sees no problems
  43         d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=true",
  44                                            method="POST"))
  45         def _got_results(out):
  46             self.failUnless("<span>Healthy : Healthy</span>" in out, out)
  47             self.failUnless("Recoverable Versions: 10*seq1-" in out, out)
  48             self.failIf("Not Healthy!" in out, out)
  49             self.failIf("Unhealthy" in out, out)
  50             self.failIf("Corrupt Shares" in out, out)
  51         d.addCallback(_got_results)
  52         d.addErrback(self.explain_web_error)
  53         return d
  54
  55     def test_corrupt(self):
  56         self.basedir = "deepcheck/MutableChecker/corrupt"
  57         self.set_up_grid()
  58         CONTENTS = "a little bit of data"
  59         d = self.g.clients[0].create_mutable_file(CONTENTS)
  60         def _stash_and_corrupt(node):
  61             self.node = node
  62             self.fileurl = "uri/" + urllib.quote(node.get_uri())
  63             self.corrupt_shares_numbered(node.get_uri(), [0],
  64                                          _corrupt_mutable_share_data)
  65         d.addCallback(_stash_and_corrupt)
  66         # now make sure the webapi verifier notices it
  67         d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=true",
  68                                            method="POST"))
  69         def _got_results(out):
  70             self.failUnless("Not Healthy!" in out, out)
  71             self.failUnless("Unhealthy: best version has only 9 shares (encoding is 3-of-10)" in out, out)
  72             self.failUnless("Corrupt Shares:" in out, out)
  73         d.addCallback(_got_results)
  74
  75         # now make sure the webapi repairer can fix it
  76         d.addCallback(lambda ign:
  77                       self.GET(self.fileurl+"?t=check&verify=true&repair=true",
  78                                method="POST"))
  79         def _got_repair_results(out):
  80             self.failUnless("<div>Repair successful</div>" in out, out)
  81         d.addCallback(_got_repair_results)
  82         d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=true",
  83                                            method="POST"))
  84         def _got_postrepair_results(out):
  85             self.failIf("Not Healthy!" in out, out)
  86             self.failUnless("Recoverable Versions: 10*seq" in out, out)
  87         d.addCallback(_got_postrepair_results)
  88         d.addErrback(self.explain_web_error)
  89
  90         return d
  91
  92     def test_delete_share(self):
  93         self.basedir = "deepcheck/MutableChecker/delete_share"
  94         self.set_up_grid()
  95         CONTENTS = "a little bit of data"
  96         d = self.g.clients[0].create_mutable_file(CONTENTS)
  97         def _stash_and_delete(node):
  98             self.node = node
  99             self.fileurl = "uri/" + urllib.quote(node.get_uri())
 100             self.delete_shares_numbered(node.get_uri(), [0])
 101         d.addCallback(_stash_and_delete)
 102         # now make sure the webapi checker notices it
 103         d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=false",
 104                                            method="POST"))
 105         def _got_results(out):
 106             self.failUnless("Not Healthy!" in out, out)
 107             self.failUnless("Unhealthy: best version has only 9 shares (encoding is 3-of-10)" in out, out)
 108             self.failIf("Corrupt Shares" in out, out)
 109         d.addCallback(_got_results)
 110
 111         # now make sure the webapi repairer can fix it
 112         d.addCallback(lambda ign:
 113                       self.GET(self.fileurl+"?t=check&verify=false&repair=true",
 114                                method="POST"))
 115         def _got_repair_results(out):
 116             self.failUnless("Repair successful" in out)
 117         d.addCallback(_got_repair_results)
 118         d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=false",
 119                                            method="POST"))
 120         def _got_postrepair_results(out):
 121             self.failIf("Not Healthy!" in out, out)
 122             self.failUnless("Recoverable Versions: 10*seq" in out)
 123         d.addCallback(_got_postrepair_results)
 124         d.addErrback(self.explain_web_error)
 125
 126         return d
 127
 128
 129 class DeepCheckBase(GridTestMixin, ErrorMixin, StallMixin, ShouldFailMixin):
 130
 131     def web_json(self, n, **kwargs):
 132         kwargs["output"] = "json"
 133         d = self.web(n, "POST", **kwargs)
 134         d.addCallback(self.decode_json)
 135         return d
 136
 137     def decode_json(self, (s,url)):
 138         try:
 139             data = simplejson.loads(s)
 140         except ValueError:
 141             self.fail("%s: not JSON: '%s'" % (url, s))
 142         return data
 143
 144     def parse_streamed_json(self, s):
 145         for unit in s.split("\n"):
 146             if not unit:
 147                 # stream should end with a newline, so split returns ""
 148                 continue
 149             try:
 150                 yield simplejson.loads(unit)
 151             except ValueError, le:
 152                 le.args = tuple(le.args + (unit,))
 153                 raise
 154
 155     def web(self, n, method="GET", **kwargs):
 156         # returns (data, url)
 157         url = (self.client_baseurls[0] + "uri/%s" % urllib.quote(n.get_uri())
 158                + "?" + "&".join(["%s=%s" % (k,v) for (k,v) in kwargs.items()]))
 159         d = getPage(url, method=method)
 160         d.addCallback(lambda data: (data,url))
 161         return d
 162
 163     def wait_for_operation(self, ignored, ophandle):
 164         url = self.client_baseurls[0] + "operations/" + ophandle
 165         url += "?t=status&output=JSON"
 166         d = getPage(url)
 167         def _got(res):
 168             try:
 169                 data = simplejson.loads(res)
 170             except ValueError:
 171                 self.fail("%s: not JSON: '%s'" % (url, res))
 172             if not data["finished"]:
 173                 d = self.stall(delay=1.0)
 174                 d.addCallback(self.wait_for_operation, ophandle)
 175                 return d
 176             return data
 177         d.addCallback(_got)
 178         return d
 179
 180     def get_operation_results(self, ignored, ophandle, output=None):
 181         url = self.client_baseurls[0] + "operations/" + ophandle
 182         url += "?t=status"
 183         if output:
 184             url += "&output=" + output
 185         d = getPage(url)
 186         def _got(res):
 187             if output and output.lower() == "json":
 188                 try:
 189                     return simplejson.loads(res)
 190                 except ValueError:
 191                     self.fail("%s: not JSON: '%s'" % (url, res))
 192             return res
 193         d.addCallback(_got)
 194         return d
 195
 196     def slow_web(self, n, output=None, **kwargs):
 197         # use ophandle=
 198         handle = base32.b2a(os.urandom(4))
 199         d = self.web(n, "POST", ophandle=handle, **kwargs)
 200         d.addCallback(self.wait_for_operation, handle)
 201         d.addCallback(self.get_operation_results, handle, output=output)
 202         return d
 203
 204
 205 class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
 206     # construct a small directory tree (with one dir, one immutable file, one
 207     # mutable file, two LIT files, one DIR2:LIT empty dir, one DIR2:LIT tiny
 208     # dir, and a loop), and then check/examine it in various ways.
 209
 210     def set_up_tree(self):
 211         # 2.9s
 212
 213         c0 = self.g.clients[0]
 214         d = c0.create_dirnode()
 215         def _created_root(n):
 216             self.root = n
 217             self.root_uri = n.get_uri()
 218         d.addCallback(_created_root)
 219         d.addCallback(lambda ign: c0.create_mutable_file("mutable file contents"))
 220         d.addCallback(lambda n: self.root.set_node(u"mutable", n))
 221         def _created_mutable(n):
 222             self.mutable = n
 223             self.mutable_uri = n.get_uri()
 224         d.addCallback(_created_mutable)
 225
 226         large = upload.Data("Lots of data\n" * 1000, None)
 227         d.addCallback(lambda ign: self.root.add_file(u"large", large))
 228         def _created_large(n):
 229             self.large = n
 230             self.large_uri = n.get_uri()
 231         d.addCallback(_created_large)
 232
 233         small = upload.Data("Small enough for a LIT", None)
 234         d.addCallback(lambda ign: self.root.add_file(u"small", small))
 235         def _created_small(n):
 236             self.small = n
 237             self.small_uri = n.get_uri()
 238         d.addCallback(_created_small)
 239
 240         small2 = upload.Data("Small enough for a LIT too", None)
 241         d.addCallback(lambda ign: self.root.add_file(u"small2", small2))
 242         def _created_small2(n):
 243             self.small2 = n
 244             self.small2_uri = n.get_uri()
 245         d.addCallback(_created_small2)
 246
 247         empty_litdir_uri = "URI:DIR2-LIT:"
 248         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
 249
 250         d.addCallback(lambda ign: self.root._create_and_validate_node(None, empty_litdir_uri, name=u"test_deepcheck empty_lit_dir"))
 251         def _created_empty_lit_dir(n):
 252             self.empty_lit_dir = n
 253             self.empty_lit_dir_uri = n.get_uri()
 254             self.root.set_node(u"empty_lit_dir", n)
 255         d.addCallback(_created_empty_lit_dir)
 256
 257         d.addCallback(lambda ign: self.root._create_and_validate_node(None, tiny_litdir_uri, name=u"test_deepcheck tiny_lit_dir"))
 258         def _created_tiny_lit_dir(n):
 259             self.tiny_lit_dir = n
 260             self.tiny_lit_dir_uri = n.get_uri()
 261             self.root.set_node(u"tiny_lit_dir", n)
 262         d.addCallback(_created_tiny_lit_dir)
 263
 264         d.addCallback(lambda ign: self.root.set_node(u"loop", self.root))
 265         return d
 266
 267     def check_is_healthy(self, cr, n, where, incomplete=False):
 268         self.failUnless(ICheckResults.providedBy(cr), where)
 269         self.failUnless(cr.is_healthy(), where)
 270         self.failUnlessEqual(cr.get_storage_index(), n.get_storage_index(),
 271                              where)
 272         self.failUnlessEqual(cr.get_storage_index_string(),
 273                              base32.b2a(n.get_storage_index()), where)
 274         num_servers = len(self.g.all_servers)
 275         needs_rebalancing = bool( num_servers < 10 )
 276         if not incomplete:
 277             self.failUnlessEqual(cr.needs_rebalancing(), needs_rebalancing,
 278                                  str((where, cr, cr.get_data())))
 279         d = cr.get_data()
 280         self.failUnlessEqual(d["count-shares-good"], 10, where)
 281         self.failUnlessEqual(d["count-shares-needed"], 3, where)
 282         self.failUnlessEqual(d["count-shares-expected"], 10, where)
 283         if not incomplete:
 284             self.failUnlessEqual(d["count-good-share-hosts"], num_servers,
 285                                  where)
 286         self.failUnlessEqual(d["count-corrupt-shares"], 0, where)
 287         self.failUnlessEqual(d["list-corrupt-shares"], [], where)
 288         if not incomplete:
 289             self.failUnlessEqual(sorted(d["servers-responding"]),
 290                                  sorted(self.g.servers_by_id.keys()),
 291                                  where)
 292             self.failUnless("sharemap" in d, str((where, d)))
 293             all_serverids = set()
 294             for (shareid, serverids) in d["sharemap"].items():
 295                 all_serverids.update(serverids)
 296             self.failUnlessEqual(sorted(all_serverids),
 297                                  sorted(self.g.servers_by_id.keys()),
 298                                  where)
 299
 300         self.failUnlessEqual(d["count-wrong-shares"], 0, where)
 301         self.failUnlessEqual(d["count-recoverable-versions"], 1, where)
 302         self.failUnlessEqual(d["count-unrecoverable-versions"], 0, where)
 303
 304
 305     def check_and_repair_is_healthy(self, cr, n, where, incomplete=False):
 306         self.failUnless(ICheckAndRepairResults.providedBy(cr), (where, cr))
 307         self.failUnless(cr.get_pre_repair_results().is_healthy(), where)
 308         self.check_is_healthy(cr.get_pre_repair_results(), n, where, incomplete)
 309         self.failUnless(cr.get_post_repair_results().is_healthy(), where)
 310         self.check_is_healthy(cr.get_post_repair_results(), n, where, incomplete)
 311         self.failIf(cr.get_repair_attempted(), where)
 312
 313     def deep_check_is_healthy(self, cr, num_healthy, where):
 314         self.failUnless(IDeepCheckResults.providedBy(cr))
 315         self.failUnlessEqual(cr.get_counters()["count-objects-healthy"],
 316                              num_healthy, where)
 317
 318     def deep_check_and_repair_is_healthy(self, cr, num_healthy, where):
 319         self.failUnless(IDeepCheckAndRepairResults.providedBy(cr), where)
 320         c = cr.get_counters()
 321         self.failUnlessEqual(c["count-objects-healthy-pre-repair"],
 322                              num_healthy, where)
 323         self.failUnlessEqual(c["count-objects-healthy-post-repair"],
 324                              num_healthy, where)
 325         self.failUnlessEqual(c["count-repairs-attempted"], 0, where)
 326
 327     def test_good(self):
 328         self.basedir = "deepcheck/DeepCheckWebGood/good"
 329         self.set_up_grid()
 330         d = self.set_up_tree()
 331         d.addCallback(self.do_stats)
 332         d.addCallback(self.do_web_stream_manifest)
 333         d.addCallback(self.do_web_stream_check)
 334         d.addCallback(self.do_test_check_good)
 335         d.addCallback(self.do_test_web_good)
 336         d.addCallback(self.do_test_cli_good)
 337         d.addErrback(self.explain_web_error)
 338         d.addErrback(self.explain_error)
 339         return d
 340
 341     def do_stats(self, ignored):
 342         d = defer.succeed(None)
 343         d.addCallback(lambda ign: self.root.start_deep_stats().when_done())
 344         d.addCallback(self.check_stats_good)
 345         return d
 346
 347     def check_stats_good(self, s):
 348         self.failUnlessEqual(s["count-directories"], 3)
 349         self.failUnlessEqual(s["count-files"], 5)
 350         self.failUnlessEqual(s["count-immutable-files"], 1)
 351         self.failUnlessEqual(s["count-literal-files"], 3)
 352         self.failUnlessEqual(s["count-mutable-files"], 1)
 353         # don't check directories: their size will vary
 354         # s["largest-directory"]
 355         # s["size-directories"]
 356         self.failUnlessEqual(s["largest-directory-children"], 7)
 357         self.failUnlessEqual(s["largest-immutable-file"], 13000)
 358         # to re-use this function for both the local
 359         # dirnode.start_deep_stats() and the webapi t=start-deep-stats, we
 360         # coerce the result into a list of tuples. dirnode.start_deep_stats()
 361         # returns a list of tuples, but JSON only knows about lists., so
 362         # t=start-deep-stats returns a list of lists.
 363         histogram = [tuple(stuff) for stuff in s["size-files-histogram"]]
 364         self.failUnlessEqual(histogram, [(4, 10, 1), (11, 31, 2),
 365                                          (10001, 31622, 1),
 366                                          ])
 367         self.failUnlessEqual(s["size-immutable-files"], 13000)
 368         self.failUnlessEqual(s["size-literal-files"], 56)
 369
 370     def do_web_stream_manifest(self, ignored):
 371         d = self.web(self.root, method="POST", t="stream-manifest")
 372         d.addCallback(lambda (output,url):
 373                       self._check_streamed_manifest(output))
 374         return d
 375
 376     def _check_streamed_manifest(self, output):
 377         units = list(self.parse_streamed_json(output))
 378         files = [u for u in units if u["type"] in ("file", "directory")]
 379         assert units[-1]["type"] == "stats"
 380         stats = units[-1]["stats"]
 381         self.failUnlessEqual(len(files), 8)
 382         # [root,mutable,large] are distributed, [small,small2,empty_litdir,tiny_litdir] are not
 383         self.failUnlessEqual(len([f for f in files
 384                                   if f["verifycap"] != ""]), 3)
 385         self.failUnlessEqual(len([f for f in files
 386                                   if f["verifycap"] == ""]), 5)
 387         self.failUnlessEqual(len([f for f in files
 388                                   if f["repaircap"] != ""]), 3)
 389         self.failUnlessEqual(len([f for f in files
 390                                   if f["repaircap"] == ""]), 5)
 391         self.failUnlessEqual(len([f for f in files
 392                                   if f["storage-index"] != ""]), 3)
 393         self.failUnlessEqual(len([f for f in files
 394                                   if f["storage-index"] == ""]), 5)
 395         # make sure that a mutable file has filecap==repaircap!=verifycap
 396         mutable = [f for f in files
 397                    if f["cap"] is not None
 398                    and f["cap"].startswith("URI:SSK:")][0]
 399         self.failUnlessEqual(mutable["cap"], self.mutable_uri)
 400         self.failIfEqual(mutable["cap"], mutable["verifycap"])
 401         self.failUnlessEqual(mutable["cap"], mutable["repaircap"])
 402         # for immutable file, verifycap==repaircap!=filecap
 403         large = [f for f in files
 404                    if f["cap"] is not None
 405                    and f["cap"].startswith("URI:CHK:")][0]
 406         self.failUnlessEqual(large["cap"], self.large_uri)
 407         self.failIfEqual(large["cap"], large["verifycap"])
 408         self.failUnlessEqual(large["verifycap"], large["repaircap"])
 409         self.check_stats_good(stats)
 410
 411     def do_web_stream_check(self, ignored):
 412         # TODO
 413         return
 414         d = self.web(self.root, t="stream-deep-check")
 415         def _check(res):
 416             units = list(self.parse_streamed_json(res))
 417             #files = [u for u in units if u["type"] in ("file", "directory")]
 418             assert units[-1]["type"] == "stats"
 419             #stats = units[-1]["stats"]
 420             # ...
 421         d.addCallback(_check)
 422         return d
 423
 424     def do_test_check_good(self, ignored):
 425         d = defer.succeed(None)
 426         # check the individual items
 427         d.addCallback(lambda ign: self.root.check(Monitor()))
 428         d.addCallback(self.check_is_healthy, self.root, "root")
 429         d.addCallback(lambda ign: self.mutable.check(Monitor()))
 430         d.addCallback(self.check_is_healthy, self.mutable, "mutable")
 431         d.addCallback(lambda ign: self.large.check(Monitor()))
 432         d.addCallback(self.check_is_healthy, self.large, "large")
 433         d.addCallback(lambda ign: self.small.check(Monitor()))
 434         d.addCallback(self.failUnlessEqual, None, "small")
 435         d.addCallback(lambda ign: self.small2.check(Monitor()))
 436         d.addCallback(self.failUnlessEqual, None, "small2")
 437         d.addCallback(lambda ign: self.empty_lit_dir.check(Monitor()))
 438         d.addCallback(self.failUnlessEqual, None, "empty_lit_dir")
 439         d.addCallback(lambda ign: self.tiny_lit_dir.check(Monitor()))
 440         d.addCallback(self.failUnlessEqual, None, "tiny_lit_dir")
 441
 442         # and again with verify=True
 443         d.addCallback(lambda ign: self.root.check(Monitor(), verify=True))
 444         d.addCallback(self.check_is_healthy, self.root, "root")
 445         d.addCallback(lambda ign: self.mutable.check(Monitor(), verify=True))
 446         d.addCallback(self.check_is_healthy, self.mutable, "mutable")
 447         d.addCallback(lambda ign: self.large.check(Monitor(), verify=True))
 448         d.addCallback(self.check_is_healthy, self.large, "large", incomplete=True)
 449         d.addCallback(lambda ign: self.small.check(Monitor(), verify=True))
 450         d.addCallback(self.failUnlessEqual, None, "small")
 451         d.addCallback(lambda ign: self.small2.check(Monitor(), verify=True))
 452         d.addCallback(self.failUnlessEqual, None, "small2")
 453         d.addCallback(lambda ign: self.empty_lit_dir.check(Monitor(), verify=True))
 454         d.addCallback(self.failUnlessEqual, None, "empty_lit_dir")
 455         d.addCallback(lambda ign: self.tiny_lit_dir.check(Monitor(), verify=True))
 456         d.addCallback(self.failUnlessEqual, None, "tiny_lit_dir")
 457
 458         # and check_and_repair(), which should be a nop
 459         d.addCallback(lambda ign: self.root.check_and_repair(Monitor()))
 460         d.addCallback(self.check_and_repair_is_healthy, self.root, "root")
 461         d.addCallback(lambda ign: self.mutable.check_and_repair(Monitor()))
 462         d.addCallback(self.check_and_repair_is_healthy, self.mutable, "mutable")
 463         d.addCallback(lambda ign: self.large.check_and_repair(Monitor()))
 464         d.addCallback(self.check_and_repair_is_healthy, self.large, "large")
 465         d.addCallback(lambda ign: self.small.check_and_repair(Monitor()))
 466         d.addCallback(self.failUnlessEqual, None, "small")
 467         d.addCallback(lambda ign: self.small2.check_and_repair(Monitor()))
 468         d.addCallback(self.failUnlessEqual, None, "small2")
 469         d.addCallback(lambda ign: self.empty_lit_dir.check_and_repair(Monitor()))
 470         d.addCallback(self.failUnlessEqual, None, "empty_lit_dir")
 471         d.addCallback(lambda ign: self.tiny_lit_dir.check_and_repair(Monitor()))
 472
 473         # check_and_repair(verify=True)
 474         d.addCallback(lambda ign: self.root.check_and_repair(Monitor(), verify=True))
 475         d.addCallback(self.check_and_repair_is_healthy, self.root, "root")
 476         d.addCallback(lambda ign: self.mutable.check_and_repair(Monitor(), verify=True))
 477         d.addCallback(self.check_and_repair_is_healthy, self.mutable, "mutable")
 478         d.addCallback(lambda ign: self.large.check_and_repair(Monitor(), verify=True))
 479         d.addCallback(self.check_and_repair_is_healthy, self.large, "large", incomplete=True)
 480         d.addCallback(lambda ign: self.small.check_and_repair(Monitor(), verify=True))
 481         d.addCallback(self.failUnlessEqual, None, "small")
 482         d.addCallback(lambda ign: self.small2.check_and_repair(Monitor(), verify=True))
 483         d.addCallback(self.failUnlessEqual, None, "small2")
 484         d.addCallback(self.failUnlessEqual, None, "small2")
 485         d.addCallback(lambda ign: self.empty_lit_dir.check_and_repair(Monitor(), verify=True))
 486         d.addCallback(self.failUnlessEqual, None, "empty_lit_dir")
 487         d.addCallback(lambda ign: self.tiny_lit_dir.check_and_repair(Monitor(), verify=True))
 488
 489
 490         # now deep-check the root, with various verify= and repair= options
 491         d.addCallback(lambda ign:
 492                       self.root.start_deep_check().when_done())
 493         d.addCallback(self.deep_check_is_healthy, 3, "root")
 494         d.addCallback(lambda ign:
 495                       self.root.start_deep_check(verify=True).when_done())
 496         d.addCallback(self.deep_check_is_healthy, 3, "root")
 497         d.addCallback(lambda ign:
 498                       self.root.start_deep_check_and_repair().when_done())
 499         d.addCallback(self.deep_check_and_repair_is_healthy, 3, "root")
 500         d.addCallback(lambda ign:
 501                       self.root.start_deep_check_and_repair(verify=True).when_done())
 502         d.addCallback(self.deep_check_and_repair_is_healthy, 3, "root")
 503
 504         # and finally, start a deep-check, but then cancel it.
 505         d.addCallback(lambda ign: self.root.start_deep_check())
 506         def _checking(monitor):
 507             monitor.cancel()
 508             d = monitor.when_done()
 509             # this should fire as soon as the next dirnode.list finishes.
 510             # TODO: add a counter to measure how many list() calls are made,
 511             # assert that no more than one gets to run before the cancel()
 512             # takes effect.
 513             def _finished_normally(res):
 514                 self.fail("this was supposed to fail, not finish normally")
 515             def _cancelled(f):
 516                 f.trap(OperationCancelledError)
 517             d.addCallbacks(_finished_normally, _cancelled)
 518             return d
 519         d.addCallback(_checking)
 520
 521         return d
 522
 523     def json_check_is_healthy(self, data, n, where, incomplete=False):
 524
 525         self.failUnlessEqual(data["storage-index"],
 526                              base32.b2a(n.get_storage_index()), where)
 527         self.failUnless("summary" in data, (where, data))
 528         self.failUnlessEqual(data["summary"].lower(), "healthy",
 529                              "%s: '%s'" % (where, data["summary"]))
 530         r = data["results"]
 531         self.failUnlessEqual(r["healthy"], True, where)
 532         num_servers = len(self.g.all_servers)
 533         needs_rebalancing = bool( num_servers < 10 )
 534         if not incomplete:
 535             self.failUnlessEqual(r["needs-rebalancing"], needs_rebalancing,
 536                                  where)
 537         self.failUnlessEqual(r["count-shares-good"], 10, where)
 538         self.failUnlessEqual(r["count-shares-needed"], 3, where)
 539         self.failUnlessEqual(r["count-shares-expected"], 10, where)
 540         if not incomplete:
 541             self.failUnlessEqual(r["count-good-share-hosts"], num_servers,
 542                                  where)
 543         self.failUnlessEqual(r["count-corrupt-shares"], 0, where)
 544         self.failUnlessEqual(r["list-corrupt-shares"], [], where)
 545         if not incomplete:
 546             self.failUnlessEqual(sorted(r["servers-responding"]),
 547                                  sorted([idlib.nodeid_b2a(sid)
 548                                          for sid in self.g.servers_by_id]),
 549                                  where)
 550             self.failUnless("sharemap" in r, where)
 551             all_serverids = set()
 552             for (shareid, serverids_s) in r["sharemap"].items():
 553                 all_serverids.update(serverids_s)
 554             self.failUnlessEqual(sorted(all_serverids),
 555                                  sorted([idlib.nodeid_b2a(sid)
 556                                          for sid in self.g.servers_by_id]),
 557                                  where)
 558         self.failUnlessEqual(r["count-wrong-shares"], 0, where)
 559         self.failUnlessEqual(r["count-recoverable-versions"], 1, where)
 560         self.failUnlessEqual(r["count-unrecoverable-versions"], 0, where)
 561
 562     def json_check_and_repair_is_healthy(self, data, n, where, incomplete=False):
 563         self.failUnlessEqual(data["storage-index"],
 564                              base32.b2a(n.get_storage_index()), where)
 565         self.failUnlessEqual(data["repair-attempted"], False, where)
 566         self.json_check_is_healthy(data["pre-repair-results"],
 567                                    n, where, incomplete)
 568         self.json_check_is_healthy(data["post-repair-results"],
 569                                    n, where, incomplete)
 570
 571     def json_full_deepcheck_is_healthy(self, data, n, where):
 572         self.failUnlessEqual(data["root-storage-index"],
 573                              base32.b2a(n.get_storage_index()), where)
 574         self.failUnlessEqual(data["count-objects-checked"], 3, where)
 575         self.failUnlessEqual(data["count-objects-healthy"], 3, where)
 576         self.failUnlessEqual(data["count-objects-unhealthy"], 0, where)
 577         self.failUnlessEqual(data["count-corrupt-shares"], 0, where)
 578         self.failUnlessEqual(data["list-corrupt-shares"], [], where)
 579         self.failUnlessEqual(data["list-unhealthy-files"], [], where)
 580         self.json_check_stats_good(data["stats"], where)
 581
 582     def json_full_deepcheck_and_repair_is_healthy(self, data, n, where):
 583         self.failUnlessEqual(data["root-storage-index"],
 584                              base32.b2a(n.get_storage_index()), where)
 585         self.failUnlessEqual(data["count-objects-checked"], 3, where)
 586
 587         self.failUnlessEqual(data["count-objects-healthy-pre-repair"], 3, where)
 588         self.failUnlessEqual(data["count-objects-unhealthy-pre-repair"], 0, where)
 589         self.failUnlessEqual(data["count-corrupt-shares-pre-repair"], 0, where)
 590
 591         self.failUnlessEqual(data["count-objects-healthy-post-repair"], 3, where)
 592         self.failUnlessEqual(data["count-objects-unhealthy-post-repair"], 0, where)
 593         self.failUnlessEqual(data["count-corrupt-shares-post-repair"], 0, where)
 594
 595         self.failUnlessEqual(data["list-corrupt-shares"], [], where)
 596         self.failUnlessEqual(data["list-remaining-corrupt-shares"], [], where)
 597         self.failUnlessEqual(data["list-unhealthy-files"], [], where)
 598
 599         self.failUnlessEqual(data["count-repairs-attempted"], 0, where)
 600         self.failUnlessEqual(data["count-repairs-successful"], 0, where)
 601         self.failUnlessEqual(data["count-repairs-unsuccessful"], 0, where)
 602
 603
 604     def json_check_lit(self, data, n, where):
 605         self.failUnlessEqual(data["storage-index"], "", where)
 606         self.failUnlessEqual(data["results"]["healthy"], True, where)
 607
 608     def json_check_stats_good(self, data, where):
 609         self.check_stats_good(data)
 610
 611     def do_test_web_good(self, ignored):
 612         d = defer.succeed(None)
 613
 614         # stats
 615         d.addCallback(lambda ign:
 616                       self.slow_web(self.root,
 617                                     t="start-deep-stats", output="json"))
 618         d.addCallback(self.json_check_stats_good, "deep-stats")
 619
 620         # check, no verify
 621         d.addCallback(lambda ign: self.web_json(self.root, t="check"))
 622         d.addCallback(self.json_check_is_healthy, self.root, "root")
 623         d.addCallback(lambda ign: self.web_json(self.mutable, t="check"))
 624         d.addCallback(self.json_check_is_healthy, self.mutable, "mutable")
 625         d.addCallback(lambda ign: self.web_json(self.large, t="check"))
 626         d.addCallback(self.json_check_is_healthy, self.large, "large")
 627         d.addCallback(lambda ign: self.web_json(self.small, t="check"))
 628         d.addCallback(self.json_check_lit, self.small, "small")
 629         d.addCallback(lambda ign: self.web_json(self.small2, t="check"))
 630         d.addCallback(self.json_check_lit, self.small2, "small2")
 631         d.addCallback(lambda ign: self.web_json(self.empty_lit_dir, t="check"))
 632         d.addCallback(self.json_check_lit, self.empty_lit_dir, "empty_lit_dir")
 633         d.addCallback(lambda ign: self.web_json(self.tiny_lit_dir, t="check"))
 634         d.addCallback(self.json_check_lit, self.tiny_lit_dir, "tiny_lit_dir")
 635
 636         # check and verify
 637         d.addCallback(lambda ign:
 638                       self.web_json(self.root, t="check", verify="true"))
 639         d.addCallback(self.json_check_is_healthy, self.root, "root+v")
 640         d.addCallback(lambda ign:
 641                       self.web_json(self.mutable, t="check", verify="true"))
 642         d.addCallback(self.json_check_is_healthy, self.mutable, "mutable+v")
 643         d.addCallback(lambda ign:
 644                       self.web_json(self.large, t="check", verify="true"))
 645         d.addCallback(self.json_check_is_healthy, self.large, "large+v",
 646                       incomplete=True)
 647         d.addCallback(lambda ign:
 648                       self.web_json(self.small, t="check", verify="true"))
 649         d.addCallback(self.json_check_lit, self.small, "small+v")
 650         d.addCallback(lambda ign:
 651                       self.web_json(self.small2, t="check", verify="true"))
 652         d.addCallback(self.json_check_lit, self.small2, "small2+v")
 653         d.addCallback(lambda ign: self.web_json(self.empty_lit_dir, t="check", verify="true"))
 654         d.addCallback(self.json_check_lit, self.empty_lit_dir, "empty_lit_dir+v")
 655         d.addCallback(lambda ign: self.web_json(self.tiny_lit_dir, t="check", verify="true"))
 656         d.addCallback(self.json_check_lit, self.tiny_lit_dir, "tiny_lit_dir+v")
 657
 658         # check and repair, no verify
 659         d.addCallback(lambda ign:
 660                       self.web_json(self.root, t="check", repair="true"))
 661         d.addCallback(self.json_check_and_repair_is_healthy, self.root, "root+r")
 662         d.addCallback(lambda ign:
 663                       self.web_json(self.mutable, t="check", repair="true"))
 664         d.addCallback(self.json_check_and_repair_is_healthy, self.mutable, "mutable+r")
 665         d.addCallback(lambda ign:
 666                       self.web_json(self.large, t="check", repair="true"))
 667         d.addCallback(self.json_check_and_repair_is_healthy, self.large, "large+r")
 668         d.addCallback(lambda ign:
 669                       self.web_json(self.small, t="check", repair="true"))
 670         d.addCallback(self.json_check_lit, self.small, "small+r")
 671         d.addCallback(lambda ign:
 672                       self.web_json(self.small2, t="check", repair="true"))
 673         d.addCallback(self.json_check_lit, self.small2, "small2+r")
 674         d.addCallback(lambda ign: self.web_json(self.empty_lit_dir, t="check", repair="true"))
 675         d.addCallback(self.json_check_lit, self.empty_lit_dir, "empty_lit_dir+r")
 676         d.addCallback(lambda ign: self.web_json(self.tiny_lit_dir, t="check", repair="true"))
 677         d.addCallback(self.json_check_lit, self.tiny_lit_dir, "tiny_lit_dir+r")
 678
 679         # check+verify+repair
 680         d.addCallback(lambda ign:
 681                       self.web_json(self.root, t="check", repair="true", verify="true"))
 682         d.addCallback(self.json_check_and_repair_is_healthy, self.root, "root+vr")
 683         d.addCallback(lambda ign:
 684                       self.web_json(self.mutable, t="check", repair="true", verify="true"))
 685         d.addCallback(self.json_check_and_repair_is_healthy, self.mutable, "mutable+vr")
 686         d.addCallback(lambda ign:
 687                       self.web_json(self.large, t="check", repair="true", verify="true"))
 688         d.addCallback(self.json_check_and_repair_is_healthy, self.large, "large+vr", incomplete=True)
 689         d.addCallback(lambda ign:
 690                       self.web_json(self.small, t="check", repair="true", verify="true"))
 691         d.addCallback(self.json_check_lit, self.small, "small+vr")
 692         d.addCallback(lambda ign:
 693                       self.web_json(self.small2, t="check", repair="true", verify="true"))
 694         d.addCallback(self.json_check_lit, self.small2, "small2+vr")
 695         d.addCallback(lambda ign: self.web_json(self.empty_lit_dir, t="check", repair="true", verify=True))
 696         d.addCallback(self.json_check_lit, self.empty_lit_dir, "empty_lit_dir+vr")
 697         d.addCallback(lambda ign: self.web_json(self.tiny_lit_dir, t="check", repair="true", verify=True))
 698         d.addCallback(self.json_check_lit, self.tiny_lit_dir, "tiny_lit_dir+vr")
 699
 700         # now run a deep-check, with various verify= and repair= flags
 701         d.addCallback(lambda ign:
 702                       self.slow_web(self.root, t="start-deep-check", output="json"))
 703         d.addCallback(self.json_full_deepcheck_is_healthy, self.root, "root+d")
 704         d.addCallback(lambda ign:
 705                       self.slow_web(self.root, t="start-deep-check", verify="true",
 706                                     output="json"))
 707         d.addCallback(self.json_full_deepcheck_is_healthy, self.root, "root+dv")
 708         d.addCallback(lambda ign:
 709                       self.slow_web(self.root, t="start-deep-check", repair="true",
 710                                     output="json"))
 711         d.addCallback(self.json_full_deepcheck_and_repair_is_healthy, self.root, "root+dr")
 712         d.addCallback(lambda ign:
 713                       self.slow_web(self.root, t="start-deep-check", verify="true", repair="true", output="json"))
 714         d.addCallback(self.json_full_deepcheck_and_repair_is_healthy, self.root, "root+dvr")
 715
 716         # now look at t=info
 717         d.addCallback(lambda ign: self.web(self.root, t="info"))
 718         # TODO: examine the output
 719         d.addCallback(lambda ign: self.web(self.mutable, t="info"))
 720         d.addCallback(lambda ign: self.web(self.large, t="info"))
 721         d.addCallback(lambda ign: self.web(self.small, t="info"))
 722         d.addCallback(lambda ign: self.web(self.small2, t="info"))
 723         d.addCallback(lambda ign: self.web(self.empty_lit_dir, t="info"))
 724         d.addCallback(lambda ign: self.web(self.tiny_lit_dir, t="info"))
 725
 726         return d
 727
 728     def _run_cli(self, argv, stdin=""):
 729         #print "CLI:", argv
 730         stdout, stderr = StringIO(), StringIO()
 731         d = threads.deferToThread(runner.runner, argv, run_by_human=False,
 732                                   stdin=StringIO(stdin),
 733                                   stdout=stdout, stderr=stderr)
 734         def _done(res):
 735             return stdout.getvalue(), stderr.getvalue()
 736         d.addCallback(_done)
 737         return d
 738
 739     def do_test_cli_good(self, ignored):
 740         d = defer.succeed(None)
 741         d.addCallback(lambda ign: self.do_cli_manifest_stream1())
 742         d.addCallback(lambda ign: self.do_cli_manifest_stream2())
 743         d.addCallback(lambda ign: self.do_cli_manifest_stream3())
 744         d.addCallback(lambda ign: self.do_cli_manifest_stream4())
 745         d.addCallback(lambda ign: self.do_cli_manifest_stream5())
 746         d.addCallback(lambda ign: self.do_cli_stats1())
 747         d.addCallback(lambda ign: self.do_cli_stats2())
 748         return d
 749
 750     def _check_manifest_storage_index(self, out):
 751         lines = [l for l in out.split("\n") if l]
 752         self.failUnlessEqual(len(lines), 3)
 753         self.failUnless(base32.b2a(self.root.get_storage_index()) in lines)
 754         self.failUnless(base32.b2a(self.mutable.get_storage_index()) in lines)
 755         self.failUnless(base32.b2a(self.large.get_storage_index()) in lines)
 756
 757     def do_cli_manifest_stream1(self):
 758         basedir = self.get_clientdir(0)
 759         d = self._run_cli(["manifest",
 760                            "--node-directory", basedir,
 761                            self.root_uri])
 762         def _check((out,err)):
 763             self.failUnlessEqual(err, "")
 764             lines = [l for l in out.split("\n") if l]
 765             self.failUnlessEqual(len(lines), 8)
 766             caps = {}
 767             for l in lines:
 768                 try:
 769                     cap, path = l.split(None, 1)
 770                 except ValueError:
 771                     cap = l.strip()
 772                     path = ""
 773                 caps[cap] = path
 774             self.failUnless(self.root.get_uri() in caps)
 775             self.failUnlessEqual(caps[self.root.get_uri()], "")
 776             self.failUnlessEqual(caps[self.mutable.get_uri()], "mutable")
 777             self.failUnlessEqual(caps[self.large.get_uri()], "large")
 778             self.failUnlessEqual(caps[self.small.get_uri()], "small")
 779             self.failUnlessEqual(caps[self.small2.get_uri()], "small2")
 780             self.failUnlessEqual(caps[self.empty_lit_dir.get_uri()], "empty_lit_dir")
 781             self.failUnlessEqual(caps[self.tiny_lit_dir.get_uri()], "tiny_lit_dir")
 782         d.addCallback(_check)
 783         return d
 784
 785     def do_cli_manifest_stream2(self):
 786         basedir = self.get_clientdir(0)
 787         d = self._run_cli(["manifest",
 788                            "--node-directory", basedir,
 789                            "--raw",
 790                            self.root_uri])
 791         def _check((out,err)):
 792             self.failUnlessEqual(err, "")
 793             # this should be the same as the POST t=stream-manifest output
 794             self._check_streamed_manifest(out)
 795         d.addCallback(_check)
 796         return d
 797
 798     def do_cli_manifest_stream3(self):
 799         basedir = self.get_clientdir(0)
 800         d = self._run_cli(["manifest",
 801                            "--node-directory", basedir,
 802                            "--storage-index",
 803                            self.root_uri])
 804         def _check((out,err)):
 805             self.failUnlessEqual(err, "")
 806             self._check_manifest_storage_index(out)
 807         d.addCallback(_check)
 808         return d
 809
 810     def do_cli_manifest_stream4(self):
 811         basedir = self.get_clientdir(0)
 812         d = self._run_cli(["manifest",
 813                            "--node-directory", basedir,
 814                            "--verify-cap",
 815                            self.root_uri])
 816         def _check((out,err)):
 817             self.failUnlessEqual(err, "")
 818             lines = [l for l in out.split("\n") if l]
 819             self.failUnlessEqual(len(lines), 3)
 820             self.failUnless(self.root.get_verify_cap().to_string() in lines)
 821             self.failUnless(self.mutable.get_verify_cap().to_string() in lines)
 822             self.failUnless(self.large.get_verify_cap().to_string() in lines)
 823         d.addCallback(_check)
 824         return d
 825
 826     def do_cli_manifest_stream5(self):
 827         basedir = self.get_clientdir(0)
 828         d = self._run_cli(["manifest",
 829                            "--node-directory", basedir,
 830                            "--repair-cap",
 831                            self.root_uri])
 832         def _check((out,err)):
 833             self.failUnlessEqual(err, "")
 834             lines = [l for l in out.split("\n") if l]
 835             self.failUnlessEqual(len(lines), 3)
 836             self.failUnless(self.root.get_repair_cap().to_string() in lines)
 837             self.failUnless(self.mutable.get_repair_cap().to_string() in lines)
 838             self.failUnless(self.large.get_repair_cap().to_string() in lines)
 839         d.addCallback(_check)
 840         return d
 841
 842     def do_cli_stats1(self):
 843         basedir = self.get_clientdir(0)
 844         d = self._run_cli(["stats",
 845                            "--node-directory", basedir,
 846                            self.root_uri])
 847         def _check3((out,err)):
 848             lines = [l.strip() for l in out.split("\n") if l]
 849             self.failUnless("count-immutable-files: 1" in lines)
 850             self.failUnless("count-mutable-files: 1" in lines)
 851             self.failUnless("count-literal-files: 3" in lines)
 852             self.failUnless("count-files: 5" in lines)
 853             self.failUnless("count-directories: 3" in lines)
 854             self.failUnless("size-immutable-files: 13000    (13.00 kB, 12.70 kiB)" in lines, lines)
 855             self.failUnless("size-literal-files: 56" in lines, lines)
 856             self.failUnless("    4-10    : 1    (10 B, 10 B)".strip() in lines, lines)
 857             self.failUnless("   11-31    : 2    (31 B, 31 B)".strip() in lines, lines)
 858             self.failUnless("10001-31622 : 1    (31.62 kB, 30.88 kiB)".strip() in lines, lines)
 859         d.addCallback(_check3)
 860         return d
 861
 862     def do_cli_stats2(self):
 863         basedir = self.get_clientdir(0)
 864         d = self._run_cli(["stats",
 865                            "--node-directory", basedir,
 866                            "--raw",
 867                            self.root_uri])
 868         def _check4((out,err)):
 869             data = simplejson.loads(out)
 870             self.failUnlessEqual(data["count-immutable-files"], 1)
 871             self.failUnlessEqual(data["count-immutable-files"], 1)
 872             self.failUnlessEqual(data["count-mutable-files"], 1)
 873             self.failUnlessEqual(data["count-literal-files"], 3)
 874             self.failUnlessEqual(data["count-files"], 5)
 875             self.failUnlessEqual(data["count-directories"], 3)
 876             self.failUnlessEqual(data["size-immutable-files"], 13000)
 877             self.failUnlessEqual(data["size-literal-files"], 56)
 878             self.failUnless([4,10,1] in data["size-files-histogram"])
 879             self.failUnless([11,31,2] in data["size-files-histogram"])
 880             self.failUnless([10001,31622,1] in data["size-files-histogram"])
 881         d.addCallback(_check4)
 882         return d
 883
 884
 885 class DeepCheckWebBad(DeepCheckBase, unittest.TestCase):
 886     def test_bad(self):
 887         self.basedir = "deepcheck/DeepCheckWebBad/bad"
 888         self.set_up_grid()
 889         d = self.set_up_damaged_tree()
 890         d.addCallback(self.do_check)
 891         d.addCallback(self.do_deepcheck)
 892         d.addCallback(self.do_deepcheck_broken)
 893         d.addCallback(self.do_test_web_bad)
 894         d.addErrback(self.explain_web_error)
 895         d.addErrback(self.explain_error)
 896         return d
 897
 898
 899
 900     def set_up_damaged_tree(self):
 901         # 6.4s
 902
 903         # root
 904         #   mutable-good
 905         #   mutable-missing-shares
 906         #   mutable-corrupt-shares
 907         #   mutable-unrecoverable
 908         #   large-good
 909         #   large-missing-shares
 910         #   large-corrupt-shares
 911         #   large-unrecoverable
 912         # broken
 913         #   large1-good
 914         #   subdir-good
 915         #     large2-good
 916         #   subdir-unrecoverable
 917         #     large3-good
 918
 919         self.nodes = {}
 920
 921         c0 = self.g.clients[0]
 922         d = c0.create_dirnode()
 923         def _created_root(n):
 924             self.root = n
 925             self.root_uri = n.get_uri()
 926         d.addCallback(_created_root)
 927         d.addCallback(self.create_mangled, "mutable-good")
 928         d.addCallback(self.create_mangled, "mutable-missing-shares")
 929         d.addCallback(self.create_mangled, "mutable-corrupt-shares")
 930         d.addCallback(self.create_mangled, "mutable-unrecoverable")
 931         d.addCallback(self.create_mangled, "large-good")
 932         d.addCallback(self.create_mangled, "large-missing-shares")
 933         d.addCallback(self.create_mangled, "large-corrupt-shares")
 934         d.addCallback(self.create_mangled, "large-unrecoverable")
 935         d.addCallback(lambda ignored: c0.create_dirnode())
 936         d.addCallback(self._stash_node, "broken")
 937         large1 = upload.Data("Lots of data\n" * 1000 + "large1" + "\n", None)
 938         d.addCallback(lambda ignored:
 939                       self.nodes["broken"].add_file(u"large1", large1))
 940         d.addCallback(lambda ignored:
 941                       self.nodes["broken"].create_subdirectory(u"subdir-good"))
 942         large2 = upload.Data("Lots of data\n" * 1000 + "large2" + "\n", None)
 943         d.addCallback(lambda subdir: subdir.add_file(u"large2-good", large2))
 944         d.addCallback(lambda ignored:
 945                       self.nodes["broken"].create_subdirectory(u"subdir-unrecoverable"))
 946         d.addCallback(self._stash_node, "subdir-unrecoverable")
 947         large3 = upload.Data("Lots of data\n" * 1000 + "large3" + "\n", None)
 948         d.addCallback(lambda subdir: subdir.add_file(u"large3-good", large3))
 949         d.addCallback(lambda ignored:
 950                       self._delete_most_shares(self.nodes["broken"]))
 951         return d
 952
 953     def _stash_node(self, node, name):
 954         self.nodes[name] = node
 955         return node
 956
 957     def create_mangled(self, ignored, name):
 958         nodetype, mangletype = name.split("-", 1)
 959         if nodetype == "mutable":
 960             d = self.g.clients[0].create_mutable_file("mutable file contents")
 961             d.addCallback(lambda n: self.root.set_node(unicode(name), n))
 962         elif nodetype == "large":
 963             large = upload.Data("Lots of data\n" * 1000 + name + "\n", None)
 964             d = self.root.add_file(unicode(name), large)
 965         elif nodetype == "small":
 966             small = upload.Data("Small enough for a LIT", None)
 967             d = self.root.add_file(unicode(name), small)
 968
 969         d.addCallback(self._stash_node, name)
 970
 971         if mangletype == "good":
 972             pass
 973         elif mangletype == "missing-shares":
 974             d.addCallback(self._delete_some_shares)
 975         elif mangletype == "corrupt-shares":
 976             d.addCallback(self._corrupt_some_shares)
 977         else:
 978             assert mangletype == "unrecoverable"
 979             d.addCallback(self._delete_most_shares)
 980
 981         return d
 982
 983     def _run_cli(self, argv):
 984         stdout, stderr = StringIO(), StringIO()
 985         # this can only do synchronous operations
 986         assert argv[0] == "debug"
 987         runner.runner(argv, run_by_human=False, stdout=stdout, stderr=stderr)
 988         return stdout.getvalue()
 989
 990     def _delete_some_shares(self, node):
 991         self.delete_shares_numbered(node.get_uri(), [0,1])
 992
 993     def _corrupt_some_shares(self, node):
 994         for (shnum, serverid, sharefile) in self.find_shares(node.get_uri()):
 995             if shnum in (0,1):
 996                 self._run_cli(["debug", "corrupt-share", sharefile])
 997
 998     def _delete_most_shares(self, node):
 999         self.delete_shares_numbered(node.get_uri(), range(1,10))
1000
1001
1002     def check_is_healthy(self, cr, where):
1003         try:
1004             self.failUnless(ICheckResults.providedBy(cr), (cr, type(cr), where))
1005             self.failUnless(cr.is_healthy(), (cr.get_report(), cr.is_healthy(), cr.get_summary(), where))
1006             self.failUnless(cr.is_recoverable(), where)
1007             d = cr.get_data()
1008             self.failUnlessEqual(d["count-recoverable-versions"], 1, where)
1009             self.failUnlessEqual(d["count-unrecoverable-versions"], 0, where)
1010             return cr
1011         except Exception, le:
1012             le.args = tuple(le.args + (where,))
1013             raise
1014
1015     def check_is_missing_shares(self, cr, where):
1016         self.failUnless(ICheckResults.providedBy(cr), where)
1017         self.failIf(cr.is_healthy(), where)
1018         self.failUnless(cr.is_recoverable(), where)
1019         d = cr.get_data()
1020         self.failUnlessEqual(d["count-recoverable-versions"], 1, where)
1021         self.failUnlessEqual(d["count-unrecoverable-versions"], 0, where)
1022         return cr
1023
1024     def check_has_corrupt_shares(self, cr, where):
1025         # by "corrupt-shares" we mean the file is still recoverable
1026         self.failUnless(ICheckResults.providedBy(cr), where)
1027         d = cr.get_data()
1028         self.failIf(cr.is_healthy(), (where, cr))
1029         self.failUnless(cr.is_recoverable(), where)
1030         d = cr.get_data()
1031         self.failUnless(d["count-shares-good"] < 10, where)
1032         self.failUnless(d["count-corrupt-shares"], where)
1033         self.failUnless(d["list-corrupt-shares"], where)
1034         return cr
1035
1036     def check_is_unrecoverable(self, cr, where):
1037         self.failUnless(ICheckResults.providedBy(cr), where)
1038         d = cr.get_data()
1039         self.failIf(cr.is_healthy(), where)
1040         self.failIf(cr.is_recoverable(), where)
1041         self.failUnless(d["count-shares-good"] < d["count-shares-needed"], (d["count-shares-good"], d["count-shares-needed"], where))
1042         self.failUnlessEqual(d["count-recoverable-versions"], 0, where)
1043         self.failUnlessEqual(d["count-unrecoverable-versions"], 1, where)
1044         return cr
1045
1046     def do_check(self, ignored):
1047         d = defer.succeed(None)
1048
1049         # check the individual items, without verification. This will not
1050         # detect corrupt shares.
1051         def _check(which, checker):
1052             d = self.nodes[which].check(Monitor())
1053             d.addCallback(checker, which + "--check")
1054             return d
1055
1056         d.addCallback(lambda ign: _check("mutable-good", self.check_is_healthy))
1057         d.addCallback(lambda ign: _check("mutable-missing-shares",
1058                                          self.check_is_missing_shares))
1059         d.addCallback(lambda ign: _check("mutable-corrupt-shares",
1060                                          self.check_is_healthy))
1061         d.addCallback(lambda ign: _check("mutable-unrecoverable",
1062                                          self.check_is_unrecoverable))
1063         d.addCallback(lambda ign: _check("large-good", self.check_is_healthy))
1064         d.addCallback(lambda ign: _check("large-missing-shares",
1065                                          self.check_is_missing_shares))
1066         d.addCallback(lambda ign: _check("large-corrupt-shares",
1067                                          self.check_is_healthy))
1068         d.addCallback(lambda ign: _check("large-unrecoverable",
1069                                          self.check_is_unrecoverable))
1070
1071         # and again with verify=True, which *does* detect corrupt shares.
1072         def _checkv(which, checker):
1073             d = self.nodes[which].check(Monitor(), verify=True)
1074             d.addCallback(checker, which + "--check-and-verify")
1075             return d
1076
1077         d.addCallback(lambda ign: _checkv("mutable-good", self.check_is_healthy))
1078         d.addCallback(lambda ign: _checkv("mutable-missing-shares",
1079                                          self.check_is_missing_shares))
1080         d.addCallback(lambda ign: _checkv("mutable-corrupt-shares",
1081                                          self.check_has_corrupt_shares))
1082         d.addCallback(lambda ign: _checkv("mutable-unrecoverable",
1083                                          self.check_is_unrecoverable))
1084         d.addCallback(lambda ign: _checkv("large-good", self.check_is_healthy))
1085         d.addCallback(lambda ign: _checkv("large-missing-shares", self.check_is_missing_shares))
1086         d.addCallback(lambda ign: _checkv("large-corrupt-shares", self.check_has_corrupt_shares))
1087         d.addCallback(lambda ign: _checkv("large-unrecoverable",
1088                                          self.check_is_unrecoverable))
1089
1090         return d
1091
1092     def do_deepcheck(self, ignored):
1093         d = defer.succeed(None)
1094
1095         # now deep-check the root, with various verify= and repair= options
1096         d.addCallback(lambda ign:
1097                       self.root.start_deep_check().when_done())
1098         def _check1(cr):
1099             self.failUnless(IDeepCheckResults.providedBy(cr))
1100             c = cr.get_counters()
1101             self.failUnlessEqual(c["count-objects-checked"], 9)
1102             self.failUnlessEqual(c["count-objects-healthy"], 5)
1103             self.failUnlessEqual(c["count-objects-unhealthy"], 4)
1104             self.failUnlessEqual(c["count-objects-unrecoverable"], 2)
1105         d.addCallback(_check1)
1106
1107         d.addCallback(lambda ign:
1108                       self.root.start_deep_check(verify=True).when_done())
1109         def _check2(cr):
1110             self.failUnless(IDeepCheckResults.providedBy(cr))
1111             c = cr.get_counters()
1112             self.failUnlessEqual(c["count-objects-checked"], 9)
1113             self.failUnlessEqual(c["count-objects-healthy"], 3)
1114             self.failUnlessEqual(c["count-objects-unhealthy"], 6)
1115             self.failUnlessEqual(c["count-objects-healthy"], 3) # root, mutable good, large good
1116             self.failUnlessEqual(c["count-objects-unrecoverable"], 2) # mutable unrecoverable, large unrecoverable
1117         d.addCallback(_check2)
1118
1119         return d
1120
1121     def do_deepcheck_broken(self, ignored):
1122         # deep-check on the broken directory should fail, because of the
1123         # untraversable subdir
1124         def _do_deep_check():
1125             return self.nodes["broken"].start_deep_check().when_done()
1126         d = self.shouldFail(UnrecoverableFileError, "do_deep_check",
1127                             "no recoverable versions",
1128                             _do_deep_check)
1129         return d
1130
1131     def json_is_healthy(self, data, where):
1132         r = data["results"]
1133         self.failUnless(r["healthy"], where)
1134         self.failUnless(r["recoverable"], where)
1135         self.failUnlessEqual(r["count-recoverable-versions"], 1, where)
1136         self.failUnlessEqual(r["count-unrecoverable-versions"], 0, where)
1137
1138     def json_is_missing_shares(self, data, where):
1139         r = data["results"]
1140         self.failIf(r["healthy"], where)
1141         self.failUnless(r["recoverable"], where)
1142         self.failUnlessEqual(r["count-recoverable-versions"], 1, where)
1143         self.failUnlessEqual(r["count-unrecoverable-versions"], 0, where)
1144
1145     def json_has_corrupt_shares(self, data, where):
1146         # by "corrupt-shares" we mean the file is still recoverable
1147         r = data["results"]
1148         self.failIf(r["healthy"], where)
1149         self.failUnless(r["recoverable"], where)
1150         self.failUnless(r["count-shares-good"] < 10, where)
1151         self.failUnless(r["count-corrupt-shares"], where)
1152         self.failUnless(r["list-corrupt-shares"], where)
1153
1154     def json_is_unrecoverable(self, data, where):
1155         r = data["results"]
1156         self.failIf(r["healthy"], where)
1157         self.failIf(r["recoverable"], where)
1158         self.failUnless(r["count-shares-good"] < r["count-shares-needed"],
1159                         where)
1160         self.failUnlessEqual(r["count-recoverable-versions"], 0, where)
1161         self.failUnlessEqual(r["count-unrecoverable-versions"], 1, where)
1162
1163     def do_test_web_bad(self, ignored):
1164         d = defer.succeed(None)
1165
1166         # check, no verify
1167         def _check(which, checker):
1168             d = self.web_json(self.nodes[which], t="check")
1169             d.addCallback(checker, which + "--webcheck")
1170             return d
1171
1172         d.addCallback(lambda ign: _check("mutable-good",
1173                                          self.json_is_healthy))
1174         d.addCallback(lambda ign: _check("mutable-missing-shares",
1175                                          self.json_is_missing_shares))
1176         d.addCallback(lambda ign: _check("mutable-corrupt-shares",
1177                                          self.json_is_healthy))
1178         d.addCallback(lambda ign: _check("mutable-unrecoverable",
1179                                          self.json_is_unrecoverable))
1180         d.addCallback(lambda ign: _check("large-good",
1181                                          self.json_is_healthy))
1182         d.addCallback(lambda ign: _check("large-missing-shares",
1183                                          self.json_is_missing_shares))
1184         d.addCallback(lambda ign: _check("large-corrupt-shares",
1185                                          self.json_is_healthy))
1186         d.addCallback(lambda ign: _check("large-unrecoverable",
1187                                          self.json_is_unrecoverable))
1188
1189         # check and verify
1190         def _checkv(which, checker):
1191             d = self.web_json(self.nodes[which], t="check", verify="true")
1192             d.addCallback(checker, which + "--webcheck-and-verify")
1193             return d
1194
1195         d.addCallback(lambda ign: _checkv("mutable-good",
1196                                           self.json_is_healthy))
1197         d.addCallback(lambda ign: _checkv("mutable-missing-shares",
1198                                          self.json_is_missing_shares))
1199         d.addCallback(lambda ign: _checkv("mutable-corrupt-shares",
1200                                          self.json_has_corrupt_shares))
1201         d.addCallback(lambda ign: _checkv("mutable-unrecoverable",
1202                                          self.json_is_unrecoverable))
1203         d.addCallback(lambda ign: _checkv("large-good",
1204                                           self.json_is_healthy))
1205         d.addCallback(lambda ign: _checkv("large-missing-shares", self.json_is_missing_shares))
1206         d.addCallback(lambda ign: _checkv("large-corrupt-shares", self.json_has_corrupt_shares))
1207         d.addCallback(lambda ign: _checkv("large-unrecoverable",
1208                                          self.json_is_unrecoverable))
1209
1210         return d
1211
1212 class Large(DeepCheckBase, unittest.TestCase):
1213     def test_lots_of_lits(self):
1214         self.basedir = "deepcheck/Large/lots_of_lits"
1215         self.set_up_grid()
1216         # create the following directory structure:
1217         #  root/
1218         #   subdir/
1219         #    000-large (CHK)
1220         #    001-small (LIT)
1221         #    002-small
1222         #    ...
1223         #    399-small
1224         # then do a deepcheck and make sure it doesn't cause a
1225         # Deferred-tail-recursion stack overflow
1226
1227         COUNT = 400
1228         c0 = self.g.clients[0]
1229         d = c0.create_dirnode()
1230         self.stash = {}
1231         def _created_root(n):
1232             self.root = n
1233             return n
1234         d.addCallback(_created_root)
1235         d.addCallback(lambda root: root.create_subdirectory(u"subdir"))
1236         def _add_children(subdir_node):
1237             self.subdir_node = subdir_node
1238             kids = {}
1239             for i in range(1, COUNT):
1240                 litcap = LiteralFileURI("%03d-data" % i).to_string()
1241                 kids[u"%03d-small" % i] = (litcap, litcap)
1242             return subdir_node.set_children(kids)
1243         d.addCallback(_add_children)
1244         up = upload.Data("large enough for CHK" * 100, "")
1245         d.addCallback(lambda ign: self.subdir_node.add_file(u"0000-large", up))
1246
1247         def _start_deepcheck(ignored):
1248             return self.web(self.root, method="POST", t="stream-deep-check")
1249         d.addCallback(_start_deepcheck)
1250         def _check( (output, url) ):
1251             units = list(self.parse_streamed_json(output))
1252             self.failUnlessEqual(len(units), 2+COUNT+1)
1253         d.addCallback(_check)
1254
1255         return d