2 import os, simplejson, urllib
3 from cStringIO import StringIO
4 from twisted.trial import unittest
5 from twisted.internet import defer
6 from twisted.internet import threads # CLI tests use deferToThread
7 from allmydata.immutable import upload
8 from allmydata.mutable.common import UnrecoverableFileError
9 from allmydata.util import idlib
10 from allmydata.util import base32
11 from allmydata.scripts import runner
12 from allmydata.interfaces import ICheckResults, ICheckAndRepairResults, \
13 IDeepCheckResults, IDeepCheckAndRepairResults
14 from allmydata.monitor import Monitor, OperationCancelledError
15 from allmydata.uri import LiteralFileURI
16 from twisted.web.client import getPage
18 from allmydata.test.common import ErrorMixin, _corrupt_mutable_share_data, \
20 from allmydata.test.common_util import StallMixin
21 from allmydata.test.no_network import GridTestMixin
23 timeout = 2400 # One of these took 1046.091s on Zandr's ARM box.
25 class MutableChecker(GridTestMixin, unittest.TestCase, ErrorMixin):
26 def _run_cli(self, argv):
27 stdout, stderr = StringIO(), StringIO()
28 # this can only do synchronous operations
29 assert argv[0] == "debug"
30 runner.runner(argv, run_by_human=False, stdout=stdout, stderr=stderr)
31 return stdout.getvalue()
34 self.basedir = "deepcheck/MutableChecker/good"
36 CONTENTS = "a little bit of data"
37 d = self.g.clients[0].create_mutable_file(CONTENTS)
40 self.fileurl = "uri/" + urllib.quote(node.get_uri())
41 si = self.node.get_storage_index()
42 d.addCallback(_created)
43 # now make sure the webapi verifier sees no problems
44 d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=true",
46 def _got_results(out):
47 self.failUnless("<span>Healthy : Healthy</span>" in out, out)
48 self.failUnless("Recoverable Versions: 10*seq1-" in out, out)
49 self.failIf("Not Healthy!" in out, out)
50 self.failIf("Unhealthy" in out, out)
51 self.failIf("Corrupt Shares" in out, out)
52 d.addCallback(_got_results)
53 d.addErrback(self.explain_web_error)
56 def test_corrupt(self):
57 self.basedir = "deepcheck/MutableChecker/corrupt"
59 CONTENTS = "a little bit of data"
60 d = self.g.clients[0].create_mutable_file(CONTENTS)
61 def _stash_and_corrupt(node):
63 self.fileurl = "uri/" + urllib.quote(node.get_uri())
64 self.corrupt_shares_numbered(node.get_uri(), [0],
65 _corrupt_mutable_share_data)
66 d.addCallback(_stash_and_corrupt)
67 # now make sure the webapi verifier notices it
68 d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=true",
70 def _got_results(out):
71 self.failUnless("Not Healthy!" in out, out)
72 self.failUnless("Unhealthy: best version has only 9 shares (encoding is 3-of-10)" in out, out)
73 self.failUnless("Corrupt Shares:" in out, out)
74 d.addCallback(_got_results)
76 # now make sure the webapi repairer can fix it
77 d.addCallback(lambda ign:
78 self.GET(self.fileurl+"?t=check&verify=true&repair=true",
80 def _got_repair_results(out):
81 self.failUnless("<div>Repair successful</div>" in out, out)
82 d.addCallback(_got_repair_results)
83 d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=true",
85 def _got_postrepair_results(out):
86 self.failIf("Not Healthy!" in out, out)
87 self.failUnless("Recoverable Versions: 10*seq" in out, out)
88 d.addCallback(_got_postrepair_results)
89 d.addErrback(self.explain_web_error)
93 def test_delete_share(self):
94 self.basedir = "deepcheck/MutableChecker/delete_share"
96 CONTENTS = "a little bit of data"
97 d = self.g.clients[0].create_mutable_file(CONTENTS)
98 def _stash_and_delete(node):
100 self.fileurl = "uri/" + urllib.quote(node.get_uri())
101 self.delete_shares_numbered(node.get_uri(), [0])
102 d.addCallback(_stash_and_delete)
103 # now make sure the webapi checker notices it
104 d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=false",
106 def _got_results(out):
107 self.failUnless("Not Healthy!" in out, out)
108 self.failUnless("Unhealthy: best version has only 9 shares (encoding is 3-of-10)" in out, out)
109 self.failIf("Corrupt Shares" in out, out)
110 d.addCallback(_got_results)
112 # now make sure the webapi repairer can fix it
113 d.addCallback(lambda ign:
114 self.GET(self.fileurl+"?t=check&verify=false&repair=true",
116 def _got_repair_results(out):
117 self.failUnless("Repair successful" in out)
118 d.addCallback(_got_repair_results)
119 d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=false",
121 def _got_postrepair_results(out):
122 self.failIf("Not Healthy!" in out, out)
123 self.failUnless("Recoverable Versions: 10*seq" in out)
124 d.addCallback(_got_postrepair_results)
125 d.addErrback(self.explain_web_error)
130 class DeepCheckBase(GridTestMixin, ErrorMixin, StallMixin, ShouldFailMixin):
132 def web_json(self, n, **kwargs):
133 kwargs["output"] = "json"
134 d = self.web(n, "POST", **kwargs)
135 d.addCallback(self.decode_json)
138 def decode_json(self, (s,url)):
140 data = simplejson.loads(s)
142 self.fail("%s: not JSON: '%s'" % (url, s))
145 def parse_streamed_json(self, s):
146 for unit in s.split("\n"):
148 # stream should end with a newline, so split returns ""
150 yield simplejson.loads(unit)
152 def web(self, n, method="GET", **kwargs):
153 # returns (data, url)
154 url = (self.client_baseurls[0] + "uri/%s" % urllib.quote(n.get_uri())
155 + "?" + "&".join(["%s=%s" % (k,v) for (k,v) in kwargs.items()]))
156 d = getPage(url, method=method)
157 d.addCallback(lambda data: (data,url))
160 def wait_for_operation(self, ignored, ophandle):
161 url = self.client_baseurls[0] + "operations/" + ophandle
162 url += "?t=status&output=JSON"
166 data = simplejson.loads(res)
168 self.fail("%s: not JSON: '%s'" % (url, res))
169 if not data["finished"]:
170 d = self.stall(delay=1.0)
171 d.addCallback(self.wait_for_operation, ophandle)
177 def get_operation_results(self, ignored, ophandle, output=None):
178 url = self.client_baseurls[0] + "operations/" + ophandle
181 url += "&output=" + output
184 if output and output.lower() == "json":
186 return simplejson.loads(res)
188 self.fail("%s: not JSON: '%s'" % (url, res))
193 def slow_web(self, n, output=None, **kwargs):
195 handle = base32.b2a(os.urandom(4))
196 d = self.web(n, "POST", ophandle=handle, **kwargs)
197 d.addCallback(self.wait_for_operation, handle)
198 d.addCallback(self.get_operation_results, handle, output=output)
202 class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
203 # construct a small directory tree (with one dir, one immutable file, one
204 # mutable file, one LIT file, and a loop), and then check/examine it in
207 def set_up_tree(self):
216 c0 = self.g.clients[0]
217 d = c0.create_dirnode()
218 def _created_root(n):
220 self.root_uri = n.get_uri()
221 d.addCallback(_created_root)
222 d.addCallback(lambda ign: c0.create_mutable_file("mutable file contents"))
223 d.addCallback(lambda n: self.root.set_node(u"mutable", n))
224 def _created_mutable(n):
226 self.mutable_uri = n.get_uri()
227 d.addCallback(_created_mutable)
229 large = upload.Data("Lots of data\n" * 1000, None)
230 d.addCallback(lambda ign: self.root.add_file(u"large", large))
231 def _created_large(n):
233 self.large_uri = n.get_uri()
234 d.addCallback(_created_large)
236 small = upload.Data("Small enough for a LIT", None)
237 d.addCallback(lambda ign: self.root.add_file(u"small", small))
238 def _created_small(n):
240 self.small_uri = n.get_uri()
241 d.addCallback(_created_small)
243 small2 = upload.Data("Small enough for a LIT too", None)
244 d.addCallback(lambda ign: self.root.add_file(u"small2", small2))
245 def _created_small2(n):
247 self.small2_uri = n.get_uri()
248 d.addCallback(_created_small2)
250 d.addCallback(lambda ign: self.root.set_node(u"loop", self.root))
253 def check_is_healthy(self, cr, n, where, incomplete=False):
254 self.failUnless(ICheckResults.providedBy(cr), where)
255 self.failUnless(cr.is_healthy(), where)
256 self.failUnlessEqual(cr.get_storage_index(), n.get_storage_index(),
258 self.failUnlessEqual(cr.get_storage_index_string(),
259 base32.b2a(n.get_storage_index()), where)
260 num_servers = len(self.g.all_servers)
261 needs_rebalancing = bool( num_servers < 10 )
263 self.failUnlessEqual(cr.needs_rebalancing(), needs_rebalancing,
264 str((where, cr, cr.get_data())))
266 self.failUnlessEqual(d["count-shares-good"], 10, where)
267 self.failUnlessEqual(d["count-shares-needed"], 3, where)
268 self.failUnlessEqual(d["count-shares-expected"], 10, where)
270 self.failUnlessEqual(d["count-good-share-hosts"], num_servers,
272 self.failUnlessEqual(d["count-corrupt-shares"], 0, where)
273 self.failUnlessEqual(d["list-corrupt-shares"], [], where)
275 self.failUnlessEqual(sorted(d["servers-responding"]),
276 sorted(self.g.servers_by_id.keys()),
278 self.failUnless("sharemap" in d, str((where, d)))
279 all_serverids = set()
280 for (shareid, serverids) in d["sharemap"].items():
281 all_serverids.update(serverids)
282 self.failUnlessEqual(sorted(all_serverids),
283 sorted(self.g.servers_by_id.keys()),
286 self.failUnlessEqual(d["count-wrong-shares"], 0, where)
287 self.failUnlessEqual(d["count-recoverable-versions"], 1, where)
288 self.failUnlessEqual(d["count-unrecoverable-versions"], 0, where)
291 def check_and_repair_is_healthy(self, cr, n, where, incomplete=False):
292 self.failUnless(ICheckAndRepairResults.providedBy(cr), (where, cr))
293 self.failUnless(cr.get_pre_repair_results().is_healthy(), where)
294 self.check_is_healthy(cr.get_pre_repair_results(), n, where, incomplete)
295 self.failUnless(cr.get_post_repair_results().is_healthy(), where)
296 self.check_is_healthy(cr.get_post_repair_results(), n, where, incomplete)
297 self.failIf(cr.get_repair_attempted(), where)
299 def deep_check_is_healthy(self, cr, num_healthy, where):
300 self.failUnless(IDeepCheckResults.providedBy(cr))
301 self.failUnlessEqual(cr.get_counters()["count-objects-healthy"],
304 def deep_check_and_repair_is_healthy(self, cr, num_healthy, where):
305 self.failUnless(IDeepCheckAndRepairResults.providedBy(cr), where)
306 c = cr.get_counters()
307 self.failUnlessEqual(c["count-objects-healthy-pre-repair"],
309 self.failUnlessEqual(c["count-objects-healthy-post-repair"],
311 self.failUnlessEqual(c["count-repairs-attempted"], 0, where)
314 self.basedir = "deepcheck/DeepCheckWebGood/good"
316 d = self.set_up_tree()
317 d.addCallback(self.do_stats)
318 d.addCallback(self.do_web_stream_manifest)
319 d.addCallback(self.do_web_stream_check)
320 d.addCallback(self.do_test_check_good)
321 d.addCallback(self.do_test_web_good)
322 d.addCallback(self.do_test_cli_good)
323 d.addErrback(self.explain_web_error)
324 d.addErrback(self.explain_error)
327 def do_stats(self, ignored):
328 d = defer.succeed(None)
329 d.addCallback(lambda ign: self.root.start_deep_stats().when_done())
330 d.addCallback(self.check_stats_good)
333 def check_stats_good(self, s):
334 self.failUnlessEqual(s["count-directories"], 1)
335 self.failUnlessEqual(s["count-files"], 4)
336 self.failUnlessEqual(s["count-immutable-files"], 1)
337 self.failUnlessEqual(s["count-literal-files"], 2)
338 self.failUnlessEqual(s["count-mutable-files"], 1)
339 # don't check directories: their size will vary
340 # s["largest-directory"]
341 # s["size-directories"]
342 self.failUnlessEqual(s["largest-directory-children"], 5)
343 self.failUnlessEqual(s["largest-immutable-file"], 13000)
344 # to re-use this function for both the local
345 # dirnode.start_deep_stats() and the webapi t=start-deep-stats, we
346 # coerce the result into a list of tuples. dirnode.start_deep_stats()
347 # returns a list of tuples, but JSON only knows about lists., so
348 # t=start-deep-stats returns a list of lists.
349 histogram = [tuple(stuff) for stuff in s["size-files-histogram"]]
350 self.failUnlessEqual(histogram, [(11, 31, 2),
353 self.failUnlessEqual(s["size-immutable-files"], 13000)
354 self.failUnlessEqual(s["size-literal-files"], 48)
356 def do_web_stream_manifest(self, ignored):
357 d = self.web(self.root, method="POST", t="stream-manifest")
358 d.addCallback(lambda (output,url):
359 self._check_streamed_manifest(output))
362 def _check_streamed_manifest(self, output):
363 units = list(self.parse_streamed_json(output))
364 files = [u for u in units if u["type"] in ("file", "directory")]
365 assert units[-1]["type"] == "stats"
366 stats = units[-1]["stats"]
367 self.failUnlessEqual(len(files), 5)
368 # [root,mutable,large] are distributed, [small,small2] are not
369 self.failUnlessEqual(len([f for f in files
370 if f["verifycap"] is not None]), 3)
371 self.failUnlessEqual(len([f for f in files
372 if f["verifycap"] is None]), 2)
373 self.failUnlessEqual(len([f for f in files
374 if f["repaircap"] is not None]), 3)
375 self.failUnlessEqual(len([f for f in files
376 if f["repaircap"] is None]), 2)
377 self.failUnlessEqual(len([f for f in files
378 if f["storage-index"] is not None]), 3)
379 self.failUnlessEqual(len([f for f in files
380 if f["storage-index"] is None]), 2)
381 # make sure that a mutable file has filecap==repaircap!=verifycap
382 mutable = [f for f in files
383 if f["cap"] is not None
384 and f["cap"].startswith("URI:SSK:")][0]
385 self.failUnlessEqual(mutable["cap"], self.mutable_uri)
386 self.failIfEqual(mutable["cap"], mutable["verifycap"])
387 self.failUnlessEqual(mutable["cap"], mutable["repaircap"])
388 # for immutable file, verifycap==repaircap!=filecap
389 large = [f for f in files
390 if f["cap"] is not None
391 and f["cap"].startswith("URI:CHK:")][0]
392 self.failUnlessEqual(large["cap"], self.large_uri)
393 self.failIfEqual(large["cap"], large["verifycap"])
394 self.failUnlessEqual(large["verifycap"], large["repaircap"])
395 self.check_stats_good(stats)
397 def do_web_stream_check(self, ignored):
399 d = self.web(self.root, t="stream-deep-check")
401 units = list(self.parse_streamed_json(res))
402 files = [u for u in units if u["type"] in ("file", "directory")]
403 assert units[-1]["type"] == "stats"
404 stats = units[-1]["stats"]
406 d.addCallback(_check)
409 def do_test_check_good(self, ignored):
410 d = defer.succeed(None)
411 # check the individual items
412 d.addCallback(lambda ign: self.root.check(Monitor()))
413 d.addCallback(self.check_is_healthy, self.root, "root")
414 d.addCallback(lambda ign: self.mutable.check(Monitor()))
415 d.addCallback(self.check_is_healthy, self.mutable, "mutable")
416 d.addCallback(lambda ign: self.large.check(Monitor()))
417 d.addCallback(self.check_is_healthy, self.large, "large")
418 d.addCallback(lambda ign: self.small.check(Monitor()))
419 d.addCallback(self.failUnlessEqual, None, "small")
420 d.addCallback(lambda ign: self.small2.check(Monitor()))
421 d.addCallback(self.failUnlessEqual, None, "small2")
423 # and again with verify=True
424 d.addCallback(lambda ign: self.root.check(Monitor(), verify=True))
425 d.addCallback(self.check_is_healthy, self.root, "root")
426 d.addCallback(lambda ign: self.mutable.check(Monitor(), verify=True))
427 d.addCallback(self.check_is_healthy, self.mutable, "mutable")
428 d.addCallback(lambda ign: self.large.check(Monitor(), verify=True))
429 d.addCallback(self.check_is_healthy, self.large, "large", incomplete=True)
430 d.addCallback(lambda ign: self.small.check(Monitor(), verify=True))
431 d.addCallback(self.failUnlessEqual, None, "small")
432 d.addCallback(lambda ign: self.small2.check(Monitor(), verify=True))
433 d.addCallback(self.failUnlessEqual, None, "small2")
435 # and check_and_repair(), which should be a nop
436 d.addCallback(lambda ign: self.root.check_and_repair(Monitor()))
437 d.addCallback(self.check_and_repair_is_healthy, self.root, "root")
438 d.addCallback(lambda ign: self.mutable.check_and_repair(Monitor()))
439 d.addCallback(self.check_and_repair_is_healthy, self.mutable, "mutable")
440 #TODO d.addCallback(lambda ign: self.large.check_and_repair(Monitor()))
441 #TODO d.addCallback(self.check_and_repair_is_healthy, self.large, "large")
442 #TODO d.addCallback(lambda ign: self.small.check_and_repair(Monitor()))
443 #TODO d.addCallback(self.failUnlessEqual, None, "small")
444 #TODO d.addCallback(lambda ign: self.small2.check_and_repair(Monitor()))
445 #TODO d.addCallback(self.failUnlessEqual, None, "small2")
447 # check_and_repair(verify=True)
448 d.addCallback(lambda ign: self.root.check_and_repair(Monitor(), verify=True))
449 d.addCallback(self.check_and_repair_is_healthy, self.root, "root")
450 d.addCallback(lambda ign: self.mutable.check_and_repair(Monitor(), verify=True))
451 d.addCallback(self.check_and_repair_is_healthy, self.mutable, "mutable")
452 #TODO d.addCallback(lambda ign: self.large.check_and_repair(Monitor(), verify=True))
453 #TODO d.addCallback(self.check_and_repair_is_healthy, self.large, "large",
454 #TODO incomplete=True)
455 #TODO d.addCallback(lambda ign: self.small.check_and_repair(Monitor(), verify=True))
456 #TODO d.addCallback(self.failUnlessEqual, None, "small")
457 #TODO d.addCallback(lambda ign: self.small2.check_and_repair(Monitor(), verify=True))
458 #TODO d.addCallback(self.failUnlessEqual, None, "small2")
461 # now deep-check the root, with various verify= and repair= options
462 d.addCallback(lambda ign:
463 self.root.start_deep_check().when_done())
464 d.addCallback(self.deep_check_is_healthy, 3, "root")
465 d.addCallback(lambda ign:
466 self.root.start_deep_check(verify=True).when_done())
467 d.addCallback(self.deep_check_is_healthy, 3, "root")
468 d.addCallback(lambda ign:
469 self.root.start_deep_check_and_repair().when_done())
470 d.addCallback(self.deep_check_and_repair_is_healthy, 3, "root")
471 d.addCallback(lambda ign:
472 self.root.start_deep_check_and_repair(verify=True).when_done())
473 d.addCallback(self.deep_check_and_repair_is_healthy, 3, "root")
475 # and finally, start a deep-check, but then cancel it.
476 d.addCallback(lambda ign: self.root.start_deep_check())
477 def _checking(monitor):
479 d = monitor.when_done()
480 # this should fire as soon as the next dirnode.list finishes.
481 # TODO: add a counter to measure how many list() calls are made,
482 # assert that no more than one gets to run before the cancel()
484 def _finished_normally(res):
485 self.fail("this was supposed to fail, not finish normally")
487 f.trap(OperationCancelledError)
488 d.addCallbacks(_finished_normally, _cancelled)
490 d.addCallback(_checking)
494 def json_check_is_healthy(self, data, n, where, incomplete=False):
496 self.failUnlessEqual(data["storage-index"],
497 base32.b2a(n.get_storage_index()), where)
498 self.failUnless("summary" in data, (where, data))
499 self.failUnlessEqual(data["summary"].lower(), "healthy",
500 "%s: '%s'" % (where, data["summary"]))
502 self.failUnlessEqual(r["healthy"], True, where)
503 num_servers = len(self.g.all_servers)
504 needs_rebalancing = bool( num_servers < 10 )
506 self.failUnlessEqual(r["needs-rebalancing"], needs_rebalancing,
508 self.failUnlessEqual(r["count-shares-good"], 10, where)
509 self.failUnlessEqual(r["count-shares-needed"], 3, where)
510 self.failUnlessEqual(r["count-shares-expected"], 10, where)
512 self.failUnlessEqual(r["count-good-share-hosts"], num_servers,
514 self.failUnlessEqual(r["count-corrupt-shares"], 0, where)
515 self.failUnlessEqual(r["list-corrupt-shares"], [], where)
517 self.failUnlessEqual(sorted(r["servers-responding"]),
518 sorted([idlib.nodeid_b2a(sid)
519 for sid in self.g.servers_by_id]),
521 self.failUnless("sharemap" in r, where)
522 all_serverids = set()
523 for (shareid, serverids_s) in r["sharemap"].items():
524 all_serverids.update(serverids_s)
525 self.failUnlessEqual(sorted(all_serverids),
526 sorted([idlib.nodeid_b2a(sid)
527 for sid in self.g.servers_by_id]),
529 self.failUnlessEqual(r["count-wrong-shares"], 0, where)
530 self.failUnlessEqual(r["count-recoverable-versions"], 1, where)
531 self.failUnlessEqual(r["count-unrecoverable-versions"], 0, where)
533 def json_check_and_repair_is_healthy(self, data, n, where, incomplete=False):
534 self.failUnlessEqual(data["storage-index"],
535 base32.b2a(n.get_storage_index()), where)
536 self.failUnlessEqual(data["repair-attempted"], False, where)
537 self.json_check_is_healthy(data["pre-repair-results"],
538 n, where, incomplete)
539 self.json_check_is_healthy(data["post-repair-results"],
540 n, where, incomplete)
542 def json_full_deepcheck_is_healthy(self, data, n, where):
543 self.failUnlessEqual(data["root-storage-index"],
544 base32.b2a(n.get_storage_index()), where)
545 self.failUnlessEqual(data["count-objects-checked"], 3, where)
546 self.failUnlessEqual(data["count-objects-healthy"], 3, where)
547 self.failUnlessEqual(data["count-objects-unhealthy"], 0, where)
548 self.failUnlessEqual(data["count-corrupt-shares"], 0, where)
549 self.failUnlessEqual(data["list-corrupt-shares"], [], where)
550 self.failUnlessEqual(data["list-unhealthy-files"], [], where)
551 self.json_check_stats_good(data["stats"], where)
553 def json_full_deepcheck_and_repair_is_healthy(self, data, n, where):
554 self.failUnlessEqual(data["root-storage-index"],
555 base32.b2a(n.get_storage_index()), where)
556 self.failUnlessEqual(data["count-objects-checked"], 3, where)
558 self.failUnlessEqual(data["count-objects-healthy-pre-repair"], 3, where)
559 self.failUnlessEqual(data["count-objects-unhealthy-pre-repair"], 0, where)
560 self.failUnlessEqual(data["count-corrupt-shares-pre-repair"], 0, where)
562 self.failUnlessEqual(data["count-objects-healthy-post-repair"], 3, where)
563 self.failUnlessEqual(data["count-objects-unhealthy-post-repair"], 0, where)
564 self.failUnlessEqual(data["count-corrupt-shares-post-repair"], 0, where)
566 self.failUnlessEqual(data["list-corrupt-shares"], [], where)
567 self.failUnlessEqual(data["list-remaining-corrupt-shares"], [], where)
568 self.failUnlessEqual(data["list-unhealthy-files"], [], where)
570 self.failUnlessEqual(data["count-repairs-attempted"], 0, where)
571 self.failUnlessEqual(data["count-repairs-successful"], 0, where)
572 self.failUnlessEqual(data["count-repairs-unsuccessful"], 0, where)
575 def json_check_lit(self, data, n, where):
576 self.failUnlessEqual(data["storage-index"], "", where)
577 self.failUnlessEqual(data["results"]["healthy"], True, where)
579 def json_check_stats_good(self, data, where):
580 self.check_stats_good(data)
582 def do_test_web_good(self, ignored):
583 d = defer.succeed(None)
586 d.addCallback(lambda ign:
587 self.slow_web(self.root,
588 t="start-deep-stats", output="json"))
589 d.addCallback(self.json_check_stats_good, "deep-stats")
592 d.addCallback(lambda ign: self.web_json(self.root, t="check"))
593 d.addCallback(self.json_check_is_healthy, self.root, "root")
594 d.addCallback(lambda ign: self.web_json(self.mutable, t="check"))
595 d.addCallback(self.json_check_is_healthy, self.mutable, "mutable")
596 d.addCallback(lambda ign: self.web_json(self.large, t="check"))
597 d.addCallback(self.json_check_is_healthy, self.large, "large")
598 d.addCallback(lambda ign: self.web_json(self.small, t="check"))
599 d.addCallback(self.json_check_lit, self.small, "small")
600 d.addCallback(lambda ign: self.web_json(self.small2, t="check"))
601 d.addCallback(self.json_check_lit, self.small2, "small2")
604 d.addCallback(lambda ign:
605 self.web_json(self.root, t="check", verify="true"))
606 d.addCallback(self.json_check_is_healthy, self.root, "root+v")
607 d.addCallback(lambda ign:
608 self.web_json(self.mutable, t="check", verify="true"))
609 d.addCallback(self.json_check_is_healthy, self.mutable, "mutable+v")
610 d.addCallback(lambda ign:
611 self.web_json(self.large, t="check", verify="true"))
612 d.addCallback(self.json_check_is_healthy, self.large, "large+v",
614 d.addCallback(lambda ign:
615 self.web_json(self.small, t="check", verify="true"))
616 d.addCallback(self.json_check_lit, self.small, "small+v")
617 d.addCallback(lambda ign:
618 self.web_json(self.small2, t="check", verify="true"))
619 d.addCallback(self.json_check_lit, self.small2, "small2+v")
621 # check and repair, no verify
622 d.addCallback(lambda ign:
623 self.web_json(self.root, t="check", repair="true"))
624 d.addCallback(self.json_check_and_repair_is_healthy, self.root, "root+r")
625 d.addCallback(lambda ign:
626 self.web_json(self.mutable, t="check", repair="true"))
627 d.addCallback(self.json_check_and_repair_is_healthy, self.mutable, "mutable+r")
628 d.addCallback(lambda ign:
629 self.web_json(self.large, t="check", repair="true"))
630 d.addCallback(self.json_check_and_repair_is_healthy, self.large, "large+r")
631 d.addCallback(lambda ign:
632 self.web_json(self.small, t="check", repair="true"))
633 d.addCallback(self.json_check_lit, self.small, "small+r")
634 d.addCallback(lambda ign:
635 self.web_json(self.small2, t="check", repair="true"))
636 d.addCallback(self.json_check_lit, self.small2, "small2+r")
638 # check+verify+repair
639 d.addCallback(lambda ign:
640 self.web_json(self.root, t="check", repair="true", verify="true"))
641 d.addCallback(self.json_check_and_repair_is_healthy, self.root, "root+vr")
642 d.addCallback(lambda ign:
643 self.web_json(self.mutable, t="check", repair="true", verify="true"))
644 d.addCallback(self.json_check_and_repair_is_healthy, self.mutable, "mutable+vr")
645 d.addCallback(lambda ign:
646 self.web_json(self.large, t="check", repair="true", verify="true"))
647 d.addCallback(self.json_check_and_repair_is_healthy, self.large, "large+vr", incomplete=True)
648 d.addCallback(lambda ign:
649 self.web_json(self.small, t="check", repair="true", verify="true"))
650 d.addCallback(self.json_check_lit, self.small, "small+vr")
651 d.addCallback(lambda ign:
652 self.web_json(self.small2, t="check", repair="true", verify="true"))
653 d.addCallback(self.json_check_lit, self.small2, "small2+vr")
655 # now run a deep-check, with various verify= and repair= flags
656 d.addCallback(lambda ign:
657 self.slow_web(self.root, t="start-deep-check", output="json"))
658 d.addCallback(self.json_full_deepcheck_is_healthy, self.root, "root+d")
659 d.addCallback(lambda ign:
660 self.slow_web(self.root, t="start-deep-check", verify="true",
662 d.addCallback(self.json_full_deepcheck_is_healthy, self.root, "root+dv")
663 d.addCallback(lambda ign:
664 self.slow_web(self.root, t="start-deep-check", repair="true",
666 d.addCallback(self.json_full_deepcheck_and_repair_is_healthy, self.root, "root+dr")
667 d.addCallback(lambda ign:
668 self.slow_web(self.root, t="start-deep-check", verify="true", repair="true", output="json"))
669 d.addCallback(self.json_full_deepcheck_and_repair_is_healthy, self.root, "root+dvr")
672 d.addCallback(lambda ign: self.web(self.root, t="info"))
673 # TODO: examine the output
674 d.addCallback(lambda ign: self.web(self.mutable, t="info"))
675 d.addCallback(lambda ign: self.web(self.large, t="info"))
676 d.addCallback(lambda ign: self.web(self.small, t="info"))
677 d.addCallback(lambda ign: self.web(self.small2, t="info"))
681 def _run_cli(self, argv, stdin=""):
683 stdout, stderr = StringIO(), StringIO()
684 d = threads.deferToThread(runner.runner, argv, run_by_human=False,
685 stdin=StringIO(stdin),
686 stdout=stdout, stderr=stderr)
688 return stdout.getvalue(), stderr.getvalue()
692 def do_test_cli_good(self, ignored):
693 d = defer.succeed(None)
694 d.addCallback(lambda ign: self.do_cli_manifest_stream1())
695 d.addCallback(lambda ign: self.do_cli_manifest_stream2())
696 d.addCallback(lambda ign: self.do_cli_manifest_stream3())
697 d.addCallback(lambda ign: self.do_cli_manifest_stream4())
698 d.addCallback(lambda ign: self.do_cli_manifest_stream5())
699 d.addCallback(lambda ign: self.do_cli_stats1())
700 d.addCallback(lambda ign: self.do_cli_stats2())
703 def _check_manifest_storage_index(self, out):
704 lines = [l for l in out.split("\n") if l]
705 self.failUnlessEqual(len(lines), 3)
706 self.failUnless(base32.b2a(self.root.get_storage_index()) in lines)
707 self.failUnless(base32.b2a(self.mutable.get_storage_index()) in lines)
708 self.failUnless(base32.b2a(self.large.get_storage_index()) in lines)
710 def do_cli_manifest_stream1(self):
711 basedir = self.get_clientdir(0)
712 d = self._run_cli(["manifest",
713 "--node-directory", basedir,
715 def _check((out,err)):
716 self.failUnlessEqual(err, "")
717 lines = [l for l in out.split("\n") if l]
718 self.failUnlessEqual(len(lines), 5)
722 cap, path = l.split(None, 1)
727 self.failUnless(self.root.get_uri() in caps)
728 self.failUnlessEqual(caps[self.root.get_uri()], "")
729 self.failUnlessEqual(caps[self.mutable.get_uri()], "mutable")
730 self.failUnlessEqual(caps[self.large.get_uri()], "large")
731 self.failUnlessEqual(caps[self.small.get_uri()], "small")
732 self.failUnlessEqual(caps[self.small2.get_uri()], "small2")
733 d.addCallback(_check)
736 def do_cli_manifest_stream2(self):
737 basedir = self.get_clientdir(0)
738 d = self._run_cli(["manifest",
739 "--node-directory", basedir,
742 def _check((out,err)):
743 self.failUnlessEqual(err, "")
744 # this should be the same as the POST t=stream-manifest output
745 self._check_streamed_manifest(out)
746 d.addCallback(_check)
749 def do_cli_manifest_stream3(self):
750 basedir = self.get_clientdir(0)
751 d = self._run_cli(["manifest",
752 "--node-directory", basedir,
755 def _check((out,err)):
756 self.failUnlessEqual(err, "")
757 self._check_manifest_storage_index(out)
758 d.addCallback(_check)
761 def do_cli_manifest_stream4(self):
762 basedir = self.get_clientdir(0)
763 d = self._run_cli(["manifest",
764 "--node-directory", basedir,
767 def _check((out,err)):
768 self.failUnlessEqual(err, "")
769 lines = [l for l in out.split("\n") if l]
770 self.failUnlessEqual(len(lines), 3)
771 self.failUnless(self.root.get_verify_cap().to_string() in lines)
772 self.failUnless(self.mutable.get_verify_cap().to_string() in lines)
773 self.failUnless(self.large.get_verify_cap().to_string() in lines)
774 d.addCallback(_check)
777 def do_cli_manifest_stream5(self):
778 basedir = self.get_clientdir(0)
779 d = self._run_cli(["manifest",
780 "--node-directory", basedir,
783 def _check((out,err)):
784 self.failUnlessEqual(err, "")
785 lines = [l for l in out.split("\n") if l]
786 self.failUnlessEqual(len(lines), 3)
787 self.failUnless(self.root.get_repair_cap().to_string() in lines)
788 self.failUnless(self.mutable.get_repair_cap().to_string() in lines)
789 self.failUnless(self.large.get_repair_cap().to_string() in lines)
790 d.addCallback(_check)
793 def do_cli_stats1(self):
794 basedir = self.get_clientdir(0)
795 d = self._run_cli(["stats",
796 "--node-directory", basedir,
798 def _check3((out,err)):
799 lines = [l.strip() for l in out.split("\n") if l]
800 self.failUnless("count-immutable-files: 1" in lines)
801 self.failUnless("count-mutable-files: 1" in lines)
802 self.failUnless("count-literal-files: 2" in lines)
803 self.failUnless("count-files: 4" in lines)
804 self.failUnless("count-directories: 1" in lines)
805 self.failUnless("size-immutable-files: 13000 (13.00 kB, 12.70 kiB)" in lines, lines)
806 self.failUnless("size-literal-files: 48" in lines)
807 self.failUnless(" 11-31 : 2 (31 B, 31 B)".strip() in lines)
808 self.failUnless("10001-31622 : 1 (31.62 kB, 30.88 kiB)".strip() in lines)
809 d.addCallback(_check3)
812 def do_cli_stats2(self):
813 basedir = self.get_clientdir(0)
814 d = self._run_cli(["stats",
815 "--node-directory", basedir,
818 def _check4((out,err)):
819 data = simplejson.loads(out)
820 self.failUnlessEqual(data["count-immutable-files"], 1)
821 self.failUnlessEqual(data["count-immutable-files"], 1)
822 self.failUnlessEqual(data["count-mutable-files"], 1)
823 self.failUnlessEqual(data["count-literal-files"], 2)
824 self.failUnlessEqual(data["count-files"], 4)
825 self.failUnlessEqual(data["count-directories"], 1)
826 self.failUnlessEqual(data["size-immutable-files"], 13000)
827 self.failUnlessEqual(data["size-literal-files"], 48)
828 self.failUnless([11,31,2] in data["size-files-histogram"])
829 self.failUnless([10001,31622,1] in data["size-files-histogram"])
830 d.addCallback(_check4)
834 class DeepCheckWebBad(DeepCheckBase, unittest.TestCase):
836 self.basedir = "deepcheck/DeepCheckWebBad/bad"
838 d = self.set_up_damaged_tree()
839 d.addCallback(self.do_check)
840 d.addCallback(self.do_deepcheck)
841 d.addCallback(self.do_deepcheck_broken)
842 d.addCallback(self.do_test_web_bad)
843 d.addErrback(self.explain_web_error)
844 d.addErrback(self.explain_error)
849 def set_up_damaged_tree(self):
854 # mutable-missing-shares
855 # mutable-corrupt-shares
856 # mutable-unrecoverable
858 # large-missing-shares
859 # large-corrupt-shares
860 # large-unrecoverable
865 # subdir-unrecoverable
870 c0 = self.g.clients[0]
871 d = c0.create_dirnode()
872 def _created_root(n):
874 self.root_uri = n.get_uri()
875 d.addCallback(_created_root)
876 d.addCallback(self.create_mangled, "mutable-good")
877 d.addCallback(self.create_mangled, "mutable-missing-shares")
878 d.addCallback(self.create_mangled, "mutable-corrupt-shares")
879 d.addCallback(self.create_mangled, "mutable-unrecoverable")
880 d.addCallback(self.create_mangled, "large-good")
881 d.addCallback(self.create_mangled, "large-missing-shares")
882 d.addCallback(self.create_mangled, "large-corrupt-shares")
883 d.addCallback(self.create_mangled, "large-unrecoverable")
884 d.addCallback(lambda ignored: c0.create_dirnode())
885 d.addCallback(self._stash_node, "broken")
886 large1 = upload.Data("Lots of data\n" * 1000 + "large1" + "\n", None)
887 d.addCallback(lambda ignored:
888 self.nodes["broken"].add_file(u"large1", large1))
889 d.addCallback(lambda ignored:
890 self.nodes["broken"].create_empty_directory(u"subdir-good"))
891 large2 = upload.Data("Lots of data\n" * 1000 + "large2" + "\n", None)
892 d.addCallback(lambda subdir: subdir.add_file(u"large2-good", large2))
893 d.addCallback(lambda ignored:
894 self.nodes["broken"].create_empty_directory(u"subdir-unrecoverable"))
895 d.addCallback(self._stash_node, "subdir-unrecoverable")
896 large3 = upload.Data("Lots of data\n" * 1000 + "large3" + "\n", None)
897 d.addCallback(lambda subdir: subdir.add_file(u"large3-good", large3))
898 d.addCallback(lambda ignored:
899 self._delete_most_shares(self.nodes["broken"]))
902 def _stash_node(self, node, name):
903 self.nodes[name] = node
906 def create_mangled(self, ignored, name):
907 nodetype, mangletype = name.split("-", 1)
908 if nodetype == "mutable":
909 d = self.g.clients[0].create_mutable_file("mutable file contents")
910 d.addCallback(lambda n: self.root.set_node(unicode(name), n))
911 elif nodetype == "large":
912 large = upload.Data("Lots of data\n" * 1000 + name + "\n", None)
913 d = self.root.add_file(unicode(name), large)
914 elif nodetype == "small":
915 small = upload.Data("Small enough for a LIT", None)
916 d = self.root.add_file(unicode(name), small)
918 d.addCallback(self._stash_node, name)
920 if mangletype == "good":
922 elif mangletype == "missing-shares":
923 d.addCallback(self._delete_some_shares)
924 elif mangletype == "corrupt-shares":
925 d.addCallback(self._corrupt_some_shares)
927 assert mangletype == "unrecoverable"
928 d.addCallback(self._delete_most_shares)
932 def _run_cli(self, argv):
933 stdout, stderr = StringIO(), StringIO()
934 # this can only do synchronous operations
935 assert argv[0] == "debug"
936 runner.runner(argv, run_by_human=False, stdout=stdout, stderr=stderr)
937 return stdout.getvalue()
939 def _delete_some_shares(self, node):
940 self.delete_shares_numbered(node.get_uri(), [0,1])
942 def _corrupt_some_shares(self, node):
943 for (shnum, serverid, sharefile) in self.find_shares(node.get_uri()):
945 self._run_cli(["debug", "corrupt-share", sharefile])
947 def _delete_most_shares(self, node):
948 self.delete_shares_numbered(node.get_uri(), range(1,10))
951 def check_is_healthy(self, cr, where):
953 self.failUnless(ICheckResults.providedBy(cr), (cr, type(cr), where))
954 self.failUnless(cr.is_healthy(), (cr.get_report(), cr.is_healthy(), cr.get_summary(), where))
955 self.failUnless(cr.is_recoverable(), where)
957 self.failUnlessEqual(d["count-recoverable-versions"], 1, where)
958 self.failUnlessEqual(d["count-unrecoverable-versions"], 0, where)
960 except Exception, le:
961 le.args = tuple(le.args + (where,))
964 def check_is_missing_shares(self, cr, where):
965 self.failUnless(ICheckResults.providedBy(cr), where)
966 self.failIf(cr.is_healthy(), where)
967 self.failUnless(cr.is_recoverable(), where)
969 self.failUnlessEqual(d["count-recoverable-versions"], 1, where)
970 self.failUnlessEqual(d["count-unrecoverable-versions"], 0, where)
973 def check_has_corrupt_shares(self, cr, where):
974 # by "corrupt-shares" we mean the file is still recoverable
975 self.failUnless(ICheckResults.providedBy(cr), where)
977 self.failIf(cr.is_healthy(), (where, cr))
978 self.failUnless(cr.is_recoverable(), where)
980 self.failUnless(d["count-shares-good"] < 10, where)
981 self.failUnless(d["count-corrupt-shares"], where)
982 self.failUnless(d["list-corrupt-shares"], where)
985 def check_is_unrecoverable(self, cr, where):
986 self.failUnless(ICheckResults.providedBy(cr), where)
988 self.failIf(cr.is_healthy(), where)
989 self.failIf(cr.is_recoverable(), where)
990 self.failUnless(d["count-shares-good"] < d["count-shares-needed"], (d["count-shares-good"], d["count-shares-needed"], where))
991 self.failUnlessEqual(d["count-recoverable-versions"], 0, where)
992 self.failUnlessEqual(d["count-unrecoverable-versions"], 1, where)
995 def do_check(self, ignored):
996 d = defer.succeed(None)
998 # check the individual items, without verification. This will not
999 # detect corrupt shares.
1000 def _check(which, checker):
1001 d = self.nodes[which].check(Monitor())
1002 d.addCallback(checker, which + "--check")
1005 d.addCallback(lambda ign: _check("mutable-good", self.check_is_healthy))
1006 d.addCallback(lambda ign: _check("mutable-missing-shares",
1007 self.check_is_missing_shares))
1008 d.addCallback(lambda ign: _check("mutable-corrupt-shares",
1009 self.check_is_healthy))
1010 d.addCallback(lambda ign: _check("mutable-unrecoverable",
1011 self.check_is_unrecoverable))
1012 d.addCallback(lambda ign: _check("large-good", self.check_is_healthy))
1013 d.addCallback(lambda ign: _check("large-missing-shares",
1014 self.check_is_missing_shares))
1015 d.addCallback(lambda ign: _check("large-corrupt-shares",
1016 self.check_is_healthy))
1017 d.addCallback(lambda ign: _check("large-unrecoverable",
1018 self.check_is_unrecoverable))
1020 # and again with verify=True, which *does* detect corrupt shares.
1021 def _checkv(which, checker):
1022 d = self.nodes[which].check(Monitor(), verify=True)
1023 d.addCallback(checker, which + "--check-and-verify")
1026 d.addCallback(lambda ign: _checkv("mutable-good", self.check_is_healthy))
1027 d.addCallback(lambda ign: _checkv("mutable-missing-shares",
1028 self.check_is_missing_shares))
1029 d.addCallback(lambda ign: _checkv("mutable-corrupt-shares",
1030 self.check_has_corrupt_shares))
1031 d.addCallback(lambda ign: _checkv("mutable-unrecoverable",
1032 self.check_is_unrecoverable))
1033 d.addCallback(lambda ign: _checkv("large-good", self.check_is_healthy))
1034 d.addCallback(lambda ign: _checkv("large-missing-shares", self.check_is_missing_shares))
1035 d.addCallback(lambda ign: _checkv("large-corrupt-shares", self.check_has_corrupt_shares))
1036 d.addCallback(lambda ign: _checkv("large-unrecoverable",
1037 self.check_is_unrecoverable))
1041 def do_deepcheck(self, ignored):
1042 d = defer.succeed(None)
1044 # now deep-check the root, with various verify= and repair= options
1045 d.addCallback(lambda ign:
1046 self.root.start_deep_check().when_done())
1048 self.failUnless(IDeepCheckResults.providedBy(cr))
1049 c = cr.get_counters()
1050 self.failUnlessEqual(c["count-objects-checked"], 9)
1051 self.failUnlessEqual(c["count-objects-healthy"], 5)
1052 self.failUnlessEqual(c["count-objects-unhealthy"], 4)
1053 self.failUnlessEqual(c["count-objects-unrecoverable"], 2)
1054 d.addCallback(_check1)
1056 d.addCallback(lambda ign:
1057 self.root.start_deep_check(verify=True).when_done())
1059 self.failUnless(IDeepCheckResults.providedBy(cr))
1060 c = cr.get_counters()
1061 self.failUnlessEqual(c["count-objects-checked"], 9)
1062 self.failUnlessEqual(c["count-objects-healthy"], 3)
1063 self.failUnlessEqual(c["count-objects-unhealthy"], 6)
1064 self.failUnlessEqual(c["count-objects-healthy"], 3) # root, mutable good, large good
1065 self.failUnlessEqual(c["count-objects-unrecoverable"], 2) # mutable unrecoverable, large unrecoverable
1066 d.addCallback(_check2)
1070 def do_deepcheck_broken(self, ignored):
1071 # deep-check on the broken directory should fail, because of the
1072 # untraversable subdir
1073 def _do_deep_check():
1074 return self.nodes["broken"].start_deep_check().when_done()
1075 d = self.shouldFail(UnrecoverableFileError, "do_deep_check",
1076 "no recoverable versions",
1080 def json_is_healthy(self, data, where):
1082 self.failUnless(r["healthy"], where)
1083 self.failUnless(r["recoverable"], where)
1084 self.failUnlessEqual(r["count-recoverable-versions"], 1, where)
1085 self.failUnlessEqual(r["count-unrecoverable-versions"], 0, where)
1087 def json_is_missing_shares(self, data, where):
1089 self.failIf(r["healthy"], where)
1090 self.failUnless(r["recoverable"], where)
1091 self.failUnlessEqual(r["count-recoverable-versions"], 1, where)
1092 self.failUnlessEqual(r["count-unrecoverable-versions"], 0, where)
1094 def json_has_corrupt_shares(self, data, where):
1095 # by "corrupt-shares" we mean the file is still recoverable
1097 self.failIf(r["healthy"], where)
1098 self.failUnless(r["recoverable"], where)
1099 self.failUnless(r["count-shares-good"] < 10, where)
1100 self.failUnless(r["count-corrupt-shares"], where)
1101 self.failUnless(r["list-corrupt-shares"], where)
1103 def json_is_unrecoverable(self, data, where):
1105 self.failIf(r["healthy"], where)
1106 self.failIf(r["recoverable"], where)
1107 self.failUnless(r["count-shares-good"] < r["count-shares-needed"],
1109 self.failUnlessEqual(r["count-recoverable-versions"], 0, where)
1110 self.failUnlessEqual(r["count-unrecoverable-versions"], 1, where)
1112 def do_test_web_bad(self, ignored):
1113 d = defer.succeed(None)
1116 def _check(which, checker):
1117 d = self.web_json(self.nodes[which], t="check")
1118 d.addCallback(checker, which + "--webcheck")
1121 d.addCallback(lambda ign: _check("mutable-good",
1122 self.json_is_healthy))
1123 d.addCallback(lambda ign: _check("mutable-missing-shares",
1124 self.json_is_missing_shares))
1125 d.addCallback(lambda ign: _check("mutable-corrupt-shares",
1126 self.json_is_healthy))
1127 d.addCallback(lambda ign: _check("mutable-unrecoverable",
1128 self.json_is_unrecoverable))
1129 d.addCallback(lambda ign: _check("large-good",
1130 self.json_is_healthy))
1131 d.addCallback(lambda ign: _check("large-missing-shares",
1132 self.json_is_missing_shares))
1133 d.addCallback(lambda ign: _check("large-corrupt-shares",
1134 self.json_is_healthy))
1135 d.addCallback(lambda ign: _check("large-unrecoverable",
1136 self.json_is_unrecoverable))
1139 def _checkv(which, checker):
1140 d = self.web_json(self.nodes[which], t="check", verify="true")
1141 d.addCallback(checker, which + "--webcheck-and-verify")
1144 d.addCallback(lambda ign: _checkv("mutable-good",
1145 self.json_is_healthy))
1146 d.addCallback(lambda ign: _checkv("mutable-missing-shares",
1147 self.json_is_missing_shares))
1148 d.addCallback(lambda ign: _checkv("mutable-corrupt-shares",
1149 self.json_has_corrupt_shares))
1150 d.addCallback(lambda ign: _checkv("mutable-unrecoverable",
1151 self.json_is_unrecoverable))
1152 d.addCallback(lambda ign: _checkv("large-good",
1153 self.json_is_healthy))
1154 d.addCallback(lambda ign: _checkv("large-missing-shares", self.json_is_missing_shares))
1155 d.addCallback(lambda ign: _checkv("large-corrupt-shares", self.json_has_corrupt_shares))
1156 d.addCallback(lambda ign: _checkv("large-unrecoverable",
1157 self.json_is_unrecoverable))
1161 class Large(DeepCheckBase, unittest.TestCase):
1162 def test_lots_of_lits(self):
1163 self.basedir = "deepcheck/Large/lots_of_lits"
1165 # create the following directory structure:
1173 # then do a deepcheck and make sure it doesn't cause a
1174 # Deferred-tail-recursion stack overflow
1177 c0 = self.g.clients[0]
1178 d = c0.create_dirnode()
1180 def _created_root(n):
1183 d.addCallback(_created_root)
1184 d.addCallback(lambda root: root.create_empty_directory(u"subdir"))
1185 def _add_children(subdir_node):
1186 self.subdir_node = subdir_node
1188 for i in range(1, COUNT):
1189 litcap = LiteralFileURI("%03d-data" % i).to_string()
1190 kids.append( (u"%03d-small" % i, litcap, litcap) )
1191 return subdir_node.set_children(kids)
1192 d.addCallback(_add_children)
1193 up = upload.Data("large enough for CHK" * 100, "")
1194 d.addCallback(lambda ign: self.subdir_node.add_file(u"0000-large", up))
1196 def _start_deepcheck(ignored):
1197 return self.web(self.root, method="POST", t="stream-deep-check")
1198 d.addCallback(_start_deepcheck)
1199 def _check( (output, url) ):
1200 units = list(self.parse_streamed_json(output))
1201 self.failUnlessEqual(len(units), 2+COUNT+1)
1202 d.addCallback(_check)