4 from nevow import rend, inevow, tags as T
5 from twisted.web import http, html
6 from allmydata.web.common import getxmlfile, get_arg, get_root, WebError
7 from allmydata.web.operations import ReloadMixin
8 from allmydata.interfaces import ICheckAndRepairResults, ICheckResults
9 from allmydata.util import base32, idlib, dictutil
11 def json_check_counts(r):
12 d = {"count-shares-good": r.get_share_counter_good(),
13 "count-shares-needed": r.get_encoding_needed(),
14 "count-shares-expected": r.get_encoding_expected(),
15 "count-good-share-hosts": r.get_host_counter_good_shares(),
16 "count-corrupt-shares": len(r.get_corrupt_shares()),
17 "list-corrupt-shares": [ (idlib.nodeid_b2a(serverid),
18 base32.b2a(si), shnum)
19 for (serverid, si, shnum)
20 in r.get_corrupt_shares() ],
21 "servers-responding": [idlib.nodeid_b2a(serverid)
22 for serverid in r.get_servers_responding()],
23 "sharemap": dict([(shareid,
24 sorted([s.get_longname() for s in servers]))
25 for (shareid, servers)
26 in r.get_sharemap().items()]),
27 "count-wrong-shares": r.get_share_counter_wrong(),
28 "count-recoverable-versions": r.get_version_counter_recoverable(),
29 "count-unrecoverable-versions": r.get_version_counter_unrecoverable(),
33 def json_check_results(r):
36 data = {"storage-index": "",
37 "results": {"healthy": True},
41 data["storage-index"] = r.get_storage_index_string()
42 data["summary"] = r.get_summary()
43 data["results"] = json_check_counts(r)
44 data["results"]["needs-rebalancing"] = r.needs_rebalancing()
45 data["results"]["healthy"] = r.is_healthy()
46 data["results"]["recoverable"] = r.is_recoverable()
49 def json_check_and_repair_results(r):
52 data = {"storage-index": "",
53 "repair-attempted": False,
57 data["storage-index"] = r.get_storage_index_string()
58 data["repair-attempted"] = r.get_repair_attempted()
59 data["repair-successful"] = r.get_repair_successful()
60 pre = r.get_pre_repair_results()
61 data["pre-repair-results"] = json_check_results(pre)
62 post = r.get_post_repair_results()
63 data["post-repair-results"] = json_check_results(post)
67 # self.client must point to the Client, so we can get nicknames and
68 # determine the permuted peer order
70 def _join_pathstring(self, path):
72 pathstring = "/".join(self._html(path))
77 def _render_results(self, ctx, cr):
78 assert ICheckResults(cr)
80 sb = c.get_storage_broker()
83 r.append(T.li[name + ": ", value])
85 add("Report", T.pre["\n".join(self._html(cr.get_report()))])
87 "need %d-of-%d, have %d" % (cr.get_encoding_needed(),
88 cr.get_encoding_expected(),
89 cr.get_share_counter_good()))
90 add("Hosts with good shares", cr.get_host_counter_good_shares())
92 if cr.get_corrupt_shares():
94 for (serverid, si, shnum) in cr.get_corrupt_shares():
95 nickname = sb.get_nickname_for_serverid(serverid)
96 badsharemap.append(T.tr[T.td["sh#%d" % shnum],
97 T.td[T.div(class_="nickname")[nickname],
98 T.div(class_="nodeid")[T.tt[base32.b2a(serverid)]]],
100 add("Corrupt shares", T.table()[
101 T.tr[T.th["Share ID"],
102 T.th(class_="nickname-and-peerid")[T.div["Nickname"], T.div(class_="nodeid")["Node ID"]]],
105 add("Corrupt shares", "none")
107 add("Wrong Shares", cr.get_share_counter_wrong())
110 shares_on_server = dictutil.DictOfSets()
112 # FIXME: The two tables below contain nickname-and-nodeid table column markup which is duplicated with each other, introducer.xhtml, and deep-check-results.xhtml. All of these (and any other presentations of nickname-and-nodeid) should be combined.
114 for shareid in sorted(cr.get_sharemap().keys()):
115 servers = sorted(cr.get_sharemap()[shareid],
116 key=lambda s: s.get_longname())
117 for i,s in enumerate(servers):
118 shares_on_server.add(s, shareid)
122 d = T.tr[T.td[shareid_s],
123 T.td[T.div(class_="nickname")[s.get_nickname()],
124 T.div(class_="nodeid")[T.tt[s.get_name()]]]
126 sharemap_data.append(d)
127 add("Good Shares (sorted in share order)",
128 T.table()[T.tr[T.th["Share ID"], T.th(class_="nickname-and-peerid")[T.div["Nickname"], T.div(class_="nodeid")["Node ID"]]],
132 add("Recoverable Versions", cr.get_version_counter_recoverable())
133 add("Unrecoverable Versions", cr.get_version_counter_unrecoverable())
135 # this table is sorted by permuted order
136 permuted_servers = [s
138 in sb.get_servers_for_psi(cr.get_storage_index())]
140 num_shares_left = sum([len(shareids)
141 for shareids in shares_on_server.values()])
143 for s in permuted_servers:
144 shareids = list(shares_on_server.get(s, []))
146 shareids_s = [ T.tt[shareid, " "] for shareid in sorted(shareids) ]
147 d = T.tr[T.td[T.div(class_="nickname")[s.get_nickname()],
148 T.div(class_="nodeid")[T.tt[s.get_name()]]],
152 num_shares_left -= len(shareids)
153 if not num_shares_left:
155 add("Share Balancing (servers in permuted order)",
156 T.table()[T.tr[T.th(class_="nickname-and-peerid")[T.div["Nickname"], T.div(class_="nodeid")["Node ID"]], T.th["Share IDs"]],
162 if isinstance(s, (str, unicode)):
163 return html.escape(s)
164 assert isinstance(s, (list, tuple))
165 return [html.escape(w) for w in s]
167 def want_json(self, ctx):
168 output = get_arg(inevow.IRequest(ctx), "output", "").lower()
169 if output.lower() == "json":
173 def _render_si_link(self, ctx, storage_index):
174 si_s = base32.b2a(storage_index)
175 req = inevow.IRequest(ctx)
176 ophandle = req.prepath[-1]
177 target = "%s/operations/%s/%s" % (get_root(ctx), ophandle, si_s)
178 output = get_arg(ctx, "output")
180 target = target + "?output=%s" % output
181 return T.a(href=target)[si_s]
183 class LiteralCheckResultsRenderer(rend.Page, ResultsBase):
184 docFactory = getxmlfile("literal-check-results.xhtml")
186 def __init__(self, client):
188 rend.Page.__init__(self, client)
190 def renderHTTP(self, ctx):
191 if self.want_json(ctx):
192 return self.json(ctx)
193 return rend.Page.renderHTTP(self, ctx)
196 inevow.IRequest(ctx).setHeader("content-type", "text/plain")
197 data = json_check_results(None)
198 return simplejson.dumps(data, indent=1) + "\n"
200 def render_return(self, ctx, data):
201 req = inevow.IRequest(ctx)
202 return_to = get_arg(req, "return_to", None)
204 return T.div[T.a(href=return_to)["Return to file."]]
209 def renderHTTP(self, ctx):
210 if self.want_json(ctx):
211 return self.json(ctx)
212 return rend.Page.renderHTTP(self, ctx)
214 def render_storage_index(self, ctx, data):
215 return self.r.get_storage_index_string()
217 def render_return(self, ctx, data):
218 req = inevow.IRequest(ctx)
219 return_to = get_arg(req, "return_to", None)
221 return T.div[T.a(href=return_to)["Return to file/directory."]]
224 class CheckResultsRenderer(CheckerBase, rend.Page, ResultsBase):
225 docFactory = getxmlfile("check-results.xhtml")
227 def __init__(self, client, results):
229 self.r = ICheckResults(results)
230 rend.Page.__init__(self, results)
233 inevow.IRequest(ctx).setHeader("content-type", "text/plain")
234 data = json_check_results(self.r)
235 return simplejson.dumps(data, indent=1) + "\n"
237 def render_summary(self, ctx, data):
239 if data.is_healthy():
240 results.append("Healthy")
241 elif data.is_recoverable():
242 results.append("Not Healthy!")
244 results.append("Not Recoverable!")
245 results.append(" : ")
246 results.append(self._html(data.get_summary()))
247 return ctx.tag[results]
249 def render_repair(self, ctx, data):
250 if data.is_healthy():
252 repair = T.form(action=".", method="post",
253 enctype="multipart/form-data")[
255 T.input(type="hidden", name="t", value="check"),
256 T.input(type="hidden", name="repair", value="true"),
257 T.input(type="submit", value="Repair"),
259 return "" # repair button disabled until we make it work correctly,
260 # see #622 for details
261 return ctx.tag[repair]
263 def render_results(self, ctx, data):
264 cr = self._render_results(ctx, data)
267 class CheckAndRepairResultsRenderer(CheckerBase, rend.Page, ResultsBase):
268 docFactory = getxmlfile("check-and-repair-results.xhtml")
270 def __init__(self, client, results):
274 self.r = ICheckAndRepairResults(results)
275 rend.Page.__init__(self, results)
278 inevow.IRequest(ctx).setHeader("content-type", "text/plain")
279 data = json_check_and_repair_results(self.r)
280 return simplejson.dumps(data, indent=1) + "\n"
282 def render_summary(self, ctx, data):
283 cr = data.get_post_repair_results()
286 results.append("Healthy")
287 elif cr.is_recoverable():
288 results.append("Not Healthy!")
290 results.append("Not Recoverable!")
291 results.append(" : ")
292 results.append(self._html(cr.get_summary()))
293 return ctx.tag[results]
295 def render_repair_results(self, ctx, data):
296 if data.get_repair_attempted():
297 if data.get_repair_successful():
298 return ctx.tag["Repair successful"]
300 return ctx.tag["Repair unsuccessful"]
301 return ctx.tag["No repair necessary"]
303 def render_post_repair_results(self, ctx, data):
304 cr = self._render_results(ctx, data.get_post_repair_results())
305 return ctx.tag[T.div["Post-Repair Checker Results:"], cr]
307 def render_maybe_pre_repair_results(self, ctx, data):
308 if data.get_repair_attempted():
309 cr = self._render_results(ctx, data.get_pre_repair_results())
310 return ctx.tag[T.div["Pre-Repair Checker Results:"], cr]
314 class DeepCheckResultsRenderer(rend.Page, ResultsBase, ReloadMixin):
315 docFactory = getxmlfile("deep-check-results.xhtml")
317 def __init__(self, client, monitor):
319 self.monitor = monitor
321 def childFactory(self, ctx, name):
324 # /operation/$OPHANDLE/$STORAGEINDEX provides detailed information
325 # about a specific file or directory that was checked
326 si = base32.a2b(name)
327 r = self.monitor.get_status()
329 return CheckResultsRenderer(self.client,
330 r.get_results_for_storage_index(si))
332 raise WebError("No detailed results for SI %s" % html.escape(name),
335 def renderHTTP(self, ctx):
336 if self.want_json(ctx):
337 return self.json(ctx)
338 return rend.Page.renderHTTP(self, ctx)
341 inevow.IRequest(ctx).setHeader("content-type", "text/plain")
343 data["finished"] = self.monitor.is_finished()
344 res = self.monitor.get_status()
345 data["root-storage-index"] = res.get_root_storage_index_string()
346 c = res.get_counters()
347 data["count-objects-checked"] = c["count-objects-checked"]
348 data["count-objects-healthy"] = c["count-objects-healthy"]
349 data["count-objects-unhealthy"] = c["count-objects-unhealthy"]
350 data["count-corrupt-shares"] = c["count-corrupt-shares"]
351 data["list-corrupt-shares"] = [ (idlib.nodeid_b2a(serverid),
352 base32.b2a(storage_index),
354 for (serverid, storage_index, shnum)
355 in res.get_corrupt_shares() ]
356 data["list-unhealthy-files"] = [ (path_t, json_check_results(r))
358 in res.get_all_results().items()
359 if not r.is_healthy() ]
360 data["stats"] = res.get_stats()
361 return simplejson.dumps(data, indent=1) + "\n"
363 def render_root_storage_index(self, ctx, data):
364 return self.monitor.get_status().get_root_storage_index_string()
366 def data_objects_checked(self, ctx, data):
367 return self.monitor.get_status().get_counters()["count-objects-checked"]
368 def data_objects_healthy(self, ctx, data):
369 return self.monitor.get_status().get_counters()["count-objects-healthy"]
370 def data_objects_unhealthy(self, ctx, data):
371 return self.monitor.get_status().get_counters()["count-objects-unhealthy"]
372 def data_objects_unrecoverable(self, ctx, data):
373 return self.monitor.get_status().get_counters()["count-objects-unrecoverable"]
375 def data_count_corrupt_shares(self, ctx, data):
376 return self.monitor.get_status().get_counters()["count-corrupt-shares"]
378 def render_problems_p(self, ctx, data):
379 c = self.monitor.get_status().get_counters()
380 if c["count-objects-unhealthy"]:
384 def data_problems(self, ctx, data):
385 all_objects = self.monitor.get_status().get_all_results()
386 for path in sorted(all_objects.keys()):
387 cr = all_objects[path]
388 assert ICheckResults.providedBy(cr)
389 if not cr.is_healthy():
392 def render_problem(self, ctx, data):
395 summary = cr.get_summary()
397 summary_text = ": " + summary
398 summary_text += " [SI: %s]" % cr.get_storage_index_string()
399 return ctx.tag[self._join_pathstring(path), self._html(summary_text)]
402 def render_servers_with_corrupt_shares_p(self, ctx, data):
403 if self.monitor.get_status().get_counters()["count-corrupt-shares"]:
407 def data_servers_with_corrupt_shares(self, ctx, data):
409 for (serverid, storage_index, sharenum)
410 in self.monitor.get_status().get_corrupt_shares()]
414 def render_server_problem(self, ctx, data):
416 data = [idlib.shortnodeid_b2a(serverid)]
417 sb = self.client.get_storage_broker()
418 nickname = sb.get_nickname_for_serverid(serverid)
420 data.append(" (%s)" % self._html(nickname))
424 def render_corrupt_shares_p(self, ctx, data):
425 if self.monitor.get_status().get_counters()["count-corrupt-shares"]:
428 def data_corrupt_shares(self, ctx, data):
429 return self.monitor.get_status().get_corrupt_shares()
430 def render_share_problem(self, ctx, data):
431 serverid, storage_index, sharenum = data
432 sb = self.client.get_storage_broker()
433 nickname = sb.get_nickname_for_serverid(serverid)
434 ctx.fillSlots("serverid", idlib.shortnodeid_b2a(serverid))
436 ctx.fillSlots("nickname", self._html(nickname))
437 ctx.fillSlots("si", self._render_si_link(ctx, storage_index))
438 ctx.fillSlots("shnum", str(sharenum))
441 def render_return(self, ctx, data):
442 req = inevow.IRequest(ctx)
443 return_to = get_arg(req, "return_to", None)
445 return T.div[T.a(href=return_to)["Return to file/directory."]]
448 def data_all_objects(self, ctx, data):
449 r = self.monitor.get_status().get_all_results()
450 for path in sorted(r.keys()):
451 yield (path, r[path])
453 def render_object(self, ctx, data):
455 ctx.fillSlots("path", self._join_pathstring(path))
456 ctx.fillSlots("healthy", str(r.is_healthy()))
457 ctx.fillSlots("recoverable", str(r.is_recoverable()))
458 storage_index = r.get_storage_index()
459 ctx.fillSlots("storage_index", self._render_si_link(ctx, storage_index))
460 ctx.fillSlots("summary", self._html(r.get_summary()))
463 def render_runtime(self, ctx, data):
464 req = inevow.IRequest(ctx)
465 runtime = time.time() - req.processing_started_timestamp
466 return ctx.tag["runtime: %s seconds" % runtime]
468 class DeepCheckAndRepairResultsRenderer(rend.Page, ResultsBase, ReloadMixin):
469 docFactory = getxmlfile("deep-check-and-repair-results.xhtml")
471 def __init__(self, client, monitor):
473 self.monitor = monitor
475 def childFactory(self, ctx, name):
478 # /operation/$OPHANDLE/$STORAGEINDEX provides detailed information
479 # about a specific file or directory that was checked
480 si = base32.a2b(name)
481 s = self.monitor.get_status()
483 results = s.get_results_for_storage_index(si)
484 return CheckAndRepairResultsRenderer(self.client, results)
486 raise WebError("No detailed results for SI %s" % html.escape(name),
489 def renderHTTP(self, ctx):
490 if self.want_json(ctx):
491 return self.json(ctx)
492 return rend.Page.renderHTTP(self, ctx)
495 inevow.IRequest(ctx).setHeader("content-type", "text/plain")
496 res = self.monitor.get_status()
498 data["finished"] = self.monitor.is_finished()
499 data["root-storage-index"] = res.get_root_storage_index_string()
500 c = res.get_counters()
501 data["count-objects-checked"] = c["count-objects-checked"]
503 data["count-objects-healthy-pre-repair"] = c["count-objects-healthy-pre-repair"]
504 data["count-objects-unhealthy-pre-repair"] = c["count-objects-unhealthy-pre-repair"]
505 data["count-objects-healthy-post-repair"] = c["count-objects-healthy-post-repair"]
506 data["count-objects-unhealthy-post-repair"] = c["count-objects-unhealthy-post-repair"]
508 data["count-repairs-attempted"] = c["count-repairs-attempted"]
509 data["count-repairs-successful"] = c["count-repairs-successful"]
510 data["count-repairs-unsuccessful"] = c["count-repairs-unsuccessful"]
512 data["count-corrupt-shares-pre-repair"] = c["count-corrupt-shares-pre-repair"]
513 data["count-corrupt-shares-post-repair"] = c["count-corrupt-shares-pre-repair"]
515 data["list-corrupt-shares"] = [ (idlib.nodeid_b2a(serverid),
516 base32.b2a(storage_index),
518 for (serverid, storage_index, shnum)
519 in res.get_corrupt_shares() ]
521 remaining_corrupt = [ (idlib.nodeid_b2a(serverid),
522 base32.b2a(storage_index),
524 for (serverid, storage_index, shnum)
525 in res.get_remaining_corrupt_shares() ]
526 data["list-remaining-corrupt-shares"] = remaining_corrupt
528 unhealthy = [ (path_t,
529 json_check_results(crr.get_pre_repair_results()))
531 in res.get_all_results().items()
532 if not crr.get_pre_repair_results().is_healthy() ]
533 data["list-unhealthy-files"] = unhealthy
534 data["stats"] = res.get_stats()
535 return simplejson.dumps(data, indent=1) + "\n"
537 def render_root_storage_index(self, ctx, data):
538 return self.monitor.get_status().get_root_storage_index_string()
540 def data_objects_checked(self, ctx, data):
541 return self.monitor.get_status().get_counters()["count-objects-checked"]
543 def data_objects_healthy(self, ctx, data):
544 return self.monitor.get_status().get_counters()["count-objects-healthy-pre-repair"]
545 def data_objects_unhealthy(self, ctx, data):
546 return self.monitor.get_status().get_counters()["count-objects-unhealthy-pre-repair"]
547 def data_corrupt_shares(self, ctx, data):
548 return self.monitor.get_status().get_counters()["count-corrupt-shares-pre-repair"]
550 def data_repairs_attempted(self, ctx, data):
551 return self.monitor.get_status().get_counters()["count-repairs-attempted"]
552 def data_repairs_successful(self, ctx, data):
553 return self.monitor.get_status().get_counters()["count-repairs-successful"]
554 def data_repairs_unsuccessful(self, ctx, data):
555 return self.monitor.get_status().get_counters()["count-repairs-unsuccessful"]
557 def data_objects_healthy_post(self, ctx, data):
558 return self.monitor.get_status().get_counters()["count-objects-healthy-post-repair"]
559 def data_objects_unhealthy_post(self, ctx, data):
560 return self.monitor.get_status().get_counters()["count-objects-unhealthy-post-repair"]
561 def data_corrupt_shares_post(self, ctx, data):
562 return self.monitor.get_status().get_counters()["count-corrupt-shares-post-repair"]
564 def render_pre_repair_problems_p(self, ctx, data):
565 c = self.monitor.get_status().get_counters()
566 if c["count-objects-unhealthy-pre-repair"]:
570 def data_pre_repair_problems(self, ctx, data):
571 all_objects = self.monitor.get_status().get_all_results()
572 for path in sorted(all_objects.keys()):
573 r = all_objects[path]
574 assert ICheckAndRepairResults.providedBy(r)
575 cr = r.get_pre_repair_results()
576 if not cr.is_healthy():
579 def render_problem(self, ctx, data):
581 return ctx.tag[self._join_pathstring(path), ": ",
582 self._html(cr.get_summary())]
584 def render_post_repair_problems_p(self, ctx, data):
585 c = self.monitor.get_status().get_counters()
586 if (c["count-objects-unhealthy-post-repair"]
587 or c["count-corrupt-shares-post-repair"]):
591 def data_post_repair_problems(self, ctx, data):
592 all_objects = self.monitor.get_status().get_all_results()
593 for path in sorted(all_objects.keys()):
594 r = all_objects[path]
595 assert ICheckAndRepairResults.providedBy(r)
596 cr = r.get_post_repair_results()
597 if not cr.is_healthy():
600 def render_servers_with_corrupt_shares_p(self, ctx, data):
601 if self.monitor.get_status().get_counters()["count-corrupt-shares-pre-repair"]:
604 def data_servers_with_corrupt_shares(self, ctx, data):
606 def render_server_problem(self, ctx, data):
610 def render_remaining_corrupt_shares_p(self, ctx, data):
611 if self.monitor.get_status().get_counters()["count-corrupt-shares-post-repair"]:
614 def data_post_repair_corrupt_shares(self, ctx, data):
617 def render_share_problem(self, ctx, data):
621 def render_return(self, ctx, data):
622 req = inevow.IRequest(ctx)
623 return_to = get_arg(req, "return_to", None)
625 return T.div[T.a(href=return_to)["Return to file/directory."]]
628 def data_all_objects(self, ctx, data):
629 r = self.monitor.get_status().get_all_results()
630 for path in sorted(r.keys()):
631 yield (path, r[path])
633 def render_object(self, ctx, data):
635 ctx.fillSlots("path", self._join_pathstring(path))
636 ctx.fillSlots("healthy_pre_repair",
637 str(r.get_pre_repair_results().is_healthy()))
638 ctx.fillSlots("recoverable_pre_repair",
639 str(r.get_pre_repair_results().is_recoverable()))
640 ctx.fillSlots("healthy_post_repair",
641 str(r.get_post_repair_results().is_healthy()))
642 storage_index = r.get_storage_index()
643 ctx.fillSlots("storage_index",
644 self._render_si_link(ctx, storage_index))
645 ctx.fillSlots("summary",
646 self._html(r.get_pre_repair_results().get_summary()))
649 def render_runtime(self, ctx, data):
650 req = inevow.IRequest(ctx)
651 runtime = time.time() - req.processing_started_timestamp
652 return ctx.tag["runtime: %s seconds" % runtime]