4 from nevow import rend, inevow, tags as T
5 from twisted.web import http, html
6 from allmydata.web.common import getxmlfile, get_arg, get_root, WebError
7 from allmydata.web.operations import ReloadMixin
8 from allmydata.interfaces import ICheckAndRepairResults, ICheckResults
9 from allmydata.util import base32, idlib
11 def json_check_counts(d):
13 r["count-shares-good"] = d["count-shares-good"]
14 r["count-shares-needed"] = d["count-shares-needed"]
15 r["count-shares-expected"] = d["count-shares-expected"]
16 r["count-good-share-hosts"] = d["count-good-share-hosts"]
17 r["count-corrupt-shares"] = d["count-corrupt-shares"]
18 r["list-corrupt-shares"] = [ (idlib.nodeid_b2a(serverid),
19 base32.b2a(si), shnum)
20 for (serverid, si, shnum)
21 in d["list-corrupt-shares"] ]
22 r["servers-responding"] = [idlib.nodeid_b2a(serverid)
23 for serverid in d["servers-responding"]]
25 for (shareid, serverids) in d["sharemap"].items():
26 sharemap[shareid] = [idlib.nodeid_b2a(serverid)
27 for serverid in serverids]
28 r["sharemap"] = sharemap
30 r["count-wrong-shares"] = d["count-wrong-shares"]
31 r["count-recoverable-versions"] = d["count-recoverable-versions"]
32 r["count-unrecoverable-versions"] = d["count-unrecoverable-versions"]
36 def json_check_results(r):
39 data = {"storage-index": "",
40 "results": {"healthy": True},
44 data["storage-index"] = r.get_storage_index_string()
45 data["summary"] = r.get_summary()
46 data["results"] = json_check_counts(r.get_data())
47 data["results"]["needs-rebalancing"] = r.needs_rebalancing()
48 data["results"]["healthy"] = r.is_healthy()
49 data["results"]["recoverable"] = r.is_recoverable()
52 def json_check_and_repair_results(r):
55 data = {"storage-index": "",
56 "repair-attempted": False,
60 data["storage-index"] = r.get_storage_index_string()
61 data["repair-attempted"] = r.get_repair_attempted()
62 data["repair-successful"] = r.get_repair_successful()
63 pre = r.get_pre_repair_results()
64 data["pre-repair-results"] = json_check_results(pre)
65 post = r.get_post_repair_results()
66 data["post-repair-results"] = json_check_results(post)
70 # self.client must point to the Client, so we can get nicknames and
71 # determine the permuted peer order
73 def _join_pathstring(self, path):
75 pathstring = "/".join(self._html(path))
80 def _render_results(self, ctx, cr):
81 assert ICheckResults(cr)
86 r.append(T.li[name + ": ", value])
88 add("Report", T.pre["\n".join(self._html(cr.get_report()))])
90 "need %d-of-%d, have %d" % (data["count-shares-needed"],
91 data["count-shares-expected"],
92 data["count-shares-good"]))
93 add("Hosts with good shares", data["count-good-share-hosts"])
95 if data["list-corrupt-shares"]:
97 for (serverid, si, shnum) in data["list-corrupt-shares"]:
98 nickname = c.get_nickname_for_peerid(serverid)
99 badsharemap.append(T.tr[T.td["sh#%d" % shnum],
100 T.td[T.tt[base32.b2a(serverid)],
101 " (", nickname, ")"],
103 add("Corrupt shares", T.table(border="1")[badsharemap])
105 add("Corrupt shares", "none")
107 add("Wrong Shares", data["count-wrong-shares"])
112 for shareid in sorted(data["sharemap"].keys()):
113 serverids = data["sharemap"][shareid]
114 for i,serverid in enumerate(serverids):
115 if serverid not in servers:
116 servers[serverid] = []
117 servers[serverid].append(shareid)
121 nickname = c.get_nickname_for_peerid(serverid)
122 sharemap.append(T.tr[T.td[shareid_s],
123 T.td[T.tt[base32.b2a(serverid)],
124 " (", nickname, ")"],
126 add("Good Shares (sorted in share order)",
127 T.table(border="1")[sharemap])
130 add("Recoverable Versions", data["count-recoverable-versions"])
131 add("Unrecoverable Versions", data["count-unrecoverable-versions"])
133 # this table is sorted by permuted order
134 permuted_peer_ids = [peerid
136 in c.get_permuted_peers("storage",
137 cr.get_storage_index())]
139 num_shares_left = sum([len(shares) for shares in servers.values()])
141 for serverid in permuted_peer_ids:
142 nickname = c.get_nickname_for_peerid(serverid)
143 shareids = servers.get(serverid, [])
145 shareids_s = [ T.tt[shareid, " "] for shareid in shareids ]
146 servermap.append(T.tr[T.td[T.tt[base32.b2a(serverid)],
147 " (", nickname, ")"],
149 num_shares_left -= len(shareids)
150 if not num_shares_left:
152 add("Share Balancing (servers in permuted order)",
153 T.table(border="1")[servermap])
158 if isinstance(s, (str, unicode)):
159 return html.escape(s)
160 assert isinstance(s, (list, tuple))
161 return [html.escape(w) for w in s]
163 def want_json(self, ctx):
164 output = get_arg(inevow.IRequest(ctx), "output", "").lower()
165 if output.lower() == "json":
169 def _render_si_link(self, ctx, storage_index):
170 si_s = base32.b2a(storage_index)
172 req = inevow.IRequest(ctx)
173 ophandle = req.prepath[-1]
174 target = "%s/operations/%s/%s" % (get_root(ctx), ophandle, si_s)
175 output = get_arg(ctx, "output")
177 target = target + "?output=%s" % output
178 return T.a(href=target)[si_s]
180 class LiteralCheckResults(rend.Page, ResultsBase):
181 docFactory = getxmlfile("literal-check-results.xhtml")
183 def __init__(self, client):
185 rend.Page.__init__(self, client)
187 def renderHTTP(self, ctx):
188 if self.want_json(ctx):
189 return self.json(ctx)
190 return rend.Page.renderHTTP(self, ctx)
193 inevow.IRequest(ctx).setHeader("content-type", "text/plain")
194 data = {"storage-index": "",
195 "results": {"healthy": True},
197 return simplejson.dumps(data, indent=1) + "\n"
199 def render_return(self, ctx, data):
200 req = inevow.IRequest(ctx)
201 return_to = get_arg(req, "return_to", None)
203 return T.div[T.a(href=return_to)["Return to parent directory"]]
208 def renderHTTP(self, ctx):
209 if self.want_json(ctx):
210 return self.json(ctx)
211 return rend.Page.renderHTTP(self, ctx)
213 def render_storage_index(self, ctx, data):
214 return self.r.get_storage_index_string()
216 def render_return(self, ctx, data):
217 req = inevow.IRequest(ctx)
218 return_to = get_arg(req, "return_to", None)
220 return T.div[T.a(href=return_to)["Return to parent directory"]]
223 class CheckResults(CheckerBase, rend.Page, ResultsBase):
224 docFactory = getxmlfile("check-results.xhtml")
226 def __init__(self, client, results):
228 self.r = ICheckResults(results)
229 rend.Page.__init__(self, results)
232 inevow.IRequest(ctx).setHeader("content-type", "text/plain")
233 data = json_check_results(self.r)
234 return simplejson.dumps(data, indent=1) + "\n"
236 def render_summary(self, ctx, data):
238 if data.is_healthy():
239 results.append("Healthy")
240 elif data.is_recoverable():
241 results.append("Not Healthy!")
243 results.append("Not Recoverable!")
244 results.append(" : ")
245 results.append(self._html(data.get_summary()))
246 return ctx.tag[results]
248 def render_repair(self, ctx, data):
249 if data.is_healthy():
251 repair = T.form(action=".", method="post",
252 enctype="multipart/form-data")[
254 T.input(type="hidden", name="t", value="check"),
255 T.input(type="hidden", name="repair", value="true"),
256 T.input(type="submit", value="Repair"),
258 return ctx.tag[repair]
260 def render_results(self, ctx, data):
261 cr = self._render_results(ctx, data)
264 class CheckAndRepairResults(CheckerBase, rend.Page, ResultsBase):
265 docFactory = getxmlfile("check-and-repair-results.xhtml")
267 def __init__(self, client, results):
269 self.r = ICheckAndRepairResults(results)
270 rend.Page.__init__(self, results)
273 inevow.IRequest(ctx).setHeader("content-type", "text/plain")
274 data = json_check_and_repair_results(self.r)
275 return simplejson.dumps(data, indent=1) + "\n"
277 def render_summary(self, ctx, data):
278 cr = data.get_post_repair_results()
281 results.append("Healthy")
282 elif cr.is_recoverable():
283 results.append("Not Healthy!")
285 results.append("Not Recoverable!")
286 results.append(" : ")
287 results.append(self._html(cr.get_summary()))
288 return ctx.tag[results]
290 def render_repair_results(self, ctx, data):
291 if data.get_repair_attempted():
292 if data.get_repair_successful():
293 return ctx.tag["Repair successful"]
295 return ctx.tag["Repair unsuccessful"]
296 return ctx.tag["No repair necessary"]
298 def render_post_repair_results(self, ctx, data):
299 cr = self._render_results(ctx, data.get_post_repair_results())
300 return ctx.tag[T.div["Post-Repair Checker Results:"], cr]
302 def render_maybe_pre_repair_results(self, ctx, data):
303 if data.get_repair_attempted():
304 cr = self._render_results(ctx, data.get_pre_repair_results())
305 return ctx.tag[T.div["Pre-Repair Checker Results:"], cr]
309 class DeepCheckResults(rend.Page, ResultsBase, ReloadMixin):
310 docFactory = getxmlfile("deep-check-results.xhtml")
312 def __init__(self, client, monitor):
314 self.monitor = monitor
316 def childFactory(self, ctx, name):
319 # /operation/$OPHANDLE/$STORAGEINDEX provides detailed information
320 # about a specific file or directory that was checked
321 si = base32.a2b(name)
322 r = self.monitor.get_status()
324 return CheckResults(self.client,
325 r.get_results_for_storage_index(si))
327 raise WebError("No detailed results for SI %s" % html.escape(name),
330 def renderHTTP(self, ctx):
331 if self.want_json(ctx):
332 return self.json(ctx)
333 return rend.Page.renderHTTP(self, ctx)
336 inevow.IRequest(ctx).setHeader("content-type", "text/plain")
338 data["finished"] = self.monitor.is_finished()
339 res = self.monitor.get_status()
340 data["root-storage-index"] = res.get_root_storage_index_string()
341 c = res.get_counters()
342 data["count-objects-checked"] = c["count-objects-checked"]
343 data["count-objects-healthy"] = c["count-objects-healthy"]
344 data["count-objects-unhealthy"] = c["count-objects-unhealthy"]
345 data["count-corrupt-shares"] = c["count-corrupt-shares"]
346 data["list-corrupt-shares"] = [ (idlib.nodeid_b2a(serverid),
347 base32.b2a(storage_index),
349 for (serverid, storage_index, shnum)
350 in res.get_corrupt_shares() ]
351 data["list-unhealthy-files"] = [ (path_t, json_check_results(r))
353 in res.get_all_results().items()
354 if not r.is_healthy() ]
355 data["stats"] = res.get_stats()
356 return simplejson.dumps(data, indent=1) + "\n"
358 def render_root_storage_index(self, ctx, data):
359 return self.monitor.get_status().get_root_storage_index_string()
361 def data_objects_checked(self, ctx, data):
362 return self.monitor.get_status().get_counters()["count-objects-checked"]
363 def data_objects_healthy(self, ctx, data):
364 return self.monitor.get_status().get_counters()["count-objects-healthy"]
365 def data_objects_unhealthy(self, ctx, data):
366 return self.monitor.get_status().get_counters()["count-objects-unhealthy"]
367 def data_objects_unrecoverable(self, ctx, data):
368 return self.monitor.get_status().get_counters()["count-objects-unrecoverable"]
370 def data_count_corrupt_shares(self, ctx, data):
371 return self.monitor.get_status().get_counters()["count-corrupt-shares"]
373 def render_problems_p(self, ctx, data):
374 c = self.monitor.get_status().get_counters()
375 if c["count-objects-unhealthy"]:
379 def data_problems(self, ctx, data):
380 all_objects = self.monitor.get_status().get_all_results()
381 for path in sorted(all_objects.keys()):
382 cr = all_objects[path]
383 assert ICheckResults.providedBy(cr)
384 if not cr.is_healthy():
387 def render_problem(self, ctx, data):
390 summary = cr.get_summary()
392 summary_text = ": " + summary
393 summary_text += " [SI: %s]" % cr.get_storage_index_string()
394 return ctx.tag[self._join_pathstring(path), self._html(summary_text)]
397 def render_servers_with_corrupt_shares_p(self, ctx, data):
398 if self.monitor.get_status().get_counters()["count-corrupt-shares"]:
402 def data_servers_with_corrupt_shares(self, ctx, data):
404 for (serverid, storage_index, sharenum)
405 in self.monitor.get_status().get_corrupt_shares()]
409 def render_server_problem(self, ctx, data):
411 data = [idlib.shortnodeid_b2a(serverid)]
412 nickname = self.client.get_nickname_for_peerid(serverid)
414 data.append(" (%s)" % self._html(nickname))
418 def render_corrupt_shares_p(self, ctx, data):
419 if self.monitor.get_status().get_counters()["count-corrupt-shares"]:
422 def data_corrupt_shares(self, ctx, data):
423 return self.monitor.get_status().get_corrupt_shares()
424 def render_share_problem(self, ctx, data):
425 serverid, storage_index, sharenum = data
426 nickname = self.client.get_nickname_for_peerid(serverid)
427 ctx.fillSlots("serverid", idlib.shortnodeid_b2a(serverid))
429 ctx.fillSlots("nickname", self._html(nickname))
430 ctx.fillSlots("si", self._render_si_link(ctx, storage_index))
431 ctx.fillSlots("shnum", str(sharenum))
434 def render_return(self, ctx, data):
435 req = inevow.IRequest(ctx)
436 return_to = get_arg(req, "return_to", None)
438 return T.div[T.a(href=return_to)["Return to parent directory"]]
441 def data_all_objects(self, ctx, data):
442 r = self.monitor.get_status().get_all_results()
443 for path in sorted(r.keys()):
444 yield (path, r[path])
446 def render_object(self, ctx, data):
448 ctx.fillSlots("path", self._join_pathstring(path))
449 ctx.fillSlots("healthy", str(r.is_healthy()))
450 ctx.fillSlots("recoverable", str(r.is_recoverable()))
451 storage_index = r.get_storage_index()
452 ctx.fillSlots("storage_index", self._render_si_link(ctx, storage_index))
453 ctx.fillSlots("summary", self._html(r.get_summary()))
456 def render_runtime(self, ctx, data):
457 req = inevow.IRequest(ctx)
458 runtime = time.time() - req.processing_started_timestamp
459 return ctx.tag["runtime: %s seconds" % runtime]
461 class DeepCheckAndRepairResults(rend.Page, ResultsBase, ReloadMixin):
462 docFactory = getxmlfile("deep-check-and-repair-results.xhtml")
464 def __init__(self, client, monitor):
466 self.monitor = monitor
468 def childFactory(self, ctx, name):
471 # /operation/$OPHANDLE/$STORAGEINDEX provides detailed information
472 # about a specific file or directory that was checked
473 si = base32.a2b(name)
474 r = self.monitor.get_status()
476 return CheckAndRepairResults(self.client,
477 r.get_results_for_storage_index(si))
479 raise WebError("No detailed results for SI %s" % html.escape(name),
482 def renderHTTP(self, ctx):
483 if self.want_json(ctx):
484 return self.json(ctx)
485 return rend.Page.renderHTTP(self, ctx)
488 inevow.IRequest(ctx).setHeader("content-type", "text/plain")
489 res = self.monitor.get_status()
491 data["finished"] = self.monitor.is_finished()
492 data["root-storage-index"] = res.get_root_storage_index_string()
493 c = res.get_counters()
494 data["count-objects-checked"] = c["count-objects-checked"]
496 data["count-objects-healthy-pre-repair"] = c["count-objects-healthy-pre-repair"]
497 data["count-objects-unhealthy-pre-repair"] = c["count-objects-unhealthy-pre-repair"]
498 data["count-objects-healthy-post-repair"] = c["count-objects-healthy-post-repair"]
499 data["count-objects-unhealthy-post-repair"] = c["count-objects-unhealthy-post-repair"]
501 data["count-repairs-attempted"] = c["count-repairs-attempted"]
502 data["count-repairs-successful"] = c["count-repairs-successful"]
503 data["count-repairs-unsuccessful"] = c["count-repairs-unsuccessful"]
505 data["count-corrupt-shares-pre-repair"] = c["count-corrupt-shares-pre-repair"]
506 data["count-corrupt-shares-post-repair"] = c["count-corrupt-shares-pre-repair"]
508 data["list-corrupt-shares"] = [ (idlib.nodeid_b2a(serverid),
509 base32.b2a(storage_index),
511 for (serverid, storage_index, shnum)
512 in res.get_corrupt_shares() ]
514 remaining_corrupt = [ (idlib.nodeid_b2a(serverid),
515 base32.b2a(storage_index),
517 for (serverid, storage_index, shnum)
518 in res.get_remaining_corrupt_shares() ]
519 data["list-remaining-corrupt-shares"] = remaining_corrupt
521 unhealthy = [ (path_t,
522 json_check_results(crr.get_pre_repair_results()))
524 in res.get_all_results().items()
525 if not crr.get_pre_repair_results().is_healthy() ]
526 data["list-unhealthy-files"] = unhealthy
527 data["stats"] = res.get_stats()
528 return simplejson.dumps(data, indent=1) + "\n"
530 def render_root_storage_index(self, ctx, data):
531 return self.monitor.get_status().get_root_storage_index_string()
533 def data_objects_checked(self, ctx, data):
534 return self.monitor.get_status().get_counters()["count-objects-checked"]
536 def data_objects_healthy(self, ctx, data):
537 return self.monitor.get_status().get_counters()["count-objects-healthy-pre-repair"]
538 def data_objects_unhealthy(self, ctx, data):
539 return self.monitor.get_status().get_counters()["count-objects-unhealthy-pre-repair"]
540 def data_corrupt_shares(self, ctx, data):
541 return self.monitor.get_status().get_counters()["count-corrupt-shares-pre-repair"]
543 def data_repairs_attempted(self, ctx, data):
544 return self.monitor.get_status().get_counters()["count-repairs-attempted"]
545 def data_repairs_successful(self, ctx, data):
546 return self.monitor.get_status().get_counters()["count-repairs-successful"]
547 def data_repairs_unsuccessful(self, ctx, data):
548 return self.monitor.get_status().get_counters()["count-repairs-unsuccessful"]
550 def data_objects_healthy_post(self, ctx, data):
551 return self.monitor.get_status().get_counters()["count-objects-healthy-post-repair"]
552 def data_objects_unhealthy_post(self, ctx, data):
553 return self.monitor.get_status().get_counters()["count-objects-unhealthy-post-repair"]
554 def data_corrupt_shares_post(self, ctx, data):
555 return self.monitor.get_status().get_counters()["count-corrupt-shares-post-repair"]
557 def render_pre_repair_problems_p(self, ctx, data):
558 c = self.monitor.get_status().get_counters()
559 if c["count-objects-unhealthy-pre-repair"]:
563 def data_pre_repair_problems(self, ctx, data):
564 all_objects = self.monitor.get_status().get_all_results()
565 for path in sorted(all_objects.keys()):
566 r = all_objects[path]
567 assert ICheckAndRepairResults.providedBy(r)
568 cr = r.get_pre_repair_results()
569 if not cr.is_healthy():
572 def render_problem(self, ctx, data):
574 return ctx.tag[self._join_pathstring(path), ": ",
575 self._html(cr.get_summary())]
577 def render_post_repair_problems_p(self, ctx, data):
578 c = self.monitor.get_status().get_counters()
579 if (c["count-objects-unhealthy-post-repair"]
580 or c["count-corrupt-shares-post-repair"]):
584 def data_post_repair_problems(self, ctx, data):
585 all_objects = self.monitor.get_status().get_all_results()
586 for path in sorted(all_objects.keys()):
587 r = all_objects[path]
588 assert ICheckAndRepairResults.providedBy(r)
589 cr = r.get_post_repair_results()
590 if not cr.is_healthy():
593 def render_servers_with_corrupt_shares_p(self, ctx, data):
594 if self.monitor.get_status().get_counters()["count-corrupt-shares-pre-repair"]:
597 def data_servers_with_corrupt_shares(self, ctx, data):
599 def render_server_problem(self, ctx, data):
603 def render_remaining_corrupt_shares_p(self, ctx, data):
604 if self.monitor.get_status().get_counters()["count-corrupt-shares-post-repair"]:
607 def data_post_repair_corrupt_shares(self, ctx, data):
610 def render_share_problem(self, ctx, data):
614 def render_return(self, ctx, data):
615 req = inevow.IRequest(ctx)
616 return_to = get_arg(req, "return_to", None)
618 return T.div[T.a(href=return_to)["Return to parent directory"]]
621 def data_all_objects(self, ctx, data):
622 r = self.monitor.get_status().get_all_results()
623 for path in sorted(r.keys()):
624 yield (path, r[path])
626 def render_object(self, ctx, data):
628 ctx.fillSlots("path", self._join_pathstring(path))
629 ctx.fillSlots("healthy_pre_repair",
630 str(r.get_pre_repair_results().is_healthy()))
631 ctx.fillSlots("recoverable_pre_repair",
632 str(r.get_pre_repair_results().is_recoverable()))
633 ctx.fillSlots("healthy_post_repair",
634 str(r.get_post_repair_results().is_healthy()))
635 storage_index = r.get_storage_index()
636 ctx.fillSlots("storage_index",
637 self._render_si_link(ctx, storage_index))
638 ctx.fillSlots("summary",
639 self._html(r.get_pre_repair_results().get_summary()))
642 def render_runtime(self, ctx, data):
643 req = inevow.IRequest(ctx)
644 runtime = time.time() - req.processing_started_timestamp
645 return ctx.tag["runtime: %s seconds" % runtime]