4 from nevow import rend, inevow, tags as T
5 from twisted.web import http, html
6 from allmydata.web.common import getxmlfile, get_arg, get_root, WebError
7 from allmydata.web.operations import ReloadMixin
8 from allmydata.interfaces import ICheckAndRepairResults, ICheckResults
9 from allmydata.util import base32, idlib
11 def json_check_counts(d):
13 r["count-shares-good"] = d["count-shares-good"]
14 r["count-shares-needed"] = d["count-shares-needed"]
15 r["count-shares-expected"] = d["count-shares-expected"]
16 r["count-good-share-hosts"] = d["count-good-share-hosts"]
17 r["count-corrupt-shares"] = d["count-corrupt-shares"]
18 r["list-corrupt-shares"] = [ (idlib.nodeid_b2a(serverid),
19 base32.b2a(si), shnum)
20 for (serverid, si, shnum)
21 in d["list-corrupt-shares"] ]
22 r["servers-responding"] = [idlib.nodeid_b2a(serverid)
23 for serverid in d["servers-responding"]]
25 for (shareid, serverids) in d["sharemap"].items():
26 sharemap[shareid] = [idlib.nodeid_b2a(serverid)
27 for serverid in serverids]
28 r["sharemap"] = sharemap
30 r["count-wrong-shares"] = d["count-wrong-shares"]
31 r["count-recoverable-versions"] = d["count-recoverable-versions"]
32 r["count-unrecoverable-versions"] = d["count-unrecoverable-versions"]
36 def json_check_results(r):
39 data = {"storage-index": "",
40 "results": {"healthy": True},
44 data["storage-index"] = r.get_storage_index_string()
45 data["summary"] = r.get_summary()
46 data["results"] = json_check_counts(r.get_data())
47 data["results"]["needs-rebalancing"] = r.needs_rebalancing()
48 data["results"]["healthy"] = r.is_healthy()
49 data["results"]["recoverable"] = r.is_recoverable()
52 def json_check_and_repair_results(r):
55 data = {"storage-index": "",
56 "repair-attempted": False,
60 data["storage-index"] = r.get_storage_index_string()
61 data["repair-attempted"] = r.get_repair_attempted()
62 data["repair-successful"] = r.get_repair_successful()
63 pre = r.get_pre_repair_results()
64 data["pre-repair-results"] = json_check_results(pre)
65 post = r.get_post_repair_results()
66 data["post-repair-results"] = json_check_results(post)
70 # self.client must point to the Client, so we can get nicknames and
71 # determine the permuted peer order
73 def _join_pathstring(self, path):
75 pathstring = "/".join(self._html(path))
80 def _render_results(self, ctx, cr):
81 assert ICheckResults(cr)
83 sb = c.get_storage_broker()
87 r.append(T.li[name + ": ", value])
89 add("Report", T.pre["\n".join(self._html(cr.get_report()))])
91 "need %d-of-%d, have %d" % (data["count-shares-needed"],
92 data["count-shares-expected"],
93 data["count-shares-good"]))
94 add("Hosts with good shares", data["count-good-share-hosts"])
96 if data["list-corrupt-shares"]:
98 for (serverid, si, shnum) in data["list-corrupt-shares"]:
99 nickname = sb.get_nickname_for_serverid(serverid)
100 badsharemap.append(T.tr[T.td["sh#%d" % shnum],
101 T.td[T.div(class_="nickname")[nickname],
102 T.div(class_="nodeid")[T.tt[base32.b2a(serverid)]]],
104 add("Corrupt shares", T.table()[
105 T.tr[T.th["Share ID"],
106 T.th(class_="nickname-and-peerid")[T.div["Nickname"], T.div(class_="nodeid")["Node ID"]]],
109 add("Corrupt shares", "none")
111 add("Wrong Shares", data["count-wrong-shares"])
116 # FIXME: The two tables below contain nickname-and-nodeid table column markup which is duplicated with each other, introducer.xhtml, and deep-check-results.xhtml. All of these (and any other presentations of nickname-and-nodeid) should be combined.
118 for shareid in sorted(data["sharemap"].keys()):
119 serverids = data["sharemap"][shareid]
120 for i,serverid in enumerate(serverids):
121 if serverid not in servers:
122 servers[serverid] = []
123 servers[serverid].append(shareid)
127 nickname = sb.get_nickname_for_serverid(serverid)
128 sharemap.append(T.tr[T.td[shareid_s],
129 T.td[T.div(class_="nickname")[nickname],
130 T.div(class_="nodeid")[T.tt[base32.b2a(serverid)]]]
132 add("Good Shares (sorted in share order)",
133 T.table()[T.tr[T.th["Share ID"], T.th(class_="nickname-and-peerid")[T.div["Nickname"], T.div(class_="nodeid")["Node ID"]]],
137 add("Recoverable Versions", data["count-recoverable-versions"])
138 add("Unrecoverable Versions", data["count-unrecoverable-versions"])
140 # this table is sorted by permuted order
141 sb = c.get_storage_broker()
142 permuted_servers = [s
144 in sb.get_servers_for_psi(cr.get_storage_index())]
146 num_shares_left = sum([len(shares) for shares in servers.values()])
148 for s in permuted_servers:
149 nickname = s.get_nickname()
150 shareids = servers.get(s.get_serverid(), [])
152 shareids_s = [ T.tt[shareid, " "] for shareid in sorted(shareids) ]
153 servermap.append(T.tr[T.td[T.div(class_="nickname")[nickname],
154 T.div(class_="nodeid")[T.tt[s.get_name()]]],
157 num_shares_left -= len(shareids)
158 if not num_shares_left:
160 add("Share Balancing (servers in permuted order)",
161 T.table()[T.tr[T.th(class_="nickname-and-peerid")[T.div["Nickname"], T.div(class_="nodeid")["Node ID"]], T.th["Share IDs"]],
167 if isinstance(s, (str, unicode)):
168 return html.escape(s)
169 assert isinstance(s, (list, tuple))
170 return [html.escape(w) for w in s]
172 def want_json(self, ctx):
173 output = get_arg(inevow.IRequest(ctx), "output", "").lower()
174 if output.lower() == "json":
178 def _render_si_link(self, ctx, storage_index):
179 si_s = base32.b2a(storage_index)
180 req = inevow.IRequest(ctx)
181 ophandle = req.prepath[-1]
182 target = "%s/operations/%s/%s" % (get_root(ctx), ophandle, si_s)
183 output = get_arg(ctx, "output")
185 target = target + "?output=%s" % output
186 return T.a(href=target)[si_s]
188 class LiteralCheckResultsRenderer(rend.Page, ResultsBase):
189 docFactory = getxmlfile("literal-check-results.xhtml")
191 def __init__(self, client):
193 rend.Page.__init__(self, client)
195 def renderHTTP(self, ctx):
196 if self.want_json(ctx):
197 return self.json(ctx)
198 return rend.Page.renderHTTP(self, ctx)
201 inevow.IRequest(ctx).setHeader("content-type", "text/plain")
202 data = json_check_results(None)
203 return simplejson.dumps(data, indent=1) + "\n"
205 def render_return(self, ctx, data):
206 req = inevow.IRequest(ctx)
207 return_to = get_arg(req, "return_to", None)
209 return T.div[T.a(href=return_to)["Return to file."]]
214 def renderHTTP(self, ctx):
215 if self.want_json(ctx):
216 return self.json(ctx)
217 return rend.Page.renderHTTP(self, ctx)
219 def render_storage_index(self, ctx, data):
220 return self.r.get_storage_index_string()
222 def render_return(self, ctx, data):
223 req = inevow.IRequest(ctx)
224 return_to = get_arg(req, "return_to", None)
226 return T.div[T.a(href=return_to)["Return to file/directory."]]
229 class CheckResultsRenderer(CheckerBase, rend.Page, ResultsBase):
230 docFactory = getxmlfile("check-results.xhtml")
232 def __init__(self, client, results):
234 self.r = ICheckResults(results)
235 rend.Page.__init__(self, results)
238 inevow.IRequest(ctx).setHeader("content-type", "text/plain")
239 data = json_check_results(self.r)
240 return simplejson.dumps(data, indent=1) + "\n"
242 def render_summary(self, ctx, data):
244 if data.is_healthy():
245 results.append("Healthy")
246 elif data.is_recoverable():
247 results.append("Not Healthy!")
249 results.append("Not Recoverable!")
250 results.append(" : ")
251 results.append(self._html(data.get_summary()))
252 return ctx.tag[results]
254 def render_repair(self, ctx, data):
255 if data.is_healthy():
257 repair = T.form(action=".", method="post",
258 enctype="multipart/form-data")[
260 T.input(type="hidden", name="t", value="check"),
261 T.input(type="hidden", name="repair", value="true"),
262 T.input(type="submit", value="Repair"),
264 return "" # repair button disabled until we make it work correctly,
265 # see #622 for details
266 return ctx.tag[repair]
268 def render_results(self, ctx, data):
269 cr = self._render_results(ctx, data)
272 class CheckAndRepairResultsRenderer(CheckerBase, rend.Page, ResultsBase):
273 docFactory = getxmlfile("check-and-repair-results.xhtml")
275 def __init__(self, client, results):
279 self.r = ICheckAndRepairResults(results)
280 rend.Page.__init__(self, results)
283 inevow.IRequest(ctx).setHeader("content-type", "text/plain")
284 data = json_check_and_repair_results(self.r)
285 return simplejson.dumps(data, indent=1) + "\n"
287 def render_summary(self, ctx, data):
288 cr = data.get_post_repair_results()
291 results.append("Healthy")
292 elif cr.is_recoverable():
293 results.append("Not Healthy!")
295 results.append("Not Recoverable!")
296 results.append(" : ")
297 results.append(self._html(cr.get_summary()))
298 return ctx.tag[results]
300 def render_repair_results(self, ctx, data):
301 if data.get_repair_attempted():
302 if data.get_repair_successful():
303 return ctx.tag["Repair successful"]
305 return ctx.tag["Repair unsuccessful"]
306 return ctx.tag["No repair necessary"]
308 def render_post_repair_results(self, ctx, data):
309 cr = self._render_results(ctx, data.get_post_repair_results())
310 return ctx.tag[T.div["Post-Repair Checker Results:"], cr]
312 def render_maybe_pre_repair_results(self, ctx, data):
313 if data.get_repair_attempted():
314 cr = self._render_results(ctx, data.get_pre_repair_results())
315 return ctx.tag[T.div["Pre-Repair Checker Results:"], cr]
319 class DeepCheckResultsRenderer(rend.Page, ResultsBase, ReloadMixin):
320 docFactory = getxmlfile("deep-check-results.xhtml")
322 def __init__(self, client, monitor):
324 self.monitor = monitor
326 def childFactory(self, ctx, name):
329 # /operation/$OPHANDLE/$STORAGEINDEX provides detailed information
330 # about a specific file or directory that was checked
331 si = base32.a2b(name)
332 r = self.monitor.get_status()
334 return CheckResultsRenderer(self.client,
335 r.get_results_for_storage_index(si))
337 raise WebError("No detailed results for SI %s" % html.escape(name),
340 def renderHTTP(self, ctx):
341 if self.want_json(ctx):
342 return self.json(ctx)
343 return rend.Page.renderHTTP(self, ctx)
346 inevow.IRequest(ctx).setHeader("content-type", "text/plain")
348 data["finished"] = self.monitor.is_finished()
349 res = self.monitor.get_status()
350 data["root-storage-index"] = res.get_root_storage_index_string()
351 c = res.get_counters()
352 data["count-objects-checked"] = c["count-objects-checked"]
353 data["count-objects-healthy"] = c["count-objects-healthy"]
354 data["count-objects-unhealthy"] = c["count-objects-unhealthy"]
355 data["count-corrupt-shares"] = c["count-corrupt-shares"]
356 data["list-corrupt-shares"] = [ (idlib.nodeid_b2a(serverid),
357 base32.b2a(storage_index),
359 for (serverid, storage_index, shnum)
360 in res.get_corrupt_shares() ]
361 data["list-unhealthy-files"] = [ (path_t, json_check_results(r))
363 in res.get_all_results().items()
364 if not r.is_healthy() ]
365 data["stats"] = res.get_stats()
366 return simplejson.dumps(data, indent=1) + "\n"
368 def render_root_storage_index(self, ctx, data):
369 return self.monitor.get_status().get_root_storage_index_string()
371 def data_objects_checked(self, ctx, data):
372 return self.monitor.get_status().get_counters()["count-objects-checked"]
373 def data_objects_healthy(self, ctx, data):
374 return self.monitor.get_status().get_counters()["count-objects-healthy"]
375 def data_objects_unhealthy(self, ctx, data):
376 return self.monitor.get_status().get_counters()["count-objects-unhealthy"]
377 def data_objects_unrecoverable(self, ctx, data):
378 return self.monitor.get_status().get_counters()["count-objects-unrecoverable"]
380 def data_count_corrupt_shares(self, ctx, data):
381 return self.monitor.get_status().get_counters()["count-corrupt-shares"]
383 def render_problems_p(self, ctx, data):
384 c = self.monitor.get_status().get_counters()
385 if c["count-objects-unhealthy"]:
389 def data_problems(self, ctx, data):
390 all_objects = self.monitor.get_status().get_all_results()
391 for path in sorted(all_objects.keys()):
392 cr = all_objects[path]
393 assert ICheckResults.providedBy(cr)
394 if not cr.is_healthy():
397 def render_problem(self, ctx, data):
400 summary = cr.get_summary()
402 summary_text = ": " + summary
403 summary_text += " [SI: %s]" % cr.get_storage_index_string()
404 return ctx.tag[self._join_pathstring(path), self._html(summary_text)]
407 def render_servers_with_corrupt_shares_p(self, ctx, data):
408 if self.monitor.get_status().get_counters()["count-corrupt-shares"]:
412 def data_servers_with_corrupt_shares(self, ctx, data):
414 for (serverid, storage_index, sharenum)
415 in self.monitor.get_status().get_corrupt_shares()]
419 def render_server_problem(self, ctx, data):
421 data = [idlib.shortnodeid_b2a(serverid)]
422 sb = self.client.get_storage_broker()
423 nickname = sb.get_nickname_for_serverid(serverid)
425 data.append(" (%s)" % self._html(nickname))
429 def render_corrupt_shares_p(self, ctx, data):
430 if self.monitor.get_status().get_counters()["count-corrupt-shares"]:
433 def data_corrupt_shares(self, ctx, data):
434 return self.monitor.get_status().get_corrupt_shares()
435 def render_share_problem(self, ctx, data):
436 serverid, storage_index, sharenum = data
437 sb = self.client.get_storage_broker()
438 nickname = sb.get_nickname_for_serverid(serverid)
439 ctx.fillSlots("serverid", idlib.shortnodeid_b2a(serverid))
441 ctx.fillSlots("nickname", self._html(nickname))
442 ctx.fillSlots("si", self._render_si_link(ctx, storage_index))
443 ctx.fillSlots("shnum", str(sharenum))
446 def render_return(self, ctx, data):
447 req = inevow.IRequest(ctx)
448 return_to = get_arg(req, "return_to", None)
450 return T.div[T.a(href=return_to)["Return to file/directory."]]
453 def data_all_objects(self, ctx, data):
454 r = self.monitor.get_status().get_all_results()
455 for path in sorted(r.keys()):
456 yield (path, r[path])
458 def render_object(self, ctx, data):
460 ctx.fillSlots("path", self._join_pathstring(path))
461 ctx.fillSlots("healthy", str(r.is_healthy()))
462 ctx.fillSlots("recoverable", str(r.is_recoverable()))
463 storage_index = r.get_storage_index()
464 ctx.fillSlots("storage_index", self._render_si_link(ctx, storage_index))
465 ctx.fillSlots("summary", self._html(r.get_summary()))
468 def render_runtime(self, ctx, data):
469 req = inevow.IRequest(ctx)
470 runtime = time.time() - req.processing_started_timestamp
471 return ctx.tag["runtime: %s seconds" % runtime]
473 class DeepCheckAndRepairResultsRenderer(rend.Page, ResultsBase, ReloadMixin):
474 docFactory = getxmlfile("deep-check-and-repair-results.xhtml")
476 def __init__(self, client, monitor):
478 self.monitor = monitor
480 def childFactory(self, ctx, name):
483 # /operation/$OPHANDLE/$STORAGEINDEX provides detailed information
484 # about a specific file or directory that was checked
485 si = base32.a2b(name)
486 s = self.monitor.get_status()
488 results = s.get_results_for_storage_index(si)
489 return CheckAndRepairResultsRenderer(self.client, results)
491 raise WebError("No detailed results for SI %s" % html.escape(name),
494 def renderHTTP(self, ctx):
495 if self.want_json(ctx):
496 return self.json(ctx)
497 return rend.Page.renderHTTP(self, ctx)
500 inevow.IRequest(ctx).setHeader("content-type", "text/plain")
501 res = self.monitor.get_status()
503 data["finished"] = self.monitor.is_finished()
504 data["root-storage-index"] = res.get_root_storage_index_string()
505 c = res.get_counters()
506 data["count-objects-checked"] = c["count-objects-checked"]
508 data["count-objects-healthy-pre-repair"] = c["count-objects-healthy-pre-repair"]
509 data["count-objects-unhealthy-pre-repair"] = c["count-objects-unhealthy-pre-repair"]
510 data["count-objects-healthy-post-repair"] = c["count-objects-healthy-post-repair"]
511 data["count-objects-unhealthy-post-repair"] = c["count-objects-unhealthy-post-repair"]
513 data["count-repairs-attempted"] = c["count-repairs-attempted"]
514 data["count-repairs-successful"] = c["count-repairs-successful"]
515 data["count-repairs-unsuccessful"] = c["count-repairs-unsuccessful"]
517 data["count-corrupt-shares-pre-repair"] = c["count-corrupt-shares-pre-repair"]
518 data["count-corrupt-shares-post-repair"] = c["count-corrupt-shares-pre-repair"]
520 data["list-corrupt-shares"] = [ (idlib.nodeid_b2a(serverid),
521 base32.b2a(storage_index),
523 for (serverid, storage_index, shnum)
524 in res.get_corrupt_shares() ]
526 remaining_corrupt = [ (idlib.nodeid_b2a(serverid),
527 base32.b2a(storage_index),
529 for (serverid, storage_index, shnum)
530 in res.get_remaining_corrupt_shares() ]
531 data["list-remaining-corrupt-shares"] = remaining_corrupt
533 unhealthy = [ (path_t,
534 json_check_results(crr.get_pre_repair_results()))
536 in res.get_all_results().items()
537 if not crr.get_pre_repair_results().is_healthy() ]
538 data["list-unhealthy-files"] = unhealthy
539 data["stats"] = res.get_stats()
540 return simplejson.dumps(data, indent=1) + "\n"
542 def render_root_storage_index(self, ctx, data):
543 return self.monitor.get_status().get_root_storage_index_string()
545 def data_objects_checked(self, ctx, data):
546 return self.monitor.get_status().get_counters()["count-objects-checked"]
548 def data_objects_healthy(self, ctx, data):
549 return self.monitor.get_status().get_counters()["count-objects-healthy-pre-repair"]
550 def data_objects_unhealthy(self, ctx, data):
551 return self.monitor.get_status().get_counters()["count-objects-unhealthy-pre-repair"]
552 def data_corrupt_shares(self, ctx, data):
553 return self.monitor.get_status().get_counters()["count-corrupt-shares-pre-repair"]
555 def data_repairs_attempted(self, ctx, data):
556 return self.monitor.get_status().get_counters()["count-repairs-attempted"]
557 def data_repairs_successful(self, ctx, data):
558 return self.monitor.get_status().get_counters()["count-repairs-successful"]
559 def data_repairs_unsuccessful(self, ctx, data):
560 return self.monitor.get_status().get_counters()["count-repairs-unsuccessful"]
562 def data_objects_healthy_post(self, ctx, data):
563 return self.monitor.get_status().get_counters()["count-objects-healthy-post-repair"]
564 def data_objects_unhealthy_post(self, ctx, data):
565 return self.monitor.get_status().get_counters()["count-objects-unhealthy-post-repair"]
566 def data_corrupt_shares_post(self, ctx, data):
567 return self.monitor.get_status().get_counters()["count-corrupt-shares-post-repair"]
569 def render_pre_repair_problems_p(self, ctx, data):
570 c = self.monitor.get_status().get_counters()
571 if c["count-objects-unhealthy-pre-repair"]:
575 def data_pre_repair_problems(self, ctx, data):
576 all_objects = self.monitor.get_status().get_all_results()
577 for path in sorted(all_objects.keys()):
578 r = all_objects[path]
579 assert ICheckAndRepairResults.providedBy(r)
580 cr = r.get_pre_repair_results()
581 if not cr.is_healthy():
584 def render_problem(self, ctx, data):
586 return ctx.tag[self._join_pathstring(path), ": ",
587 self._html(cr.get_summary())]
589 def render_post_repair_problems_p(self, ctx, data):
590 c = self.monitor.get_status().get_counters()
591 if (c["count-objects-unhealthy-post-repair"]
592 or c["count-corrupt-shares-post-repair"]):
596 def data_post_repair_problems(self, ctx, data):
597 all_objects = self.monitor.get_status().get_all_results()
598 for path in sorted(all_objects.keys()):
599 r = all_objects[path]
600 assert ICheckAndRepairResults.providedBy(r)
601 cr = r.get_post_repair_results()
602 if not cr.is_healthy():
605 def render_servers_with_corrupt_shares_p(self, ctx, data):
606 if self.monitor.get_status().get_counters()["count-corrupt-shares-pre-repair"]:
609 def data_servers_with_corrupt_shares(self, ctx, data):
611 def render_server_problem(self, ctx, data):
615 def render_remaining_corrupt_shares_p(self, ctx, data):
616 if self.monitor.get_status().get_counters()["count-corrupt-shares-post-repair"]:
619 def data_post_repair_corrupt_shares(self, ctx, data):
622 def render_share_problem(self, ctx, data):
626 def render_return(self, ctx, data):
627 req = inevow.IRequest(ctx)
628 return_to = get_arg(req, "return_to", None)
630 return T.div[T.a(href=return_to)["Return to file/directory."]]
633 def data_all_objects(self, ctx, data):
634 r = self.monitor.get_status().get_all_results()
635 for path in sorted(r.keys()):
636 yield (path, r[path])
638 def render_object(self, ctx, data):
640 ctx.fillSlots("path", self._join_pathstring(path))
641 ctx.fillSlots("healthy_pre_repair",
642 str(r.get_pre_repair_results().is_healthy()))
643 ctx.fillSlots("recoverable_pre_repair",
644 str(r.get_pre_repair_results().is_recoverable()))
645 ctx.fillSlots("healthy_post_repair",
646 str(r.get_post_repair_results().is_healthy()))
647 storage_index = r.get_storage_index()
648 ctx.fillSlots("storage_index",
649 self._render_si_link(ctx, storage_index))
650 ctx.fillSlots("summary",
651 self._html(r.get_pre_repair_results().get_summary()))
654 def render_runtime(self, ctx, data):
655 req = inevow.IRequest(ctx)
656 runtime = time.time() - req.processing_started_timestamp
657 return ctx.tag["runtime: %s seconds" % runtime]