4 from nevow import rend, inevow, tags as T
5 from twisted.web import http, html
6 from allmydata.web.common import getxmlfile, get_arg, get_root, WebError
7 from allmydata.web.operations import ReloadMixin
8 from allmydata.interfaces import ICheckAndRepairResults, ICheckResults
9 from allmydata.util import base32, idlib
11 def json_check_counts(d):
13 r["count-shares-good"] = d["count-shares-good"]
14 r["count-shares-needed"] = d["count-shares-needed"]
15 r["count-shares-expected"] = d["count-shares-expected"]
16 r["count-good-share-hosts"] = d["count-good-share-hosts"]
17 r["count-corrupt-shares"] = d["count-corrupt-shares"]
18 r["list-corrupt-shares"] = [ (idlib.nodeid_b2a(serverid),
19 base32.b2a(si), shnum)
20 for (serverid, si, shnum)
21 in d["list-corrupt-shares"] ]
22 r["servers-responding"] = [idlib.nodeid_b2a(serverid)
23 for serverid in d["servers-responding"]]
25 for (shareid, serverids) in d["sharemap"].items():
26 sharemap[shareid] = [idlib.nodeid_b2a(serverid)
27 for serverid in serverids]
28 r["sharemap"] = sharemap
30 r["count-wrong-shares"] = d["count-wrong-shares"]
31 r["count-recoverable-versions"] = d["count-recoverable-versions"]
32 r["count-unrecoverable-versions"] = d["count-unrecoverable-versions"]
36 def json_check_results(r):
39 data = {"storage-index": "",
40 "results": {"healthy": True},
44 data["storage-index"] = r.get_storage_index_string()
45 data["summary"] = r.get_summary()
46 data["results"] = json_check_counts(r.get_data())
47 data["results"]["needs-rebalancing"] = r.needs_rebalancing()
48 data["results"]["healthy"] = r.is_healthy()
49 data["results"]["recoverable"] = r.is_recoverable()
52 def json_check_and_repair_results(r):
55 data = {"storage-index": "",
56 "repair-attempted": False,
60 data["storage-index"] = r.get_storage_index_string()
61 data["repair-attempted"] = r.get_repair_attempted()
62 data["repair-successful"] = r.get_repair_successful()
63 pre = r.get_pre_repair_results()
64 data["pre-repair-results"] = json_check_results(pre)
65 post = r.get_post_repair_results()
66 data["post-repair-results"] = json_check_results(post)
70 # self.client must point to the Client, so we can get nicknames and
71 # determine the permuted peer order
73 def _join_pathstring(self, path):
75 pathstring = "/".join(self._html(path))
80 def _render_results(self, ctx, cr):
81 assert ICheckResults(cr)
86 r.append(T.li[name + ": ", value])
88 add("Report", T.pre["\n".join(self._html(cr.get_report()))])
90 "need %d-of-%d, have %d" % (data["count-shares-needed"],
91 data["count-shares-expected"],
92 data["count-shares-good"]))
93 add("Hosts with good shares", data["count-good-share-hosts"])
95 if data["list-corrupt-shares"]:
97 for (serverid, si, shnum) in data["list-corrupt-shares"]:
98 nickname = c.get_nickname_for_peerid(serverid)
99 badsharemap.append(T.tr[T.td["sh#%d" % shnum],
100 T.td[T.tt[base32.b2a(serverid)],
101 " (", nickname, ")"],
103 add("Corrupt shares", T.table(border="1")[badsharemap])
105 add("Corrupt shares", "none")
107 add("Wrong Shares", data["count-wrong-shares"])
112 # FIXME: The two tables below contain nickname-and-nodeid table column markup which is duplicated with each other, introducer.xhtml, and deep-check-results.xhtml. All of these (and any other presentations of nickname-and-nodeid) should be combined.
114 for shareid in sorted(data["sharemap"].keys()):
115 serverids = data["sharemap"][shareid]
116 for i,serverid in enumerate(serverids):
117 if serverid not in servers:
118 servers[serverid] = []
119 servers[serverid].append(shareid)
123 nickname = c.get_nickname_for_peerid(serverid)
124 sharemap.append(T.tr[T.td[shareid_s],
125 T.td[T.tt[base32.b2a(serverid)],
126 " (", nickname, ")"],
128 add("Good Shares (sorted in share order)",
129 T.table(border="1")[sharemap])
132 add("Recoverable Versions", data["count-recoverable-versions"])
133 add("Unrecoverable Versions", data["count-unrecoverable-versions"])
135 # this table is sorted by permuted order
136 permuted_peer_ids = [peerid
138 in c.get_permuted_peers("storage",
139 cr.get_storage_index())]
141 num_shares_left = sum([len(shares) for shares in servers.values()])
143 for serverid in permuted_peer_ids:
144 nickname = c.get_nickname_for_peerid(serverid)
145 shareids = servers.get(serverid, [])
147 shareids_s = [ T.tt[shareid, " "] for shareid in sorted(shareids) ]
148 servermap.append(T.tr[T.td[T.tt[base32.b2a(serverid)],
149 " (", nickname, ")"],
151 num_shares_left -= len(shareids)
152 if not num_shares_left:
154 add("Share Balancing (servers in permuted order)",
155 T.table(border="1")[servermap])
160 if isinstance(s, (str, unicode)):
161 return html.escape(s)
162 assert isinstance(s, (list, tuple))
163 return [html.escape(w) for w in s]
165 def want_json(self, ctx):
166 output = get_arg(inevow.IRequest(ctx), "output", "").lower()
167 if output.lower() == "json":
171 def _render_si_link(self, ctx, storage_index):
172 si_s = base32.b2a(storage_index)
174 req = inevow.IRequest(ctx)
175 ophandle = req.prepath[-1]
176 target = "%s/operations/%s/%s" % (get_root(ctx), ophandle, si_s)
177 output = get_arg(ctx, "output")
179 target = target + "?output=%s" % output
180 return T.a(href=target)[si_s]
182 class LiteralCheckResults(rend.Page, ResultsBase):
183 docFactory = getxmlfile("literal-check-results.xhtml")
185 def __init__(self, client):
187 rend.Page.__init__(self, client)
189 def renderHTTP(self, ctx):
190 if self.want_json(ctx):
191 return self.json(ctx)
192 return rend.Page.renderHTTP(self, ctx)
195 inevow.IRequest(ctx).setHeader("content-type", "text/plain")
196 data = json_check_results(None)
197 return simplejson.dumps(data, indent=1) + "\n"
199 def render_return(self, ctx, data):
200 req = inevow.IRequest(ctx)
201 return_to = get_arg(req, "return_to", None)
203 return T.div[T.a(href=return_to)["Return to file."]]
208 def renderHTTP(self, ctx):
209 if self.want_json(ctx):
210 return self.json(ctx)
211 return rend.Page.renderHTTP(self, ctx)
213 def render_storage_index(self, ctx, data):
214 return self.r.get_storage_index_string()
216 def render_return(self, ctx, data):
217 req = inevow.IRequest(ctx)
218 return_to = get_arg(req, "return_to", None)
220 return T.div[T.a(href=return_to)["Return to file/directory."]]
223 class CheckResults(CheckerBase, rend.Page, ResultsBase):
224 docFactory = getxmlfile("check-results.xhtml")
226 def __init__(self, client, results):
228 self.r = ICheckResults(results)
229 rend.Page.__init__(self, results)
232 inevow.IRequest(ctx).setHeader("content-type", "text/plain")
233 data = json_check_results(self.r)
234 return simplejson.dumps(data, indent=1) + "\n"
236 def render_summary(self, ctx, data):
238 if data.is_healthy():
239 results.append("Healthy")
240 elif data.is_recoverable():
241 results.append("Not Healthy!")
243 results.append("Not Recoverable!")
244 results.append(" : ")
245 results.append(self._html(data.get_summary()))
246 return ctx.tag[results]
248 def render_repair(self, ctx, data):
249 if data.is_healthy():
251 repair = T.form(action=".", method="post",
252 enctype="multipart/form-data")[
254 T.input(type="hidden", name="t", value="check"),
255 T.input(type="hidden", name="repair", value="true"),
256 T.input(type="submit", value="Repair"),
258 return "" # repair button disabled until we make it work correctly,
259 # see #622 for details
260 return ctx.tag[repair]
262 def render_results(self, ctx, data):
263 cr = self._render_results(ctx, data)
266 class CheckAndRepairResults(CheckerBase, rend.Page, ResultsBase):
267 docFactory = getxmlfile("check-and-repair-results.xhtml")
269 def __init__(self, client, results):
273 self.r = ICheckAndRepairResults(results)
274 rend.Page.__init__(self, results)
277 inevow.IRequest(ctx).setHeader("content-type", "text/plain")
278 data = json_check_and_repair_results(self.r)
279 return simplejson.dumps(data, indent=1) + "\n"
281 def render_summary(self, ctx, data):
282 cr = data.get_post_repair_results()
285 results.append("Healthy")
286 elif cr.is_recoverable():
287 results.append("Not Healthy!")
289 results.append("Not Recoverable!")
290 results.append(" : ")
291 results.append(self._html(cr.get_summary()))
292 return ctx.tag[results]
294 def render_repair_results(self, ctx, data):
295 if data.get_repair_attempted():
296 if data.get_repair_successful():
297 return ctx.tag["Repair successful"]
299 return ctx.tag["Repair unsuccessful"]
300 return ctx.tag["No repair necessary"]
302 def render_post_repair_results(self, ctx, data):
303 cr = self._render_results(ctx, data.get_post_repair_results())
304 return ctx.tag[T.div["Post-Repair Checker Results:"], cr]
306 def render_maybe_pre_repair_results(self, ctx, data):
307 if data.get_repair_attempted():
308 cr = self._render_results(ctx, data.get_pre_repair_results())
309 return ctx.tag[T.div["Pre-Repair Checker Results:"], cr]
313 class DeepCheckResults(rend.Page, ResultsBase, ReloadMixin):
314 docFactory = getxmlfile("deep-check-results.xhtml")
316 def __init__(self, client, monitor):
318 self.monitor = monitor
320 def childFactory(self, ctx, name):
323 # /operation/$OPHANDLE/$STORAGEINDEX provides detailed information
324 # about a specific file or directory that was checked
325 si = base32.a2b(name)
326 r = self.monitor.get_status()
328 return CheckResults(self.client,
329 r.get_results_for_storage_index(si))
331 raise WebError("No detailed results for SI %s" % html.escape(name),
334 def renderHTTP(self, ctx):
335 if self.want_json(ctx):
336 return self.json(ctx)
337 return rend.Page.renderHTTP(self, ctx)
340 inevow.IRequest(ctx).setHeader("content-type", "text/plain")
342 data["finished"] = self.monitor.is_finished()
343 res = self.monitor.get_status()
344 data["root-storage-index"] = res.get_root_storage_index_string()
345 c = res.get_counters()
346 data["count-objects-checked"] = c["count-objects-checked"]
347 data["count-objects-healthy"] = c["count-objects-healthy"]
348 data["count-objects-unhealthy"] = c["count-objects-unhealthy"]
349 data["count-corrupt-shares"] = c["count-corrupt-shares"]
350 data["list-corrupt-shares"] = [ (idlib.nodeid_b2a(serverid),
351 base32.b2a(storage_index),
353 for (serverid, storage_index, shnum)
354 in res.get_corrupt_shares() ]
355 data["list-unhealthy-files"] = [ (path_t, json_check_results(r))
357 in res.get_all_results().items()
358 if not r.is_healthy() ]
359 data["stats"] = res.get_stats()
360 return simplejson.dumps(data, indent=1) + "\n"
362 def render_root_storage_index(self, ctx, data):
363 return self.monitor.get_status().get_root_storage_index_string()
365 def data_objects_checked(self, ctx, data):
366 return self.monitor.get_status().get_counters()["count-objects-checked"]
367 def data_objects_healthy(self, ctx, data):
368 return self.monitor.get_status().get_counters()["count-objects-healthy"]
369 def data_objects_unhealthy(self, ctx, data):
370 return self.monitor.get_status().get_counters()["count-objects-unhealthy"]
371 def data_objects_unrecoverable(self, ctx, data):
372 return self.monitor.get_status().get_counters()["count-objects-unrecoverable"]
374 def data_count_corrupt_shares(self, ctx, data):
375 return self.monitor.get_status().get_counters()["count-corrupt-shares"]
377 def render_problems_p(self, ctx, data):
378 c = self.monitor.get_status().get_counters()
379 if c["count-objects-unhealthy"]:
383 def data_problems(self, ctx, data):
384 all_objects = self.monitor.get_status().get_all_results()
385 for path in sorted(all_objects.keys()):
386 cr = all_objects[path]
387 assert ICheckResults.providedBy(cr)
388 if not cr.is_healthy():
391 def render_problem(self, ctx, data):
394 summary = cr.get_summary()
396 summary_text = ": " + summary
397 summary_text += " [SI: %s]" % cr.get_storage_index_string()
398 return ctx.tag[self._join_pathstring(path), self._html(summary_text)]
401 def render_servers_with_corrupt_shares_p(self, ctx, data):
402 if self.monitor.get_status().get_counters()["count-corrupt-shares"]:
406 def data_servers_with_corrupt_shares(self, ctx, data):
408 for (serverid, storage_index, sharenum)
409 in self.monitor.get_status().get_corrupt_shares()]
413 def render_server_problem(self, ctx, data):
415 data = [idlib.shortnodeid_b2a(serverid)]
416 nickname = self.client.get_nickname_for_peerid(serverid)
418 data.append(" (%s)" % self._html(nickname))
422 def render_corrupt_shares_p(self, ctx, data):
423 if self.monitor.get_status().get_counters()["count-corrupt-shares"]:
426 def data_corrupt_shares(self, ctx, data):
427 return self.monitor.get_status().get_corrupt_shares()
428 def render_share_problem(self, ctx, data):
429 serverid, storage_index, sharenum = data
430 nickname = self.client.get_nickname_for_peerid(serverid)
431 ctx.fillSlots("serverid", idlib.shortnodeid_b2a(serverid))
433 ctx.fillSlots("nickname", self._html(nickname))
434 ctx.fillSlots("si", self._render_si_link(ctx, storage_index))
435 ctx.fillSlots("shnum", str(sharenum))
438 def render_return(self, ctx, data):
439 req = inevow.IRequest(ctx)
440 return_to = get_arg(req, "return_to", None)
442 return T.div[T.a(href=return_to)["Return to file/directory."]]
445 def data_all_objects(self, ctx, data):
446 r = self.monitor.get_status().get_all_results()
447 for path in sorted(r.keys()):
448 yield (path, r[path])
450 def render_object(self, ctx, data):
452 ctx.fillSlots("path", self._join_pathstring(path))
453 ctx.fillSlots("healthy", str(r.is_healthy()))
454 ctx.fillSlots("recoverable", str(r.is_recoverable()))
455 storage_index = r.get_storage_index()
456 ctx.fillSlots("storage_index", self._render_si_link(ctx, storage_index))
457 ctx.fillSlots("summary", self._html(r.get_summary()))
460 def render_runtime(self, ctx, data):
461 req = inevow.IRequest(ctx)
462 runtime = time.time() - req.processing_started_timestamp
463 return ctx.tag["runtime: %s seconds" % runtime]
465 class DeepCheckAndRepairResults(rend.Page, ResultsBase, ReloadMixin):
466 docFactory = getxmlfile("deep-check-and-repair-results.xhtml")
468 def __init__(self, client, monitor):
470 self.monitor = monitor
472 def childFactory(self, ctx, name):
475 # /operation/$OPHANDLE/$STORAGEINDEX provides detailed information
476 # about a specific file or directory that was checked
477 si = base32.a2b(name)
478 r = self.monitor.get_status()
480 return CheckAndRepairResults(self.client,
481 r.get_results_for_storage_index(si))
483 raise WebError("No detailed results for SI %s" % html.escape(name),
486 def renderHTTP(self, ctx):
487 if self.want_json(ctx):
488 return self.json(ctx)
489 return rend.Page.renderHTTP(self, ctx)
492 inevow.IRequest(ctx).setHeader("content-type", "text/plain")
493 res = self.monitor.get_status()
495 data["finished"] = self.monitor.is_finished()
496 data["root-storage-index"] = res.get_root_storage_index_string()
497 c = res.get_counters()
498 data["count-objects-checked"] = c["count-objects-checked"]
500 data["count-objects-healthy-pre-repair"] = c["count-objects-healthy-pre-repair"]
501 data["count-objects-unhealthy-pre-repair"] = c["count-objects-unhealthy-pre-repair"]
502 data["count-objects-healthy-post-repair"] = c["count-objects-healthy-post-repair"]
503 data["count-objects-unhealthy-post-repair"] = c["count-objects-unhealthy-post-repair"]
505 data["count-repairs-attempted"] = c["count-repairs-attempted"]
506 data["count-repairs-successful"] = c["count-repairs-successful"]
507 data["count-repairs-unsuccessful"] = c["count-repairs-unsuccessful"]
509 data["count-corrupt-shares-pre-repair"] = c["count-corrupt-shares-pre-repair"]
510 data["count-corrupt-shares-post-repair"] = c["count-corrupt-shares-pre-repair"]
512 data["list-corrupt-shares"] = [ (idlib.nodeid_b2a(serverid),
513 base32.b2a(storage_index),
515 for (serverid, storage_index, shnum)
516 in res.get_corrupt_shares() ]
518 remaining_corrupt = [ (idlib.nodeid_b2a(serverid),
519 base32.b2a(storage_index),
521 for (serverid, storage_index, shnum)
522 in res.get_remaining_corrupt_shares() ]
523 data["list-remaining-corrupt-shares"] = remaining_corrupt
525 unhealthy = [ (path_t,
526 json_check_results(crr.get_pre_repair_results()))
528 in res.get_all_results().items()
529 if not crr.get_pre_repair_results().is_healthy() ]
530 data["list-unhealthy-files"] = unhealthy
531 data["stats"] = res.get_stats()
532 return simplejson.dumps(data, indent=1) + "\n"
534 def render_root_storage_index(self, ctx, data):
535 return self.monitor.get_status().get_root_storage_index_string()
537 def data_objects_checked(self, ctx, data):
538 return self.monitor.get_status().get_counters()["count-objects-checked"]
540 def data_objects_healthy(self, ctx, data):
541 return self.monitor.get_status().get_counters()["count-objects-healthy-pre-repair"]
542 def data_objects_unhealthy(self, ctx, data):
543 return self.monitor.get_status().get_counters()["count-objects-unhealthy-pre-repair"]
544 def data_corrupt_shares(self, ctx, data):
545 return self.monitor.get_status().get_counters()["count-corrupt-shares-pre-repair"]
547 def data_repairs_attempted(self, ctx, data):
548 return self.monitor.get_status().get_counters()["count-repairs-attempted"]
549 def data_repairs_successful(self, ctx, data):
550 return self.monitor.get_status().get_counters()["count-repairs-successful"]
551 def data_repairs_unsuccessful(self, ctx, data):
552 return self.monitor.get_status().get_counters()["count-repairs-unsuccessful"]
554 def data_objects_healthy_post(self, ctx, data):
555 return self.monitor.get_status().get_counters()["count-objects-healthy-post-repair"]
556 def data_objects_unhealthy_post(self, ctx, data):
557 return self.monitor.get_status().get_counters()["count-objects-unhealthy-post-repair"]
558 def data_corrupt_shares_post(self, ctx, data):
559 return self.monitor.get_status().get_counters()["count-corrupt-shares-post-repair"]
561 def render_pre_repair_problems_p(self, ctx, data):
562 c = self.monitor.get_status().get_counters()
563 if c["count-objects-unhealthy-pre-repair"]:
567 def data_pre_repair_problems(self, ctx, data):
568 all_objects = self.monitor.get_status().get_all_results()
569 for path in sorted(all_objects.keys()):
570 r = all_objects[path]
571 assert ICheckAndRepairResults.providedBy(r)
572 cr = r.get_pre_repair_results()
573 if not cr.is_healthy():
576 def render_problem(self, ctx, data):
578 return ctx.tag[self._join_pathstring(path), ": ",
579 self._html(cr.get_summary())]
581 def render_post_repair_problems_p(self, ctx, data):
582 c = self.monitor.get_status().get_counters()
583 if (c["count-objects-unhealthy-post-repair"]
584 or c["count-corrupt-shares-post-repair"]):
588 def data_post_repair_problems(self, ctx, data):
589 all_objects = self.monitor.get_status().get_all_results()
590 for path in sorted(all_objects.keys()):
591 r = all_objects[path]
592 assert ICheckAndRepairResults.providedBy(r)
593 cr = r.get_post_repair_results()
594 if not cr.is_healthy():
597 def render_servers_with_corrupt_shares_p(self, ctx, data):
598 if self.monitor.get_status().get_counters()["count-corrupt-shares-pre-repair"]:
601 def data_servers_with_corrupt_shares(self, ctx, data):
603 def render_server_problem(self, ctx, data):
607 def render_remaining_corrupt_shares_p(self, ctx, data):
608 if self.monitor.get_status().get_counters()["count-corrupt-shares-post-repair"]:
611 def data_post_repair_corrupt_shares(self, ctx, data):
614 def render_share_problem(self, ctx, data):
618 def render_return(self, ctx, data):
619 req = inevow.IRequest(ctx)
620 return_to = get_arg(req, "return_to", None)
622 return T.div[T.a(href=return_to)["Return to file/directory."]]
625 def data_all_objects(self, ctx, data):
626 r = self.monitor.get_status().get_all_results()
627 for path in sorted(r.keys()):
628 yield (path, r[path])
630 def render_object(self, ctx, data):
632 ctx.fillSlots("path", self._join_pathstring(path))
633 ctx.fillSlots("healthy_pre_repair",
634 str(r.get_pre_repair_results().is_healthy()))
635 ctx.fillSlots("recoverable_pre_repair",
636 str(r.get_pre_repair_results().is_recoverable()))
637 ctx.fillSlots("healthy_post_repair",
638 str(r.get_post_repair_results().is_healthy()))
639 storage_index = r.get_storage_index()
640 ctx.fillSlots("storage_index",
641 self._render_si_link(ctx, storage_index))
642 ctx.fillSlots("summary",
643 self._html(r.get_pre_repair_results().get_summary()))
646 def render_runtime(self, ctx, data):
647 req = inevow.IRequest(ctx)
648 runtime = time.time() - req.processing_started_timestamp
649 return ctx.tag["runtime: %s seconds" % runtime]