4 from nevow import rend, inevow, tags as T
5 from twisted.web import http, html
6 from allmydata.web.common import getxmlfile, get_arg, get_root, WebError
7 from allmydata.web.operations import ReloadMixin
8 from allmydata.interfaces import ICheckAndRepairResults, ICheckResults
9 from allmydata.util import base32, dictutil
11 def json_check_counts(r):
12 d = {"count-shares-good": r.get_share_counter_good(),
13 "count-shares-needed": r.get_encoding_needed(),
14 "count-shares-expected": r.get_encoding_expected(),
15 "count-good-share-hosts": r.get_host_counter_good_shares(),
16 "count-corrupt-shares": len(r.get_corrupt_shares()),
17 "list-corrupt-shares": [ (s.get_longname(), base32.b2a(si), shnum)
19 in r.get_corrupt_shares() ],
20 "servers-responding": [s.get_longname()
21 for s in r.get_servers_responding()],
22 "sharemap": dict([(shareid,
23 sorted([s.get_longname() for s in servers]))
24 for (shareid, servers)
25 in r.get_sharemap().items()]),
26 "count-wrong-shares": r.get_share_counter_wrong(),
27 "count-recoverable-versions": r.get_version_counter_recoverable(),
28 "count-unrecoverable-versions": r.get_version_counter_unrecoverable(),
32 def json_check_results(r):
35 data = {"storage-index": "",
36 "results": {"healthy": True},
40 data["storage-index"] = r.get_storage_index_string()
41 data["summary"] = r.get_summary()
42 data["results"] = json_check_counts(r)
43 data["results"]["needs-rebalancing"] = r.needs_rebalancing()
44 data["results"]["healthy"] = r.is_healthy()
45 data["results"]["recoverable"] = r.is_recoverable()
48 def json_check_and_repair_results(r):
51 data = {"storage-index": "",
52 "repair-attempted": False,
56 data["storage-index"] = r.get_storage_index_string()
57 data["repair-attempted"] = r.get_repair_attempted()
58 data["repair-successful"] = r.get_repair_successful()
59 pre = r.get_pre_repair_results()
60 data["pre-repair-results"] = json_check_results(pre)
61 post = r.get_post_repair_results()
62 data["post-repair-results"] = json_check_results(post)
66 # self.client must point to the Client, so we can get nicknames and
67 # determine the permuted peer order
69 def _join_pathstring(self, path):
71 pathstring = "/".join(self._html(path))
76 def _render_results(self, ctx, cr):
77 assert ICheckResults(cr)
79 sb = c.get_storage_broker()
82 r.append(T.li[name + ": ", value])
84 add("Report", T.pre["\n".join(self._html(cr.get_report()))])
86 "need %d-of-%d, have %d" % (cr.get_encoding_needed(),
87 cr.get_encoding_expected(),
88 cr.get_share_counter_good()))
89 add("Hosts with good shares", cr.get_host_counter_good_shares())
91 if cr.get_corrupt_shares():
93 for (s, si, shnum) in cr.get_corrupt_shares():
94 d = T.tr[T.td["sh#%d" % shnum],
95 T.td[T.div(class_="nickname")[s.get_nickname()],
96 T.div(class_="nodeid")[T.tt[s.get_name()]]],
99 add("Corrupt shares", T.table()[
100 T.tr[T.th["Share ID"],
101 T.th(class_="nickname-and-peerid")[T.div["Nickname"], T.div(class_="nodeid")["Node ID"]]],
104 add("Corrupt shares", "none")
106 add("Wrong Shares", cr.get_share_counter_wrong())
109 shares_on_server = dictutil.DictOfSets()
111 # FIXME: The two tables below contain nickname-and-nodeid table column markup which is duplicated with each other, introducer.xhtml, and deep-check-results.xhtml. All of these (and any other presentations of nickname-and-nodeid) should be combined.
113 for shareid in sorted(cr.get_sharemap().keys()):
114 servers = sorted(cr.get_sharemap()[shareid],
115 key=lambda s: s.get_longname())
116 for i,s in enumerate(servers):
117 shares_on_server.add(s, shareid)
121 d = T.tr[T.td[shareid_s],
122 T.td[T.div(class_="nickname")[s.get_nickname()],
123 T.div(class_="nodeid")[T.tt[s.get_name()]]]
125 sharemap_data.append(d)
126 add("Good Shares (sorted in share order)",
127 T.table()[T.tr[T.th["Share ID"], T.th(class_="nickname-and-peerid")[T.div["Nickname"], T.div(class_="nodeid")["Node ID"]]],
131 add("Recoverable Versions", cr.get_version_counter_recoverable())
132 add("Unrecoverable Versions", cr.get_version_counter_unrecoverable())
134 # this table is sorted by permuted order
135 permuted_servers = [s
137 in sb.get_servers_for_psi(cr.get_storage_index())]
139 num_shares_left = sum([len(shareids)
140 for shareids in shares_on_server.values()])
142 for s in permuted_servers:
143 shareids = list(shares_on_server.get(s, []))
145 shareids_s = [ T.tt[shareid, " "] for shareid in sorted(shareids) ]
146 d = T.tr[T.td[T.div(class_="nickname")[s.get_nickname()],
147 T.div(class_="nodeid")[T.tt[s.get_name()]]],
151 num_shares_left -= len(shareids)
152 if not num_shares_left:
154 add("Share Balancing (servers in permuted order)",
155 T.table()[T.tr[T.th(class_="nickname-and-peerid")[T.div["Nickname"], T.div(class_="nodeid")["Node ID"]], T.th["Share IDs"]],
161 if isinstance(s, (str, unicode)):
162 return html.escape(s)
163 assert isinstance(s, (list, tuple))
164 return [html.escape(w) for w in s]
166 def want_json(self, ctx):
167 output = get_arg(inevow.IRequest(ctx), "output", "").lower()
168 if output.lower() == "json":
172 def _render_si_link(self, ctx, storage_index):
173 si_s = base32.b2a(storage_index)
174 req = inevow.IRequest(ctx)
175 ophandle = req.prepath[-1]
176 target = "%s/operations/%s/%s" % (get_root(ctx), ophandle, si_s)
177 output = get_arg(ctx, "output")
179 target = target + "?output=%s" % output
180 return T.a(href=target)[si_s]
182 class LiteralCheckResultsRenderer(rend.Page, ResultsBase):
183 docFactory = getxmlfile("literal-check-results.xhtml")
185 def __init__(self, client):
187 rend.Page.__init__(self, client)
189 def renderHTTP(self, ctx):
190 if self.want_json(ctx):
191 return self.json(ctx)
192 return rend.Page.renderHTTP(self, ctx)
195 inevow.IRequest(ctx).setHeader("content-type", "text/plain")
196 data = json_check_results(None)
197 return simplejson.dumps(data, indent=1) + "\n"
199 def render_return(self, ctx, data):
200 req = inevow.IRequest(ctx)
201 return_to = get_arg(req, "return_to", None)
203 return T.div[T.a(href=return_to)["Return to file."]]
208 def renderHTTP(self, ctx):
209 if self.want_json(ctx):
210 return self.json(ctx)
211 return rend.Page.renderHTTP(self, ctx)
213 def render_storage_index(self, ctx, data):
214 return self.r.get_storage_index_string()
216 def render_return(self, ctx, data):
217 req = inevow.IRequest(ctx)
218 return_to = get_arg(req, "return_to", None)
220 return T.div[T.a(href=return_to)["Return to file/directory."]]
223 class CheckResultsRenderer(CheckerBase, rend.Page, ResultsBase):
224 docFactory = getxmlfile("check-results.xhtml")
226 def __init__(self, client, results):
228 self.r = ICheckResults(results)
229 rend.Page.__init__(self, results)
232 inevow.IRequest(ctx).setHeader("content-type", "text/plain")
233 data = json_check_results(self.r)
234 return simplejson.dumps(data, indent=1) + "\n"
236 def render_summary(self, ctx, data):
238 if data.is_healthy():
239 results.append("Healthy")
240 elif data.is_recoverable():
241 results.append("Not Healthy!")
243 results.append("Not Recoverable!")
244 results.append(" : ")
245 results.append(self._html(data.get_summary()))
246 return ctx.tag[results]
248 def render_repair(self, ctx, data):
249 if data.is_healthy():
251 repair = T.form(action=".", method="post",
252 enctype="multipart/form-data")[
254 T.input(type="hidden", name="t", value="check"),
255 T.input(type="hidden", name="repair", value="true"),
256 T.input(type="submit", value="Repair"),
258 return "" # repair button disabled until we make it work correctly,
259 # see #622 for details
260 return ctx.tag[repair]
262 def render_results(self, ctx, data):
263 cr = self._render_results(ctx, data)
266 class CheckAndRepairResultsRenderer(CheckerBase, rend.Page, ResultsBase):
267 docFactory = getxmlfile("check-and-repair-results.xhtml")
269 def __init__(self, client, results):
273 self.r = ICheckAndRepairResults(results)
274 rend.Page.__init__(self, results)
277 inevow.IRequest(ctx).setHeader("content-type", "text/plain")
278 data = json_check_and_repair_results(self.r)
279 return simplejson.dumps(data, indent=1) + "\n"
281 def render_summary(self, ctx, data):
282 cr = data.get_post_repair_results()
285 results.append("Healthy")
286 elif cr.is_recoverable():
287 results.append("Not Healthy!")
289 results.append("Not Recoverable!")
290 results.append(" : ")
291 results.append(self._html(cr.get_summary()))
292 return ctx.tag[results]
294 def render_repair_results(self, ctx, data):
295 if data.get_repair_attempted():
296 if data.get_repair_successful():
297 return ctx.tag["Repair successful"]
299 return ctx.tag["Repair unsuccessful"]
300 return ctx.tag["No repair necessary"]
302 def render_post_repair_results(self, ctx, data):
303 cr = self._render_results(ctx, data.get_post_repair_results())
304 return ctx.tag[T.div["Post-Repair Checker Results:"], cr]
306 def render_maybe_pre_repair_results(self, ctx, data):
307 if data.get_repair_attempted():
308 cr = self._render_results(ctx, data.get_pre_repair_results())
309 return ctx.tag[T.div["Pre-Repair Checker Results:"], cr]
313 class DeepCheckResultsRenderer(rend.Page, ResultsBase, ReloadMixin):
314 docFactory = getxmlfile("deep-check-results.xhtml")
316 def __init__(self, client, monitor):
318 self.monitor = monitor
320 def childFactory(self, ctx, name):
323 # /operation/$OPHANDLE/$STORAGEINDEX provides detailed information
324 # about a specific file or directory that was checked
325 si = base32.a2b(name)
326 r = self.monitor.get_status()
328 return CheckResultsRenderer(self.client,
329 r.get_results_for_storage_index(si))
331 raise WebError("No detailed results for SI %s" % html.escape(name),
334 def renderHTTP(self, ctx):
335 if self.want_json(ctx):
336 return self.json(ctx)
337 return rend.Page.renderHTTP(self, ctx)
340 inevow.IRequest(ctx).setHeader("content-type", "text/plain")
342 data["finished"] = self.monitor.is_finished()
343 res = self.monitor.get_status()
344 data["root-storage-index"] = res.get_root_storage_index_string()
345 c = res.get_counters()
346 data["count-objects-checked"] = c["count-objects-checked"]
347 data["count-objects-healthy"] = c["count-objects-healthy"]
348 data["count-objects-unhealthy"] = c["count-objects-unhealthy"]
349 data["count-corrupt-shares"] = c["count-corrupt-shares"]
350 data["list-corrupt-shares"] = [ (s.get_longname(),
351 base32.b2a(storage_index),
353 for (s, storage_index, shnum)
354 in res.get_corrupt_shares() ]
355 data["list-unhealthy-files"] = [ (path_t, json_check_results(r))
357 in res.get_all_results().items()
358 if not r.is_healthy() ]
359 data["stats"] = res.get_stats()
360 return simplejson.dumps(data, indent=1) + "\n"
362 def render_root_storage_index(self, ctx, data):
363 return self.monitor.get_status().get_root_storage_index_string()
365 def data_objects_checked(self, ctx, data):
366 return self.monitor.get_status().get_counters()["count-objects-checked"]
367 def data_objects_healthy(self, ctx, data):
368 return self.monitor.get_status().get_counters()["count-objects-healthy"]
369 def data_objects_unhealthy(self, ctx, data):
370 return self.monitor.get_status().get_counters()["count-objects-unhealthy"]
371 def data_objects_unrecoverable(self, ctx, data):
372 return self.monitor.get_status().get_counters()["count-objects-unrecoverable"]
374 def data_count_corrupt_shares(self, ctx, data):
375 return self.monitor.get_status().get_counters()["count-corrupt-shares"]
377 def render_problems_p(self, ctx, data):
378 c = self.monitor.get_status().get_counters()
379 if c["count-objects-unhealthy"]:
383 def data_problems(self, ctx, data):
384 all_objects = self.monitor.get_status().get_all_results()
385 for path in sorted(all_objects.keys()):
386 cr = all_objects[path]
387 assert ICheckResults.providedBy(cr)
388 if not cr.is_healthy():
391 def render_problem(self, ctx, data):
394 summary = cr.get_summary()
396 summary_text = ": " + summary
397 summary_text += " [SI: %s]" % cr.get_storage_index_string()
398 return ctx.tag[self._join_pathstring(path), self._html(summary_text)]
401 def render_servers_with_corrupt_shares_p(self, ctx, data):
402 if self.monitor.get_status().get_counters()["count-corrupt-shares"]:
406 def data_servers_with_corrupt_shares(self, ctx, data):
408 for (s, storage_index, sharenum)
409 in self.monitor.get_status().get_corrupt_shares()]
410 servers.sort(key=lambda s: s.get_longname())
413 def render_server_problem(self, ctx, server):
414 data = [server.get_name()]
415 nickname = server.get_nickname()
417 data.append(" (%s)" % self._html(nickname))
421 def render_corrupt_shares_p(self, ctx, data):
422 if self.monitor.get_status().get_counters()["count-corrupt-shares"]:
425 def data_corrupt_shares(self, ctx, data):
426 return self.monitor.get_status().get_corrupt_shares()
427 def render_share_problem(self, ctx, data):
428 server, storage_index, sharenum = data
429 nickname = server.get_nickname()
430 ctx.fillSlots("serverid", server.get_name())
432 ctx.fillSlots("nickname", self._html(nickname))
433 ctx.fillSlots("si", self._render_si_link(ctx, storage_index))
434 ctx.fillSlots("shnum", str(sharenum))
437 def render_return(self, ctx, data):
438 req = inevow.IRequest(ctx)
439 return_to = get_arg(req, "return_to", None)
441 return T.div[T.a(href=return_to)["Return to file/directory."]]
444 def data_all_objects(self, ctx, data):
445 r = self.monitor.get_status().get_all_results()
446 for path in sorted(r.keys()):
447 yield (path, r[path])
449 def render_object(self, ctx, data):
451 ctx.fillSlots("path", self._join_pathstring(path))
452 ctx.fillSlots("healthy", str(r.is_healthy()))
453 ctx.fillSlots("recoverable", str(r.is_recoverable()))
454 storage_index = r.get_storage_index()
455 ctx.fillSlots("storage_index", self._render_si_link(ctx, storage_index))
456 ctx.fillSlots("summary", self._html(r.get_summary()))
459 def render_runtime(self, ctx, data):
460 req = inevow.IRequest(ctx)
461 runtime = time.time() - req.processing_started_timestamp
462 return ctx.tag["runtime: %s seconds" % runtime]
464 class DeepCheckAndRepairResultsRenderer(rend.Page, ResultsBase, ReloadMixin):
465 docFactory = getxmlfile("deep-check-and-repair-results.xhtml")
467 def __init__(self, client, monitor):
469 self.monitor = monitor
471 def childFactory(self, ctx, name):
474 # /operation/$OPHANDLE/$STORAGEINDEX provides detailed information
475 # about a specific file or directory that was checked
476 si = base32.a2b(name)
477 s = self.monitor.get_status()
479 results = s.get_results_for_storage_index(si)
480 return CheckAndRepairResultsRenderer(self.client, results)
482 raise WebError("No detailed results for SI %s" % html.escape(name),
485 def renderHTTP(self, ctx):
486 if self.want_json(ctx):
487 return self.json(ctx)
488 return rend.Page.renderHTTP(self, ctx)
491 inevow.IRequest(ctx).setHeader("content-type", "text/plain")
492 res = self.monitor.get_status()
494 data["finished"] = self.monitor.is_finished()
495 data["root-storage-index"] = res.get_root_storage_index_string()
496 c = res.get_counters()
497 data["count-objects-checked"] = c["count-objects-checked"]
499 data["count-objects-healthy-pre-repair"] = c["count-objects-healthy-pre-repair"]
500 data["count-objects-unhealthy-pre-repair"] = c["count-objects-unhealthy-pre-repair"]
501 data["count-objects-healthy-post-repair"] = c["count-objects-healthy-post-repair"]
502 data["count-objects-unhealthy-post-repair"] = c["count-objects-unhealthy-post-repair"]
504 data["count-repairs-attempted"] = c["count-repairs-attempted"]
505 data["count-repairs-successful"] = c["count-repairs-successful"]
506 data["count-repairs-unsuccessful"] = c["count-repairs-unsuccessful"]
508 data["count-corrupt-shares-pre-repair"] = c["count-corrupt-shares-pre-repair"]
509 data["count-corrupt-shares-post-repair"] = c["count-corrupt-shares-pre-repair"]
511 data["list-corrupt-shares"] = [ (s.get_longname(),
512 base32.b2a(storage_index),
514 for (s, storage_index, shnum)
515 in res.get_corrupt_shares() ]
517 remaining_corrupt = [ (s.get_longname(), base32.b2a(storage_index),
519 for (s, storage_index, shnum)
520 in res.get_remaining_corrupt_shares() ]
521 data["list-remaining-corrupt-shares"] = remaining_corrupt
523 unhealthy = [ (path_t,
524 json_check_results(crr.get_pre_repair_results()))
526 in res.get_all_results().items()
527 if not crr.get_pre_repair_results().is_healthy() ]
528 data["list-unhealthy-files"] = unhealthy
529 data["stats"] = res.get_stats()
530 return simplejson.dumps(data, indent=1) + "\n"
532 def render_root_storage_index(self, ctx, data):
533 return self.monitor.get_status().get_root_storage_index_string()
535 def data_objects_checked(self, ctx, data):
536 return self.monitor.get_status().get_counters()["count-objects-checked"]
538 def data_objects_healthy(self, ctx, data):
539 return self.monitor.get_status().get_counters()["count-objects-healthy-pre-repair"]
540 def data_objects_unhealthy(self, ctx, data):
541 return self.monitor.get_status().get_counters()["count-objects-unhealthy-pre-repair"]
542 def data_corrupt_shares(self, ctx, data):
543 return self.monitor.get_status().get_counters()["count-corrupt-shares-pre-repair"]
545 def data_repairs_attempted(self, ctx, data):
546 return self.monitor.get_status().get_counters()["count-repairs-attempted"]
547 def data_repairs_successful(self, ctx, data):
548 return self.monitor.get_status().get_counters()["count-repairs-successful"]
549 def data_repairs_unsuccessful(self, ctx, data):
550 return self.monitor.get_status().get_counters()["count-repairs-unsuccessful"]
552 def data_objects_healthy_post(self, ctx, data):
553 return self.monitor.get_status().get_counters()["count-objects-healthy-post-repair"]
554 def data_objects_unhealthy_post(self, ctx, data):
555 return self.monitor.get_status().get_counters()["count-objects-unhealthy-post-repair"]
556 def data_corrupt_shares_post(self, ctx, data):
557 return self.monitor.get_status().get_counters()["count-corrupt-shares-post-repair"]
559 def render_pre_repair_problems_p(self, ctx, data):
560 c = self.monitor.get_status().get_counters()
561 if c["count-objects-unhealthy-pre-repair"]:
565 def data_pre_repair_problems(self, ctx, data):
566 all_objects = self.monitor.get_status().get_all_results()
567 for path in sorted(all_objects.keys()):
568 r = all_objects[path]
569 assert ICheckAndRepairResults.providedBy(r)
570 cr = r.get_pre_repair_results()
571 if not cr.is_healthy():
574 def render_problem(self, ctx, data):
576 return ctx.tag[self._join_pathstring(path), ": ",
577 self._html(cr.get_summary())]
579 def render_post_repair_problems_p(self, ctx, data):
580 c = self.monitor.get_status().get_counters()
581 if (c["count-objects-unhealthy-post-repair"]
582 or c["count-corrupt-shares-post-repair"]):
586 def data_post_repair_problems(self, ctx, data):
587 all_objects = self.monitor.get_status().get_all_results()
588 for path in sorted(all_objects.keys()):
589 r = all_objects[path]
590 assert ICheckAndRepairResults.providedBy(r)
591 cr = r.get_post_repair_results()
592 if not cr.is_healthy():
595 def render_servers_with_corrupt_shares_p(self, ctx, data):
596 if self.monitor.get_status().get_counters()["count-corrupt-shares-pre-repair"]:
599 def data_servers_with_corrupt_shares(self, ctx, data):
601 def render_server_problem(self, ctx, data):
605 def render_remaining_corrupt_shares_p(self, ctx, data):
606 if self.monitor.get_status().get_counters()["count-corrupt-shares-post-repair"]:
609 def data_post_repair_corrupt_shares(self, ctx, data):
612 def render_share_problem(self, ctx, data):
616 def render_return(self, ctx, data):
617 req = inevow.IRequest(ctx)
618 return_to = get_arg(req, "return_to", None)
620 return T.div[T.a(href=return_to)["Return to file/directory."]]
623 def data_all_objects(self, ctx, data):
624 r = self.monitor.get_status().get_all_results()
625 for path in sorted(r.keys()):
626 yield (path, r[path])
628 def render_object(self, ctx, data):
630 ctx.fillSlots("path", self._join_pathstring(path))
631 ctx.fillSlots("healthy_pre_repair",
632 str(r.get_pre_repair_results().is_healthy()))
633 ctx.fillSlots("recoverable_pre_repair",
634 str(r.get_pre_repair_results().is_recoverable()))
635 ctx.fillSlots("healthy_post_repair",
636 str(r.get_post_repair_results().is_healthy()))
637 storage_index = r.get_storage_index()
638 ctx.fillSlots("storage_index",
639 self._render_si_link(ctx, storage_index))
640 ctx.fillSlots("summary",
641 self._html(r.get_pre_repair_results().get_summary()))
644 def render_runtime(self, ctx, data):
645 req = inevow.IRequest(ctx)
646 runtime = time.time() - req.processing_started_timestamp
647 return ctx.tag["runtime: %s seconds" % runtime]