]> git.rkrishnan.org Git - tahoe-lafs/tahoe-lafs.git/blob - src/allmydata/web/check_results.py
rename "checker results" to "check results", because it is more parallel to "check...
[tahoe-lafs/tahoe-lafs.git] / src / allmydata / web / check_results.py
1
2 import time
3 import simplejson
4 from nevow import rend, inevow, tags as T
5 from twisted.web import http, html
6 from allmydata.web.common import getxmlfile, get_arg, get_root, \
7      IClient, WebError
8 from allmydata.web.operations import ReloadMixin
9 from allmydata.interfaces import ICheckAndRepairResults, ICheckResults
10 from allmydata.util import base32, idlib
11
12 class ResultsBase:
13     def _render_results(self, ctx, cr):
14         assert ICheckResults(cr)
15         c = IClient(ctx)
16         data = cr.get_data()
17         r = []
18         def add(name, value):
19             r.append(T.li[name + ": ", value])
20
21         add("Report", T.pre["\n".join(self._html(cr.get_report()))])
22         add("Share Counts",
23             "need %d-of-%d, have %d" % (data["count-shares-needed"],
24                                         data["count-shares-expected"],
25                                         data["count-shares-good"]))
26         add("Hosts with good shares", data["count-good-share-hosts"])
27
28         if data["list-corrupt-shares"]:
29             badsharemap = []
30             for (serverid, si, shnum) in data["list-corrupt-shares"]:
31                 nickname = c.get_nickname_for_peerid(serverid)
32                 badsharemap.append(T.tr[T.td["sh#%d" % shnum],
33                                         T.td[T.tt[base32.b2a(serverid)],
34                                              " (", nickname, ")"],
35                                         ])
36             add("Corrupt shares", T.table(border="1")[badsharemap])
37         else:
38             add("Corrupt shares", "none")
39
40         add("Wrong Shares", data["count-wrong-shares"])
41
42         sharemap = []
43         servers = {}
44
45         for shareid in sorted(data["sharemap"].keys()):
46             serverids = data["sharemap"][shareid]
47             for i,serverid in enumerate(serverids):
48                 if serverid not in servers:
49                     servers[serverid] = []
50                 servers[serverid].append(shareid)
51                 shareid_s = ""
52                 if i == 0:
53                     shareid_s = shareid
54                 nickname = c.get_nickname_for_peerid(serverid)
55                 sharemap.append(T.tr[T.td[shareid_s],
56                                      T.td[T.tt[base32.b2a(serverid)],
57                                           " (", nickname, ")"],
58                                      ])
59         add("Good Shares (sorted in share order)",
60             T.table(border="1")[sharemap])
61
62
63         add("Recoverable Versions", data["count-recoverable-versions"])
64         add("Unrecoverable Versions", data["count-unrecoverable-versions"])
65
66         # this table is sorted by permuted order
67         permuted_peer_ids = [peerid
68                              for (peerid, rref)
69                              in c.get_permuted_peers("storage",
70                                                      cr.get_storage_index())]
71
72         num_shares_left = sum([len(shares) for shares in servers.values()])
73         servermap = []
74         for serverid in permuted_peer_ids:
75             nickname = c.get_nickname_for_peerid(serverid)
76             shareids = servers.get(serverid, [])
77             shareids.reverse()
78             shareids_s = [ T.tt[shareid, " "] for shareid in shareids ]
79             servermap.append(T.tr[T.td[T.tt[base32.b2a(serverid)],
80                                        " (", nickname, ")"],
81                                   T.td[shareids_s] ])
82             num_shares_left -= len(shareids)
83             if not num_shares_left:
84                 break
85         add("Share Balancing (servers in permuted order)",
86             T.table(border="1")[servermap])
87
88         return T.ul[r]
89
90     def _json_check_and_repair_results(self, r):
91         data = {}
92         data["storage-index"] = r.get_storage_index_string()
93         data["repair-attempted"] = r.get_repair_attempted()
94         data["repair-successful"] = r.get_repair_successful()
95         pre = r.get_pre_repair_results()
96         data["pre-repair-results"] = self._json_check_results(pre)
97         post = r.get_post_repair_results()
98         data["post-repair-results"] = self._json_check_results(post)
99         return data
100
101     def _json_check_results(self, r):
102         data = {}
103         data["storage-index"] = r.get_storage_index_string()
104         data["summary"] = r.get_summary()
105         data["results"] = self._json_check_counts(r.get_data())
106         data["results"]["needs-rebalancing"] = r.needs_rebalancing()
107         data["results"]["healthy"] = r.is_healthy()
108         data["results"]["recoverable"] = r.is_recoverable()
109         return data
110
111     def _json_check_counts(self, d):
112         r = {}
113         r["count-shares-good"] = d["count-shares-good"]
114         r["count-shares-needed"] = d["count-shares-needed"]
115         r["count-shares-expected"] = d["count-shares-expected"]
116         r["count-good-share-hosts"] = d["count-good-share-hosts"]
117         r["count-corrupt-shares"] = d["count-corrupt-shares"]
118         r["list-corrupt-shares"] = [ (idlib.nodeid_b2a(serverid),
119                                       base32.b2a(si), shnum)
120                                      for (serverid, si, shnum)
121                                      in d["list-corrupt-shares"] ]
122         r["servers-responding"] = [idlib.nodeid_b2a(serverid)
123                                    for serverid in d["servers-responding"]]
124         sharemap = {}
125         for (shareid, serverids) in d["sharemap"].items():
126             sharemap[shareid] = [idlib.nodeid_b2a(serverid)
127                                  for serverid in serverids]
128         r["sharemap"] = sharemap
129
130         r["count-wrong-shares"] = d["count-wrong-shares"]
131         r["count-recoverable-versions"] = d["count-recoverable-versions"]
132         r["count-unrecoverable-versions"] = d["count-unrecoverable-versions"]
133
134         return r
135
136     def _html(self, s):
137         if isinstance(s, (str, unicode)):
138             return html.escape(s)
139         assert isinstance(s, (list, tuple))
140         return [html.escape(w) for w in s]
141
142     def want_json(self, ctx):
143         output = get_arg(inevow.IRequest(ctx), "output", "").lower()
144         if output.lower() == "json":
145             return True
146         return False
147
148     def _render_si_link(self, ctx, storage_index):
149         si_s = base32.b2a(storage_index)
150         root = get_root(ctx)
151         req = inevow.IRequest(ctx)
152         ophandle = req.prepath[-1]
153         target = "%s/operations/%s/%s" % (get_root(ctx), ophandle, si_s)
154         output = get_arg(ctx, "output")
155         if output:
156             target = target + "?output=%s" % output
157         return T.a(href=target)[si_s]
158
159 class LiteralCheckerResults(rend.Page, ResultsBase):
160     docFactory = getxmlfile("literal-check-results.xhtml")
161
162     def renderHTTP(self, ctx):
163         if self.want_json(ctx):
164             return self.json(ctx)
165         return rend.Page.renderHTTP(self, ctx)
166
167     def json(self, ctx):
168         inevow.IRequest(ctx).setHeader("content-type", "text/plain")
169         data = {"storage-index": "",
170                 "results": {"healthy": True},
171                 }
172         return simplejson.dumps(data, indent=1) + "\n"
173
174 class CheckerBase:
175
176     def renderHTTP(self, ctx):
177         if self.want_json(ctx):
178             return self.json(ctx)
179         return rend.Page.renderHTTP(self, ctx)
180
181     def render_storage_index(self, ctx, data):
182         return self.r.get_storage_index_string()
183
184     def render_return(self, ctx, data):
185         req = inevow.IRequest(ctx)
186         return_to = get_arg(req, "return_to", None)
187         if return_to:
188             return T.div[T.a(href=return_to)["Return to parent directory"]]
189         return ""
190
191 class CheckerResults(CheckerBase, rend.Page, ResultsBase):
192     docFactory = getxmlfile("check-results.xhtml")
193
194     def __init__(self, results):
195         self.r = ICheckResults(results)
196
197     def json(self, ctx):
198         inevow.IRequest(ctx).setHeader("content-type", "text/plain")
199         data = self._json_check_results(self.r)
200         return simplejson.dumps(data, indent=1) + "\n"
201
202     def render_summary(self, ctx, data):
203         results = []
204         if self.r.is_healthy():
205             results.append("Healthy")
206         elif self.r.is_recoverable():
207             results.append("Not Healthy!")
208         else:
209             results.append("Not Recoverable!")
210         results.append(" : ")
211         results.append(self._html(self.r.get_summary()))
212         return ctx.tag[results]
213
214     def render_repair(self, ctx, data):
215         if self.r.is_healthy():
216             return ""
217         repair = T.form(action=".", method="post",
218                         enctype="multipart/form-data")[
219             T.fieldset[
220             T.input(type="hidden", name="t", value="check"),
221             T.input(type="hidden", name="repair", value="true"),
222             T.input(type="submit", value="Repair"),
223             ]]
224         return ctx.tag[repair]
225
226     def render_rebalance(self, ctx, data):
227         if self.r.needs_rebalancing():
228             return ctx.tag["(needs rebalancing)"]
229         return ctx.tag["(does not need rebalancing)"]
230
231     def render_results(self, ctx, data):
232         cr = self._render_results(ctx, self.r)
233         return ctx.tag[cr]
234
235 class CheckAndRepairResults(CheckerBase, rend.Page, ResultsBase):
236     docFactory = getxmlfile("check-and-repair-results.xhtml")
237
238     def __init__(self, results):
239         self.r = ICheckAndRepairResults(results)
240
241     def json(self, ctx):
242         inevow.IRequest(ctx).setHeader("content-type", "text/plain")
243         data = self._json_check_and_repair_results(self.r)
244         return simplejson.dumps(data, indent=1) + "\n"
245
246     def render_summary(self, ctx, data):
247         cr = self.r.get_post_repair_results()
248         results = []
249         if cr.is_healthy():
250             results.append("Healthy")
251         elif cr.is_recoverable():
252             results.append("Not Healthy!")
253         else:
254             results.append("Not Recoverable!")
255         results.append(" : ")
256         results.append(self._html(cr.get_summary()))
257         return ctx.tag[results]
258
259     def render_repair_results(self, ctx, data):
260         if self.r.get_repair_attempted():
261             if self.r.get_repair_successful():
262                 return ctx.tag["Repair successful"]
263             else:
264                 return ctx.tag["Repair unsuccessful"]
265         return ctx.tag["No repair necessary"]
266
267     def render_post_repair_results(self, ctx, data):
268         cr = self._render_results(ctx, self.r.get_post_repair_results())
269         return ctx.tag[cr]
270
271     def render_maybe_pre_repair_results(self, ctx, data):
272         if self.r.get_repair_attempted():
273             cr = self._render_results(ctx, self.r.get_pre_repair_results())
274             return ctx.tag[T.div["Pre-Repair Checker Results:"], cr]
275         return ""
276
277
278 class DeepCheckResults(rend.Page, ResultsBase, ReloadMixin):
279     docFactory = getxmlfile("deep-check-results.xhtml")
280
281     def __init__(self, monitor):
282         self.monitor = monitor
283
284     def childFactory(self, ctx, name):
285         if not name:
286             return self
287         # /operation/$OPHANDLE/$STORAGEINDEX provides detailed information
288         # about a specific file or directory that was checked
289         si = base32.a2b(name)
290         r = self.monitor.get_status()
291         try:
292             return CheckerResults(r.get_results_for_storage_index(si))
293         except KeyError:
294             raise WebError("No detailed results for SI %s" % html.escape(name),
295                            http.NOT_FOUND)
296
297     def renderHTTP(self, ctx):
298         if self.want_json(ctx):
299             return self.json(ctx)
300         return rend.Page.renderHTTP(self, ctx)
301
302     def json(self, ctx):
303         inevow.IRequest(ctx).setHeader("content-type", "text/plain")
304         data = {}
305         data["finished"] = self.monitor.is_finished()
306         res = self.monitor.get_status()
307         data["root-storage-index"] = res.get_root_storage_index_string()
308         c = res.get_counters()
309         data["count-objects-checked"] = c["count-objects-checked"]
310         data["count-objects-healthy"] = c["count-objects-healthy"]
311         data["count-objects-unhealthy"] = c["count-objects-unhealthy"]
312         data["count-corrupt-shares"] = c["count-corrupt-shares"]
313         data["list-corrupt-shares"] = [ (idlib.nodeid_b2a(serverid),
314                                          base32.b2a(storage_index),
315                                          shnum)
316                                         for (serverid, storage_index, shnum)
317                                         in res.get_corrupt_shares() ]
318         data["list-unhealthy-files"] = [ (path_t, self._json_check_results(r))
319                                          for (path_t, r)
320                                          in res.get_all_results().items()
321                                          if not r.is_healthy() ]
322         data["stats"] = res.get_stats()
323         return simplejson.dumps(data, indent=1) + "\n"
324
325     def render_root_storage_index(self, ctx, data):
326         return self.monitor.get_status().get_root_storage_index_string()
327
328     def data_objects_checked(self, ctx, data):
329         return self.monitor.get_status().get_counters()["count-objects-checked"]
330     def data_objects_healthy(self, ctx, data):
331         return self.monitor.get_status().get_counters()["count-objects-healthy"]
332     def data_objects_unhealthy(self, ctx, data):
333         return self.monitor.get_status().get_counters()["count-objects-unhealthy"]
334     def data_objects_unrecoverable(self, ctx, data):
335         return self.monitor.get_status().get_counters()["count-objects-unrecoverable"]
336
337     def data_count_corrupt_shares(self, ctx, data):
338         return self.monitor.get_status().get_counters()["count-corrupt-shares"]
339
340     def render_problems_p(self, ctx, data):
341         c = self.monitor.get_status().get_counters()
342         if c["count-objects-unhealthy"]:
343             return ctx.tag
344         return ""
345
346     def data_problems(self, ctx, data):
347         all_objects = self.monitor.get_status().get_all_results()
348         for path in sorted(all_objects.keys()):
349             cr = all_objects[path]
350             assert ICheckResults.providedBy(cr)
351             if not cr.is_healthy():
352                 yield path, cr
353
354     def render_problem(self, ctx, data):
355         path, cr = data
356         summary_text = ""
357         summary = cr.get_summary()
358         if summary:
359             summary_text = ": " + summary
360         summary_text += " [SI: %s]" % cr.get_storage_index_string()
361         return ctx.tag["/".join(self._html(path)), self._html(summary_text)]
362
363
364     def render_servers_with_corrupt_shares_p(self, ctx, data):
365         if self.monitor.get_status().get_counters()["count-corrupt-shares"]:
366             return ctx.tag
367         return ""
368
369     def data_servers_with_corrupt_shares(self, ctx, data):
370         servers = [serverid
371                    for (serverid, storage_index, sharenum)
372                    in self.monitor.get_status().get_corrupt_shares()]
373         servers.sort()
374         return servers
375
376     def render_server_problem(self, ctx, data):
377         serverid = data
378         data = [idlib.shortnodeid_b2a(serverid)]
379         c = IClient(ctx)
380         nickname = c.get_nickname_for_peerid(serverid)
381         if nickname:
382             data.append(" (%s)" % self._html(nickname))
383         return ctx.tag[data]
384
385
386     def render_corrupt_shares_p(self, ctx, data):
387         if self.monitor.get_status().get_counters()["count-corrupt-shares"]:
388             return ctx.tag
389         return ""
390     def data_corrupt_shares(self, ctx, data):
391         return self.monitor.get_status().get_corrupt_shares()
392     def render_share_problem(self, ctx, data):
393         serverid, storage_index, sharenum = data
394         nickname = IClient(ctx).get_nickname_for_peerid(serverid)
395         ctx.fillSlots("serverid", idlib.shortnodeid_b2a(serverid))
396         if nickname:
397             ctx.fillSlots("nickname", self._html(nickname))
398         ctx.fillSlots("si", self._render_si_link(ctx, storage_index))
399         ctx.fillSlots("shnum", str(sharenum))
400         return ctx.tag
401
402     def render_return(self, ctx, data):
403         req = inevow.IRequest(ctx)
404         return_to = get_arg(req, "return_to", None)
405         if return_to:
406             return T.div[T.a(href=return_to)["Return to parent directory"]]
407         return ""
408
409     def data_all_objects(self, ctx, data):
410         r = self.monitor.get_status().get_all_results()
411         for path in sorted(r.keys()):
412             yield (path, r[path])
413
414     def render_object(self, ctx, data):
415         path, r = data
416         if path:
417             pathstring = "/".join(self._html(path))
418         else:
419             pathstring = "<root>"
420         ctx.fillSlots("path", pathstring)
421         ctx.fillSlots("healthy", str(r.is_healthy()))
422         ctx.fillSlots("recoverable", str(r.is_recoverable()))
423         storage_index = r.get_storage_index()
424         ctx.fillSlots("storage_index", self._render_si_link(ctx, storage_index))
425         ctx.fillSlots("summary", self._html(r.get_summary()))
426         return ctx.tag
427
428     def render_runtime(self, ctx, data):
429         req = inevow.IRequest(ctx)
430         runtime = time.time() - req.processing_started_timestamp
431         return ctx.tag["runtime: %s seconds" % runtime]
432
433 class DeepCheckAndRepairResults(rend.Page, ResultsBase, ReloadMixin):
434     docFactory = getxmlfile("deep-check-and-repair-results.xhtml")
435
436     def __init__(self, monitor):
437         #assert IDeepCheckAndRepairResults(results)
438         #self.r = results
439         self.monitor = monitor
440
441     def renderHTTP(self, ctx):
442         if self.want_json(ctx):
443             return self.json(ctx)
444         return rend.Page.renderHTTP(self, ctx)
445
446     def json(self, ctx):
447         inevow.IRequest(ctx).setHeader("content-type", "text/plain")
448         res = self.monitor.get_status()
449         data = {}
450         data["finished"] = self.monitor.is_finished()
451         data["root-storage-index"] = res.get_root_storage_index_string()
452         c = res.get_counters()
453         data["count-objects-checked"] = c["count-objects-checked"]
454
455         data["count-objects-healthy-pre-repair"] = c["count-objects-healthy-pre-repair"]
456         data["count-objects-unhealthy-pre-repair"] = c["count-objects-unhealthy-pre-repair"]
457         data["count-objects-healthy-post-repair"] = c["count-objects-healthy-post-repair"]
458         data["count-objects-unhealthy-post-repair"] = c["count-objects-unhealthy-post-repair"]
459
460         data["count-repairs-attempted"] = c["count-repairs-attempted"]
461         data["count-repairs-successful"] = c["count-repairs-successful"]
462         data["count-repairs-unsuccessful"] = c["count-repairs-unsuccessful"]
463
464         data["count-corrupt-shares-pre-repair"] = c["count-corrupt-shares-pre-repair"]
465         data["count-corrupt-shares-post-repair"] = c["count-corrupt-shares-pre-repair"]
466
467         data["list-corrupt-shares"] = [ (idlib.nodeid_b2a(serverid),
468                                          base32.b2a(storage_index),
469                                          shnum)
470                                         for (serverid, storage_index, shnum)
471                                         in res.get_corrupt_shares() ]
472
473         remaining_corrupt = [ (idlib.nodeid_b2a(serverid),
474                                base32.b2a(storage_index),
475                                shnum)
476                               for (serverid, storage_index, shnum)
477                               in res.get_remaining_corrupt_shares() ]
478         data["list-remaining-corrupt-shares"] = remaining_corrupt
479
480         unhealthy = [ (path_t,
481                        self._json_check_results(crr.get_pre_repair_results()))
482                       for (path_t, crr)
483                       in res.get_all_results().items()
484                       if not crr.get_pre_repair_results().is_healthy() ]
485         data["list-unhealthy-files"] = unhealthy
486         data["stats"] = res.get_stats()
487         return simplejson.dumps(data, indent=1) + "\n"
488
489     def render_root_storage_index(self, ctx, data):
490         return self.monitor.get_status().get_root_storage_index_string()
491
492     def data_objects_checked(self, ctx, data):
493         return self.monitor.get_status().get_counters()["count-objects-checked"]
494
495     def data_objects_healthy(self, ctx, data):
496         return self.monitor.get_status().get_counters()["count-objects-healthy-pre-repair"]
497     def data_objects_unhealthy(self, ctx, data):
498         return self.monitor.get_status().get_counters()["count-objects-unhealthy-pre-repair"]
499     def data_corrupt_shares(self, ctx, data):
500         return self.monitor.get_status().get_counters()["count-corrupt-shares-pre-repair"]
501
502     def data_repairs_attempted(self, ctx, data):
503         return self.monitor.get_status().get_counters()["count-repairs-attempted"]
504     def data_repairs_successful(self, ctx, data):
505         return self.monitor.get_status().get_counters()["count-repairs-successful"]
506     def data_repairs_unsuccessful(self, ctx, data):
507         return self.monitor.get_status().get_counters()["count-repairs-unsuccessful"]
508
509     def data_objects_healthy_post(self, ctx, data):
510         return self.monitor.get_status().get_counters()["count-objects-healthy-post-repair"]
511     def data_objects_unhealthy_post(self, ctx, data):
512         return self.monitor.get_status().get_counters()["count-objects-unhealthy-post-repair"]
513     def data_corrupt_shares_post(self, ctx, data):
514         return self.monitor.get_status().get_counters()["count-corrupt-shares-post-repair"]
515
516     def render_pre_repair_problems_p(self, ctx, data):
517         c = self.monitor.get_status().get_counters()
518         if c["count-objects-unhealthy-pre-repair"]:
519             return ctx.tag
520         return ""
521
522     def data_pre_repair_problems(self, ctx, data):
523         all_objects = self.monitor.get_status().get_all_results()
524         for path in sorted(all_objects.keys()):
525             r = all_objects[path]
526             assert ICheckAndRepairResults.providedBy(r)
527             cr = r.get_pre_repair_results()
528             if not cr.is_healthy():
529                 yield path, cr
530
531     def render_problem(self, ctx, data):
532         path, cr = data
533         return ["/".join(self._html(path)), ": ", self._html(cr.get_summary())]
534
535     def render_post_repair_problems_p(self, ctx, data):
536         c = self.monitor.get_status().get_counters()
537         if (c["count-objects-unhealthy-post-repair"]
538             or c["count-corrupt-shares-post-repair"]):
539             return ctx.tag
540         return ""
541
542     def data_post_repair_problems(self, ctx, data):
543         all_objects = self.monitor.get_status().get_all_results()
544         for path in sorted(all_objects.keys()):
545             r = all_objects[path]
546             assert ICheckAndRepairResults.providedBy(r)
547             cr = r.get_post_repair_results()
548             if not cr.is_healthy():
549                 yield path, cr
550
551     def render_servers_with_corrupt_shares_p(self, ctx, data):
552         if self.monitor.get_status().get_counters()["count-corrupt-shares-pre-repair"]:
553             return ctx.tag
554         return ""
555     def data_servers_with_corrupt_shares(self, ctx, data):
556         return [] # TODO
557     def render_server_problem(self, ctx, data):
558         pass
559
560
561     def render_remaining_corrupt_shares_p(self, ctx, data):
562         if self.monitor.get_status().get_counters()["count-corrupt-shares-post-repair"]:
563             return ctx.tag
564         return ""
565     def data_post_repair_corrupt_shares(self, ctx, data):
566         return [] # TODO
567
568     def render_share_problem(self, ctx, data):
569         pass
570
571
572     def render_return(self, ctx, data):
573         req = inevow.IRequest(ctx)
574         return_to = get_arg(req, "return_to", None)
575         if return_to:
576             return T.div[T.a(href=return_to)["Return to parent directory"]]
577         return ""
578
579     def data_all_objects(self, ctx, data):
580         r = self.monitor.get_status().get_all_results()
581         for path in sorted(r.keys()):
582             yield (path, r[path])
583
584     def render_object(self, ctx, data):
585         path, r = data
586         ctx.fillSlots("path", "/".join(self._html(path)))
587         ctx.fillSlots("healthy_pre_repair",
588                       str(r.get_pre_repair_results().is_healthy()))
589         ctx.fillSlots("healthy_post_repair",
590                       str(r.get_post_repair_results().is_healthy()))
591         ctx.fillSlots("summary",
592                       self._html(r.get_pre_repair_results().get_summary()))
593         return ctx.tag
594
595     def render_runtime(self, ctx, data):
596         req = inevow.IRequest(ctx)
597         runtime = time.time() - req.processing_started_timestamp
598         return ctx.tag["runtime: %s seconds" % runtime]