4 from twisted.protocols.basic import LineOnlyReceiver
5 from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \
7 from allmydata.scripts.common_http import do_http, format_http_error
8 from allmydata.util.encodingutil import quote_output, quote_path
13 def _quote_serverid_index_share(serverid, storage_index, sharenum):
14 return "server %s, SI %s, shnum %r" % (quote_output(serverid, quotemarks=False),
15 quote_output(storage_index, quotemarks=False),
19 stdout = options.stdout
20 stderr = options.stderr
21 nodeurl = options['node-url']
22 if not nodeurl.endswith("/"):
26 rootcap, path = get_alias(options.aliases, where, DEFAULT_ALIAS)
27 except UnknownAliasError, e:
32 url = nodeurl + "uri/%s" % urllib.quote(rootcap)
34 url += "/" + escape_path(path)
35 # todo: should it end with a slash?
36 url += "?t=check&output=JSON"
41 if options["add-lease"]:
42 url += "&add-lease=true"
44 resp = do_http("POST", url)
45 if resp.status != 200:
46 print >>stderr, format_http_error("ERROR", resp)
49 if options.get("raw"):
53 data = simplejson.loads(jdata)
57 if data["pre-repair-results"]["results"]["healthy"]:
60 summary = "not healthy"
61 stdout.write("Summary: %s\n" % summary)
62 cr = data["pre-repair-results"]["results"]
63 stdout.write(" storage index: %s\n" % quote_output(data["storage-index"], quotemarks=False))
64 stdout.write(" good-shares: %r (encoding is %r-of-%r)\n"
65 % (cr["count-shares-good"],
66 cr["count-shares-needed"],
67 cr["count-shares-expected"]))
68 stdout.write(" wrong-shares: %r\n" % cr["count-wrong-shares"])
69 corrupt = cr["list-corrupt-shares"]
71 stdout.write(" corrupt shares:\n")
72 for (serverid, storage_index, sharenum) in corrupt:
73 stdout.write(" %s\n" % _quote_serverid_index_share(serverid, storage_index, sharenum))
74 if data["repair-attempted"]:
75 if data["repair-successful"]:
76 stdout.write(" repair successful\n")
78 stdout.write(" repair failed\n")
80 # LIT files and directories do not have a "summary" field.
81 summary = data.get("summary", "Healthy (LIT)")
82 stdout.write("Summary: %s\n" % quote_output(summary, quotemarks=False))
84 stdout.write(" storage index: %s\n" % quote_output(data["storage-index"], quotemarks=False))
86 if all([field in cr for field in ("count-shares-good", "count-shares-needed",
87 "count-shares-expected", "count-wrong-shares")]):
88 stdout.write(" good-shares: %r (encoding is %r-of-%r)\n"
89 % (cr["count-shares-good"],
90 cr["count-shares-needed"],
91 cr["count-shares-expected"]))
92 stdout.write(" wrong-shares: %r\n" % cr["count-wrong-shares"])
94 corrupt = cr.get("list-corrupt-shares", [])
96 stdout.write(" corrupt shares:\n")
97 for (serverid, storage_index, sharenum) in corrupt:
98 stdout.write(" %s\n" % _quote_serverid_index_share(serverid, storage_index, sharenum))
104 disconnecting = False
106 class DeepCheckOutput(LineOnlyReceiver):
108 def __init__(self, streamer, options):
109 self.streamer = streamer
110 self.transport = FakeTransport()
112 self.verbose = bool(options["verbose"])
113 self.stdout = options.stdout
114 self.stderr = options.stderr
116 self.files_healthy = 0
117 self.files_unhealthy = 0
118 self.in_error = False
120 def lineReceived(self, line):
122 print >>self.stderr, quote_output(line, quotemarks=False)
124 if line.startswith("ERROR:"):
127 print >>self.stderr, quote_output(line, quotemarks=False)
130 d = simplejson.loads(line)
132 if d["type"] not in ("file", "directory"):
134 self.num_objects += 1
135 # non-verbose means print a progress marker every 100 files
136 if self.num_objects % 100 == 0:
137 print >>stdout, "%d objects checked.." % self.num_objects
138 cr = d["check-results"]
139 if cr["results"]["healthy"]:
140 self.files_healthy += 1
142 self.files_unhealthy += 1
144 # verbose means also print one line per file
149 # LIT files and directories do not have a "summary" field.
150 summary = cr.get("summary", "Healthy (LIT)")
151 print >>stdout, "%s: %s" % (quote_path(path), quote_output(summary, quotemarks=False))
153 # always print out corrupt shares
154 for shareloc in cr["results"].get("list-corrupt-shares", []):
155 (serverid, storage_index, sharenum) = shareloc
156 print >>stdout, " corrupt: %s" % _quote_serverid_index_share(serverid, storage_index, sharenum)
162 print >>stdout, "done: %d objects checked, %d healthy, %d unhealthy" \
163 % (self.num_objects, self.files_healthy, self.files_unhealthy)
165 class DeepCheckAndRepairOutput(LineOnlyReceiver):
167 def __init__(self, streamer, options):
168 self.streamer = streamer
169 self.transport = FakeTransport()
171 self.verbose = bool(options["verbose"])
172 self.stdout = options.stdout
173 self.stderr = options.stderr
175 self.pre_repair_files_healthy = 0
176 self.pre_repair_files_unhealthy = 0
177 self.repairs_attempted = 0
178 self.repairs_successful = 0
179 self.post_repair_files_healthy = 0
180 self.post_repair_files_unhealthy = 0
181 self.in_error = False
183 def lineReceived(self, line):
185 print >>self.stderr, quote_output(line, quotemarks=False)
187 if line.startswith("ERROR:"):
190 print >>self.stderr, quote_output(line, quotemarks=False)
193 d = simplejson.loads(line)
195 if d["type"] not in ("file", "directory"):
197 self.num_objects += 1
198 # non-verbose means print a progress marker every 100 files
199 if self.num_objects % 100 == 0:
200 print >>stdout, "%d objects checked.." % self.num_objects
201 crr = d["check-and-repair-results"]
202 if d["storage-index"]:
203 if crr["pre-repair-results"]["results"]["healthy"]:
205 self.pre_repair_files_healthy += 1
208 self.pre_repair_files_unhealthy += 1
209 if crr["post-repair-results"]["results"]["healthy"]:
210 self.post_repair_files_healthy += 1
212 self.post_repair_files_unhealthy += 1
216 self.pre_repair_files_healthy += 1
217 self.post_repair_files_healthy += 1
218 if crr["repair-attempted"]:
219 self.repairs_attempted += 1
220 if crr["repair-successful"]:
221 self.repairs_successful += 1
223 # verbose means also print one line per file
227 # we don't seem to have a summary available, so build one
231 summary = "not healthy"
232 print >>stdout, "%s: %s" % (quote_path(path), summary)
234 # always print out corrupt shares
235 prr = crr.get("pre-repair-results", {})
236 for shareloc in prr.get("results", {}).get("list-corrupt-shares", []):
237 (serverid, storage_index, sharenum) = shareloc
238 print >>stdout, " corrupt: %s" % _quote_serverid_index_share(serverid, storage_index, sharenum)
240 # always print out repairs
241 if crr["repair-attempted"]:
242 if crr["repair-successful"]:
243 print >>stdout, " repair successful"
245 print >>stdout, " repair failed"
251 print >>stdout, "done: %d objects checked" % self.num_objects
252 print >>stdout, " pre-repair: %d healthy, %d unhealthy" \
253 % (self.pre_repair_files_healthy,
254 self.pre_repair_files_unhealthy)
255 print >>stdout, " %d repairs attempted, %d successful, %d failed" \
256 % (self.repairs_attempted,
257 self.repairs_successful,
258 (self.repairs_attempted - self.repairs_successful))
259 print >>stdout, " post-repair: %d healthy, %d unhealthy" \
260 % (self.post_repair_files_healthy,
261 self.post_repair_files_unhealthy)
263 class DeepCheckStreamer(LineOnlyReceiver):
265 def run(self, options):
266 stdout = options.stdout
267 stderr = options.stderr
269 self.options = options
270 nodeurl = options['node-url']
271 if not nodeurl.endswith("/"):
273 self.nodeurl = nodeurl
274 where = options.where
276 rootcap, path = get_alias(options.aliases, where, DEFAULT_ALIAS)
277 except UnknownAliasError, e:
282 url = nodeurl + "uri/%s" % urllib.quote(rootcap)
284 url += "/" + escape_path(path)
285 # todo: should it end with a slash?
286 url += "?t=stream-deep-check"
287 if options["verify"]:
288 url += "&verify=true"
289 if options["repair"]:
290 url += "&repair=true"
291 output = DeepCheckAndRepairOutput(self, options)
293 output = DeepCheckOutput(self, options)
294 if options["add-lease"]:
295 url += "&add-lease=true"
296 resp = do_http("POST", url)
297 if resp.status not in (200, 302):
298 print >>stderr, format_http_error("ERROR", resp)
301 # use Twisted to split this into lines
303 chunk = resp.read(100)
306 if self.options["raw"]:
309 output.dataReceived(chunk)
310 if not self.options["raw"]:
314 def deepcheck(options):
315 return DeepCheckStreamer().run(options)