4 from twisted.protocols.basic import LineOnlyReceiver
5 from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \
7 from allmydata.scripts.common_http import do_http, format_http_error
8 from allmydata.util.encodingutil import quote_output, quote_path
13 def _quote_serverid_index_share(serverid, storage_index, sharenum):
14 return "server %s, SI %s, shnum %r" % (quote_output(serverid, quotemarks=False),
15 quote_output(storage_index, quotemarks=False),
18 def check_location(options, where):
19 stdout = options.stdout
20 stderr = options.stderr
21 nodeurl = options['node-url']
22 if not nodeurl.endswith("/"):
25 rootcap, path = get_alias(options.aliases, where, DEFAULT_ALIAS)
26 except UnknownAliasError, e:
31 url = nodeurl + "uri/%s" % urllib.quote(rootcap)
33 url += "/" + escape_path(path)
34 # todo: should it end with a slash?
35 url += "?t=check&output=JSON"
40 if options["add-lease"]:
41 url += "&add-lease=true"
43 resp = do_http("POST", url)
44 if resp.status != 200:
45 print >>stderr, format_http_error("ERROR", resp)
48 if options.get("raw"):
52 data = simplejson.loads(jdata)
56 if data["pre-repair-results"]["results"]["healthy"]:
59 summary = "not healthy"
60 stdout.write("Summary: %s\n" % summary)
61 cr = data["pre-repair-results"]["results"]
62 stdout.write(" storage index: %s\n" % quote_output(data["storage-index"], quotemarks=False))
63 stdout.write(" good-shares: %r (encoding is %r-of-%r)\n"
64 % (cr["count-shares-good"],
65 cr["count-shares-needed"],
66 cr["count-shares-expected"]))
67 stdout.write(" wrong-shares: %r\n" % cr["count-wrong-shares"])
68 corrupt = cr["list-corrupt-shares"]
70 stdout.write(" corrupt shares:\n")
71 for (serverid, storage_index, sharenum) in corrupt:
72 stdout.write(" %s\n" % _quote_serverid_index_share(serverid, storage_index, sharenum))
73 if data["repair-attempted"]:
74 if data["repair-successful"]:
75 stdout.write(" repair successful\n")
77 stdout.write(" repair failed\n")
79 # LIT files and directories do not have a "summary" field.
80 summary = data.get("summary", "Healthy (LIT)")
81 stdout.write("Summary: %s\n" % quote_output(summary, quotemarks=False))
83 stdout.write(" storage index: %s\n" % quote_output(data["storage-index"], quotemarks=False))
85 if all([field in cr for field in ("count-shares-good", "count-shares-needed",
86 "count-shares-expected", "count-wrong-shares")]):
87 stdout.write(" good-shares: %r (encoding is %r-of-%r)\n"
88 % (cr["count-shares-good"],
89 cr["count-shares-needed"],
90 cr["count-shares-expected"]))
91 stdout.write(" wrong-shares: %r\n" % cr["count-wrong-shares"])
93 corrupt = cr.get("list-corrupt-shares", [])
95 stdout.write(" corrupt shares:\n")
96 for (serverid, storage_index, sharenum) in corrupt:
97 stdout.write(" %s\n" % _quote_serverid_index_share(serverid, storage_index, sharenum))
102 if len(options.locations) == 0:
103 errno = check_location(options, unicode())
107 for location in options.locations:
108 errno = check_location(options, location)
114 disconnecting = False
116 class DeepCheckOutput(LineOnlyReceiver):
118 def __init__(self, streamer, options):
119 self.streamer = streamer
120 self.transport = FakeTransport()
122 self.verbose = bool(options["verbose"])
123 self.stdout = options.stdout
124 self.stderr = options.stderr
126 self.files_healthy = 0
127 self.files_unhealthy = 0
128 self.in_error = False
130 def lineReceived(self, line):
132 print >>self.stderr, quote_output(line, quotemarks=False)
134 if line.startswith("ERROR:"):
137 print >>self.stderr, quote_output(line, quotemarks=False)
140 d = simplejson.loads(line)
142 if d["type"] not in ("file", "directory"):
144 self.num_objects += 1
145 # non-verbose means print a progress marker every 100 files
146 if self.num_objects % 100 == 0:
147 print >>stdout, "%d objects checked.." % self.num_objects
148 cr = d["check-results"]
149 if cr["results"]["healthy"]:
150 self.files_healthy += 1
152 self.files_unhealthy += 1
154 # verbose means also print one line per file
159 # LIT files and directories do not have a "summary" field.
160 summary = cr.get("summary", "Healthy (LIT)")
161 print >>stdout, "%s: %s" % (quote_path(path), quote_output(summary, quotemarks=False))
163 # always print out corrupt shares
164 for shareloc in cr["results"].get("list-corrupt-shares", []):
165 (serverid, storage_index, sharenum) = shareloc
166 print >>stdout, " corrupt: %s" % _quote_serverid_index_share(serverid, storage_index, sharenum)
172 print >>stdout, "done: %d objects checked, %d healthy, %d unhealthy" \
173 % (self.num_objects, self.files_healthy, self.files_unhealthy)
175 class DeepCheckAndRepairOutput(LineOnlyReceiver):
177 def __init__(self, streamer, options):
178 self.streamer = streamer
179 self.transport = FakeTransport()
181 self.verbose = bool(options["verbose"])
182 self.stdout = options.stdout
183 self.stderr = options.stderr
185 self.pre_repair_files_healthy = 0
186 self.pre_repair_files_unhealthy = 0
187 self.repairs_attempted = 0
188 self.repairs_successful = 0
189 self.post_repair_files_healthy = 0
190 self.post_repair_files_unhealthy = 0
191 self.in_error = False
193 def lineReceived(self, line):
195 print >>self.stderr, quote_output(line, quotemarks=False)
197 if line.startswith("ERROR:"):
200 print >>self.stderr, quote_output(line, quotemarks=False)
203 d = simplejson.loads(line)
205 if d["type"] not in ("file", "directory"):
207 self.num_objects += 1
208 # non-verbose means print a progress marker every 100 files
209 if self.num_objects % 100 == 0:
210 print >>stdout, "%d objects checked.." % self.num_objects
211 crr = d["check-and-repair-results"]
212 if d["storage-index"]:
213 if crr["pre-repair-results"]["results"]["healthy"]:
215 self.pre_repair_files_healthy += 1
218 self.pre_repair_files_unhealthy += 1
219 if crr["post-repair-results"]["results"]["healthy"]:
220 self.post_repair_files_healthy += 1
222 self.post_repair_files_unhealthy += 1
226 self.pre_repair_files_healthy += 1
227 self.post_repair_files_healthy += 1
228 if crr["repair-attempted"]:
229 self.repairs_attempted += 1
230 if crr["repair-successful"]:
231 self.repairs_successful += 1
233 # verbose means also print one line per file
237 # we don't seem to have a summary available, so build one
241 summary = "not healthy"
242 print >>stdout, "%s: %s" % (quote_path(path), summary)
244 # always print out corrupt shares
245 prr = crr.get("pre-repair-results", {})
246 for shareloc in prr.get("results", {}).get("list-corrupt-shares", []):
247 (serverid, storage_index, sharenum) = shareloc
248 print >>stdout, " corrupt: %s" % _quote_serverid_index_share(serverid, storage_index, sharenum)
250 # always print out repairs
251 if crr["repair-attempted"]:
252 if crr["repair-successful"]:
253 print >>stdout, " repair successful"
255 print >>stdout, " repair failed"
261 print >>stdout, "done: %d objects checked" % self.num_objects
262 print >>stdout, " pre-repair: %d healthy, %d unhealthy" \
263 % (self.pre_repair_files_healthy,
264 self.pre_repair_files_unhealthy)
265 print >>stdout, " %d repairs attempted, %d successful, %d failed" \
266 % (self.repairs_attempted,
267 self.repairs_successful,
268 (self.repairs_attempted - self.repairs_successful))
269 print >>stdout, " post-repair: %d healthy, %d unhealthy" \
270 % (self.post_repair_files_healthy,
271 self.post_repair_files_unhealthy)
273 class DeepCheckStreamer(LineOnlyReceiver):
275 def deepcheck_location(self, options, where):
276 stdout = options.stdout
277 stderr = options.stderr
279 self.options = options
280 nodeurl = options['node-url']
281 if not nodeurl.endswith("/"):
283 self.nodeurl = nodeurl
286 rootcap, path = get_alias(options.aliases, where, DEFAULT_ALIAS)
287 except UnknownAliasError, e:
292 url = nodeurl + "uri/%s" % urllib.quote(rootcap)
294 url += "/" + escape_path(path)
295 # todo: should it end with a slash?
296 url += "?t=stream-deep-check"
297 if options["verify"]:
298 url += "&verify=true"
299 if options["repair"]:
300 url += "&repair=true"
301 output = DeepCheckAndRepairOutput(self, options)
303 output = DeepCheckOutput(self, options)
304 if options["add-lease"]:
305 url += "&add-lease=true"
306 resp = do_http("POST", url)
307 if resp.status not in (200, 302):
308 print >>stderr, format_http_error("ERROR", resp)
311 # use Twisted to split this into lines
313 chunk = resp.read(100)
316 if self.options["raw"]:
319 output.dataReceived(chunk)
320 if not self.options["raw"]:
325 def run(self, options):
326 if len(options.locations) == 0:
327 errno = self.deepcheck_location(options, unicode())
331 for location in options.locations:
332 errno = self.deepcheck_location(options, location)
337 def deepcheck(options):
338 return DeepCheckStreamer().run(options)