2 # do not import any allmydata modules at this level. Do that from inside
3 # individual functions instead.
4 import struct, time, os, sys
5 from twisted.python import usage, failure
6 from twisted.internet import defer
7 from twisted.scripts import trial as twisted_trial
10 class DumpOptions(usage.Options):
11 def getSynopsis(self):
12 return "Usage: tahoe debug dump-share SHARE_FILENAME"
15 ["offsets", None, "Display a table of section offsets."],
16 ["leases-only", None, "Dump leases but not CHK contents."],
19 def getUsage(self, width=None):
20 t = usage.Options.getUsage(self, width)
22 Print lots of information about the given share, by parsing the share's
23 contents. This includes share type, lease information, encoding parameters,
24 hash-tree roots, public keys, and segment sizes. This command also emits a
25 verify-cap for the file that uses the share.
27 tahoe debug dump-share testgrid/node-3/storage/shares/4v/4vozh77tsrw7mdhnj7qvp5ky74/0
32 def parseArgs(self, filename):
33 from allmydata.util.encodingutil import argv_to_abspath
34 self['filename'] = argv_to_abspath(filename)
36 def dump_share(options):
37 from allmydata.storage.mutable import MutableShareFile
38 from allmydata.util.encodingutil import quote_output
42 # check the version, to see if we have a mutable or immutable share
43 print >>out, "share filename: %s" % quote_output(options['filename'])
45 f = open(options['filename'], "rb")
48 if prefix == MutableShareFile.MAGIC:
49 return dump_mutable_share(options)
50 # otherwise assume it's immutable
51 return dump_immutable_share(options)
53 def dump_immutable_share(options):
54 from allmydata.storage.immutable import ShareFile
57 f = ShareFile(options['filename'])
58 if not options["leases-only"]:
59 dump_immutable_chk_share(f, out, options)
60 dump_immutable_lease_info(f, out)
64 def dump_immutable_chk_share(f, out, options):
65 from allmydata import uri
66 from allmydata.util import base32
67 from allmydata.immutable.layout import ReadBucketProxy
68 from allmydata.util.encodingutil import quote_output, to_str
70 # use a ReadBucketProxy to parse the bucket and find the uri extension
71 bp = ReadBucketProxy(None, None, '')
72 offsets = bp._parse_offsets(f.read_share_data(0, 0x44))
73 print >>out, "%20s: %d" % ("version", bp._version)
74 seek = offsets['uri_extension']
75 length = struct.unpack(bp._fieldstruct,
76 f.read_share_data(seek, bp._fieldsize))[0]
78 UEB_data = f.read_share_data(seek, length)
80 unpacked = uri.unpack_extension_readable(UEB_data)
81 keys1 = ("size", "num_segments", "segment_size",
82 "needed_shares", "total_shares")
83 keys2 = ("codec_name", "codec_params", "tail_codec_params")
84 keys3 = ("plaintext_hash", "plaintext_root_hash",
85 "crypttext_hash", "crypttext_root_hash",
86 "share_root_hash", "UEB_hash")
87 display_keys = {"size": "file_size"}
90 dk = display_keys.get(k, k)
91 print >>out, "%20s: %s" % (dk, unpacked[k])
95 dk = display_keys.get(k, k)
96 print >>out, "%20s: %s" % (dk, unpacked[k])
100 dk = display_keys.get(k, k)
101 print >>out, "%20s: %s" % (dk, unpacked[k])
103 leftover = set(unpacked.keys()) - set(keys1 + keys2 + keys3)
106 print >>out, "LEFTOVER:"
107 for k in sorted(leftover):
108 print >>out, "%20s: %s" % (k, unpacked[k])
110 # the storage index isn't stored in the share itself, so we depend upon
111 # knowing the parent directory name to get it
112 pieces = options['filename'].split(os.sep)
114 piece = to_str(pieces[-2])
115 if base32.could_be_base32_encoded(piece):
116 storage_index = base32.a2b(piece)
117 uri_extension_hash = base32.a2b(unpacked["UEB_hash"])
118 u = uri.CHKFileVerifierURI(storage_index, uri_extension_hash,
119 unpacked["needed_shares"],
120 unpacked["total_shares"], unpacked["size"])
121 verify_cap = u.to_string()
122 print >>out, "%20s: %s" % ("verify-cap", quote_output(verify_cap, quotemarks=False))
125 sizes['data'] = (offsets['plaintext_hash_tree'] -
127 sizes['validation'] = (offsets['uri_extension'] -
128 offsets['plaintext_hash_tree'])
129 sizes['uri-extension'] = len(UEB_data)
131 print >>out, " Size of data within the share:"
132 for k in sorted(sizes):
133 print >>out, "%20s: %s" % (k, sizes[k])
135 if options['offsets']:
137 print >>out, " Section Offsets:"
138 print >>out, "%20s: %s" % ("share data", f._data_offset)
139 for k in ["data", "plaintext_hash_tree", "crypttext_hash_tree",
140 "block_hashes", "share_hashes", "uri_extension"]:
141 name = {"data": "block data"}.get(k,k)
142 offset = f._data_offset + offsets[k]
143 print >>out, " %20s: %s (0x%x)" % (name, offset, offset)
144 print >>out, "%20s: %s" % ("leases", f._lease_offset)
146 def dump_immutable_lease_info(f, out):
147 # display lease information too
149 leases = list(f.get_leases())
151 for i,lease in enumerate(leases):
152 when = format_expiration_time(lease.expiration_time)
153 print >>out, " Lease #%d: owner=%d, expire in %s" \
154 % (i, lease.owner_num, when)
156 print >>out, " No leases."
158 def format_expiration_time(expiration_time):
160 remains = expiration_time - now
161 when = "%ds" % remains
162 if remains > 24*3600:
163 when += " (%d days)" % (remains / (24*3600))
165 when += " (%d hours)" % (remains / 3600)
169 def dump_mutable_share(options):
170 from allmydata.storage.mutable import MutableShareFile
171 from allmydata.util import base32, idlib
173 m = MutableShareFile(options['filename'])
174 f = open(options['filename'], "rb")
175 WE, nodeid = m._read_write_enabler_and_nodeid(f)
176 num_extra_leases = m._read_num_extra_leases(f)
177 data_length = m._read_data_length(f)
178 extra_lease_offset = m._read_extra_lease_offset(f)
179 container_size = extra_lease_offset - m.DATA_OFFSET
180 leases = list(m._enumerate_leases(f))
182 share_type = "unknown"
183 f.seek(m.DATA_OFFSET)
185 if version == "\x00":
186 # this slot contains an SMDF share
188 elif version == "\x01":
193 print >>out, "Mutable slot found:"
194 print >>out, " share_type: %s" % share_type
195 print >>out, " write_enabler: %s" % base32.b2a(WE)
196 print >>out, " WE for nodeid: %s" % idlib.nodeid_b2a(nodeid)
197 print >>out, " num_extra_leases: %d" % num_extra_leases
198 print >>out, " container_size: %d" % container_size
199 print >>out, " data_length: %d" % data_length
201 for (leasenum, lease) in leases:
203 print >>out, " Lease #%d:" % leasenum
204 print >>out, " ownerid: %d" % lease.owner_num
205 when = format_expiration_time(lease.expiration_time)
206 print >>out, " expires in %s" % when
207 print >>out, " renew_secret: %s" % base32.b2a(lease.renew_secret)
208 print >>out, " cancel_secret: %s" % base32.b2a(lease.cancel_secret)
209 print >>out, " secrets are for nodeid: %s" % idlib.nodeid_b2a(lease.nodeid)
211 print >>out, "No leases."
214 if share_type == "SDMF":
215 dump_SDMF_share(m, data_length, options)
216 elif share_type == "MDMF":
217 dump_MDMF_share(m, data_length, options)
221 def dump_SDMF_share(m, length, options):
222 from allmydata.mutable.layout import unpack_share, unpack_header
223 from allmydata.mutable.common import NeedMoreDataError
224 from allmydata.util import base32, hashutil
225 from allmydata.uri import SSKVerifierURI
226 from allmydata.util.encodingutil import quote_output, to_str
228 offset = m.DATA_OFFSET
232 f = open(options['filename'], "rb")
234 data = f.read(min(length, 2000))
238 pieces = unpack_share(data)
239 except NeedMoreDataError, e:
240 # retry once with the larger size
241 size = e.needed_bytes
242 f = open(options['filename'], "rb")
244 data = f.read(min(length, size))
246 pieces = unpack_share(data)
248 (seqnum, root_hash, IV, k, N, segsize, datalen,
249 pubkey, signature, share_hash_chain, block_hash_tree,
250 share_data, enc_privkey) = pieces
251 (ig_version, ig_seqnum, ig_roothash, ig_IV, ig_k, ig_N, ig_segsize,
252 ig_datalen, offsets) = unpack_header(data)
254 print >>out, " SDMF contents:"
255 print >>out, " seqnum: %d" % seqnum
256 print >>out, " root_hash: %s" % base32.b2a(root_hash)
257 print >>out, " IV: %s" % base32.b2a(IV)
258 print >>out, " required_shares: %d" % k
259 print >>out, " total_shares: %d" % N
260 print >>out, " segsize: %d" % segsize
261 print >>out, " datalen: %d" % datalen
262 print >>out, " enc_privkey: %d bytes" % len(enc_privkey)
263 print >>out, " pubkey: %d bytes" % len(pubkey)
264 print >>out, " signature: %d bytes" % len(signature)
265 share_hash_ids = ",".join(sorted([str(hid)
266 for hid in share_hash_chain.keys()]))
267 print >>out, " share_hash_chain: %s" % share_hash_ids
268 print >>out, " block_hash_tree: %d nodes" % len(block_hash_tree)
270 # the storage index isn't stored in the share itself, so we depend upon
271 # knowing the parent directory name to get it
272 pieces = options['filename'].split(os.sep)
274 piece = to_str(pieces[-2])
275 if base32.could_be_base32_encoded(piece):
276 storage_index = base32.a2b(piece)
277 fingerprint = hashutil.ssk_pubkey_fingerprint_hash(pubkey)
278 u = SSKVerifierURI(storage_index, fingerprint)
279 verify_cap = u.to_string()
280 print >>out, " verify-cap:", quote_output(verify_cap, quotemarks=False)
282 if options['offsets']:
283 # NOTE: this offset-calculation code is fragile, and needs to be
284 # merged with MutableShareFile's internals.
286 print >>out, " Section Offsets:"
287 def printoffset(name, value, shift=0):
288 print >>out, "%s%20s: %s (0x%x)" % (" "*shift, name, value, value)
289 printoffset("first lease", m.HEADER_SIZE)
290 printoffset("share data", m.DATA_OFFSET)
291 o_seqnum = m.DATA_OFFSET + struct.calcsize(">B")
292 printoffset("seqnum", o_seqnum, 2)
293 o_root_hash = m.DATA_OFFSET + struct.calcsize(">BQ")
294 printoffset("root_hash", o_root_hash, 2)
295 for k in ["signature", "share_hash_chain", "block_hash_tree",
297 "enc_privkey", "EOF"]:
298 name = {"share_data": "block data",
299 "EOF": "end of share data"}.get(k,k)
300 offset = m.DATA_OFFSET + offsets[k]
301 printoffset(name, offset, 2)
302 f = open(options['filename'], "rb")
303 printoffset("extra leases", m._read_extra_lease_offset(f) + 4)
308 def dump_MDMF_share(m, length, options):
309 from allmydata.mutable.layout import MDMFSlotReadProxy
310 from allmydata.mutable.common import NeedMoreDataError
311 from allmydata.util import base32, hashutil
312 from allmydata.uri import MDMFVerifierURI
313 from allmydata.util.encodingutil import quote_output, to_str
315 offset = m.DATA_OFFSET
319 f = open(options['filename'], "rb")
320 storage_index = None; shnum = 0
322 class ShareDumper(MDMFSlotReadProxy):
323 def _read(self, readvs, force_remote=False, queue=False):
325 for (where,length) in readvs:
327 data.append(f.read(length))
328 return defer.succeed({shnum: data})
330 # assume 2kB will be enough
331 p = ShareDumper(None, storage_index, shnum)
335 # these methods return Deferreds, but we happen to know that they run
336 # synchronously when not actually talking to a remote server
338 d.addCallback(stash.append)
341 verinfo = extract(p.get_verinfo)
342 encprivkey = extract(p.get_encprivkey)
343 signature = extract(p.get_signature)
344 pubkey = extract(p.get_verification_key)
345 block_hash_tree = extract(p.get_blockhashes)
346 share_hash_chain = extract(p.get_sharehashes)
349 (seqnum, root_hash, salt_to_use, segsize, datalen, k, N, prefix,
352 print >>out, " MDMF contents:"
353 print >>out, " seqnum: %d" % seqnum
354 print >>out, " root_hash: %s" % base32.b2a(root_hash)
355 #print >>out, " IV: %s" % base32.b2a(IV)
356 print >>out, " required_shares: %d" % k
357 print >>out, " total_shares: %d" % N
358 print >>out, " segsize: %d" % segsize
359 print >>out, " datalen: %d" % datalen
360 print >>out, " enc_privkey: %d bytes" % len(encprivkey)
361 print >>out, " pubkey: %d bytes" % len(pubkey)
362 print >>out, " signature: %d bytes" % len(signature)
363 share_hash_ids = ",".join([str(hid)
364 for hid in sorted(share_hash_chain.keys())])
365 print >>out, " share_hash_chain: %s" % share_hash_ids
366 print >>out, " block_hash_tree: %d nodes" % len(block_hash_tree)
368 # the storage index isn't stored in the share itself, so we depend upon
369 # knowing the parent directory name to get it
370 pieces = options['filename'].split(os.sep)
372 piece = to_str(pieces[-2])
373 if base32.could_be_base32_encoded(piece):
374 storage_index = base32.a2b(piece)
375 fingerprint = hashutil.ssk_pubkey_fingerprint_hash(pubkey)
376 hints = [str(k), str(segsize)]
377 u = MDMFVerifierURI(storage_index, fingerprint, hints)
378 verify_cap = u.to_string()
379 print >>out, " verify-cap:", quote_output(verify_cap, quotemarks=False)
381 if options['offsets']:
382 # NOTE: this offset-calculation code is fragile, and needs to be
383 # merged with MutableShareFile's internals.
386 print >>out, " Section Offsets:"
387 def printoffset(name, value, shift=0):
388 print >>out, "%s%.20s: %s (0x%x)" % (" "*shift, name, value, value)
389 printoffset("first lease", m.HEADER_SIZE, 2)
390 printoffset("share data", m.DATA_OFFSET, 2)
391 o_seqnum = m.DATA_OFFSET + struct.calcsize(">B")
392 printoffset("seqnum", o_seqnum, 4)
393 o_root_hash = m.DATA_OFFSET + struct.calcsize(">BQ")
394 printoffset("root_hash", o_root_hash, 4)
395 for k in ["enc_privkey", "share_hash_chain", "signature",
396 "verification_key", "verification_key_end",
397 "share_data", "block_hash_tree", "EOF"]:
398 name = {"share_data": "block data",
399 "verification_key": "pubkey",
400 "verification_key_end": "end of pubkey",
401 "EOF": "end of share data"}.get(k,k)
402 offset = m.DATA_OFFSET + offsets[k]
403 printoffset(name, offset, 4)
404 f = open(options['filename'], "rb")
405 printoffset("extra leases", m._read_extra_lease_offset(f) + 4, 2)
412 class DumpCapOptions(usage.Options):
413 def getSynopsis(self):
414 return "Usage: tahoe debug dump-cap [options] FILECAP"
417 None, "Specify the storage server nodeid (ASCII), to construct WE and secrets."],
418 ["client-secret", "c", None,
419 "Specify the client's base secret (ASCII), to construct secrets."],
420 ["client-dir", "d", None,
421 "Specify the client's base directory, from which a -c secret will be read."],
423 def parseArgs(self, cap):
426 def getUsage(self, width=None):
427 t = usage.Options.getUsage(self, width)
429 Print information about the given cap-string (aka: URI, file-cap, dir-cap,
430 read-cap, write-cap). The URI string is parsed and unpacked. This prints the
431 type of the cap, its storage index, and any derived keys.
433 tahoe debug dump-cap URI:SSK-Verifier:4vozh77tsrw7mdhnj7qvp5ky74:q7f3dwz76sjys4kqfdt3ocur2pay3a6rftnkqmi2uxu3vqsdsofq
435 This may be useful to determine if a read-cap and a write-cap refer to the
436 same time, or to extract the storage-index from a file-cap (to then use with
439 If additional information is provided (storage server nodeid and/or client
440 base secret), this command will compute the shared secrets used for the
441 write-enabler and for lease-renewal.
446 def dump_cap(options):
447 from allmydata import uri
448 from allmydata.util import base32
449 from base64 import b32decode
450 import urlparse, urllib
455 if options['nodeid']:
456 nodeid = b32decode(options['nodeid'].upper())
458 if options['client-secret']:
459 secret = base32.a2b(options['client-secret'])
460 elif options['client-dir']:
461 secretfile = os.path.join(options['client-dir'], "private", "secret")
463 secret = base32.a2b(open(secretfile, "r").read().strip())
464 except EnvironmentError:
467 if cap.startswith("http"):
468 scheme, netloc, path, params, query, fragment = urlparse.urlparse(cap)
469 assert path.startswith("/uri/")
470 cap = urllib.unquote(path[len("/uri/"):])
472 u = uri.from_string(cap)
475 dump_uri_instance(u, nodeid, secret, out)
477 def _dump_secrets(storage_index, secret, nodeid, out):
478 from allmydata.util import hashutil
479 from allmydata.util import base32
482 crs = hashutil.my_renewal_secret_hash(secret)
483 print >>out, " client renewal secret:", base32.b2a(crs)
484 frs = hashutil.file_renewal_secret_hash(crs, storage_index)
485 print >>out, " file renewal secret:", base32.b2a(frs)
487 renew = hashutil.bucket_renewal_secret_hash(frs, nodeid)
488 print >>out, " lease renewal secret:", base32.b2a(renew)
489 ccs = hashutil.my_cancel_secret_hash(secret)
490 print >>out, " client cancel secret:", base32.b2a(ccs)
491 fcs = hashutil.file_cancel_secret_hash(ccs, storage_index)
492 print >>out, " file cancel secret:", base32.b2a(fcs)
494 cancel = hashutil.bucket_cancel_secret_hash(fcs, nodeid)
495 print >>out, " lease cancel secret:", base32.b2a(cancel)
497 def dump_uri_instance(u, nodeid, secret, out, show_header=True):
498 from allmydata import uri
499 from allmydata.storage.server import si_b2a
500 from allmydata.util import base32, hashutil
501 from allmydata.util.encodingutil import quote_output
503 if isinstance(u, uri.CHKFileURI):
505 print >>out, "CHK File:"
506 print >>out, " key:", base32.b2a(u.key)
507 print >>out, " UEB hash:", base32.b2a(u.uri_extension_hash)
508 print >>out, " size:", u.size
509 print >>out, " k/N: %d/%d" % (u.needed_shares, u.total_shares)
510 print >>out, " storage index:", si_b2a(u.get_storage_index())
511 _dump_secrets(u.get_storage_index(), secret, nodeid, out)
512 elif isinstance(u, uri.CHKFileVerifierURI):
514 print >>out, "CHK Verifier URI:"
515 print >>out, " UEB hash:", base32.b2a(u.uri_extension_hash)
516 print >>out, " size:", u.size
517 print >>out, " k/N: %d/%d" % (u.needed_shares, u.total_shares)
518 print >>out, " storage index:", si_b2a(u.get_storage_index())
520 elif isinstance(u, uri.LiteralFileURI):
522 print >>out, "Literal File URI:"
523 print >>out, " data:", quote_output(u.data)
525 elif isinstance(u, uri.WriteableSSKFileURI): # SDMF
527 print >>out, "SDMF Writeable URI:"
528 print >>out, " writekey:", base32.b2a(u.writekey)
529 print >>out, " readkey:", base32.b2a(u.readkey)
530 print >>out, " storage index:", si_b2a(u.get_storage_index())
531 print >>out, " fingerprint:", base32.b2a(u.fingerprint)
534 we = hashutil.ssk_write_enabler_hash(u.writekey, nodeid)
535 print >>out, " write_enabler:", base32.b2a(we)
537 _dump_secrets(u.get_storage_index(), secret, nodeid, out)
538 elif isinstance(u, uri.ReadonlySSKFileURI):
540 print >>out, "SDMF Read-only URI:"
541 print >>out, " readkey:", base32.b2a(u.readkey)
542 print >>out, " storage index:", si_b2a(u.get_storage_index())
543 print >>out, " fingerprint:", base32.b2a(u.fingerprint)
544 elif isinstance(u, uri.SSKVerifierURI):
546 print >>out, "SDMF Verifier URI:"
547 print >>out, " storage index:", si_b2a(u.get_storage_index())
548 print >>out, " fingerprint:", base32.b2a(u.fingerprint)
550 elif isinstance(u, uri.WriteableMDMFFileURI): # MDMF
552 print >>out, "MDMF Writeable URI:"
553 print >>out, " writekey:", base32.b2a(u.writekey)
554 print >>out, " readkey:", base32.b2a(u.readkey)
555 print >>out, " storage index:", si_b2a(u.get_storage_index())
556 print >>out, " fingerprint:", base32.b2a(u.fingerprint)
559 we = hashutil.ssk_write_enabler_hash(u.writekey, nodeid)
560 print >>out, " write_enabler:", base32.b2a(we)
562 _dump_secrets(u.get_storage_index(), secret, nodeid, out)
563 elif isinstance(u, uri.ReadonlyMDMFFileURI):
565 print >>out, "MDMF Read-only URI:"
566 print >>out, " readkey:", base32.b2a(u.readkey)
567 print >>out, " storage index:", si_b2a(u.get_storage_index())
568 print >>out, " fingerprint:", base32.b2a(u.fingerprint)
569 elif isinstance(u, uri.MDMFVerifierURI):
571 print >>out, "MDMF Verifier URI:"
572 print >>out, " storage index:", si_b2a(u.get_storage_index())
573 print >>out, " fingerprint:", base32.b2a(u.fingerprint)
576 elif isinstance(u, uri.ImmutableDirectoryURI): # CHK-based directory
578 print >>out, "CHK Directory URI:"
579 dump_uri_instance(u._filenode_uri, nodeid, secret, out, False)
580 elif isinstance(u, uri.ImmutableDirectoryURIVerifier):
582 print >>out, "CHK Directory Verifier URI:"
583 dump_uri_instance(u._filenode_uri, nodeid, secret, out, False)
585 elif isinstance(u, uri.DirectoryURI): # SDMF-based directory
587 print >>out, "Directory Writeable URI:"
588 dump_uri_instance(u._filenode_uri, nodeid, secret, out, False)
589 elif isinstance(u, uri.ReadonlyDirectoryURI):
591 print >>out, "Directory Read-only URI:"
592 dump_uri_instance(u._filenode_uri, nodeid, secret, out, False)
593 elif isinstance(u, uri.DirectoryURIVerifier):
595 print >>out, "Directory Verifier URI:"
596 dump_uri_instance(u._filenode_uri, nodeid, secret, out, False)
598 elif isinstance(u, uri.MDMFDirectoryURI): # MDMF-based directory
600 print >>out, "Directory Writeable URI:"
601 dump_uri_instance(u._filenode_uri, nodeid, secret, out, False)
602 elif isinstance(u, uri.ReadonlyMDMFDirectoryURI):
604 print >>out, "Directory Read-only URI:"
605 dump_uri_instance(u._filenode_uri, nodeid, secret, out, False)
606 elif isinstance(u, uri.MDMFDirectoryURIVerifier):
608 print >>out, "Directory Verifier URI:"
609 dump_uri_instance(u._filenode_uri, nodeid, secret, out, False)
612 print >>out, "unknown cap type"
614 class FindSharesOptions(usage.Options):
615 def getSynopsis(self):
616 return "Usage: tahoe debug find-shares STORAGE_INDEX NODEDIRS.."
618 def parseArgs(self, storage_index_s, *nodedirs):
619 from allmydata.util.encodingutil import argv_to_abspath
620 self.si_s = storage_index_s
621 self.nodedirs = map(argv_to_abspath, nodedirs)
623 def getUsage(self, width=None):
624 t = usage.Options.getUsage(self, width)
626 Locate all shares for the given storage index. This command looks through one
627 or more node directories to find the shares. It returns a list of filenames,
628 one per line, for each share file found.
630 tahoe debug find-shares 4vozh77tsrw7mdhnj7qvp5ky74 testgrid/node-*
632 It may be useful during testing, when running a test grid in which all the
633 nodes are on a local disk. The share files thus located can be counted,
634 examined (with dump-share), or corrupted/deleted to test checker/repairer.
638 def find_shares(options):
639 """Given a storage index and a list of node directories, emit a list of
640 all matching shares to stdout, one per line. For example:
642 find-shares.py 44kai1tui348689nrw8fjegc8c ~/testnet/node-*
646 /home/warner/testnet/node-1/storage/shares/44k/44kai1tui348689nrw8fjegc8c/5
647 /home/warner/testnet/node-1/storage/shares/44k/44kai1tui348689nrw8fjegc8c/9
648 /home/warner/testnet/node-2/storage/shares/44k/44kai1tui348689nrw8fjegc8c/2
650 from allmydata.storage.server import si_a2b, storage_index_to_dir
651 from allmydata.util.encodingutil import listdir_unicode
654 sharedir = storage_index_to_dir(si_a2b(options.si_s))
655 for d in options.nodedirs:
656 d = os.path.join(d, "storage/shares", sharedir)
657 if os.path.exists(d):
658 for shnum in listdir_unicode(d):
659 print >>out, os.path.join(d, shnum)
664 class CatalogSharesOptions(usage.Options):
668 def parseArgs(self, *nodedirs):
669 from allmydata.util.encodingutil import argv_to_abspath
670 self.nodedirs = map(argv_to_abspath, nodedirs)
672 raise usage.UsageError("must specify at least one node directory")
674 def getSynopsis(self):
675 return "Usage: tahoe debug catalog-shares NODEDIRS.."
677 def getUsage(self, width=None):
678 t = usage.Options.getUsage(self, width)
680 Locate all shares in the given node directories, and emit a one-line summary
681 of each share. Run it like this:
683 tahoe debug catalog-shares testgrid/node-* >allshares.txt
685 The lines it emits will look like the following:
687 CHK $SI $k/$N $filesize $UEB_hash $expiration $abspath_sharefile
688 SDMF $SI $k/$N $filesize $seqnum/$roothash $expiration $abspath_sharefile
689 UNKNOWN $abspath_sharefile
691 This command can be used to build up a catalog of shares from many storage
692 servers and then sort the results to compare all shares for the same file. If
693 you see shares with the same SI but different parameters/filesize/UEB_hash,
694 then something is wrong. The misc/find-share/anomalies.py script may be
699 def call(c, *args, **kwargs):
700 # take advantage of the fact that ImmediateReadBucketProxy returns
701 # Deferreds that are already fired
704 d = defer.maybeDeferred(c, *args, **kwargs)
705 d.addCallbacks(results.append, failures.append)
707 failures[0].raiseException()
710 def describe_share(abs_sharefile, si_s, shnum_s, now, out):
711 from allmydata import uri
712 from allmydata.storage.mutable import MutableShareFile
713 from allmydata.storage.immutable import ShareFile
714 from allmydata.mutable.layout import unpack_share
715 from allmydata.mutable.common import NeedMoreDataError
716 from allmydata.immutable.layout import ReadBucketProxy
717 from allmydata.util import base32
718 from allmydata.util.encodingutil import quote_output
721 f = open(abs_sharefile, "rb")
724 if prefix == MutableShareFile.MAGIC:
726 m = MutableShareFile(abs_sharefile)
727 WE, nodeid = m._read_write_enabler_and_nodeid(f)
728 data_length = m._read_data_length(f)
729 expiration_time = min( [lease.expiration_time
730 for (i,lease) in m._enumerate_leases(f)] )
731 expiration = max(0, expiration_time - now)
733 share_type = "unknown"
734 f.seek(m.DATA_OFFSET)
735 if f.read(1) == "\x00":
736 # this slot contains an SMDF share
739 if share_type == "SDMF":
740 f.seek(m.DATA_OFFSET)
741 data = f.read(min(data_length, 2000))
744 pieces = unpack_share(data)
745 except NeedMoreDataError, e:
746 # retry once with the larger size
747 size = e.needed_bytes
748 f.seek(m.DATA_OFFSET)
749 data = f.read(min(data_length, size))
750 pieces = unpack_share(data)
751 (seqnum, root_hash, IV, k, N, segsize, datalen,
752 pubkey, signature, share_hash_chain, block_hash_tree,
753 share_data, enc_privkey) = pieces
755 print >>out, "SDMF %s %d/%d %d #%d:%s %d %s" % \
756 (si_s, k, N, datalen,
757 seqnum, base32.b2a(root_hash),
758 expiration, quote_output(abs_sharefile))
760 print >>out, "UNKNOWN mutable %s" % quote_output(abs_sharefile)
762 elif struct.unpack(">L", prefix[:4]) == (1,):
765 class ImmediateReadBucketProxy(ReadBucketProxy):
766 def __init__(self, sf):
768 ReadBucketProxy.__init__(self, None, None, "")
770 return "<ImmediateReadBucketProxy>"
771 def _read(self, offset, size):
772 return defer.succeed(sf.read_share_data(offset, size))
774 # use a ReadBucketProxy to parse the bucket and find the uri extension
775 sf = ShareFile(abs_sharefile)
776 bp = ImmediateReadBucketProxy(sf)
778 expiration_time = min( [lease.expiration_time
779 for lease in sf.get_leases()] )
780 expiration = max(0, expiration_time - now)
782 UEB_data = call(bp.get_uri_extension)
783 unpacked = uri.unpack_extension_readable(UEB_data)
785 k = unpacked["needed_shares"]
786 N = unpacked["total_shares"]
787 filesize = unpacked["size"]
788 ueb_hash = unpacked["UEB_hash"]
790 print >>out, "CHK %s %d/%d %d %s %d %s" % (si_s, k, N, filesize,
791 ueb_hash, expiration,
792 quote_output(abs_sharefile))
795 print >>out, "UNKNOWN really-unknown %s" % quote_output(abs_sharefile)
799 def catalog_shares(options):
800 from allmydata.util.encodingutil import listdir_unicode, quote_output
805 for d in options.nodedirs:
806 d = os.path.join(d, "storage/shares")
808 abbrevs = listdir_unicode(d)
809 except EnvironmentError:
810 # ignore nodes that have storage turned off altogether
813 for abbrevdir in sorted(abbrevs):
814 if abbrevdir == "incoming":
816 abbrevdir = os.path.join(d, abbrevdir)
817 # this tool may get run against bad disks, so we can't assume
818 # that listdir_unicode will always succeed. Try to catalog as much
821 sharedirs = listdir_unicode(abbrevdir)
822 for si_s in sorted(sharedirs):
823 si_dir = os.path.join(abbrevdir, si_s)
824 catalog_shares_one_abbrevdir(si_s, si_dir, now, out,err)
826 print >>err, "Error processing %s" % quote_output(abbrevdir)
827 failure.Failure().printTraceback(err)
837 def catalog_shares_one_abbrevdir(si_s, si_dir, now, out, err):
838 from allmydata.util.encodingutil import listdir_unicode, quote_output
841 for shnum_s in sorted(listdir_unicode(si_dir), key=_as_number):
842 abs_sharefile = os.path.join(si_dir, shnum_s)
843 assert os.path.isfile(abs_sharefile)
845 describe_share(abs_sharefile, si_s, shnum_s, now,
848 print >>err, "Error processing %s" % quote_output(abs_sharefile)
849 failure.Failure().printTraceback(err)
851 print >>err, "Error processing %s" % quote_output(si_dir)
852 failure.Failure().printTraceback(err)
854 class CorruptShareOptions(usage.Options):
855 def getSynopsis(self):
856 return "Usage: tahoe debug corrupt-share SHARE_FILENAME"
859 ["offset", "o", "block-random", "Specify which bit to flip."],
862 def getUsage(self, width=None):
863 t = usage.Options.getUsage(self, width)
865 Corrupt the given share by flipping a bit. This will cause a
866 verifying/downloading client to log an integrity-check failure incident, and
867 downloads will proceed with a different share.
869 The --offset parameter controls which bit should be flipped. The default is
870 to flip a single random bit of the block data.
872 tahoe debug corrupt-share testgrid/node-3/storage/shares/4v/4vozh77tsrw7mdhnj7qvp5ky74/0
874 Obviously, this command should not be used in normal operation.
877 def parseArgs(self, filename):
878 self['filename'] = filename
880 def corrupt_share(options):
882 from allmydata.storage.mutable import MutableShareFile
883 from allmydata.storage.immutable import ShareFile
884 from allmydata.mutable.layout import unpack_header
885 from allmydata.immutable.layout import ReadBucketProxy
887 fn = options['filename']
888 assert options["offset"] == "block-random", "other offsets not implemented"
889 # first, what kind of share is it?
891 def flip_bit(start, end):
892 offset = random.randrange(start, end)
893 bit = random.randrange(0, 8)
894 print >>out, "[%d..%d): %d.b%d" % (start, end, offset, bit)
898 d = chr(ord(d) ^ 0x01)
906 if prefix == MutableShareFile.MAGIC:
908 m = MutableShareFile(fn)
910 f.seek(m.DATA_OFFSET)
912 # make sure this slot contains an SMDF share
913 assert data[0] == "\x00", "non-SDMF mutable shares not supported"
916 (version, ig_seqnum, ig_roothash, ig_IV, ig_k, ig_N, ig_segsize,
917 ig_datalen, offsets) = unpack_header(data)
919 assert version == 0, "we only handle v0 SDMF files"
920 start = m.DATA_OFFSET + offsets["share_data"]
921 end = m.DATA_OFFSET + offsets["enc_privkey"]
924 # otherwise assume it's immutable
926 bp = ReadBucketProxy(None, None, '')
927 offsets = bp._parse_offsets(f.read_share_data(0, 0x24))
928 start = f._data_offset + offsets["data"]
929 end = f._data_offset + offsets["plaintext_hash_tree"]
934 class ReplOptions(usage.Options):
935 def getSynopsis(self):
936 return "Usage: tahoe debug repl"
940 return code.interact()
943 DEFAULT_TESTSUITE = 'allmydata'
945 class TrialOptions(twisted_trial.Options):
946 def getSynopsis(self):
947 return "Usage: tahoe debug trial [options] [[file|package|module|TestCase|testmethod]...]"
949 def parseOptions(self, all_subargs, *a, **kw):
950 self.trial_args = list(all_subargs)
952 # any output from the option parsing will be printed twice, but that's harmless
953 return twisted_trial.Options.parseOptions(self, all_subargs, *a, **kw)
955 def parseArgs(self, *nonoption_args):
956 if not nonoption_args:
957 self.trial_args.append(DEFAULT_TESTSUITE)
959 def getUsage(self, width=None):
960 t = twisted_trial.Options.getUsage(self, width)
962 The 'tahoe debug trial' command uses the correct imports for this instance of
963 Tahoe-LAFS. The default test suite is '%s'.
964 """ % (DEFAULT_TESTSUITE,)
968 sys.argv = ['trial'] + config.trial_args
970 # This does not return.
974 class DebugCommand(usage.Options):
976 ["dump-share", None, DumpOptions,
977 "Unpack and display the contents of a share (uri_extension and leases)."],
978 ["dump-cap", None, DumpCapOptions, "Unpack a read-cap or write-cap."],
979 ["find-shares", None, FindSharesOptions, "Locate sharefiles in node dirs."],
980 ["catalog-shares", None, CatalogSharesOptions, "Describe all shares in node dirs."],
981 ["corrupt-share", None, CorruptShareOptions, "Corrupt a share by flipping a bit."],
982 ["repl", None, ReplOptions, "Open a Python interpreter."],
983 ["trial", None, TrialOptions, "Run tests using Twisted Trial with the right imports."],
985 def postOptions(self):
986 if not hasattr(self, 'subOptions'):
987 raise usage.UsageError("must specify a subcommand")
988 def getSynopsis(self):
989 return "Usage: tahoe debug SUBCOMMAND"
990 def getUsage(self, width=None):
991 #t = usage.Options.getUsage(self, width)
994 tahoe debug dump-share Unpack and display the contents of a share.
995 tahoe debug dump-cap Unpack a read-cap or write-cap.
996 tahoe debug find-shares Locate sharefiles in node directories.
997 tahoe debug catalog-shares Describe all shares in node dirs.
998 tahoe debug corrupt-share Corrupt a share by flipping a bit.
999 tahoe debug repl Open a Python interpreter.
1000 tahoe debug trial Run tests using Twisted Trial with the right imports.
1002 Please run e.g. 'tahoe debug dump-share --help' for more details on each
1005 # See ticket #1441 for why we print different information when
1006 # run via /usr/bin/tahoe. Note that argv[0] is the full path.
1007 if sys.argv[0] == '/usr/bin/tahoe':
1009 To get branch coverage for the Tahoe test suite (on the installed copy of
1010 Tahoe), install the 'python-coverage' package and then use:
1012 python-coverage run --branch /usr/bin/tahoe debug trial
1016 Another debugging feature is that bin%stahoe allows executing an arbitrary
1017 "runner" command (typically an installed Python script, such as 'coverage'),
1018 with the Tahoe libraries on the PYTHONPATH. The runner command name is
1019 prefixed with '@', and any occurrences of '@tahoe' in its arguments are
1020 replaced by the full path to the tahoe script.
1022 For example, if 'coverage' is installed and on the PATH, you can use:
1024 bin%stahoe @coverage run --branch @tahoe debug trial
1026 to get branch coverage for the Tahoe test suite. Or, to run python with
1027 the -3 option that warns about Python 3 incompatibilities:
1029 bin%stahoe @python -3 @tahoe command [options]
1030 """ % (os.sep, os.sep, os.sep)
1034 "dump-share": dump_share,
1035 "dump-cap": dump_cap,
1036 "find-shares": find_shares,
1037 "catalog-shares": catalog_shares,
1038 "corrupt-share": corrupt_share,
1044 def do_debug(options):
1045 so = options.subOptions
1046 so.stdout = options.stdout
1047 so.stderr = options.stderr
1048 f = subDispatch[options.subCommand]
1053 ["debug", None, DebugCommand, "debug subcommands: use 'tahoe debug' for a list."],