2 # do not import any allmydata modules at this level. Do that from inside
3 # individual functions instead.
4 import struct, time, os, sys
5 from twisted.python import usage, failure
6 from twisted.internet import defer
7 from twisted.scripts import trial as twisted_trial
10 class DumpOptions(usage.Options):
11 def getSynopsis(self):
12 return "Usage: tahoe debug dump-share SHARE_FILENAME"
15 ["offsets", None, "Display a table of section offsets."],
16 ["leases-only", None, "Dump leases but not CHK contents."],
19 def getUsage(self, width=None):
20 t = usage.Options.getUsage(self, width)
22 Print lots of information about the given share, by parsing the share's
23 contents. This includes share type, lease information, encoding parameters,
24 hash-tree roots, public keys, and segment sizes. This command also emits a
25 verify-cap for the file that uses the share.
27 tahoe debug dump-share testgrid/node-3/storage/shares/4v/4vozh77tsrw7mdhnj7qvp5ky74/0
32 def parseArgs(self, filename):
33 from allmydata.util.encodingutil import argv_to_abspath
34 self['filename'] = argv_to_abspath(filename)
36 def dump_share(options):
37 from allmydata.storage.mutable import MutableShareFile
38 from allmydata.util.encodingutil import quote_output
42 # check the version, to see if we have a mutable or immutable share
43 print >>out, "share filename: %s" % quote_output(options['filename'])
45 f = open(options['filename'], "rb")
48 if prefix == MutableShareFile.MAGIC:
49 return dump_mutable_share(options)
50 # otherwise assume it's immutable
51 return dump_immutable_share(options)
53 def dump_immutable_share(options):
54 from allmydata.storage.immutable import ShareFile
57 f = ShareFile(options['filename'])
58 if not options["leases-only"]:
59 dump_immutable_chk_share(f, out, options)
60 dump_immutable_lease_info(f, out)
64 def dump_immutable_chk_share(f, out, options):
65 from allmydata import uri
66 from allmydata.util import base32
67 from allmydata.immutable.layout import ReadBucketProxy
68 from allmydata.util.encodingutil import quote_output, to_str
70 # use a ReadBucketProxy to parse the bucket and find the uri extension
71 bp = ReadBucketProxy(None, None, '')
72 offsets = bp._parse_offsets(f.read_share_data(0, 0x44))
73 print >>out, "%20s: %d" % ("version", bp._version)
74 seek = offsets['uri_extension']
75 length = struct.unpack(bp._fieldstruct,
76 f.read_share_data(seek, bp._fieldsize))[0]
78 UEB_data = f.read_share_data(seek, length)
80 unpacked = uri.unpack_extension_readable(UEB_data)
81 keys1 = ("size", "num_segments", "segment_size",
82 "needed_shares", "total_shares")
83 keys2 = ("codec_name", "codec_params", "tail_codec_params")
84 keys3 = ("plaintext_hash", "plaintext_root_hash",
85 "crypttext_hash", "crypttext_root_hash",
86 "share_root_hash", "UEB_hash")
87 display_keys = {"size": "file_size"}
90 dk = display_keys.get(k, k)
91 print >>out, "%20s: %s" % (dk, unpacked[k])
95 dk = display_keys.get(k, k)
96 print >>out, "%20s: %s" % (dk, unpacked[k])
100 dk = display_keys.get(k, k)
101 print >>out, "%20s: %s" % (dk, unpacked[k])
103 leftover = set(unpacked.keys()) - set(keys1 + keys2 + keys3)
106 print >>out, "LEFTOVER:"
107 for k in sorted(leftover):
108 print >>out, "%20s: %s" % (k, unpacked[k])
110 # the storage index isn't stored in the share itself, so we depend upon
111 # knowing the parent directory name to get it
112 pieces = options['filename'].split(os.sep)
114 piece = to_str(pieces[-2])
115 if base32.could_be_base32_encoded(piece):
116 storage_index = base32.a2b(piece)
117 uri_extension_hash = base32.a2b(unpacked["UEB_hash"])
118 u = uri.CHKFileVerifierURI(storage_index, uri_extension_hash,
119 unpacked["needed_shares"],
120 unpacked["total_shares"], unpacked["size"])
121 verify_cap = u.to_string()
122 print >>out, "%20s: %s" % ("verify-cap", quote_output(verify_cap, quotemarks=False))
125 sizes['data'] = (offsets['plaintext_hash_tree'] -
127 sizes['validation'] = (offsets['uri_extension'] -
128 offsets['plaintext_hash_tree'])
129 sizes['uri-extension'] = len(UEB_data)
131 print >>out, " Size of data within the share:"
132 for k in sorted(sizes):
133 print >>out, "%20s: %s" % (k, sizes[k])
135 if options['offsets']:
137 print >>out, " Section Offsets:"
138 print >>out, "%20s: %s" % ("share data", f._data_offset)
139 for k in ["data", "plaintext_hash_tree", "crypttext_hash_tree",
140 "block_hashes", "share_hashes", "uri_extension"]:
141 name = {"data": "block data"}.get(k,k)
142 offset = f._data_offset + offsets[k]
143 print >>out, " %20s: %s (0x%x)" % (name, offset, offset)
144 print >>out, "%20s: %s" % ("leases", f._lease_offset)
146 def dump_immutable_lease_info(f, out):
147 # display lease information too
149 leases = list(f.get_leases())
151 for i,lease in enumerate(leases):
152 when = format_expiration_time(lease.expiration_time)
153 print >>out, " Lease #%d: owner=%d, expire in %s" \
154 % (i, lease.owner_num, when)
156 print >>out, " No leases."
158 def format_expiration_time(expiration_time):
160 remains = expiration_time - now
161 when = "%ds" % remains
162 if remains > 24*3600:
163 when += " (%d days)" % (remains / (24*3600))
165 when += " (%d hours)" % (remains / 3600)
169 def dump_mutable_share(options):
170 from allmydata.storage.mutable import MutableShareFile
171 from allmydata.util import base32, idlib
173 m = MutableShareFile(options['filename'])
174 f = open(options['filename'], "rb")
175 WE, nodeid = m._read_write_enabler_and_nodeid(f)
176 num_extra_leases = m._read_num_extra_leases(f)
177 data_length = m._read_data_length(f)
178 extra_lease_offset = m._read_extra_lease_offset(f)
179 container_size = extra_lease_offset - m.DATA_OFFSET
180 leases = list(m._enumerate_leases(f))
182 share_type = "unknown"
183 f.seek(m.DATA_OFFSET)
185 if version == "\x00":
186 # this slot contains an SMDF share
188 elif version == "\x01":
193 print >>out, "Mutable slot found:"
194 print >>out, " share_type: %s" % share_type
195 print >>out, " write_enabler: %s" % base32.b2a(WE)
196 print >>out, " WE for nodeid: %s" % idlib.nodeid_b2a(nodeid)
197 print >>out, " num_extra_leases: %d" % num_extra_leases
198 print >>out, " container_size: %d" % container_size
199 print >>out, " data_length: %d" % data_length
201 for (leasenum, lease) in leases:
203 print >>out, " Lease #%d:" % leasenum
204 print >>out, " ownerid: %d" % lease.owner_num
205 when = format_expiration_time(lease.expiration_time)
206 print >>out, " expires in %s" % when
207 print >>out, " renew_secret: %s" % base32.b2a(lease.renew_secret)
208 print >>out, " cancel_secret: %s" % base32.b2a(lease.cancel_secret)
209 print >>out, " secrets are for nodeid: %s" % idlib.nodeid_b2a(lease.nodeid)
211 print >>out, "No leases."
214 if share_type == "SDMF":
215 dump_SDMF_share(m, data_length, options)
216 elif share_type == "MDMF":
217 dump_MDMF_share(m, data_length, options)
221 def dump_SDMF_share(m, length, options):
222 from allmydata.mutable.layout import unpack_share, unpack_header
223 from allmydata.mutable.common import NeedMoreDataError
224 from allmydata.util import base32, hashutil
225 from allmydata.uri import SSKVerifierURI
226 from allmydata.util.encodingutil import quote_output, to_str
228 offset = m.DATA_OFFSET
232 f = open(options['filename'], "rb")
234 data = f.read(min(length, 2000))
238 pieces = unpack_share(data)
239 except NeedMoreDataError, e:
240 # retry once with the larger size
241 size = e.needed_bytes
242 f = open(options['filename'], "rb")
244 data = f.read(min(length, size))
246 pieces = unpack_share(data)
248 (seqnum, root_hash, IV, k, N, segsize, datalen,
249 pubkey, signature, share_hash_chain, block_hash_tree,
250 share_data, enc_privkey) = pieces
251 (ig_version, ig_seqnum, ig_roothash, ig_IV, ig_k, ig_N, ig_segsize,
252 ig_datalen, offsets) = unpack_header(data)
254 print >>out, " SDMF contents:"
255 print >>out, " seqnum: %d" % seqnum
256 print >>out, " root_hash: %s" % base32.b2a(root_hash)
257 print >>out, " IV: %s" % base32.b2a(IV)
258 print >>out, " required_shares: %d" % k
259 print >>out, " total_shares: %d" % N
260 print >>out, " segsize: %d" % segsize
261 print >>out, " datalen: %d" % datalen
262 print >>out, " enc_privkey: %d bytes" % len(enc_privkey)
263 print >>out, " pubkey: %d bytes" % len(pubkey)
264 print >>out, " signature: %d bytes" % len(signature)
265 share_hash_ids = ",".join(sorted([str(hid)
266 for hid in share_hash_chain.keys()]))
267 print >>out, " share_hash_chain: %s" % share_hash_ids
268 print >>out, " block_hash_tree: %d nodes" % len(block_hash_tree)
270 # the storage index isn't stored in the share itself, so we depend upon
271 # knowing the parent directory name to get it
272 pieces = options['filename'].split(os.sep)
274 piece = to_str(pieces[-2])
275 if base32.could_be_base32_encoded(piece):
276 storage_index = base32.a2b(piece)
277 fingerprint = hashutil.ssk_pubkey_fingerprint_hash(pubkey)
278 u = SSKVerifierURI(storage_index, fingerprint)
279 verify_cap = u.to_string()
280 print >>out, " verify-cap:", quote_output(verify_cap, quotemarks=False)
282 if options['offsets']:
283 # NOTE: this offset-calculation code is fragile, and needs to be
284 # merged with MutableShareFile's internals.
286 print >>out, " Section Offsets:"
287 def printoffset(name, value, shift=0):
288 print >>out, "%s%20s: %s (0x%x)" % (" "*shift, name, value, value)
289 printoffset("first lease", m.HEADER_SIZE)
290 printoffset("share data", m.DATA_OFFSET)
291 o_seqnum = m.DATA_OFFSET + struct.calcsize(">B")
292 printoffset("seqnum", o_seqnum, 2)
293 o_root_hash = m.DATA_OFFSET + struct.calcsize(">BQ")
294 printoffset("root_hash", o_root_hash, 2)
295 for k in ["signature", "share_hash_chain", "block_hash_tree",
297 "enc_privkey", "EOF"]:
298 name = {"share_data": "block data",
299 "EOF": "end of share data"}.get(k,k)
300 offset = m.DATA_OFFSET + offsets[k]
301 printoffset(name, offset, 2)
302 f = open(options['filename'], "rb")
303 printoffset("extra leases", m._read_extra_lease_offset(f) + 4)
308 def dump_MDMF_share(m, length, options):
309 from allmydata.mutable.layout import MDMFSlotReadProxy
310 from allmydata.util import base32, hashutil
311 from allmydata.uri import MDMFVerifierURI
312 from allmydata.util.encodingutil import quote_output, to_str
314 offset = m.DATA_OFFSET
318 f = open(options['filename'], "rb")
319 storage_index = None; shnum = 0
321 class ShareDumper(MDMFSlotReadProxy):
322 def _read(self, readvs, force_remote=False, queue=False):
324 for (where,length) in readvs:
326 data.append(f.read(length))
327 return defer.succeed({shnum: data})
329 # assume 2kB will be enough
330 p = ShareDumper(None, storage_index, shnum)
334 # these methods return Deferreds, but we happen to know that they run
335 # synchronously when not actually talking to a remote server
337 d.addCallback(stash.append)
340 verinfo = extract(p.get_verinfo)
341 encprivkey = extract(p.get_encprivkey)
342 signature = extract(p.get_signature)
343 pubkey = extract(p.get_verification_key)
344 block_hash_tree = extract(p.get_blockhashes)
345 share_hash_chain = extract(p.get_sharehashes)
348 (seqnum, root_hash, salt_to_use, segsize, datalen, k, N, prefix,
351 print >>out, " MDMF contents:"
352 print >>out, " seqnum: %d" % seqnum
353 print >>out, " root_hash: %s" % base32.b2a(root_hash)
354 #print >>out, " IV: %s" % base32.b2a(IV)
355 print >>out, " required_shares: %d" % k
356 print >>out, " total_shares: %d" % N
357 print >>out, " segsize: %d" % segsize
358 print >>out, " datalen: %d" % datalen
359 print >>out, " enc_privkey: %d bytes" % len(encprivkey)
360 print >>out, " pubkey: %d bytes" % len(pubkey)
361 print >>out, " signature: %d bytes" % len(signature)
362 share_hash_ids = ",".join([str(hid)
363 for hid in sorted(share_hash_chain.keys())])
364 print >>out, " share_hash_chain: %s" % share_hash_ids
365 print >>out, " block_hash_tree: %d nodes" % len(block_hash_tree)
367 # the storage index isn't stored in the share itself, so we depend upon
368 # knowing the parent directory name to get it
369 pieces = options['filename'].split(os.sep)
371 piece = to_str(pieces[-2])
372 if base32.could_be_base32_encoded(piece):
373 storage_index = base32.a2b(piece)
374 fingerprint = hashutil.ssk_pubkey_fingerprint_hash(pubkey)
375 hints = [str(k), str(segsize)]
376 u = MDMFVerifierURI(storage_index, fingerprint, hints)
377 verify_cap = u.to_string()
378 print >>out, " verify-cap:", quote_output(verify_cap, quotemarks=False)
380 if options['offsets']:
381 # NOTE: this offset-calculation code is fragile, and needs to be
382 # merged with MutableShareFile's internals.
385 print >>out, " Section Offsets:"
386 def printoffset(name, value, shift=0):
387 print >>out, "%s%.20s: %s (0x%x)" % (" "*shift, name, value, value)
388 printoffset("first lease", m.HEADER_SIZE, 2)
389 printoffset("share data", m.DATA_OFFSET, 2)
390 o_seqnum = m.DATA_OFFSET + struct.calcsize(">B")
391 printoffset("seqnum", o_seqnum, 4)
392 o_root_hash = m.DATA_OFFSET + struct.calcsize(">BQ")
393 printoffset("root_hash", o_root_hash, 4)
394 for k in ["enc_privkey", "share_hash_chain", "signature",
395 "verification_key", "verification_key_end",
396 "share_data", "block_hash_tree", "EOF"]:
397 name = {"share_data": "block data",
398 "verification_key": "pubkey",
399 "verification_key_end": "end of pubkey",
400 "EOF": "end of share data"}.get(k,k)
401 offset = m.DATA_OFFSET + offsets[k]
402 printoffset(name, offset, 4)
403 f = open(options['filename'], "rb")
404 printoffset("extra leases", m._read_extra_lease_offset(f) + 4, 2)
411 class DumpCapOptions(usage.Options):
412 def getSynopsis(self):
413 return "Usage: tahoe debug dump-cap [options] FILECAP"
416 None, "Specify the storage server nodeid (ASCII), to construct WE and secrets."],
417 ["client-secret", "c", None,
418 "Specify the client's base secret (ASCII), to construct secrets."],
419 ["client-dir", "d", None,
420 "Specify the client's base directory, from which a -c secret will be read."],
422 def parseArgs(self, cap):
425 def getUsage(self, width=None):
426 t = usage.Options.getUsage(self, width)
428 Print information about the given cap-string (aka: URI, file-cap, dir-cap,
429 read-cap, write-cap). The URI string is parsed and unpacked. This prints the
430 type of the cap, its storage index, and any derived keys.
432 tahoe debug dump-cap URI:SSK-Verifier:4vozh77tsrw7mdhnj7qvp5ky74:q7f3dwz76sjys4kqfdt3ocur2pay3a6rftnkqmi2uxu3vqsdsofq
434 This may be useful to determine if a read-cap and a write-cap refer to the
435 same time, or to extract the storage-index from a file-cap (to then use with
438 If additional information is provided (storage server nodeid and/or client
439 base secret), this command will compute the shared secrets used for the
440 write-enabler and for lease-renewal.
445 def dump_cap(options):
446 from allmydata import uri
447 from allmydata.util import base32
448 from base64 import b32decode
449 import urlparse, urllib
454 if options['nodeid']:
455 nodeid = b32decode(options['nodeid'].upper())
457 if options['client-secret']:
458 secret = base32.a2b(options['client-secret'])
459 elif options['client-dir']:
460 secretfile = os.path.join(options['client-dir'], "private", "secret")
462 secret = base32.a2b(open(secretfile, "r").read().strip())
463 except EnvironmentError:
466 if cap.startswith("http"):
467 scheme, netloc, path, params, query, fragment = urlparse.urlparse(cap)
468 assert path.startswith("/uri/")
469 cap = urllib.unquote(path[len("/uri/"):])
471 u = uri.from_string(cap)
474 dump_uri_instance(u, nodeid, secret, out)
476 def _dump_secrets(storage_index, secret, nodeid, out):
477 from allmydata.util import hashutil
478 from allmydata.util import base32
481 crs = hashutil.my_renewal_secret_hash(secret)
482 print >>out, " client renewal secret:", base32.b2a(crs)
483 frs = hashutil.file_renewal_secret_hash(crs, storage_index)
484 print >>out, " file renewal secret:", base32.b2a(frs)
486 renew = hashutil.bucket_renewal_secret_hash(frs, nodeid)
487 print >>out, " lease renewal secret:", base32.b2a(renew)
488 ccs = hashutil.my_cancel_secret_hash(secret)
489 print >>out, " client cancel secret:", base32.b2a(ccs)
490 fcs = hashutil.file_cancel_secret_hash(ccs, storage_index)
491 print >>out, " file cancel secret:", base32.b2a(fcs)
493 cancel = hashutil.bucket_cancel_secret_hash(fcs, nodeid)
494 print >>out, " lease cancel secret:", base32.b2a(cancel)
496 def dump_uri_instance(u, nodeid, secret, out, show_header=True):
497 from allmydata import uri
498 from allmydata.storage.server import si_b2a
499 from allmydata.util import base32, hashutil
500 from allmydata.util.encodingutil import quote_output
502 if isinstance(u, uri.CHKFileURI):
504 print >>out, "CHK File:"
505 print >>out, " key:", base32.b2a(u.key)
506 print >>out, " UEB hash:", base32.b2a(u.uri_extension_hash)
507 print >>out, " size:", u.size
508 print >>out, " k/N: %d/%d" % (u.needed_shares, u.total_shares)
509 print >>out, " storage index:", si_b2a(u.get_storage_index())
510 _dump_secrets(u.get_storage_index(), secret, nodeid, out)
511 elif isinstance(u, uri.CHKFileVerifierURI):
513 print >>out, "CHK Verifier URI:"
514 print >>out, " UEB hash:", base32.b2a(u.uri_extension_hash)
515 print >>out, " size:", u.size
516 print >>out, " k/N: %d/%d" % (u.needed_shares, u.total_shares)
517 print >>out, " storage index:", si_b2a(u.get_storage_index())
519 elif isinstance(u, uri.LiteralFileURI):
521 print >>out, "Literal File URI:"
522 print >>out, " data:", quote_output(u.data)
524 elif isinstance(u, uri.WriteableSSKFileURI): # SDMF
526 print >>out, "SDMF Writeable URI:"
527 print >>out, " writekey:", base32.b2a(u.writekey)
528 print >>out, " readkey:", base32.b2a(u.readkey)
529 print >>out, " storage index:", si_b2a(u.get_storage_index())
530 print >>out, " fingerprint:", base32.b2a(u.fingerprint)
533 we = hashutil.ssk_write_enabler_hash(u.writekey, nodeid)
534 print >>out, " write_enabler:", base32.b2a(we)
536 _dump_secrets(u.get_storage_index(), secret, nodeid, out)
537 elif isinstance(u, uri.ReadonlySSKFileURI):
539 print >>out, "SDMF Read-only URI:"
540 print >>out, " readkey:", base32.b2a(u.readkey)
541 print >>out, " storage index:", si_b2a(u.get_storage_index())
542 print >>out, " fingerprint:", base32.b2a(u.fingerprint)
543 elif isinstance(u, uri.SSKVerifierURI):
545 print >>out, "SDMF Verifier URI:"
546 print >>out, " storage index:", si_b2a(u.get_storage_index())
547 print >>out, " fingerprint:", base32.b2a(u.fingerprint)
549 elif isinstance(u, uri.WriteableMDMFFileURI): # MDMF
551 print >>out, "MDMF Writeable URI:"
552 print >>out, " writekey:", base32.b2a(u.writekey)
553 print >>out, " readkey:", base32.b2a(u.readkey)
554 print >>out, " storage index:", si_b2a(u.get_storage_index())
555 print >>out, " fingerprint:", base32.b2a(u.fingerprint)
558 we = hashutil.ssk_write_enabler_hash(u.writekey, nodeid)
559 print >>out, " write_enabler:", base32.b2a(we)
561 _dump_secrets(u.get_storage_index(), secret, nodeid, out)
562 elif isinstance(u, uri.ReadonlyMDMFFileURI):
564 print >>out, "MDMF Read-only URI:"
565 print >>out, " readkey:", base32.b2a(u.readkey)
566 print >>out, " storage index:", si_b2a(u.get_storage_index())
567 print >>out, " fingerprint:", base32.b2a(u.fingerprint)
568 elif isinstance(u, uri.MDMFVerifierURI):
570 print >>out, "MDMF Verifier URI:"
571 print >>out, " storage index:", si_b2a(u.get_storage_index())
572 print >>out, " fingerprint:", base32.b2a(u.fingerprint)
575 elif isinstance(u, uri.ImmutableDirectoryURI): # CHK-based directory
577 print >>out, "CHK Directory URI:"
578 dump_uri_instance(u._filenode_uri, nodeid, secret, out, False)
579 elif isinstance(u, uri.ImmutableDirectoryURIVerifier):
581 print >>out, "CHK Directory Verifier URI:"
582 dump_uri_instance(u._filenode_uri, nodeid, secret, out, False)
584 elif isinstance(u, uri.DirectoryURI): # SDMF-based directory
586 print >>out, "Directory Writeable URI:"
587 dump_uri_instance(u._filenode_uri, nodeid, secret, out, False)
588 elif isinstance(u, uri.ReadonlyDirectoryURI):
590 print >>out, "Directory Read-only URI:"
591 dump_uri_instance(u._filenode_uri, nodeid, secret, out, False)
592 elif isinstance(u, uri.DirectoryURIVerifier):
594 print >>out, "Directory Verifier URI:"
595 dump_uri_instance(u._filenode_uri, nodeid, secret, out, False)
597 elif isinstance(u, uri.MDMFDirectoryURI): # MDMF-based directory
599 print >>out, "Directory Writeable URI:"
600 dump_uri_instance(u._filenode_uri, nodeid, secret, out, False)
601 elif isinstance(u, uri.ReadonlyMDMFDirectoryURI):
603 print >>out, "Directory Read-only URI:"
604 dump_uri_instance(u._filenode_uri, nodeid, secret, out, False)
605 elif isinstance(u, uri.MDMFDirectoryURIVerifier):
607 print >>out, "Directory Verifier URI:"
608 dump_uri_instance(u._filenode_uri, nodeid, secret, out, False)
611 print >>out, "unknown cap type"
613 class FindSharesOptions(usage.Options):
614 def getSynopsis(self):
615 return "Usage: tahoe debug find-shares STORAGE_INDEX NODEDIRS.."
617 def parseArgs(self, storage_index_s, *nodedirs):
618 from allmydata.util.encodingutil import argv_to_abspath
619 self.si_s = storage_index_s
620 self.nodedirs = map(argv_to_abspath, nodedirs)
622 def getUsage(self, width=None):
623 t = usage.Options.getUsage(self, width)
625 Locate all shares for the given storage index. This command looks through one
626 or more node directories to find the shares. It returns a list of filenames,
627 one per line, for each share file found.
629 tahoe debug find-shares 4vozh77tsrw7mdhnj7qvp5ky74 testgrid/node-*
631 It may be useful during testing, when running a test grid in which all the
632 nodes are on a local disk. The share files thus located can be counted,
633 examined (with dump-share), or corrupted/deleted to test checker/repairer.
637 def find_shares(options):
638 """Given a storage index and a list of node directories, emit a list of
639 all matching shares to stdout, one per line. For example:
641 find-shares.py 44kai1tui348689nrw8fjegc8c ~/testnet/node-*
645 /home/warner/testnet/node-1/storage/shares/44k/44kai1tui348689nrw8fjegc8c/5
646 /home/warner/testnet/node-1/storage/shares/44k/44kai1tui348689nrw8fjegc8c/9
647 /home/warner/testnet/node-2/storage/shares/44k/44kai1tui348689nrw8fjegc8c/2
649 from allmydata.storage.server import si_a2b, storage_index_to_dir
650 from allmydata.util.encodingutil import listdir_unicode
653 sharedir = storage_index_to_dir(si_a2b(options.si_s))
654 for d in options.nodedirs:
655 d = os.path.join(d, "storage/shares", sharedir)
656 if os.path.exists(d):
657 for shnum in listdir_unicode(d):
658 print >>out, os.path.join(d, shnum)
663 class CatalogSharesOptions(usage.Options):
667 def parseArgs(self, *nodedirs):
668 from allmydata.util.encodingutil import argv_to_abspath
669 self.nodedirs = map(argv_to_abspath, nodedirs)
671 raise usage.UsageError("must specify at least one node directory")
673 def getSynopsis(self):
674 return "Usage: tahoe debug catalog-shares NODEDIRS.."
676 def getUsage(self, width=None):
677 t = usage.Options.getUsage(self, width)
679 Locate all shares in the given node directories, and emit a one-line summary
680 of each share. Run it like this:
682 tahoe debug catalog-shares testgrid/node-* >allshares.txt
684 The lines it emits will look like the following:
686 CHK $SI $k/$N $filesize $UEB_hash $expiration $abspath_sharefile
687 SDMF $SI $k/$N $filesize $seqnum/$roothash $expiration $abspath_sharefile
688 UNKNOWN $abspath_sharefile
690 This command can be used to build up a catalog of shares from many storage
691 servers and then sort the results to compare all shares for the same file. If
692 you see shares with the same SI but different parameters/filesize/UEB_hash,
693 then something is wrong. The misc/find-share/anomalies.py script may be
698 def call(c, *args, **kwargs):
699 # take advantage of the fact that ImmediateReadBucketProxy returns
700 # Deferreds that are already fired
703 d = defer.maybeDeferred(c, *args, **kwargs)
704 d.addCallbacks(results.append, failures.append)
706 failures[0].raiseException()
709 def describe_share(abs_sharefile, si_s, shnum_s, now, out):
710 from allmydata import uri
711 from allmydata.storage.mutable import MutableShareFile
712 from allmydata.storage.immutable import ShareFile
713 from allmydata.mutable.layout import unpack_share
714 from allmydata.mutable.common import NeedMoreDataError
715 from allmydata.immutable.layout import ReadBucketProxy
716 from allmydata.util import base32
717 from allmydata.util.encodingutil import quote_output
720 f = open(abs_sharefile, "rb")
723 if prefix == MutableShareFile.MAGIC:
725 m = MutableShareFile(abs_sharefile)
726 WE, nodeid = m._read_write_enabler_and_nodeid(f)
727 data_length = m._read_data_length(f)
728 expiration_time = min( [lease.expiration_time
729 for (i,lease) in m._enumerate_leases(f)] )
730 expiration = max(0, expiration_time - now)
732 share_type = "unknown"
733 f.seek(m.DATA_OFFSET)
734 if f.read(1) == "\x00":
735 # this slot contains an SMDF share
738 if share_type == "SDMF":
739 f.seek(m.DATA_OFFSET)
740 data = f.read(min(data_length, 2000))
743 pieces = unpack_share(data)
744 except NeedMoreDataError, e:
745 # retry once with the larger size
746 size = e.needed_bytes
747 f.seek(m.DATA_OFFSET)
748 data = f.read(min(data_length, size))
749 pieces = unpack_share(data)
750 (seqnum, root_hash, IV, k, N, segsize, datalen,
751 pubkey, signature, share_hash_chain, block_hash_tree,
752 share_data, enc_privkey) = pieces
754 print >>out, "SDMF %s %d/%d %d #%d:%s %d %s" % \
755 (si_s, k, N, datalen,
756 seqnum, base32.b2a(root_hash),
757 expiration, quote_output(abs_sharefile))
759 print >>out, "UNKNOWN mutable %s" % quote_output(abs_sharefile)
761 elif struct.unpack(">L", prefix[:4]) == (1,):
764 class ImmediateReadBucketProxy(ReadBucketProxy):
765 def __init__(self, sf):
767 ReadBucketProxy.__init__(self, None, None, "")
769 return "<ImmediateReadBucketProxy>"
770 def _read(self, offset, size):
771 return defer.succeed(sf.read_share_data(offset, size))
773 # use a ReadBucketProxy to parse the bucket and find the uri extension
774 sf = ShareFile(abs_sharefile)
775 bp = ImmediateReadBucketProxy(sf)
777 expiration_time = min( [lease.expiration_time
778 for lease in sf.get_leases()] )
779 expiration = max(0, expiration_time - now)
781 UEB_data = call(bp.get_uri_extension)
782 unpacked = uri.unpack_extension_readable(UEB_data)
784 k = unpacked["needed_shares"]
785 N = unpacked["total_shares"]
786 filesize = unpacked["size"]
787 ueb_hash = unpacked["UEB_hash"]
789 print >>out, "CHK %s %d/%d %d %s %d %s" % (si_s, k, N, filesize,
790 ueb_hash, expiration,
791 quote_output(abs_sharefile))
794 print >>out, "UNKNOWN really-unknown %s" % quote_output(abs_sharefile)
798 def catalog_shares(options):
799 from allmydata.util.encodingutil import listdir_unicode, quote_output
804 for d in options.nodedirs:
805 d = os.path.join(d, "storage/shares")
807 abbrevs = listdir_unicode(d)
808 except EnvironmentError:
809 # ignore nodes that have storage turned off altogether
812 for abbrevdir in sorted(abbrevs):
813 if abbrevdir == "incoming":
815 abbrevdir = os.path.join(d, abbrevdir)
816 # this tool may get run against bad disks, so we can't assume
817 # that listdir_unicode will always succeed. Try to catalog as much
820 sharedirs = listdir_unicode(abbrevdir)
821 for si_s in sorted(sharedirs):
822 si_dir = os.path.join(abbrevdir, si_s)
823 catalog_shares_one_abbrevdir(si_s, si_dir, now, out,err)
825 print >>err, "Error processing %s" % quote_output(abbrevdir)
826 failure.Failure().printTraceback(err)
836 def catalog_shares_one_abbrevdir(si_s, si_dir, now, out, err):
837 from allmydata.util.encodingutil import listdir_unicode, quote_output
840 for shnum_s in sorted(listdir_unicode(si_dir), key=_as_number):
841 abs_sharefile = os.path.join(si_dir, shnum_s)
842 assert os.path.isfile(abs_sharefile)
844 describe_share(abs_sharefile, si_s, shnum_s, now,
847 print >>err, "Error processing %s" % quote_output(abs_sharefile)
848 failure.Failure().printTraceback(err)
850 print >>err, "Error processing %s" % quote_output(si_dir)
851 failure.Failure().printTraceback(err)
853 class CorruptShareOptions(usage.Options):
854 def getSynopsis(self):
855 return "Usage: tahoe debug corrupt-share SHARE_FILENAME"
858 ["offset", "o", "block-random", "Specify which bit to flip."],
861 def getUsage(self, width=None):
862 t = usage.Options.getUsage(self, width)
864 Corrupt the given share by flipping a bit. This will cause a
865 verifying/downloading client to log an integrity-check failure incident, and
866 downloads will proceed with a different share.
868 The --offset parameter controls which bit should be flipped. The default is
869 to flip a single random bit of the block data.
871 tahoe debug corrupt-share testgrid/node-3/storage/shares/4v/4vozh77tsrw7mdhnj7qvp5ky74/0
873 Obviously, this command should not be used in normal operation.
876 def parseArgs(self, filename):
877 self['filename'] = filename
879 def corrupt_share(options):
881 from allmydata.storage.mutable import MutableShareFile
882 from allmydata.storage.immutable import ShareFile
883 from allmydata.mutable.layout import unpack_header
884 from allmydata.immutable.layout import ReadBucketProxy
886 fn = options['filename']
887 assert options["offset"] == "block-random", "other offsets not implemented"
888 # first, what kind of share is it?
890 def flip_bit(start, end):
891 offset = random.randrange(start, end)
892 bit = random.randrange(0, 8)
893 print >>out, "[%d..%d): %d.b%d" % (start, end, offset, bit)
897 d = chr(ord(d) ^ 0x01)
905 if prefix == MutableShareFile.MAGIC:
907 m = MutableShareFile(fn)
909 f.seek(m.DATA_OFFSET)
911 # make sure this slot contains an SMDF share
912 assert data[0] == "\x00", "non-SDMF mutable shares not supported"
915 (version, ig_seqnum, ig_roothash, ig_IV, ig_k, ig_N, ig_segsize,
916 ig_datalen, offsets) = unpack_header(data)
918 assert version == 0, "we only handle v0 SDMF files"
919 start = m.DATA_OFFSET + offsets["share_data"]
920 end = m.DATA_OFFSET + offsets["enc_privkey"]
923 # otherwise assume it's immutable
925 bp = ReadBucketProxy(None, None, '')
926 offsets = bp._parse_offsets(f.read_share_data(0, 0x24))
927 start = f._data_offset + offsets["data"]
928 end = f._data_offset + offsets["plaintext_hash_tree"]
933 class ReplOptions(usage.Options):
934 def getSynopsis(self):
935 return "Usage: tahoe debug repl"
939 return code.interact()
942 DEFAULT_TESTSUITE = 'allmydata'
944 class TrialOptions(twisted_trial.Options):
945 def getSynopsis(self):
946 return "Usage: tahoe debug trial [options] [[file|package|module|TestCase|testmethod]...]"
948 def parseOptions(self, all_subargs, *a, **kw):
949 self.trial_args = list(all_subargs)
951 # any output from the option parsing will be printed twice, but that's harmless
952 return twisted_trial.Options.parseOptions(self, all_subargs, *a, **kw)
954 def parseArgs(self, *nonoption_args):
955 if not nonoption_args:
956 self.trial_args.append(DEFAULT_TESTSUITE)
958 def getUsage(self, width=None):
959 t = twisted_trial.Options.getUsage(self, width)
961 The 'tahoe debug trial' command uses the correct imports for this instance of
962 Tahoe-LAFS. The default test suite is '%s'.
963 """ % (DEFAULT_TESTSUITE,)
967 sys.argv = ['trial'] + config.trial_args
969 # This does not return.
973 class DebugCommand(usage.Options):
975 ["dump-share", None, DumpOptions,
976 "Unpack and display the contents of a share (uri_extension and leases)."],
977 ["dump-cap", None, DumpCapOptions, "Unpack a read-cap or write-cap."],
978 ["find-shares", None, FindSharesOptions, "Locate sharefiles in node dirs."],
979 ["catalog-shares", None, CatalogSharesOptions, "Describe all shares in node dirs."],
980 ["corrupt-share", None, CorruptShareOptions, "Corrupt a share by flipping a bit."],
981 ["repl", None, ReplOptions, "Open a Python interpreter."],
982 ["trial", None, TrialOptions, "Run tests using Twisted Trial with the right imports."],
984 def postOptions(self):
985 if not hasattr(self, 'subOptions'):
986 raise usage.UsageError("must specify a subcommand")
987 def getSynopsis(self):
988 return "Usage: tahoe debug SUBCOMMAND"
989 def getUsage(self, width=None):
990 #t = usage.Options.getUsage(self, width)
993 tahoe debug dump-share Unpack and display the contents of a share.
994 tahoe debug dump-cap Unpack a read-cap or write-cap.
995 tahoe debug find-shares Locate sharefiles in node directories.
996 tahoe debug catalog-shares Describe all shares in node dirs.
997 tahoe debug corrupt-share Corrupt a share by flipping a bit.
998 tahoe debug repl Open a Python interpreter.
999 tahoe debug trial Run tests using Twisted Trial with the right imports.
1001 Please run e.g. 'tahoe debug dump-share --help' for more details on each
1004 # See ticket #1441 for why we print different information when
1005 # run via /usr/bin/tahoe. Note that argv[0] is the full path.
1006 if sys.argv[0] == '/usr/bin/tahoe':
1008 To get branch coverage for the Tahoe test suite (on the installed copy of
1009 Tahoe), install the 'python-coverage' package and then use:
1011 python-coverage run --branch /usr/bin/tahoe debug trial
1015 Another debugging feature is that bin%stahoe allows executing an arbitrary
1016 "runner" command (typically an installed Python script, such as 'coverage'),
1017 with the Tahoe libraries on the PYTHONPATH. The runner command name is
1018 prefixed with '@', and any occurrences of '@tahoe' in its arguments are
1019 replaced by the full path to the tahoe script.
1021 For example, if 'coverage' is installed and on the PATH, you can use:
1023 bin%stahoe @coverage run --branch @tahoe debug trial
1025 to get branch coverage for the Tahoe test suite. Or, to run python with
1026 the -3 option that warns about Python 3 incompatibilities:
1028 bin%stahoe @python -3 @tahoe command [options]
1029 """ % (os.sep, os.sep, os.sep)
1033 "dump-share": dump_share,
1034 "dump-cap": dump_cap,
1035 "find-shares": find_shares,
1036 "catalog-shares": catalog_shares,
1037 "corrupt-share": corrupt_share,
1043 def do_debug(options):
1044 so = options.subOptions
1045 so.stdout = options.stdout
1046 so.stderr = options.stderr
1047 f = subDispatch[options.subCommand]
1052 ["debug", None, DebugCommand, "debug subcommands: use 'tahoe debug' for a list."],