2 # do not import any allmydata modules at this level. Do that from inside
3 # individual functions instead.
4 import struct, time, os, sys
5 from twisted.python import usage, failure
6 from twisted.internet import defer
7 from twisted.scripts import trial as twisted_trial
8 from foolscap.logging import cli as foolscap_cli
9 from allmydata.scripts.common import BaseOptions
12 class DumpOptions(BaseOptions):
13 def getSynopsis(self):
14 return "Usage: tahoe [global-opts] debug dump-share SHARE_FILENAME"
17 ["offsets", None, "Display a table of section offsets."],
18 ["leases-only", None, "Dump leases but not CHK contents."],
21 def getUsage(self, width=None):
22 t = BaseOptions.getUsage(self, width)
24 Print lots of information about the given share, by parsing the share's
25 contents. This includes share type, lease information, encoding parameters,
26 hash-tree roots, public keys, and segment sizes. This command also emits a
27 verify-cap for the file that uses the share.
29 tahoe debug dump-share testgrid/node-3/storage/shares/4v/4vozh77tsrw7mdhnj7qvp5ky74/0
34 def parseArgs(self, filename):
35 from allmydata.util.encodingutil import argv_to_abspath
36 self['filename'] = argv_to_abspath(filename)
38 def dump_share(options):
39 from allmydata.storage.mutable import MutableShareFile
40 from allmydata.util.encodingutil import quote_output
44 # check the version, to see if we have a mutable or immutable share
45 print >>out, "share filename: %s" % quote_output(options['filename'])
47 f = open(options['filename'], "rb")
50 if prefix == MutableShareFile.MAGIC:
51 return dump_mutable_share(options)
52 # otherwise assume it's immutable
53 return dump_immutable_share(options)
55 def dump_immutable_share(options):
56 from allmydata.storage.immutable import ShareFile
59 f = ShareFile(options['filename'])
60 if not options["leases-only"]:
61 dump_immutable_chk_share(f, out, options)
62 dump_immutable_lease_info(f, out)
66 def dump_immutable_chk_share(f, out, options):
67 from allmydata import uri
68 from allmydata.util import base32
69 from allmydata.immutable.layout import ReadBucketProxy
70 from allmydata.util.encodingutil import quote_output, to_str
72 # use a ReadBucketProxy to parse the bucket and find the uri extension
73 bp = ReadBucketProxy(None, None, '')
74 offsets = bp._parse_offsets(f.read_share_data(0, 0x44))
75 print >>out, "%20s: %d" % ("version", bp._version)
76 seek = offsets['uri_extension']
77 length = struct.unpack(bp._fieldstruct,
78 f.read_share_data(seek, bp._fieldsize))[0]
80 UEB_data = f.read_share_data(seek, length)
82 unpacked = uri.unpack_extension_readable(UEB_data)
83 keys1 = ("size", "num_segments", "segment_size",
84 "needed_shares", "total_shares")
85 keys2 = ("codec_name", "codec_params", "tail_codec_params")
86 keys3 = ("plaintext_hash", "plaintext_root_hash",
87 "crypttext_hash", "crypttext_root_hash",
88 "share_root_hash", "UEB_hash")
89 display_keys = {"size": "file_size"}
92 dk = display_keys.get(k, k)
93 print >>out, "%20s: %s" % (dk, unpacked[k])
97 dk = display_keys.get(k, k)
98 print >>out, "%20s: %s" % (dk, unpacked[k])
102 dk = display_keys.get(k, k)
103 print >>out, "%20s: %s" % (dk, unpacked[k])
105 leftover = set(unpacked.keys()) - set(keys1 + keys2 + keys3)
108 print >>out, "LEFTOVER:"
109 for k in sorted(leftover):
110 print >>out, "%20s: %s" % (k, unpacked[k])
112 # the storage index isn't stored in the share itself, so we depend upon
113 # knowing the parent directory name to get it
114 pieces = options['filename'].split(os.sep)
116 piece = to_str(pieces[-2])
117 if base32.could_be_base32_encoded(piece):
118 storage_index = base32.a2b(piece)
119 uri_extension_hash = base32.a2b(unpacked["UEB_hash"])
120 u = uri.CHKFileVerifierURI(storage_index, uri_extension_hash,
121 unpacked["needed_shares"],
122 unpacked["total_shares"], unpacked["size"])
123 verify_cap = u.to_string()
124 print >>out, "%20s: %s" % ("verify-cap", quote_output(verify_cap, quotemarks=False))
127 sizes['data'] = (offsets['plaintext_hash_tree'] -
129 sizes['validation'] = (offsets['uri_extension'] -
130 offsets['plaintext_hash_tree'])
131 sizes['uri-extension'] = len(UEB_data)
133 print >>out, " Size of data within the share:"
134 for k in sorted(sizes):
135 print >>out, "%20s: %s" % (k, sizes[k])
137 if options['offsets']:
139 print >>out, " Section Offsets:"
140 print >>out, "%20s: %s" % ("share data", f._data_offset)
141 for k in ["data", "plaintext_hash_tree", "crypttext_hash_tree",
142 "block_hashes", "share_hashes", "uri_extension"]:
143 name = {"data": "block data"}.get(k,k)
144 offset = f._data_offset + offsets[k]
145 print >>out, " %20s: %s (0x%x)" % (name, offset, offset)
146 print >>out, "%20s: %s" % ("leases", f._lease_offset)
148 def dump_immutable_lease_info(f, out):
149 # display lease information too
151 leases = list(f.get_leases())
153 for i,lease in enumerate(leases):
154 when = format_expiration_time(lease.expiration_time)
155 print >>out, " Lease #%d: owner=%d, expire in %s" \
156 % (i, lease.owner_num, when)
158 print >>out, " No leases."
160 def format_expiration_time(expiration_time):
162 remains = expiration_time - now
163 when = "%ds" % remains
164 if remains > 24*3600:
165 when += " (%d days)" % (remains / (24*3600))
167 when += " (%d hours)" % (remains / 3600)
171 def dump_mutable_share(options):
172 from allmydata.storage.mutable import MutableShareFile
173 from allmydata.util import base32, idlib
175 m = MutableShareFile(options['filename'])
176 f = open(options['filename'], "rb")
177 WE, nodeid = m._read_write_enabler_and_nodeid(f)
178 num_extra_leases = m._read_num_extra_leases(f)
179 data_length = m._read_data_length(f)
180 extra_lease_offset = m._read_extra_lease_offset(f)
181 container_size = extra_lease_offset - m.DATA_OFFSET
182 leases = list(m._enumerate_leases(f))
184 share_type = "unknown"
185 f.seek(m.DATA_OFFSET)
187 if version == "\x00":
188 # this slot contains an SMDF share
190 elif version == "\x01":
195 print >>out, "Mutable slot found:"
196 print >>out, " share_type: %s" % share_type
197 print >>out, " write_enabler: %s" % base32.b2a(WE)
198 print >>out, " WE for nodeid: %s" % idlib.nodeid_b2a(nodeid)
199 print >>out, " num_extra_leases: %d" % num_extra_leases
200 print >>out, " container_size: %d" % container_size
201 print >>out, " data_length: %d" % data_length
203 for (leasenum, lease) in leases:
205 print >>out, " Lease #%d:" % leasenum
206 print >>out, " ownerid: %d" % lease.owner_num
207 when = format_expiration_time(lease.expiration_time)
208 print >>out, " expires in %s" % when
209 print >>out, " renew_secret: %s" % base32.b2a(lease.renew_secret)
210 print >>out, " cancel_secret: %s" % base32.b2a(lease.cancel_secret)
211 print >>out, " secrets are for nodeid: %s" % idlib.nodeid_b2a(lease.nodeid)
213 print >>out, "No leases."
216 if share_type == "SDMF":
217 dump_SDMF_share(m, data_length, options)
218 elif share_type == "MDMF":
219 dump_MDMF_share(m, data_length, options)
223 def dump_SDMF_share(m, length, options):
224 from allmydata.mutable.layout import unpack_share, unpack_header
225 from allmydata.mutable.common import NeedMoreDataError
226 from allmydata.util import base32, hashutil
227 from allmydata.uri import SSKVerifierURI
228 from allmydata.util.encodingutil import quote_output, to_str
230 offset = m.DATA_OFFSET
234 f = open(options['filename'], "rb")
236 data = f.read(min(length, 2000))
240 pieces = unpack_share(data)
241 except NeedMoreDataError, e:
242 # retry once with the larger size
243 size = e.needed_bytes
244 f = open(options['filename'], "rb")
246 data = f.read(min(length, size))
248 pieces = unpack_share(data)
250 (seqnum, root_hash, IV, k, N, segsize, datalen,
251 pubkey, signature, share_hash_chain, block_hash_tree,
252 share_data, enc_privkey) = pieces
253 (ig_version, ig_seqnum, ig_roothash, ig_IV, ig_k, ig_N, ig_segsize,
254 ig_datalen, offsets) = unpack_header(data)
256 print >>out, " SDMF contents:"
257 print >>out, " seqnum: %d" % seqnum
258 print >>out, " root_hash: %s" % base32.b2a(root_hash)
259 print >>out, " IV: %s" % base32.b2a(IV)
260 print >>out, " required_shares: %d" % k
261 print >>out, " total_shares: %d" % N
262 print >>out, " segsize: %d" % segsize
263 print >>out, " datalen: %d" % datalen
264 print >>out, " enc_privkey: %d bytes" % len(enc_privkey)
265 print >>out, " pubkey: %d bytes" % len(pubkey)
266 print >>out, " signature: %d bytes" % len(signature)
267 share_hash_ids = ",".join(sorted([str(hid)
268 for hid in share_hash_chain.keys()]))
269 print >>out, " share_hash_chain: %s" % share_hash_ids
270 print >>out, " block_hash_tree: %d nodes" % len(block_hash_tree)
272 # the storage index isn't stored in the share itself, so we depend upon
273 # knowing the parent directory name to get it
274 pieces = options['filename'].split(os.sep)
276 piece = to_str(pieces[-2])
277 if base32.could_be_base32_encoded(piece):
278 storage_index = base32.a2b(piece)
279 fingerprint = hashutil.ssk_pubkey_fingerprint_hash(pubkey)
280 u = SSKVerifierURI(storage_index, fingerprint)
281 verify_cap = u.to_string()
282 print >>out, " verify-cap:", quote_output(verify_cap, quotemarks=False)
284 if options['offsets']:
285 # NOTE: this offset-calculation code is fragile, and needs to be
286 # merged with MutableShareFile's internals.
288 print >>out, " Section Offsets:"
289 def printoffset(name, value, shift=0):
290 print >>out, "%s%20s: %s (0x%x)" % (" "*shift, name, value, value)
291 printoffset("first lease", m.HEADER_SIZE)
292 printoffset("share data", m.DATA_OFFSET)
293 o_seqnum = m.DATA_OFFSET + struct.calcsize(">B")
294 printoffset("seqnum", o_seqnum, 2)
295 o_root_hash = m.DATA_OFFSET + struct.calcsize(">BQ")
296 printoffset("root_hash", o_root_hash, 2)
297 for k in ["signature", "share_hash_chain", "block_hash_tree",
299 "enc_privkey", "EOF"]:
300 name = {"share_data": "block data",
301 "EOF": "end of share data"}.get(k,k)
302 offset = m.DATA_OFFSET + offsets[k]
303 printoffset(name, offset, 2)
304 f = open(options['filename'], "rb")
305 printoffset("extra leases", m._read_extra_lease_offset(f) + 4)
310 def dump_MDMF_share(m, length, options):
311 from allmydata.mutable.layout import MDMFSlotReadProxy
312 from allmydata.util import base32, hashutil
313 from allmydata.uri import MDMFVerifierURI
314 from allmydata.util.encodingutil import quote_output, to_str
316 offset = m.DATA_OFFSET
319 f = open(options['filename'], "rb")
320 storage_index = None; shnum = 0
322 class ShareDumper(MDMFSlotReadProxy):
323 def _read(self, readvs, force_remote=False, queue=False):
325 for (where,length) in readvs:
327 data.append(f.read(length))
328 return defer.succeed({shnum: data})
330 p = ShareDumper(None, storage_index, shnum)
333 # these methods return Deferreds, but we happen to know that they run
334 # synchronously when not actually talking to a remote server
336 d.addCallback(stash.append)
339 verinfo = extract(p.get_verinfo)
340 encprivkey = extract(p.get_encprivkey)
341 signature = extract(p.get_signature)
342 pubkey = extract(p.get_verification_key)
343 block_hash_tree = extract(p.get_blockhashes)
344 share_hash_chain = extract(p.get_sharehashes)
347 (seqnum, root_hash, salt_to_use, segsize, datalen, k, N, prefix,
350 print >>out, " MDMF contents:"
351 print >>out, " seqnum: %d" % seqnum
352 print >>out, " root_hash: %s" % base32.b2a(root_hash)
353 #print >>out, " IV: %s" % base32.b2a(IV)
354 print >>out, " required_shares: %d" % k
355 print >>out, " total_shares: %d" % N
356 print >>out, " segsize: %d" % segsize
357 print >>out, " datalen: %d" % datalen
358 print >>out, " enc_privkey: %d bytes" % len(encprivkey)
359 print >>out, " pubkey: %d bytes" % len(pubkey)
360 print >>out, " signature: %d bytes" % len(signature)
361 share_hash_ids = ",".join([str(hid)
362 for hid in sorted(share_hash_chain.keys())])
363 print >>out, " share_hash_chain: %s" % share_hash_ids
364 print >>out, " block_hash_tree: %d nodes" % len(block_hash_tree)
366 # the storage index isn't stored in the share itself, so we depend upon
367 # knowing the parent directory name to get it
368 pieces = options['filename'].split(os.sep)
370 piece = to_str(pieces[-2])
371 if base32.could_be_base32_encoded(piece):
372 storage_index = base32.a2b(piece)
373 fingerprint = hashutil.ssk_pubkey_fingerprint_hash(pubkey)
374 u = MDMFVerifierURI(storage_index, fingerprint)
375 verify_cap = u.to_string()
376 print >>out, " verify-cap:", quote_output(verify_cap, quotemarks=False)
378 if options['offsets']:
379 # NOTE: this offset-calculation code is fragile, and needs to be
380 # merged with MutableShareFile's internals.
383 print >>out, " Section Offsets:"
384 def printoffset(name, value, shift=0):
385 print >>out, "%s%.20s: %s (0x%x)" % (" "*shift, name, value, value)
386 printoffset("first lease", m.HEADER_SIZE, 2)
387 printoffset("share data", m.DATA_OFFSET, 2)
388 o_seqnum = m.DATA_OFFSET + struct.calcsize(">B")
389 printoffset("seqnum", o_seqnum, 4)
390 o_root_hash = m.DATA_OFFSET + struct.calcsize(">BQ")
391 printoffset("root_hash", o_root_hash, 4)
392 for k in ["enc_privkey", "share_hash_chain", "signature",
393 "verification_key", "verification_key_end",
394 "share_data", "block_hash_tree", "EOF"]:
395 name = {"share_data": "block data",
396 "verification_key": "pubkey",
397 "verification_key_end": "end of pubkey",
398 "EOF": "end of share data"}.get(k,k)
399 offset = m.DATA_OFFSET + offsets[k]
400 printoffset(name, offset, 4)
401 f = open(options['filename'], "rb")
402 printoffset("extra leases", m._read_extra_lease_offset(f) + 4, 2)
409 class DumpCapOptions(BaseOptions):
410 def getSynopsis(self):
411 return "Usage: tahoe [global-opts] debug dump-cap [options] FILECAP"
414 None, "Specify the storage server nodeid (ASCII), to construct WE and secrets."],
415 ["client-secret", "c", None,
416 "Specify the client's base secret (ASCII), to construct secrets."],
417 ["client-dir", "d", None,
418 "Specify the client's base directory, from which a -c secret will be read."],
420 def parseArgs(self, cap):
423 def getUsage(self, width=None):
424 t = BaseOptions.getUsage(self, width)
426 Print information about the given cap-string (aka: URI, file-cap, dir-cap,
427 read-cap, write-cap). The URI string is parsed and unpacked. This prints the
428 type of the cap, its storage index, and any derived keys.
430 tahoe debug dump-cap URI:SSK-Verifier:4vozh77tsrw7mdhnj7qvp5ky74:q7f3dwz76sjys4kqfdt3ocur2pay3a6rftnkqmi2uxu3vqsdsofq
432 This may be useful to determine if a read-cap and a write-cap refer to the
433 same time, or to extract the storage-index from a file-cap (to then use with
436 If additional information is provided (storage server nodeid and/or client
437 base secret), this command will compute the shared secrets used for the
438 write-enabler and for lease-renewal.
443 def dump_cap(options):
444 from allmydata import uri
445 from allmydata.util import base32
446 from base64 import b32decode
447 import urlparse, urllib
452 if options['nodeid']:
453 nodeid = b32decode(options['nodeid'].upper())
455 if options['client-secret']:
456 secret = base32.a2b(options['client-secret'])
457 elif options['client-dir']:
458 secretfile = os.path.join(options['client-dir'], "private", "secret")
460 secret = base32.a2b(open(secretfile, "r").read().strip())
461 except EnvironmentError:
464 if cap.startswith("http"):
465 scheme, netloc, path, params, query, fragment = urlparse.urlparse(cap)
466 assert path.startswith("/uri/")
467 cap = urllib.unquote(path[len("/uri/"):])
469 u = uri.from_string(cap)
472 dump_uri_instance(u, nodeid, secret, out)
474 def _dump_secrets(storage_index, secret, nodeid, out):
475 from allmydata.util import hashutil
476 from allmydata.util import base32
479 crs = hashutil.my_renewal_secret_hash(secret)
480 print >>out, " client renewal secret:", base32.b2a(crs)
481 frs = hashutil.file_renewal_secret_hash(crs, storage_index)
482 print >>out, " file renewal secret:", base32.b2a(frs)
484 renew = hashutil.bucket_renewal_secret_hash(frs, nodeid)
485 print >>out, " lease renewal secret:", base32.b2a(renew)
486 ccs = hashutil.my_cancel_secret_hash(secret)
487 print >>out, " client cancel secret:", base32.b2a(ccs)
488 fcs = hashutil.file_cancel_secret_hash(ccs, storage_index)
489 print >>out, " file cancel secret:", base32.b2a(fcs)
491 cancel = hashutil.bucket_cancel_secret_hash(fcs, nodeid)
492 print >>out, " lease cancel secret:", base32.b2a(cancel)
494 def dump_uri_instance(u, nodeid, secret, out, show_header=True):
495 from allmydata import uri
496 from allmydata.storage.server import si_b2a
497 from allmydata.util import base32, hashutil
498 from allmydata.util.encodingutil import quote_output
500 if isinstance(u, uri.CHKFileURI):
502 print >>out, "CHK File:"
503 print >>out, " key:", base32.b2a(u.key)
504 print >>out, " UEB hash:", base32.b2a(u.uri_extension_hash)
505 print >>out, " size:", u.size
506 print >>out, " k/N: %d/%d" % (u.needed_shares, u.total_shares)
507 print >>out, " storage index:", si_b2a(u.get_storage_index())
508 _dump_secrets(u.get_storage_index(), secret, nodeid, out)
509 elif isinstance(u, uri.CHKFileVerifierURI):
511 print >>out, "CHK Verifier URI:"
512 print >>out, " UEB hash:", base32.b2a(u.uri_extension_hash)
513 print >>out, " size:", u.size
514 print >>out, " k/N: %d/%d" % (u.needed_shares, u.total_shares)
515 print >>out, " storage index:", si_b2a(u.get_storage_index())
517 elif isinstance(u, uri.LiteralFileURI):
519 print >>out, "Literal File URI:"
520 print >>out, " data:", quote_output(u.data)
522 elif isinstance(u, uri.WriteableSSKFileURI): # SDMF
524 print >>out, "SDMF Writeable URI:"
525 print >>out, " writekey:", base32.b2a(u.writekey)
526 print >>out, " readkey:", base32.b2a(u.readkey)
527 print >>out, " storage index:", si_b2a(u.get_storage_index())
528 print >>out, " fingerprint:", base32.b2a(u.fingerprint)
531 we = hashutil.ssk_write_enabler_hash(u.writekey, nodeid)
532 print >>out, " write_enabler:", base32.b2a(we)
534 _dump_secrets(u.get_storage_index(), secret, nodeid, out)
535 elif isinstance(u, uri.ReadonlySSKFileURI):
537 print >>out, "SDMF Read-only URI:"
538 print >>out, " readkey:", base32.b2a(u.readkey)
539 print >>out, " storage index:", si_b2a(u.get_storage_index())
540 print >>out, " fingerprint:", base32.b2a(u.fingerprint)
541 elif isinstance(u, uri.SSKVerifierURI):
543 print >>out, "SDMF Verifier URI:"
544 print >>out, " storage index:", si_b2a(u.get_storage_index())
545 print >>out, " fingerprint:", base32.b2a(u.fingerprint)
547 elif isinstance(u, uri.WriteableMDMFFileURI): # MDMF
549 print >>out, "MDMF Writeable URI:"
550 print >>out, " writekey:", base32.b2a(u.writekey)
551 print >>out, " readkey:", base32.b2a(u.readkey)
552 print >>out, " storage index:", si_b2a(u.get_storage_index())
553 print >>out, " fingerprint:", base32.b2a(u.fingerprint)
556 we = hashutil.ssk_write_enabler_hash(u.writekey, nodeid)
557 print >>out, " write_enabler:", base32.b2a(we)
559 _dump_secrets(u.get_storage_index(), secret, nodeid, out)
560 elif isinstance(u, uri.ReadonlyMDMFFileURI):
562 print >>out, "MDMF Read-only URI:"
563 print >>out, " readkey:", base32.b2a(u.readkey)
564 print >>out, " storage index:", si_b2a(u.get_storage_index())
565 print >>out, " fingerprint:", base32.b2a(u.fingerprint)
566 elif isinstance(u, uri.MDMFVerifierURI):
568 print >>out, "MDMF Verifier URI:"
569 print >>out, " storage index:", si_b2a(u.get_storage_index())
570 print >>out, " fingerprint:", base32.b2a(u.fingerprint)
573 elif isinstance(u, uri.ImmutableDirectoryURI): # CHK-based directory
575 print >>out, "CHK Directory URI:"
576 dump_uri_instance(u._filenode_uri, nodeid, secret, out, False)
577 elif isinstance(u, uri.ImmutableDirectoryURIVerifier):
579 print >>out, "CHK Directory Verifier URI:"
580 dump_uri_instance(u._filenode_uri, nodeid, secret, out, False)
582 elif isinstance(u, uri.DirectoryURI): # SDMF-based directory
584 print >>out, "Directory Writeable URI:"
585 dump_uri_instance(u._filenode_uri, nodeid, secret, out, False)
586 elif isinstance(u, uri.ReadonlyDirectoryURI):
588 print >>out, "Directory Read-only URI:"
589 dump_uri_instance(u._filenode_uri, nodeid, secret, out, False)
590 elif isinstance(u, uri.DirectoryURIVerifier):
592 print >>out, "Directory Verifier URI:"
593 dump_uri_instance(u._filenode_uri, nodeid, secret, out, False)
595 elif isinstance(u, uri.MDMFDirectoryURI): # MDMF-based directory
597 print >>out, "Directory Writeable URI:"
598 dump_uri_instance(u._filenode_uri, nodeid, secret, out, False)
599 elif isinstance(u, uri.ReadonlyMDMFDirectoryURI):
601 print >>out, "Directory Read-only URI:"
602 dump_uri_instance(u._filenode_uri, nodeid, secret, out, False)
603 elif isinstance(u, uri.MDMFDirectoryURIVerifier):
605 print >>out, "Directory Verifier URI:"
606 dump_uri_instance(u._filenode_uri, nodeid, secret, out, False)
609 print >>out, "unknown cap type"
611 class FindSharesOptions(BaseOptions):
612 def getSynopsis(self):
613 return "Usage: tahoe [global-opts] debug find-shares STORAGE_INDEX NODEDIRS.."
615 def parseArgs(self, storage_index_s, *nodedirs):
616 from allmydata.util.encodingutil import argv_to_abspath
617 self.si_s = storage_index_s
618 self.nodedirs = map(argv_to_abspath, nodedirs)
620 def getUsage(self, width=None):
621 t = BaseOptions.getUsage(self, width)
623 Locate all shares for the given storage index. This command looks through one
624 or more node directories to find the shares. It returns a list of filenames,
625 one per line, for each share file found.
627 tahoe debug find-shares 4vozh77tsrw7mdhnj7qvp5ky74 testgrid/node-*
629 It may be useful during testing, when running a test grid in which all the
630 nodes are on a local disk. The share files thus located can be counted,
631 examined (with dump-share), or corrupted/deleted to test checker/repairer.
635 def find_shares(options):
636 """Given a storage index and a list of node directories, emit a list of
637 all matching shares to stdout, one per line. For example:
639 find-shares.py 44kai1tui348689nrw8fjegc8c ~/testnet/node-*
643 /home/warner/testnet/node-1/storage/shares/44k/44kai1tui348689nrw8fjegc8c/5
644 /home/warner/testnet/node-1/storage/shares/44k/44kai1tui348689nrw8fjegc8c/9
645 /home/warner/testnet/node-2/storage/shares/44k/44kai1tui348689nrw8fjegc8c/2
647 from allmydata.storage.server import si_a2b, storage_index_to_dir
648 from allmydata.util.encodingutil import listdir_unicode, quote_local_unicode_path
651 sharedir = storage_index_to_dir(si_a2b(options.si_s))
652 for d in options.nodedirs:
653 d = os.path.join(d, "storage", "shares", sharedir)
654 if os.path.exists(d):
655 for shnum in listdir_unicode(d):
656 print >>out, quote_local_unicode_path(os.path.join(d, shnum), quotemarks=False)
661 class CatalogSharesOptions(BaseOptions):
665 def parseArgs(self, *nodedirs):
666 from allmydata.util.encodingutil import argv_to_abspath
667 self.nodedirs = map(argv_to_abspath, nodedirs)
669 raise usage.UsageError("must specify at least one node directory")
671 def getSynopsis(self):
672 return "Usage: tahoe [global-opts] debug catalog-shares NODEDIRS.."
674 def getUsage(self, width=None):
675 t = BaseOptions.getUsage(self, width)
677 Locate all shares in the given node directories, and emit a one-line summary
678 of each share. Run it like this:
680 tahoe debug catalog-shares testgrid/node-* >allshares.txt
682 The lines it emits will look like the following:
684 CHK $SI $k/$N $filesize $UEB_hash $expiration $abspath_sharefile
685 SDMF $SI $k/$N $filesize $seqnum/$roothash $expiration $abspath_sharefile
686 UNKNOWN $abspath_sharefile
688 This command can be used to build up a catalog of shares from many storage
689 servers and then sort the results to compare all shares for the same file. If
690 you see shares with the same SI but different parameters/filesize/UEB_hash,
691 then something is wrong. The misc/find-share/anomalies.py script may be
696 def call(c, *args, **kwargs):
697 # take advantage of the fact that ImmediateReadBucketProxy returns
698 # Deferreds that are already fired
701 d = defer.maybeDeferred(c, *args, **kwargs)
702 d.addCallbacks(results.append, failures.append)
704 failures[0].raiseException()
707 def describe_share(abs_sharefile, si_s, shnum_s, now, out):
708 from allmydata import uri
709 from allmydata.storage.mutable import MutableShareFile
710 from allmydata.storage.immutable import ShareFile
711 from allmydata.mutable.layout import unpack_share
712 from allmydata.mutable.common import NeedMoreDataError
713 from allmydata.immutable.layout import ReadBucketProxy
714 from allmydata.util import base32
715 from allmydata.util.encodingutil import quote_output
718 f = open(abs_sharefile, "rb")
721 if prefix == MutableShareFile.MAGIC:
723 m = MutableShareFile(abs_sharefile)
724 WE, nodeid = m._read_write_enabler_and_nodeid(f)
725 data_length = m._read_data_length(f)
726 expiration_time = min( [lease.expiration_time
727 for (i,lease) in m._enumerate_leases(f)] )
728 expiration = max(0, expiration_time - now)
730 share_type = "unknown"
731 f.seek(m.DATA_OFFSET)
733 if version == "\x00":
734 # this slot contains an SMDF share
736 elif version == "\x01":
739 if share_type == "SDMF":
740 f.seek(m.DATA_OFFSET)
741 data = f.read(min(data_length, 2000))
744 pieces = unpack_share(data)
745 except NeedMoreDataError, e:
746 # retry once with the larger size
747 size = e.needed_bytes
748 f.seek(m.DATA_OFFSET)
749 data = f.read(min(data_length, size))
750 pieces = unpack_share(data)
751 (seqnum, root_hash, IV, k, N, segsize, datalen,
752 pubkey, signature, share_hash_chain, block_hash_tree,
753 share_data, enc_privkey) = pieces
755 print >>out, "SDMF %s %d/%d %d #%d:%s %d %s" % \
756 (si_s, k, N, datalen,
757 seqnum, base32.b2a(root_hash),
758 expiration, quote_output(abs_sharefile))
759 elif share_type == "MDMF":
760 from allmydata.mutable.layout import MDMFSlotReadProxy
762 # TODO: factor this out with dump_MDMF_share()
763 class ShareDumper(MDMFSlotReadProxy):
764 def _read(self, readvs, force_remote=False, queue=False):
766 for (where,length) in readvs:
767 f.seek(m.DATA_OFFSET+where)
768 data.append(f.read(length))
769 return defer.succeed({fake_shnum: data})
771 p = ShareDumper(None, "fake-si", fake_shnum)
774 # these methods return Deferreds, but we happen to know that
775 # they run synchronously when not actually talking to a
778 d.addCallback(stash.append)
781 verinfo = extract(p.get_verinfo)
782 (seqnum, root_hash, salt_to_use, segsize, datalen, k, N, prefix,
784 print >>out, "MDMF %s %d/%d %d #%d:%s %d %s" % \
785 (si_s, k, N, datalen,
786 seqnum, base32.b2a(root_hash),
787 expiration, quote_output(abs_sharefile))
789 print >>out, "UNKNOWN mutable %s" % quote_output(abs_sharefile)
791 elif struct.unpack(">L", prefix[:4]) == (1,):
794 class ImmediateReadBucketProxy(ReadBucketProxy):
795 def __init__(self, sf):
797 ReadBucketProxy.__init__(self, None, None, "")
799 return "<ImmediateReadBucketProxy>"
800 def _read(self, offset, size):
801 return defer.succeed(sf.read_share_data(offset, size))
803 # use a ReadBucketProxy to parse the bucket and find the uri extension
804 sf = ShareFile(abs_sharefile)
805 bp = ImmediateReadBucketProxy(sf)
807 expiration_time = min( [lease.expiration_time
808 for lease in sf.get_leases()] )
809 expiration = max(0, expiration_time - now)
811 UEB_data = call(bp.get_uri_extension)
812 unpacked = uri.unpack_extension_readable(UEB_data)
814 k = unpacked["needed_shares"]
815 N = unpacked["total_shares"]
816 filesize = unpacked["size"]
817 ueb_hash = unpacked["UEB_hash"]
819 print >>out, "CHK %s %d/%d %d %s %d %s" % (si_s, k, N, filesize,
820 ueb_hash, expiration,
821 quote_output(abs_sharefile))
824 print >>out, "UNKNOWN really-unknown %s" % quote_output(abs_sharefile)
828 def catalog_shares(options):
829 from allmydata.util.encodingutil import listdir_unicode, quote_output
834 for d in options.nodedirs:
835 d = os.path.join(d, "storage", "shares")
837 abbrevs = listdir_unicode(d)
838 except EnvironmentError:
839 # ignore nodes that have storage turned off altogether
842 for abbrevdir in sorted(abbrevs):
843 if abbrevdir == "incoming":
845 abbrevdir = os.path.join(d, abbrevdir)
846 # this tool may get run against bad disks, so we can't assume
847 # that listdir_unicode will always succeed. Try to catalog as much
850 sharedirs = listdir_unicode(abbrevdir)
851 for si_s in sorted(sharedirs):
852 si_dir = os.path.join(abbrevdir, si_s)
853 catalog_shares_one_abbrevdir(si_s, si_dir, now, out,err)
855 print >>err, "Error processing %s" % quote_output(abbrevdir)
856 failure.Failure().printTraceback(err)
866 def catalog_shares_one_abbrevdir(si_s, si_dir, now, out, err):
867 from allmydata.util.encodingutil import listdir_unicode, quote_output
870 for shnum_s in sorted(listdir_unicode(si_dir), key=_as_number):
871 abs_sharefile = os.path.join(si_dir, shnum_s)
872 assert os.path.isfile(abs_sharefile)
874 describe_share(abs_sharefile, si_s, shnum_s, now,
877 print >>err, "Error processing %s" % quote_output(abs_sharefile)
878 failure.Failure().printTraceback(err)
880 print >>err, "Error processing %s" % quote_output(si_dir)
881 failure.Failure().printTraceback(err)
883 class CorruptShareOptions(BaseOptions):
884 def getSynopsis(self):
885 return "Usage: tahoe [global-opts] debug corrupt-share SHARE_FILENAME"
888 ["offset", "o", "block-random", "Specify which bit to flip."],
891 def getUsage(self, width=None):
892 t = BaseOptions.getUsage(self, width)
894 Corrupt the given share by flipping a bit. This will cause a
895 verifying/downloading client to log an integrity-check failure incident, and
896 downloads will proceed with a different share.
898 The --offset parameter controls which bit should be flipped. The default is
899 to flip a single random bit of the block data.
901 tahoe debug corrupt-share testgrid/node-3/storage/shares/4v/4vozh77tsrw7mdhnj7qvp5ky74/0
903 Obviously, this command should not be used in normal operation.
906 def parseArgs(self, filename):
907 self['filename'] = filename
909 def corrupt_share(options):
911 from allmydata.storage.mutable import MutableShareFile
912 from allmydata.storage.immutable import ShareFile
913 from allmydata.mutable.layout import unpack_header
914 from allmydata.immutable.layout import ReadBucketProxy
916 fn = options['filename']
917 assert options["offset"] == "block-random", "other offsets not implemented"
918 # first, what kind of share is it?
920 def flip_bit(start, end):
921 offset = random.randrange(start, end)
922 bit = random.randrange(0, 8)
923 print >>out, "[%d..%d): %d.b%d" % (start, end, offset, bit)
927 d = chr(ord(d) ^ 0x01)
935 if prefix == MutableShareFile.MAGIC:
937 m = MutableShareFile(fn)
939 f.seek(m.DATA_OFFSET)
941 # make sure this slot contains an SMDF share
942 assert data[0] == "\x00", "non-SDMF mutable shares not supported"
945 (version, ig_seqnum, ig_roothash, ig_IV, ig_k, ig_N, ig_segsize,
946 ig_datalen, offsets) = unpack_header(data)
948 assert version == 0, "we only handle v0 SDMF files"
949 start = m.DATA_OFFSET + offsets["share_data"]
950 end = m.DATA_OFFSET + offsets["enc_privkey"]
953 # otherwise assume it's immutable
955 bp = ReadBucketProxy(None, None, '')
956 offsets = bp._parse_offsets(f.read_share_data(0, 0x24))
957 start = f._data_offset + offsets["data"]
958 end = f._data_offset + offsets["plaintext_hash_tree"]
963 class ReplOptions(BaseOptions):
964 def getSynopsis(self):
965 return "Usage: tahoe [global-opts] debug repl"
969 return code.interact()
972 DEFAULT_TESTSUITE = 'allmydata'
974 class TrialOptions(twisted_trial.Options):
975 def getSynopsis(self):
976 return "Usage: tahoe [global-opts] debug trial [options] [[file|package|module|TestCase|testmethod]...]"
978 def parseOptions(self, all_subargs, *a, **kw):
979 self.trial_args = list(all_subargs)
981 # any output from the option parsing will be printed twice, but that's harmless
982 return twisted_trial.Options.parseOptions(self, all_subargs, *a, **kw)
984 def parseArgs(self, *nonoption_args):
985 if not nonoption_args:
986 self.trial_args.append(DEFAULT_TESTSUITE)
988 def getUsage(self, width=None):
989 t = twisted_trial.Options.getUsage(self, width)
991 The 'tahoe debug trial' command uses the correct imports for this instance of
992 Tahoe-LAFS. The default test suite is '%s'.
993 """ % (DEFAULT_TESTSUITE,)
997 sys.argv = ['trial'] + config.trial_args
999 from allmydata._version import full_version
1000 if full_version.endswith("-dirty"):
1002 print >>sys.stderr, "WARNING: the source tree has been modified since the last commit."
1003 print >>sys.stderr, "(It is usually preferable to commit, then test, then amend the commit(s)"
1004 print >>sys.stderr, "if the tests fail.)"
1007 # This does not return.
1011 def fixOptionsClass( (subcmd, shortcut, OptionsClass, desc) ):
1012 class FixedOptionsClass(OptionsClass):
1013 def getSynopsis(self):
1014 t = OptionsClass.getSynopsis(self)
1015 i = t.find("Usage: flogtool ")
1017 return "Usage: tahoe [global-opts] debug flogtool " + t[i+len("Usage: flogtool "):]
1019 return "Usage: tahoe [global-opts] debug flogtool %s [options]" % (subcmd,)
1020 return (subcmd, shortcut, FixedOptionsClass, desc)
1022 class FlogtoolOptions(foolscap_cli.Options):
1024 super(FlogtoolOptions, self).__init__()
1025 self.subCommands = map(fixOptionsClass, self.subCommands)
1027 def getSynopsis(self):
1028 return "Usage: tahoe [global-opts] debug flogtool (%s) [command options]" % ("|".join([x[0] for x in self.subCommands]))
1030 def parseOptions(self, all_subargs, *a, **kw):
1031 self.flogtool_args = list(all_subargs)
1032 return super(FlogtoolOptions, self).parseOptions(self.flogtool_args, *a, **kw)
1034 def getUsage(self, width=None):
1035 t = super(FlogtoolOptions, self).getUsage(width)
1037 The 'tahoe debug flogtool' command uses the correct imports for this instance
1040 Please run 'tahoe debug flogtool SUBCOMMAND --help' for more details on each
1049 def flogtool(config):
1050 sys.argv = ['flogtool'] + config.flogtool_args
1051 return foolscap_cli.run_flogtool()
1054 class DebugCommand(BaseOptions):
1056 ["dump-share", None, DumpOptions,
1057 "Unpack and display the contents of a share (uri_extension and leases)."],
1058 ["dump-cap", None, DumpCapOptions, "Unpack a read-cap or write-cap."],
1059 ["find-shares", None, FindSharesOptions, "Locate sharefiles in node dirs."],
1060 ["catalog-shares", None, CatalogSharesOptions, "Describe all shares in node dirs."],
1061 ["corrupt-share", None, CorruptShareOptions, "Corrupt a share by flipping a bit."],
1062 ["repl", None, ReplOptions, "Open a Python interpreter."],
1063 ["trial", None, TrialOptions, "Run tests using Twisted Trial with the right imports."],
1064 ["flogtool", None, FlogtoolOptions, "Utilities to access log files."],
1066 def postOptions(self):
1067 if not hasattr(self, 'subOptions'):
1068 raise usage.UsageError("must specify a subcommand")
1069 def getSynopsis(self):
1071 def getUsage(self, width=None):
1072 #t = BaseOptions.getUsage(self, width)
1073 t = """Usage: tahoe debug SUBCOMMAND
1075 tahoe debug dump-share Unpack and display the contents of a share.
1076 tahoe debug dump-cap Unpack a read-cap or write-cap.
1077 tahoe debug find-shares Locate sharefiles in node directories.
1078 tahoe debug catalog-shares Describe all shares in node dirs.
1079 tahoe debug corrupt-share Corrupt a share by flipping a bit.
1080 tahoe debug repl Open a Python interpreter.
1081 tahoe debug trial Run tests using Twisted Trial with the right imports.
1082 tahoe debug flogtool Utilities to access log files.
1084 Please run e.g. 'tahoe debug dump-share --help' for more details on each
1087 # See ticket #1441 for why we print different information when
1088 # run via /usr/bin/tahoe. Note that argv[0] is the full path.
1089 if sys.argv[0] == '/usr/bin/tahoe':
1091 To get branch coverage for the Tahoe test suite (on the installed copy of
1092 Tahoe), install the 'python-coverage' package and then use:
1094 python-coverage run --branch /usr/bin/tahoe debug trial
1098 Another debugging feature is that bin%stahoe allows executing an arbitrary
1099 "runner" command (typically an installed Python script, such as 'coverage'),
1100 with the Tahoe libraries on the PYTHONPATH. The runner command name is
1101 prefixed with '@', and any occurrences of '@tahoe' in its arguments are
1102 replaced by the full path to the tahoe script.
1104 For example, if 'coverage' is installed and on the PATH, you can use:
1106 bin%stahoe @coverage run --branch @tahoe debug trial
1108 to get branch coverage for the Tahoe test suite. Or, to run python with
1109 the -3 option that warns about Python 3 incompatibilities:
1111 bin%stahoe @python -3 @tahoe command [options]
1112 """ % (os.sep, os.sep, os.sep)
1116 "dump-share": dump_share,
1117 "dump-cap": dump_cap,
1118 "find-shares": find_shares,
1119 "catalog-shares": catalog_shares,
1120 "corrupt-share": corrupt_share,
1123 "flogtool": flogtool,
1127 def do_debug(options):
1128 so = options.subOptions
1129 so.stdout = options.stdout
1130 so.stderr = options.stderr
1131 f = subDispatch[options.subCommand]
1136 ["debug", None, DebugCommand, "debug subcommands: use 'tahoe debug' for a list."],