2 # do not import any allmydata modules at this level. Do that from inside
3 # individual functions instead.
4 import struct, time, os, sys
5 from twisted.python import usage, failure
6 from twisted.internet import defer
7 from twisted.scripts import trial as twisted_trial
10 class DumpOptions(usage.Options):
11 def getSynopsis(self):
12 return "Usage: tahoe debug dump-share SHARE_FILENAME"
15 ["offsets", None, "Display a table of section offsets."],
16 ["leases-only", None, "Dump leases but not CHK contents."],
19 def getUsage(self, width=None):
20 t = usage.Options.getUsage(self, width)
22 Print lots of information about the given share, by parsing the share's
23 contents. This includes share type, lease information, encoding parameters,
24 hash-tree roots, public keys, and segment sizes. This command also emits a
25 verify-cap for the file that uses the share.
27 tahoe debug dump-share testgrid/node-3/storage/shares/4v/4vozh77tsrw7mdhnj7qvp5ky74/0
32 def parseArgs(self, filename):
33 from allmydata.util.encodingutil import argv_to_abspath
34 self['filename'] = argv_to_abspath(filename)
36 def dump_share(options):
37 from allmydata.storage.mutable import MutableShareFile
38 from allmydata.util.encodingutil import quote_output
42 # check the version, to see if we have a mutable or immutable share
43 print >>out, "share filename: %s" % quote_output(options['filename'])
45 f = open(options['filename'], "rb")
48 if prefix == MutableShareFile.MAGIC:
49 return dump_mutable_share(options)
50 # otherwise assume it's immutable
51 return dump_immutable_share(options)
53 def dump_immutable_share(options):
54 from allmydata.storage.immutable import ShareFile
57 f = ShareFile(options['filename'])
58 if not options["leases-only"]:
59 dump_immutable_chk_share(f, out, options)
60 dump_immutable_lease_info(f, out)
64 def dump_immutable_chk_share(f, out, options):
65 from allmydata import uri
66 from allmydata.util import base32
67 from allmydata.immutable.layout import ReadBucketProxy
68 from allmydata.util.encodingutil import quote_output, to_str
70 # use a ReadBucketProxy to parse the bucket and find the uri extension
71 bp = ReadBucketProxy(None, None, '')
72 offsets = bp._parse_offsets(f.read_share_data(0, 0x44))
73 print >>out, "%20s: %d" % ("version", bp._version)
74 seek = offsets['uri_extension']
75 length = struct.unpack(bp._fieldstruct,
76 f.read_share_data(seek, bp._fieldsize))[0]
78 UEB_data = f.read_share_data(seek, length)
80 unpacked = uri.unpack_extension_readable(UEB_data)
81 keys1 = ("size", "num_segments", "segment_size",
82 "needed_shares", "total_shares")
83 keys2 = ("codec_name", "codec_params", "tail_codec_params")
84 keys3 = ("plaintext_hash", "plaintext_root_hash",
85 "crypttext_hash", "crypttext_root_hash",
86 "share_root_hash", "UEB_hash")
87 display_keys = {"size": "file_size"}
90 dk = display_keys.get(k, k)
91 print >>out, "%20s: %s" % (dk, unpacked[k])
95 dk = display_keys.get(k, k)
96 print >>out, "%20s: %s" % (dk, unpacked[k])
100 dk = display_keys.get(k, k)
101 print >>out, "%20s: %s" % (dk, unpacked[k])
103 leftover = set(unpacked.keys()) - set(keys1 + keys2 + keys3)
106 print >>out, "LEFTOVER:"
107 for k in sorted(leftover):
108 print >>out, "%20s: %s" % (k, unpacked[k])
110 # the storage index isn't stored in the share itself, so we depend upon
111 # knowing the parent directory name to get it
112 pieces = options['filename'].split(os.sep)
114 piece = to_str(pieces[-2])
115 if base32.could_be_base32_encoded(piece):
116 storage_index = base32.a2b(piece)
117 uri_extension_hash = base32.a2b(unpacked["UEB_hash"])
118 u = uri.CHKFileVerifierURI(storage_index, uri_extension_hash,
119 unpacked["needed_shares"],
120 unpacked["total_shares"], unpacked["size"])
121 verify_cap = u.to_string()
122 print >>out, "%20s: %s" % ("verify-cap", quote_output(verify_cap, quotemarks=False))
125 sizes['data'] = (offsets['plaintext_hash_tree'] -
127 sizes['validation'] = (offsets['uri_extension'] -
128 offsets['plaintext_hash_tree'])
129 sizes['uri-extension'] = len(UEB_data)
131 print >>out, " Size of data within the share:"
132 for k in sorted(sizes):
133 print >>out, "%20s: %s" % (k, sizes[k])
135 if options['offsets']:
137 print >>out, " Section Offsets:"
138 print >>out, "%20s: %s" % ("share data", f._data_offset)
139 for k in ["data", "plaintext_hash_tree", "crypttext_hash_tree",
140 "block_hashes", "share_hashes", "uri_extension"]:
141 name = {"data": "block data"}.get(k,k)
142 offset = f._data_offset + offsets[k]
143 print >>out, " %20s: %s (0x%x)" % (name, offset, offset)
144 print >>out, "%20s: %s" % ("leases", f._lease_offset)
146 def dump_immutable_lease_info(f, out):
147 # display lease information too
149 leases = list(f.get_leases())
151 for i,lease in enumerate(leases):
152 when = format_expiration_time(lease.expiration_time)
153 print >>out, " Lease #%d: owner=%d, expire in %s" \
154 % (i, lease.owner_num, when)
156 print >>out, " No leases."
158 def format_expiration_time(expiration_time):
160 remains = expiration_time - now
161 when = "%ds" % remains
162 if remains > 24*3600:
163 when += " (%d days)" % (remains / (24*3600))
165 when += " (%d hours)" % (remains / 3600)
169 def dump_mutable_share(options):
170 from allmydata.storage.mutable import MutableShareFile
171 from allmydata.util import base32, idlib
173 m = MutableShareFile(options['filename'])
174 f = open(options['filename'], "rb")
175 WE, nodeid = m._read_write_enabler_and_nodeid(f)
176 num_extra_leases = m._read_num_extra_leases(f)
177 data_length = m._read_data_length(f)
178 extra_lease_offset = m._read_extra_lease_offset(f)
179 container_size = extra_lease_offset - m.DATA_OFFSET
180 leases = list(m._enumerate_leases(f))
182 share_type = "unknown"
183 f.seek(m.DATA_OFFSET)
184 if f.read(1) == "\x00":
185 # this slot contains an SMDF share
190 print >>out, "Mutable slot found:"
191 print >>out, " share_type: %s" % share_type
192 print >>out, " write_enabler: %s" % base32.b2a(WE)
193 print >>out, " WE for nodeid: %s" % idlib.nodeid_b2a(nodeid)
194 print >>out, " num_extra_leases: %d" % num_extra_leases
195 print >>out, " container_size: %d" % container_size
196 print >>out, " data_length: %d" % data_length
198 for (leasenum, lease) in leases:
200 print >>out, " Lease #%d:" % leasenum
201 print >>out, " ownerid: %d" % lease.owner_num
202 when = format_expiration_time(lease.expiration_time)
203 print >>out, " expires in %s" % when
204 print >>out, " renew_secret: %s" % base32.b2a(lease.renew_secret)
205 print >>out, " cancel_secret: %s" % base32.b2a(lease.cancel_secret)
206 print >>out, " secrets are for nodeid: %s" % idlib.nodeid_b2a(lease.nodeid)
208 print >>out, "No leases."
211 if share_type == "SDMF":
212 dump_SDMF_share(m, data_length, options)
216 def dump_SDMF_share(m, length, options):
217 from allmydata.mutable.layout import unpack_share, unpack_header
218 from allmydata.mutable.common import NeedMoreDataError
219 from allmydata.util import base32, hashutil
220 from allmydata.uri import SSKVerifierURI
221 from allmydata.util.encodingutil import quote_output, to_str
223 offset = m.DATA_OFFSET
227 f = open(options['filename'], "rb")
229 data = f.read(min(length, 2000))
233 pieces = unpack_share(data)
234 except NeedMoreDataError, e:
235 # retry once with the larger size
236 size = e.needed_bytes
237 f = open(options['filename'], "rb")
239 data = f.read(min(length, size))
241 pieces = unpack_share(data)
243 (seqnum, root_hash, IV, k, N, segsize, datalen,
244 pubkey, signature, share_hash_chain, block_hash_tree,
245 share_data, enc_privkey) = pieces
246 (ig_version, ig_seqnum, ig_roothash, ig_IV, ig_k, ig_N, ig_segsize,
247 ig_datalen, offsets) = unpack_header(data)
249 print >>out, " SDMF contents:"
250 print >>out, " seqnum: %d" % seqnum
251 print >>out, " root_hash: %s" % base32.b2a(root_hash)
252 print >>out, " IV: %s" % base32.b2a(IV)
253 print >>out, " required_shares: %d" % k
254 print >>out, " total_shares: %d" % N
255 print >>out, " segsize: %d" % segsize
256 print >>out, " datalen: %d" % datalen
257 print >>out, " enc_privkey: %d bytes" % len(enc_privkey)
258 print >>out, " pubkey: %d bytes" % len(pubkey)
259 print >>out, " signature: %d bytes" % len(signature)
260 share_hash_ids = ",".join(sorted([str(hid)
261 for hid in share_hash_chain.keys()]))
262 print >>out, " share_hash_chain: %s" % share_hash_ids
263 print >>out, " block_hash_tree: %d nodes" % len(block_hash_tree)
265 # the storage index isn't stored in the share itself, so we depend upon
266 # knowing the parent directory name to get it
267 pieces = options['filename'].split(os.sep)
269 piece = to_str(pieces[-2])
270 if base32.could_be_base32_encoded(piece):
271 storage_index = base32.a2b(piece)
272 fingerprint = hashutil.ssk_pubkey_fingerprint_hash(pubkey)
273 u = SSKVerifierURI(storage_index, fingerprint)
274 verify_cap = u.to_string()
275 print >>out, " verify-cap:", quote_output(verify_cap, quotemarks=False)
277 if options['offsets']:
278 # NOTE: this offset-calculation code is fragile, and needs to be
279 # merged with MutableShareFile's internals.
281 print >>out, " Section Offsets:"
282 def printoffset(name, value, shift=0):
283 print >>out, "%s%20s: %s (0x%x)" % (" "*shift, name, value, value)
284 printoffset("first lease", m.HEADER_SIZE)
285 printoffset("share data", m.DATA_OFFSET)
286 o_seqnum = m.DATA_OFFSET + struct.calcsize(">B")
287 printoffset("seqnum", o_seqnum, 2)
288 o_root_hash = m.DATA_OFFSET + struct.calcsize(">BQ")
289 printoffset("root_hash", o_root_hash, 2)
290 for k in ["signature", "share_hash_chain", "block_hash_tree",
292 "enc_privkey", "EOF"]:
293 name = {"share_data": "block data",
294 "EOF": "end of share data"}.get(k,k)
295 offset = m.DATA_OFFSET + offsets[k]
296 printoffset(name, offset, 2)
297 f = open(options['filename'], "rb")
298 printoffset("extra leases", m._read_extra_lease_offset(f) + 4)
305 class DumpCapOptions(usage.Options):
306 def getSynopsis(self):
307 return "Usage: tahoe debug dump-cap [options] FILECAP"
310 None, "Specify the storage server nodeid (ASCII), to construct WE and secrets."],
311 ["client-secret", "c", None,
312 "Specify the client's base secret (ASCII), to construct secrets."],
313 ["client-dir", "d", None,
314 "Specify the client's base directory, from which a -c secret will be read."],
316 def parseArgs(self, cap):
319 def getUsage(self, width=None):
320 t = usage.Options.getUsage(self, width)
322 Print information about the given cap-string (aka: URI, file-cap, dir-cap,
323 read-cap, write-cap). The URI string is parsed and unpacked. This prints the
324 type of the cap, its storage index, and any derived keys.
326 tahoe debug dump-cap URI:SSK-Verifier:4vozh77tsrw7mdhnj7qvp5ky74:q7f3dwz76sjys4kqfdt3ocur2pay3a6rftnkqmi2uxu3vqsdsofq
328 This may be useful to determine if a read-cap and a write-cap refer to the
329 same time, or to extract the storage-index from a file-cap (to then use with
332 If additional information is provided (storage server nodeid and/or client
333 base secret), this command will compute the shared secrets used for the
334 write-enabler and for lease-renewal.
339 def dump_cap(options):
340 from allmydata import uri
341 from allmydata.util import base32
342 from base64 import b32decode
343 import urlparse, urllib
348 if options['nodeid']:
349 nodeid = b32decode(options['nodeid'].upper())
351 if options['client-secret']:
352 secret = base32.a2b(options['client-secret'])
353 elif options['client-dir']:
354 secretfile = os.path.join(options['client-dir'], "private", "secret")
356 secret = base32.a2b(open(secretfile, "r").read().strip())
357 except EnvironmentError:
360 if cap.startswith("http"):
361 scheme, netloc, path, params, query, fragment = urlparse.urlparse(cap)
362 assert path.startswith("/uri/")
363 cap = urllib.unquote(path[len("/uri/"):])
365 u = uri.from_string(cap)
368 dump_uri_instance(u, nodeid, secret, out)
370 def _dump_secrets(storage_index, secret, nodeid, out):
371 from allmydata.util import hashutil
372 from allmydata.util import base32
375 crs = hashutil.my_renewal_secret_hash(secret)
376 print >>out, " client renewal secret:", base32.b2a(crs)
377 frs = hashutil.file_renewal_secret_hash(crs, storage_index)
378 print >>out, " file renewal secret:", base32.b2a(frs)
380 renew = hashutil.bucket_renewal_secret_hash(frs, nodeid)
381 print >>out, " lease renewal secret:", base32.b2a(renew)
382 ccs = hashutil.my_cancel_secret_hash(secret)
383 print >>out, " client cancel secret:", base32.b2a(ccs)
384 fcs = hashutil.file_cancel_secret_hash(ccs, storage_index)
385 print >>out, " file cancel secret:", base32.b2a(fcs)
387 cancel = hashutil.bucket_cancel_secret_hash(fcs, nodeid)
388 print >>out, " lease cancel secret:", base32.b2a(cancel)
390 def dump_uri_instance(u, nodeid, secret, out, show_header=True):
391 from allmydata import uri
392 from allmydata.storage.server import si_b2a
393 from allmydata.util import base32, hashutil
394 from allmydata.util.encodingutil import quote_output
396 if isinstance(u, uri.CHKFileURI):
398 print >>out, "CHK File:"
399 print >>out, " key:", base32.b2a(u.key)
400 print >>out, " UEB hash:", base32.b2a(u.uri_extension_hash)
401 print >>out, " size:", u.size
402 print >>out, " k/N: %d/%d" % (u.needed_shares, u.total_shares)
403 print >>out, " storage index:", si_b2a(u.get_storage_index())
404 _dump_secrets(u.get_storage_index(), secret, nodeid, out)
405 elif isinstance(u, uri.CHKFileVerifierURI):
407 print >>out, "CHK Verifier URI:"
408 print >>out, " UEB hash:", base32.b2a(u.uri_extension_hash)
409 print >>out, " size:", u.size
410 print >>out, " k/N: %d/%d" % (u.needed_shares, u.total_shares)
411 print >>out, " storage index:", si_b2a(u.get_storage_index())
413 elif isinstance(u, uri.LiteralFileURI):
415 print >>out, "Literal File URI:"
416 print >>out, " data:", quote_output(u.data)
418 elif isinstance(u, uri.WriteableSSKFileURI): # SDMF
420 print >>out, "SDMF Writeable URI:"
421 print >>out, " writekey:", base32.b2a(u.writekey)
422 print >>out, " readkey:", base32.b2a(u.readkey)
423 print >>out, " storage index:", si_b2a(u.get_storage_index())
424 print >>out, " fingerprint:", base32.b2a(u.fingerprint)
427 we = hashutil.ssk_write_enabler_hash(u.writekey, nodeid)
428 print >>out, " write_enabler:", base32.b2a(we)
430 _dump_secrets(u.get_storage_index(), secret, nodeid, out)
431 elif isinstance(u, uri.ReadonlySSKFileURI):
433 print >>out, "SDMF Read-only URI:"
434 print >>out, " readkey:", base32.b2a(u.readkey)
435 print >>out, " storage index:", si_b2a(u.get_storage_index())
436 print >>out, " fingerprint:", base32.b2a(u.fingerprint)
437 elif isinstance(u, uri.SSKVerifierURI):
439 print >>out, "SDMF Verifier URI:"
440 print >>out, " storage index:", si_b2a(u.get_storage_index())
441 print >>out, " fingerprint:", base32.b2a(u.fingerprint)
443 elif isinstance(u, uri.WriteableMDMFFileURI): # MDMF
445 print >>out, "MDMF Writeable URI:"
446 print >>out, " writekey:", base32.b2a(u.writekey)
447 print >>out, " readkey:", base32.b2a(u.readkey)
448 print >>out, " storage index:", si_b2a(u.get_storage_index())
449 print >>out, " fingerprint:", base32.b2a(u.fingerprint)
452 we = hashutil.ssk_write_enabler_hash(u.writekey, nodeid)
453 print >>out, " write_enabler:", base32.b2a(we)
455 _dump_secrets(u.get_storage_index(), secret, nodeid, out)
456 elif isinstance(u, uri.ReadonlyMDMFFileURI):
458 print >>out, "MDMF Read-only URI:"
459 print >>out, " readkey:", base32.b2a(u.readkey)
460 print >>out, " storage index:", si_b2a(u.get_storage_index())
461 print >>out, " fingerprint:", base32.b2a(u.fingerprint)
462 elif isinstance(u, uri.MDMFVerifierURI):
464 print >>out, "MDMF Verifier URI:"
465 print >>out, " storage index:", si_b2a(u.get_storage_index())
466 print >>out, " fingerprint:", base32.b2a(u.fingerprint)
469 elif isinstance(u, uri.ImmutableDirectoryURI): # CHK-based directory
471 print >>out, "CHK Directory URI:"
472 dump_uri_instance(u._filenode_uri, nodeid, secret, out, False)
473 elif isinstance(u, uri.ImmutableDirectoryURIVerifier):
475 print >>out, "CHK Directory Verifier URI:"
476 dump_uri_instance(u._filenode_uri, nodeid, secret, out, False)
478 elif isinstance(u, uri.DirectoryURI): # SDMF-based directory
480 print >>out, "Directory Writeable URI:"
481 dump_uri_instance(u._filenode_uri, nodeid, secret, out, False)
482 elif isinstance(u, uri.ReadonlyDirectoryURI):
484 print >>out, "Directory Read-only URI:"
485 dump_uri_instance(u._filenode_uri, nodeid, secret, out, False)
486 elif isinstance(u, uri.DirectoryURIVerifier):
488 print >>out, "Directory Verifier URI:"
489 dump_uri_instance(u._filenode_uri, nodeid, secret, out, False)
491 elif isinstance(u, uri.MDMFDirectoryURI): # MDMF-based directory
493 print >>out, "Directory Writeable URI:"
494 dump_uri_instance(u._filenode_uri, nodeid, secret, out, False)
495 elif isinstance(u, uri.ReadonlyMDMFDirectoryURI):
497 print >>out, "Directory Read-only URI:"
498 dump_uri_instance(u._filenode_uri, nodeid, secret, out, False)
499 elif isinstance(u, uri.MDMFDirectoryURIVerifier):
501 print >>out, "Directory Verifier URI:"
502 dump_uri_instance(u._filenode_uri, nodeid, secret, out, False)
505 print >>out, "unknown cap type"
507 class FindSharesOptions(usage.Options):
508 def getSynopsis(self):
509 return "Usage: tahoe debug find-shares STORAGE_INDEX NODEDIRS.."
511 def parseArgs(self, storage_index_s, *nodedirs):
512 from allmydata.util.encodingutil import argv_to_abspath
513 self.si_s = storage_index_s
514 self.nodedirs = map(argv_to_abspath, nodedirs)
516 def getUsage(self, width=None):
517 t = usage.Options.getUsage(self, width)
519 Locate all shares for the given storage index. This command looks through one
520 or more node directories to find the shares. It returns a list of filenames,
521 one per line, for each share file found.
523 tahoe debug find-shares 4vozh77tsrw7mdhnj7qvp5ky74 testgrid/node-*
525 It may be useful during testing, when running a test grid in which all the
526 nodes are on a local disk. The share files thus located can be counted,
527 examined (with dump-share), or corrupted/deleted to test checker/repairer.
531 def find_shares(options):
532 """Given a storage index and a list of node directories, emit a list of
533 all matching shares to stdout, one per line. For example:
535 find-shares.py 44kai1tui348689nrw8fjegc8c ~/testnet/node-*
539 /home/warner/testnet/node-1/storage/shares/44k/44kai1tui348689nrw8fjegc8c/5
540 /home/warner/testnet/node-1/storage/shares/44k/44kai1tui348689nrw8fjegc8c/9
541 /home/warner/testnet/node-2/storage/shares/44k/44kai1tui348689nrw8fjegc8c/2
543 from allmydata.storage.server import si_a2b, storage_index_to_dir
544 from allmydata.util.encodingutil import listdir_unicode
547 sharedir = storage_index_to_dir(si_a2b(options.si_s))
548 for d in options.nodedirs:
549 d = os.path.join(d, "storage/shares", sharedir)
550 if os.path.exists(d):
551 for shnum in listdir_unicode(d):
552 print >>out, os.path.join(d, shnum)
557 class CatalogSharesOptions(usage.Options):
561 def parseArgs(self, *nodedirs):
562 from allmydata.util.encodingutil import argv_to_abspath
563 self.nodedirs = map(argv_to_abspath, nodedirs)
565 raise usage.UsageError("must specify at least one node directory")
567 def getSynopsis(self):
568 return "Usage: tahoe debug catalog-shares NODEDIRS.."
570 def getUsage(self, width=None):
571 t = usage.Options.getUsage(self, width)
573 Locate all shares in the given node directories, and emit a one-line summary
574 of each share. Run it like this:
576 tahoe debug catalog-shares testgrid/node-* >allshares.txt
578 The lines it emits will look like the following:
580 CHK $SI $k/$N $filesize $UEB_hash $expiration $abspath_sharefile
581 SDMF $SI $k/$N $filesize $seqnum/$roothash $expiration $abspath_sharefile
582 UNKNOWN $abspath_sharefile
584 This command can be used to build up a catalog of shares from many storage
585 servers and then sort the results to compare all shares for the same file. If
586 you see shares with the same SI but different parameters/filesize/UEB_hash,
587 then something is wrong. The misc/find-share/anomalies.py script may be
592 def call(c, *args, **kwargs):
593 # take advantage of the fact that ImmediateReadBucketProxy returns
594 # Deferreds that are already fired
597 d = defer.maybeDeferred(c, *args, **kwargs)
598 d.addCallbacks(results.append, failures.append)
600 failures[0].raiseException()
603 def describe_share(abs_sharefile, si_s, shnum_s, now, out):
604 from allmydata import uri
605 from allmydata.storage.mutable import MutableShareFile
606 from allmydata.storage.immutable import ShareFile
607 from allmydata.mutable.layout import unpack_share
608 from allmydata.mutable.common import NeedMoreDataError
609 from allmydata.immutable.layout import ReadBucketProxy
610 from allmydata.util import base32
611 from allmydata.util.encodingutil import quote_output
614 f = open(abs_sharefile, "rb")
617 if prefix == MutableShareFile.MAGIC:
619 m = MutableShareFile(abs_sharefile)
620 WE, nodeid = m._read_write_enabler_and_nodeid(f)
621 data_length = m._read_data_length(f)
622 expiration_time = min( [lease.expiration_time
623 for (i,lease) in m._enumerate_leases(f)] )
624 expiration = max(0, expiration_time - now)
626 share_type = "unknown"
627 f.seek(m.DATA_OFFSET)
628 if f.read(1) == "\x00":
629 # this slot contains an SMDF share
632 if share_type == "SDMF":
633 f.seek(m.DATA_OFFSET)
634 data = f.read(min(data_length, 2000))
637 pieces = unpack_share(data)
638 except NeedMoreDataError, e:
639 # retry once with the larger size
640 size = e.needed_bytes
641 f.seek(m.DATA_OFFSET)
642 data = f.read(min(data_length, size))
643 pieces = unpack_share(data)
644 (seqnum, root_hash, IV, k, N, segsize, datalen,
645 pubkey, signature, share_hash_chain, block_hash_tree,
646 share_data, enc_privkey) = pieces
648 print >>out, "SDMF %s %d/%d %d #%d:%s %d %s" % \
649 (si_s, k, N, datalen,
650 seqnum, base32.b2a(root_hash),
651 expiration, quote_output(abs_sharefile))
653 print >>out, "UNKNOWN mutable %s" % quote_output(abs_sharefile)
655 elif struct.unpack(">L", prefix[:4]) == (1,):
658 class ImmediateReadBucketProxy(ReadBucketProxy):
659 def __init__(self, sf):
661 ReadBucketProxy.__init__(self, None, None, "")
663 return "<ImmediateReadBucketProxy>"
664 def _read(self, offset, size):
665 return defer.succeed(sf.read_share_data(offset, size))
667 # use a ReadBucketProxy to parse the bucket and find the uri extension
668 sf = ShareFile(abs_sharefile)
669 bp = ImmediateReadBucketProxy(sf)
671 expiration_time = min( [lease.expiration_time
672 for lease in sf.get_leases()] )
673 expiration = max(0, expiration_time - now)
675 UEB_data = call(bp.get_uri_extension)
676 unpacked = uri.unpack_extension_readable(UEB_data)
678 k = unpacked["needed_shares"]
679 N = unpacked["total_shares"]
680 filesize = unpacked["size"]
681 ueb_hash = unpacked["UEB_hash"]
683 print >>out, "CHK %s %d/%d %d %s %d %s" % (si_s, k, N, filesize,
684 ueb_hash, expiration,
685 quote_output(abs_sharefile))
688 print >>out, "UNKNOWN really-unknown %s" % quote_output(abs_sharefile)
692 def catalog_shares(options):
693 from allmydata.util.encodingutil import listdir_unicode, quote_output
698 for d in options.nodedirs:
699 d = os.path.join(d, "storage/shares")
701 abbrevs = listdir_unicode(d)
702 except EnvironmentError:
703 # ignore nodes that have storage turned off altogether
706 for abbrevdir in sorted(abbrevs):
707 if abbrevdir == "incoming":
709 abbrevdir = os.path.join(d, abbrevdir)
710 # this tool may get run against bad disks, so we can't assume
711 # that listdir_unicode will always succeed. Try to catalog as much
714 sharedirs = listdir_unicode(abbrevdir)
715 for si_s in sorted(sharedirs):
716 si_dir = os.path.join(abbrevdir, si_s)
717 catalog_shares_one_abbrevdir(si_s, si_dir, now, out,err)
719 print >>err, "Error processing %s" % quote_output(abbrevdir)
720 failure.Failure().printTraceback(err)
730 def catalog_shares_one_abbrevdir(si_s, si_dir, now, out, err):
731 from allmydata.util.encodingutil import listdir_unicode, quote_output
734 for shnum_s in sorted(listdir_unicode(si_dir), key=_as_number):
735 abs_sharefile = os.path.join(si_dir, shnum_s)
736 assert os.path.isfile(abs_sharefile)
738 describe_share(abs_sharefile, si_s, shnum_s, now,
741 print >>err, "Error processing %s" % quote_output(abs_sharefile)
742 failure.Failure().printTraceback(err)
744 print >>err, "Error processing %s" % quote_output(si_dir)
745 failure.Failure().printTraceback(err)
747 class CorruptShareOptions(usage.Options):
748 def getSynopsis(self):
749 return "Usage: tahoe debug corrupt-share SHARE_FILENAME"
752 ["offset", "o", "block-random", "Specify which bit to flip."],
755 def getUsage(self, width=None):
756 t = usage.Options.getUsage(self, width)
758 Corrupt the given share by flipping a bit. This will cause a
759 verifying/downloading client to log an integrity-check failure incident, and
760 downloads will proceed with a different share.
762 The --offset parameter controls which bit should be flipped. The default is
763 to flip a single random bit of the block data.
765 tahoe debug corrupt-share testgrid/node-3/storage/shares/4v/4vozh77tsrw7mdhnj7qvp5ky74/0
767 Obviously, this command should not be used in normal operation.
770 def parseArgs(self, filename):
771 self['filename'] = filename
773 def corrupt_share(options):
775 from allmydata.storage.mutable import MutableShareFile
776 from allmydata.storage.immutable import ShareFile
777 from allmydata.mutable.layout import unpack_header
778 from allmydata.immutable.layout import ReadBucketProxy
780 fn = options['filename']
781 assert options["offset"] == "block-random", "other offsets not implemented"
782 # first, what kind of share is it?
784 def flip_bit(start, end):
785 offset = random.randrange(start, end)
786 bit = random.randrange(0, 8)
787 print >>out, "[%d..%d): %d.b%d" % (start, end, offset, bit)
791 d = chr(ord(d) ^ 0x01)
799 if prefix == MutableShareFile.MAGIC:
801 m = MutableShareFile(fn)
803 f.seek(m.DATA_OFFSET)
805 # make sure this slot contains an SMDF share
806 assert data[0] == "\x00", "non-SDMF mutable shares not supported"
809 (version, ig_seqnum, ig_roothash, ig_IV, ig_k, ig_N, ig_segsize,
810 ig_datalen, offsets) = unpack_header(data)
812 assert version == 0, "we only handle v0 SDMF files"
813 start = m.DATA_OFFSET + offsets["share_data"]
814 end = m.DATA_OFFSET + offsets["enc_privkey"]
817 # otherwise assume it's immutable
819 bp = ReadBucketProxy(None, None, '')
820 offsets = bp._parse_offsets(f.read_share_data(0, 0x24))
821 start = f._data_offset + offsets["data"]
822 end = f._data_offset + offsets["plaintext_hash_tree"]
827 class ReplOptions(usage.Options):
828 def getSynopsis(self):
829 return "Usage: tahoe debug repl"
833 return code.interact()
836 DEFAULT_TESTSUITE = 'allmydata'
838 class TrialOptions(twisted_trial.Options):
839 def getSynopsis(self):
840 return "Usage: tahoe debug trial [options] [[file|package|module|TestCase|testmethod]...]"
842 def parseOptions(self, all_subargs, *a, **kw):
843 self.trial_args = list(all_subargs)
845 # any output from the option parsing will be printed twice, but that's harmless
846 return twisted_trial.Options.parseOptions(self, all_subargs, *a, **kw)
848 def parseArgs(self, *nonoption_args):
849 if not nonoption_args:
850 self.trial_args.append(DEFAULT_TESTSUITE)
852 def getUsage(self, width=None):
853 t = twisted_trial.Options.getUsage(self, width)
855 The 'tahoe debug trial' command uses the correct imports for this instance of
856 Tahoe-LAFS. The default test suite is '%s'.
857 """ % (DEFAULT_TESTSUITE,)
861 sys.argv = ['trial'] + config.trial_args
863 # This does not return.
867 class DebugCommand(usage.Options):
869 ["dump-share", None, DumpOptions,
870 "Unpack and display the contents of a share (uri_extension and leases)."],
871 ["dump-cap", None, DumpCapOptions, "Unpack a read-cap or write-cap."],
872 ["find-shares", None, FindSharesOptions, "Locate sharefiles in node dirs."],
873 ["catalog-shares", None, CatalogSharesOptions, "Describe all shares in node dirs."],
874 ["corrupt-share", None, CorruptShareOptions, "Corrupt a share by flipping a bit."],
875 ["repl", None, ReplOptions, "Open a Python interpreter."],
876 ["trial", None, TrialOptions, "Run tests using Twisted Trial with the right imports."],
878 def postOptions(self):
879 if not hasattr(self, 'subOptions'):
880 raise usage.UsageError("must specify a subcommand")
881 def getSynopsis(self):
882 return "Usage: tahoe debug SUBCOMMAND"
883 def getUsage(self, width=None):
884 #t = usage.Options.getUsage(self, width)
887 tahoe debug dump-share Unpack and display the contents of a share.
888 tahoe debug dump-cap Unpack a read-cap or write-cap.
889 tahoe debug find-shares Locate sharefiles in node directories.
890 tahoe debug catalog-shares Describe all shares in node dirs.
891 tahoe debug corrupt-share Corrupt a share by flipping a bit.
892 tahoe debug repl Open a Python interpreter.
893 tahoe debug trial Run tests using Twisted Trial with the right imports.
895 Please run e.g. 'tahoe debug dump-share --help' for more details on each
898 # See ticket #1441 for why we print different information when
899 # run via /usr/bin/tahoe. Note that argv[0] is the full path.
900 if sys.argv[0] == '/usr/bin/tahoe':
902 To get branch coverage for the Tahoe test suite (on the installed copy of
903 Tahoe), install the 'python-coverage' package and then use:
905 python-coverage run --branch /usr/bin/tahoe debug trial
909 Another debugging feature is that bin%stahoe allows executing an arbitrary
910 "runner" command (typically an installed Python script, such as 'coverage'),
911 with the Tahoe libraries on the PYTHONPATH. The runner command name is
912 prefixed with '@', and any occurrences of '@tahoe' in its arguments are
913 replaced by the full path to the tahoe script.
915 For example, if 'coverage' is installed and on the PATH, you can use:
917 bin%stahoe @coverage run --branch @tahoe debug trial
919 to get branch coverage for the Tahoe test suite. Or, to run python with
920 the -3 option that warns about Python 3 incompatibilities:
922 bin%stahoe @python -3 @tahoe command [options]
923 """ % (os.sep, os.sep, os.sep)
927 "dump-share": dump_share,
928 "dump-cap": dump_cap,
929 "find-shares": find_shares,
930 "catalog-shares": catalog_shares,
931 "corrupt-share": corrupt_share,
937 def do_debug(options):
938 so = options.subOptions
939 so.stdout = options.stdout
940 so.stderr = options.stderr
941 f = subDispatch[options.subCommand]
946 ["debug", None, DebugCommand, "debug subcommands: use 'tahoe debug' for a list."],