src/allmydata/scripts/tahoe_manifest.py

   1
   2 import urllib, simplejson
   3 from twisted.protocols.basic import LineOnlyReceiver
   4 from allmydata.util.abbreviate import abbreviate_space_both
   5 from allmydata.scripts.slow_operation import SlowOperationRunner
   6 from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \
   7                                      UnknownAliasError
   8 from allmydata.scripts.common_http import do_http, format_http_error
   9 from allmydata.util.encodingutil import quote_output, quote_path
  10
  11 class FakeTransport:
  12     disconnecting = False
  13
  14 class ManifestStreamer(LineOnlyReceiver):
  15     delimiter = "\n"
  16
  17     def __init__(self):
  18         self.transport = FakeTransport()
  19
  20     def run(self, options):
  21         self.rc = 0
  22         stdout = options.stdout
  23         stderr = options.stderr
  24         self.options = options
  25         nodeurl = options['node-url']
  26         if not nodeurl.endswith("/"):
  27             nodeurl += "/"
  28         self.nodeurl = nodeurl
  29         where = options.where
  30         try:
  31             rootcap, path = get_alias(options.aliases, where, DEFAULT_ALIAS)
  32         except UnknownAliasError, e:
  33             e.display(stderr)
  34             return 1
  35         if path == '/':
  36             path = ''
  37         url = nodeurl + "uri/%s" % urllib.quote(rootcap)
  38         if path:
  39             url += "/" + escape_path(path)
  40         # todo: should it end with a slash?
  41         url += "?t=stream-manifest"
  42         resp = do_http("POST", url)
  43         if resp.status not in (200, 302):
  44             print >>stderr, format_http_error("ERROR", resp)
  45             return 1
  46         #print "RESP", dir(resp)
  47         # use Twisted to split this into lines
  48         self.in_error = False
  49         while True:
  50             chunk = resp.read(100)
  51             if not chunk:
  52                 break
  53             if self.options["raw"]:
  54                 stdout.write(chunk)
  55             else:
  56                 self.dataReceived(chunk)
  57         return self.rc
  58
  59     def lineReceived(self, line):
  60         stdout = self.options.stdout
  61         stderr = self.options.stderr
  62         if self.in_error:
  63             print >>stderr, quote_output(line, quotemarks=False)
  64             return
  65         if line.startswith("ERROR:"):
  66             self.in_error = True
  67             self.rc = 1
  68             print >>stderr, quote_output(line, quotemarks=False)
  69             return
  70
  71         try:
  72             d = simplejson.loads(line.decode('utf-8'))
  73         except Exception, e:
  74             print >>stderr, "ERROR could not decode/parse %s\nERROR  %r" % (quote_output(line), e)
  75         else:
  76             if d["type"] in ("file", "directory"):
  77                 if self.options["storage-index"]:
  78                     si = d.get("storage-index", None)
  79                     if si:
  80                         print >>stdout, quote_output(si, quotemarks=False)
  81                 elif self.options["verify-cap"]:
  82                     vc = d.get("verifycap", None)
  83                     if vc:
  84                         print >>stdout, quote_output(vc, quotemarks=False)
  85                 elif self.options["repair-cap"]:
  86                     vc = d.get("repaircap", None)
  87                     if vc:
  88                         print >>stdout, quote_output(vc, quotemarks=False)
  89                 else:
  90                     print >>stdout, "%s %s" % (quote_output(d["cap"], quotemarks=False),
  91                                                quote_path(d["path"], quotemarks=False))
  92
  93 def manifest(options):
  94     return ManifestStreamer().run(options)
  95
  96 class StatsGrabber(SlowOperationRunner):
  97
  98     def make_url(self, base, ophandle):
  99         return base + "?t=start-deep-stats&ophandle=" + ophandle
 100
 101     def write_results(self, data):
 102         stdout = self.options.stdout
 103         keys = ("count-immutable-files",
 104                 "count-mutable-files",
 105                 "count-literal-files",
 106                 "count-files",
 107                 "count-directories",
 108                 "size-immutable-files",
 109                 "size-mutable-files",
 110                 "size-literal-files",
 111                 "size-directories",
 112                 "largest-directory",
 113                 "largest-immutable-file",
 114                 )
 115         width = max([len(k) for k in keys])
 116         print >>stdout, "Counts and Total Sizes:"
 117         for k in keys:
 118             fmt = "%" + str(width) + "s: %d"
 119             if k in data:
 120                 value = data[k]
 121                 if not k.startswith("count-") and value > 1000:
 122                     absize = abbreviate_space_both(value)
 123                     print >>stdout, fmt % (k, data[k]), "  ", absize
 124                 else:
 125                     print >>stdout, fmt % (k, data[k])
 126         if data["size-files-histogram"]:
 127             print >>stdout, "Size Histogram:"
 128             prevmax = None
 129             maxlen = max([len(str(maxsize))
 130                           for (minsize, maxsize, count)
 131                           in data["size-files-histogram"]])
 132             maxcountlen = max([len(str(count))
 133                                for (minsize, maxsize, count)
 134                                in data["size-files-histogram"]])
 135             minfmt = "%" + str(maxlen) + "d"
 136             maxfmt = "%-" + str(maxlen) + "d"
 137             countfmt = "%-" + str(maxcountlen) + "d"
 138             linefmt = minfmt + "-" + maxfmt + " : " + countfmt + "    %s"
 139             for (minsize, maxsize, count) in data["size-files-histogram"]:
 140                 if prevmax is not None and minsize != prevmax+1:
 141                     print >>stdout, " "*(maxlen-1) + "..."
 142                 prevmax = maxsize
 143                 print >>stdout, linefmt % (minsize, maxsize, count,
 144                                            abbreviate_space_both(maxsize))
 145
 146 def stats(options):
 147     return StatsGrabber().run(options)