src/allmydata/scripts/tahoe_manifest.py

   1
   2 import urllib, simplejson
   3 from twisted.protocols.basic import LineOnlyReceiver
   4 from allmydata.util.abbreviate import abbreviate_space_both
   5 from allmydata.scripts.slow_operation import SlowOperationRunner
   6 from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \
   7                                      UnknownAliasError
   8 from allmydata.scripts.common_http import do_http
   9
  10 class FakeTransport:
  11     disconnecting = False
  12
  13 class ManifestStreamer(LineOnlyReceiver):
  14     delimiter = "\n"
  15
  16     def __init__(self):
  17         self.transport = FakeTransport()
  18
  19     def run(self, options):
  20         self.rc = 0
  21         stdout = options.stdout
  22         stderr = options.stderr
  23         self.options = options
  24         nodeurl = options['node-url']
  25         if not nodeurl.endswith("/"):
  26             nodeurl += "/"
  27         self.nodeurl = nodeurl
  28         where = options.where
  29         try:
  30             rootcap, path = get_alias(options.aliases, where, DEFAULT_ALIAS)
  31         except UnknownAliasError, e:
  32             print >>stderr, "error: %s" % e.args[0]
  33             return 1
  34         if path == '/':
  35             path = ''
  36         url = nodeurl + "uri/%s" % urllib.quote(rootcap)
  37         if path:
  38             url += "/" + escape_path(path)
  39         # todo: should it end with a slash?
  40         url += "?t=stream-manifest"
  41         resp = do_http("POST", url)
  42         if resp.status not in (200, 302):
  43             print >>stderr, "ERROR", resp.status, resp.reason, resp.read()
  44             return 1
  45         #print "RESP", dir(resp)
  46         # use Twisted to split this into lines
  47         self.in_error = False
  48         while True:
  49             chunk = resp.read(100)
  50             if not chunk:
  51                 break
  52             if self.options["raw"]:
  53                 stdout.write(chunk)
  54             else:
  55                 self.dataReceived(chunk)
  56         return self.rc
  57
  58     def lineReceived(self, line):
  59         stdout = self.options.stdout
  60         stderr = self.options.stderr
  61         if self.in_error:
  62             print >>stderr, line
  63             return
  64         if line.startswith("ERROR:"):
  65             self.in_error = True
  66             self.rc = 1
  67             print >>stderr, line
  68             return
  69
  70         d = simplejson.loads(line)
  71         if d["type"] in ("file", "directory"):
  72             if self.options["storage-index"]:
  73                 si = d["storage-index"]
  74                 if si:
  75                     print >>stdout, si
  76             elif self.options["verify-cap"]:
  77                 vc = d["verifycap"]
  78                 if vc:
  79                     print >>stdout, vc
  80             elif self.options["repair-cap"]:
  81                 vc = d["repaircap"]
  82                 if vc:
  83                     print >>stdout, vc
  84             else:
  85                 try:
  86                     print >>stdout, d["cap"], "/".join(d["path"])
  87                 except UnicodeEncodeError:
  88                     print >>stdout, d["cap"], "/".join([p.encode("utf-8")
  89                                                         for p in d["path"]])
  90
  91 def manifest(options):
  92     return ManifestStreamer().run(options)
  93
  94 class StatsGrabber(SlowOperationRunner):
  95
  96     def make_url(self, base, ophandle):
  97         return base + "?t=start-deep-stats&ophandle=" + ophandle
  98
  99     def write_results(self, data):
 100         stdout = self.options.stdout
 101         keys = ("count-immutable-files",
 102                 "count-mutable-files",
 103                 "count-literal-files",
 104                 "count-files",
 105                 "count-directories",
 106                 "size-immutable-files",
 107                 "size-mutable-files",
 108                 "size-literal-files",
 109                 "size-directories",
 110                 "largest-directory",
 111                 "largest-immutable-file",
 112                 )
 113         width = max([len(k) for k in keys])
 114         print >>stdout, "Counts and Total Sizes:"
 115         for k in keys:
 116             fmt = "%" + str(width) + "s: %d"
 117             if k in data:
 118                 value = data[k]
 119                 if not k.startswith("count-") and value > 1000:
 120                     absize = abbreviate_space_both(value)
 121                     print >>stdout, fmt % (k, data[k]), "  ", absize
 122                 else:
 123                     print >>stdout, fmt % (k, data[k])
 124         if data["size-files-histogram"]:
 125             print >>stdout, "Size Histogram:"
 126             prevmax = None
 127             maxlen = max([len(str(maxsize))
 128                           for (minsize, maxsize, count)
 129                           in data["size-files-histogram"]])
 130             maxcountlen = max([len(str(count))
 131                                for (minsize, maxsize, count)
 132                                in data["size-files-histogram"]])
 133             minfmt = "%" + str(maxlen) + "d"
 134             maxfmt = "%-" + str(maxlen) + "d"
 135             countfmt = "%-" + str(maxcountlen) + "d"
 136             linefmt = minfmt + "-" + maxfmt + " : " + countfmt + "    %s"
 137             for (minsize, maxsize, count) in data["size-files-histogram"]:
 138                 if prevmax is not None and minsize != prevmax+1:
 139                     print >>stdout, " "*(maxlen-1) + "..."
 140                 prevmax = maxsize
 141                 print >>stdout, linefmt % (minsize, maxsize, count,
 142                                            abbreviate_space_both(maxsize))
 143
 144 def stats(options):
 145     return StatsGrabber().run(options)