src/allmydata/scripts/tahoe_manifest.py

   1
   2 import os, time
   3 from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path
   4 from allmydata.scripts.common_http import do_http
   5 from allmydata.util import base32
   6 from allmydata import uri
   7 import urllib
   8 import simplejson
   9
  10 class SlowOperationRunner:
  11
  12     def run(self, options):
  13         stderr = options.stderr
  14         self.options = options
  15         self.ophandle = ophandle = base32.b2a(os.urandom(16))
  16         nodeurl = options['node-url']
  17         if not nodeurl.endswith("/"):
  18             nodeurl += "/"
  19         self.nodeurl = nodeurl
  20         where = options.where
  21         rootcap, path = get_alias(options.aliases, where, DEFAULT_ALIAS)
  22         if path == '/':
  23             path = ''
  24         url = nodeurl + "uri/%s" % urllib.quote(rootcap)
  25         if path:
  26             url += "/" + escape_path(path)
  27         # todo: should it end with a slash?
  28         url += "?t=%s&ophandle=%s" % (self.operation, ophandle)
  29         resp = do_http("POST", url)
  30         if resp.status not in (200, 302):
  31             print >>stderr, "ERROR", resp.status, resp.reason, resp.read()
  32             return 1
  33         # now we poll for results. We nominally poll at t=1, 5, 10, 30, 60,
  34         # 90, k*120 seconds, but if the poll takes non-zero time, that will
  35         # be slightly longer. I'm not worried about trying to make up for
  36         # that time.
  37
  38         return self.wait_for_results()
  39
  40     def poll_times(self):
  41         for i in (1,5,10,30,60,90):
  42             yield i
  43         i = 120
  44         while True:
  45             yield i
  46             i += 120
  47
  48     def wait_for_results(self):
  49         last = 0
  50         for next in self.poll_times():
  51             delay = next - last
  52             time.sleep(delay)
  53             last = next
  54             if self.poll():
  55                 return 0
  56
  57     def poll(self):
  58         url = self.nodeurl + "operations/" + self.ophandle
  59         url += "?t=status&output=JSON&release-after-complete=true"
  60         stdout = self.options.stdout
  61         stderr = self.options.stderr
  62         resp = do_http("GET", url)
  63         if resp.status != 200:
  64             print >>stderr, "ERROR", resp.status, resp.reason, resp.read()
  65             return True
  66         data = simplejson.loads(resp.read())
  67         if not data["finished"]:
  68             return False
  69         self.write_results(data)
  70         return True
  71
  72 class ManifestGrabber(SlowOperationRunner):
  73     operation = "start-manifest"
  74
  75     def write_results(self, data):
  76         stdout = self.options.stdout
  77         stderr = self.options.stderr
  78         if self.options["storage-index"]:
  79             for (path, cap) in data["manifest"]:
  80                 u = uri.from_string(str(cap))
  81                 si = u.get_storage_index()
  82                 if si is not None:
  83                     print >>stdout, base32.b2a(si)
  84         else:
  85             for (path, cap) in data["manifest"]:
  86                 try:
  87                     print >>stdout, cap, "/".join(path)
  88                 except UnicodeEncodeError:
  89                     print >>stdout, cap, "/".join([p.encode("utf-8")
  90                                                    for p in path])
  91
  92 def manifest(options):
  93     return ManifestGrabber().run(options)
  94
  95 class StatsGrabber(SlowOperationRunner):
  96     operation = "start-deep-stats"
  97
  98     def write_results(self, data):
  99         stdout = self.options.stdout
 100         stderr = self.options.stderr
 101         keys = ("count-immutable-files",
 102                 "count-mutable-files",
 103                 "count-literal-files",
 104                 "count-files",
 105                 "count-directories",
 106                 "size-immutable-files",
 107                 "size-mutable-files",
 108                 "size-literal-files",
 109                 "size-directories",
 110                 "largest-directory",
 111                 "largest-immutable-files",
 112                 )
 113         width = max([len(k) for k in keys])
 114         print >>stdout, "Counts and Total Sizes:"
 115         for k in keys:
 116             fmt = "%" + str(width) + "s: %d"
 117             if k in data:
 118                 print >>stdout, fmt % (k, data[k])
 119         print >>stdout, "Size Histogram:"
 120         prevmax = None
 121         maxlen = max([len(str(maxsize))
 122                       for (minsize, maxsize, count)
 123                       in data["size-files-histogram"]])
 124         minfmt = "%" + str(maxlen) + "d"
 125         maxfmt = "%-" + str(maxlen) + "d"
 126         linefmt = minfmt + "-" + maxfmt + " : %d"
 127         for (minsize, maxsize, count) in data["size-files-histogram"]:
 128             if prevmax is not None and minsize != prevmax+1:
 129                 print >>stdout, " "*(maxlen-1) + "..."
 130             prevmax = maxsize
 131             print >>stdout, linefmt % (minsize, maxsize, count)
 132
 133 def stats(options):
 134     return StatsGrabber().run(options)