src/allmydata/scripts/tahoe_manifest.py

   1
   2 import urllib, simplejson
   3 from twisted.protocols.basic import LineOnlyReceiver
   4 from allmydata.util import base32
   5 from allmydata.util.abbreviate import abbreviate_space_both
   6 from allmydata import uri
   7 from allmydata.scripts.slow_operation import SlowOperationRunner
   8 from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path
   9 from allmydata.scripts.common_http import do_http
  10
  11 class FakeTransport:
  12     disconnecting = False
  13 class ManifestStreamer(LineOnlyReceiver):
  14     delimiter = "\n"
  15     def __init__(self):
  16         self.transport = FakeTransport()
  17
  18     def run(self, options):
  19         stdout = options.stdout
  20         stderr = options.stderr
  21         self.options = options
  22         nodeurl = options['node-url']
  23         if not nodeurl.endswith("/"):
  24             nodeurl += "/"
  25         self.nodeurl = nodeurl
  26         where = options.where
  27         rootcap, path = get_alias(options.aliases, where, DEFAULT_ALIAS)
  28         if path == '/':
  29             path = ''
  30         url = nodeurl + "uri/%s" % urllib.quote(rootcap)
  31         if path:
  32             url += "/" + escape_path(path)
  33         # todo: should it end with a slash?
  34         url += "?t=stream-manifest"
  35         resp = do_http("POST", url)
  36         if resp.status not in (200, 302):
  37             print >>stderr, "ERROR", resp.status, resp.reason, resp.read()
  38             return 1
  39         #print "RESP", dir(resp)
  40         # use Twisted to split this into lines
  41         while True:
  42             chunk = resp.read(100)
  43             if not chunk:
  44                 break
  45             if self.options["raw"]:
  46                 stdout.write(chunk)
  47             else:
  48                 self.dataReceived(chunk)
  49         return 0
  50
  51     def lineReceived(self, line):
  52         d = simplejson.loads(line)
  53         stdout = self.options.stdout
  54         if d["type"] in ("file", "directory"):
  55             if self.options["storage-index"]:
  56                 si = d["storage-index"]
  57                 if si:
  58                     print >>stdout, si
  59             elif self.options["verify-cap"]:
  60                 vc = d["verifycap"]
  61                 if vc:
  62                     print >>stdout, vc
  63             elif self.options["repair-cap"]:
  64                 vc = d["repaircap"]
  65                 if vc:
  66                     print >>stdout, vc
  67             else:
  68                 try:
  69                     print >>stdout, d["cap"], "/".join(d["path"])
  70                 except UnicodeEncodeError:
  71                     print >>stdout, d["cap"], "/".join([p.encode("utf-8")
  72                                                         for p in d["path"]])
  73
  74
  75
  76 class ManifestGrabber(SlowOperationRunner):
  77
  78     def make_url(self, base, ophandle):
  79         return base + "?t=start-manifest&ophandle=" + ophandle
  80
  81     def write_results(self, data):
  82         stdout = self.options.stdout
  83         stderr = self.options.stderr
  84         if self.options["storage-index"]:
  85             for (path, cap) in data["manifest"]:
  86                 u = uri.from_string(str(cap))
  87                 si = u.get_storage_index()
  88                 if si is not None:
  89                     print >>stdout, base32.b2a(si)
  90         else:
  91             for (path, cap) in data["manifest"]:
  92                 try:
  93                     print >>stdout, cap, "/".join(path)
  94                 except UnicodeEncodeError:
  95                     print >>stdout, cap, "/".join([p.encode("utf-8")
  96                                                    for p in path])
  97
  98 def manifest(options):
  99     if options["stream"]:
 100         return ManifestStreamer().run(options)
 101     else:
 102         return ManifestGrabber().run(options)
 103
 104 class StatsGrabber(SlowOperationRunner):
 105
 106     def make_url(self, base, ophandle):
 107         return base + "?t=start-deep-stats&ophandle=" + ophandle
 108
 109     def write_results(self, data):
 110         stdout = self.options.stdout
 111         stderr = self.options.stderr
 112         keys = ("count-immutable-files",
 113                 "count-mutable-files",
 114                 "count-literal-files",
 115                 "count-files",
 116                 "count-directories",
 117                 "size-immutable-files",
 118                 "size-mutable-files",
 119                 "size-literal-files",
 120                 "size-directories",
 121                 "largest-directory",
 122                 "largest-immutable-file",
 123                 )
 124         width = max([len(k) for k in keys])
 125         print >>stdout, "Counts and Total Sizes:"
 126         for k in keys:
 127             fmt = "%" + str(width) + "s: %d"
 128             if k in data:
 129                 value = data[k]
 130                 if not k.startswith("count-") and value > 1000:
 131                     absize = abbreviate_space_both(value)
 132                     print >>stdout, fmt % (k, data[k]), "  ", absize
 133                 else:
 134                     print >>stdout, fmt % (k, data[k])
 135         print >>stdout, "Size Histogram:"
 136         prevmax = None
 137         maxlen = max([len(str(maxsize))
 138                       for (minsize, maxsize, count)
 139                       in data["size-files-histogram"]])
 140         maxcountlen = max([len(str(count))
 141                            for (minsize, maxsize, count)
 142                            in data["size-files-histogram"]])
 143         minfmt = "%" + str(maxlen) + "d"
 144         maxfmt = "%-" + str(maxlen) + "d"
 145         countfmt = "%-" + str(maxcountlen) + "d"
 146         linefmt = minfmt + "-" + maxfmt + " : " + countfmt + "    %s"
 147         for (minsize, maxsize, count) in data["size-files-histogram"]:
 148             if prevmax is not None and minsize != prevmax+1:
 149                 print >>stdout, " "*(maxlen-1) + "..."
 150             prevmax = maxsize
 151             print >>stdout, linefmt % (minsize, maxsize, count,
 152                                        abbreviate_space_both(maxsize))
 153
 154 def stats(options):
 155     return StatsGrabber().run(options)