src/allmydata/web/storage.py

   1
   2 import time, simplejson
   3 from nevow import rend, tags as T, inevow
   4 from allmydata.web.common import getxmlfile, abbreviate_time, get_arg
   5 from allmydata.util.abbreviate import abbreviate_space
   6 from allmydata.util import time_format, idlib
   7
   8 def remove_prefix(s, prefix):
   9     if not s.startswith(prefix):
  10         return None
  11     return s[len(prefix):]
  12
  13 class StorageStatus(rend.Page):
  14     docFactory = getxmlfile("storage_status.xhtml")
  15     # the default 'data' argument is the StorageServer instance
  16
  17     def __init__(self, storage, nickname=""):
  18         rend.Page.__init__(self, storage)
  19         self.storage = storage
  20         self.nickname = nickname
  21
  22     def renderHTTP(self, ctx):
  23         req = inevow.IRequest(ctx)
  24         t = get_arg(req, "t")
  25         if t == "json":
  26             return self.render_JSON(req)
  27         return rend.Page.renderHTTP(self, ctx)
  28
  29     def render_JSON(self, req):
  30         req.setHeader("content-type", "text/plain")
  31         d = {"stats": self.storage.get_stats(),
  32              "bucket-counter": self.storage.bucket_counter.get_state(),
  33              "lease-checker": self.storage.lease_checker.get_state(),
  34              "lease-checker-progress": self.storage.lease_checker.get_progress(),
  35              }
  36         return simplejson.dumps(d, indent=1) + "\n"
  37
  38     def data_nickname(self, ctx, storage):
  39         return self.nickname
  40     def data_nodeid(self, ctx, storage):
  41         return idlib.nodeid_b2a(self.storage.my_nodeid)
  42
  43     def render_storage_running(self, ctx, storage):
  44         if storage:
  45             return ctx.tag
  46         else:
  47             return T.h1["No Storage Server Running"]
  48
  49     def render_bool(self, ctx, data):
  50         return {True: "Yes", False: "No"}[bool(data)]
  51
  52     def render_abbrev_space(self, ctx, size):
  53         if size is None:
  54             return "?"
  55         return abbreviate_space(size)
  56
  57     def render_space(self, ctx, size):
  58         if size is None:
  59             return "?"
  60         return "%d" % size
  61
  62     def data_stats(self, ctx, data):
  63         # FYI: 'data' appears to be self, rather than the StorageServer
  64         # object in self.original that gets passed to render_* methods. I
  65         # still don't understand Nevow.
  66
  67         # Nevow has nevow.accessors.DictionaryContainer: Any data= directive
  68         # that appears in a context in which the current data is a dictionary
  69         # will be looked up as keys in that dictionary. So if data_stats()
  70         # returns a dictionary, then we can use something like this:
  71         #
  72         #  <ul n:data="stats">
  73         #   <li>disk_total: <span n:render="abbrev" n:data="disk_total" /></li>
  74         #  </ul>
  75
  76         # to use get_stats()["storage_server.disk_total"] . However,
  77         # DictionaryContainer does a raw d[] instead of d.get(), so any
  78         # missing keys will cause an error, even if the renderer can tolerate
  79         # None values. To overcome this, we either need a dict-like object
  80         # that always returns None for unknown keys, or we must pre-populate
  81         # our dict with those missing keys, or we should get rid of data_
  82         # methods that return dicts (or find some way to override Nevow's
  83         # handling of dictionaries).
  84
  85         d = dict([ (remove_prefix(k, "storage_server."), v)
  86                    for k,v in self.storage.get_stats().items() ])
  87         d.setdefault("disk_total", None)
  88         d.setdefault("disk_used", None)
  89         d.setdefault("disk_free_for_root", None)
  90         d.setdefault("disk_free_for_nonroot", None)
  91         d.setdefault("reserved_space", None)
  92         d.setdefault("disk_avail", None)
  93         return d
  94
  95     def data_last_complete_bucket_count(self, ctx, data):
  96         s = self.storage.bucket_counter.get_state()
  97         count = s.get("last-complete-bucket-count")
  98         if count is None:
  99             return "Not computed yet"
 100         return count
 101
 102     def render_count_crawler_status(self, ctx, storage):
 103         p = self.storage.bucket_counter.get_progress()
 104         return ctx.tag[self.format_crawler_progress(p)]
 105
 106     def format_crawler_progress(self, p):
 107         cycletime = p["estimated-time-per-cycle"]
 108         cycletime_s = ""
 109         if cycletime is not None:
 110             cycletime_s = " (estimated cycle time %s)" % abbreviate_time(cycletime)
 111
 112         if p["cycle-in-progress"]:
 113             pct = p["cycle-complete-percentage"]
 114             soon = p["remaining-sleep-time"]
 115
 116             eta = p["estimated-cycle-complete-time-left"]
 117             eta_s = ""
 118             if eta is not None:
 119                 eta_s = " (ETA %ds)" % eta
 120
 121             return ["Current crawl %.1f%% complete" % pct,
 122                     eta_s,
 123                     " (next work in %s)" % abbreviate_time(soon),
 124                     cycletime_s,
 125                     ]
 126         else:
 127             soon = p["remaining-wait-time"]
 128             return ["Next crawl in %s" % abbreviate_time(soon),
 129                     cycletime_s]
 130
 131     def render_lease_expiration_enabled(self, ctx, data):
 132         lc = self.storage.lease_checker
 133         if lc.expiration_enabled:
 134             return ctx.tag["Enabled: expired leases will be removed"]
 135         else:
 136             return ctx.tag["Disabled: scan-only mode, no leases will be removed"]
 137
 138     def render_lease_expiration_mode(self, ctx, data):
 139         lc = self.storage.lease_checker
 140         if lc.mode == "age":
 141             if lc.override_lease_duration is None:
 142                 ctx.tag["Leases will expire naturally, probably 31 days after "
 143                         "creation or renewal."]
 144             else:
 145                 ctx.tag["Leases created or last renewed more than %s ago "
 146                         "will be considered expired."
 147                         % abbreviate_time(lc.override_lease_duration)]
 148         else:
 149             assert lc.mode == "cutoff-date"
 150             localizedutcdate = time.strftime("%d-%b-%Y", time.gmtime(lc.cutoff_date))
 151             isoutcdate = time_format.iso_utc_date(lc.cutoff_date)
 152             ctx.tag["Leases created or last renewed before %s (%s) UTC "
 153                     "will be considered expired." % (isoutcdate, localizedutcdate, )]
 154         if len(lc.mode) > 2:
 155             ctx.tag[" The following sharetypes will be expired: ",
 156                     " ".join(sorted(lc.sharetypes_to_expire)), "."]
 157         return ctx.tag
 158
 159     def format_recovered(self, sr, a):
 160         def maybe(d):
 161             if d is None:
 162                 return "?"
 163             return "%d" % d
 164         return "%s shares, %s buckets (%s mutable / %s immutable), %s (%s / %s)" % \
 165                (maybe(sr["%s-shares" % a]),
 166                 maybe(sr["%s-buckets" % a]),
 167                 maybe(sr["%s-buckets-mutable" % a]),
 168                 maybe(sr["%s-buckets-immutable" % a]),
 169                 abbreviate_space(sr["%s-diskbytes" % a]),
 170                 abbreviate_space(sr["%s-diskbytes-mutable" % a]),
 171                 abbreviate_space(sr["%s-diskbytes-immutable" % a]),
 172                 )
 173
 174     def render_lease_current_cycle_progress(self, ctx, data):
 175         lc = self.storage.lease_checker
 176         p = lc.get_progress()
 177         return ctx.tag[self.format_crawler_progress(p)]
 178
 179     def render_lease_current_cycle_results(self, ctx, data):
 180         lc = self.storage.lease_checker
 181         p = lc.get_progress()
 182         if not p["cycle-in-progress"]:
 183             return ""
 184         s = lc.get_state()
 185         so_far = s["cycle-to-date"]
 186         sr = so_far["space-recovered"]
 187         er = s["estimated-remaining-cycle"]
 188         esr = er["space-recovered"]
 189         ec = s["estimated-current-cycle"]
 190         ecr = ec["space-recovered"]
 191
 192         p = T.ul()
 193         def add(*pieces):
 194             p[T.li[pieces]]
 195
 196         def maybe(d):
 197             if d is None:
 198                 return "?"
 199             return "%d" % d
 200         add("So far, this cycle has examined %d shares in %d buckets"
 201             % (sr["examined-shares"], sr["examined-buckets"]),
 202             " (%d mutable / %d immutable)"
 203             % (sr["examined-buckets-mutable"], sr["examined-buckets-immutable"]),
 204             " (%s / %s)" % (abbreviate_space(sr["examined-diskbytes-mutable"]),
 205                             abbreviate_space(sr["examined-diskbytes-immutable"])),
 206             )
 207         add("and has recovered: ", self.format_recovered(sr, "actual"))
 208         if so_far["expiration-enabled"]:
 209             add("The remainder of this cycle is expected to recover: ",
 210                 self.format_recovered(esr, "actual"))
 211             add("The whole cycle is expected to examine %s shares in %s buckets"
 212                 % (maybe(ecr["examined-shares"]), maybe(ecr["examined-buckets"])))
 213             add("and to recover: ", self.format_recovered(ecr, "actual"))
 214
 215         else:
 216             add("If expiration were enabled, we would have recovered: ",
 217                 self.format_recovered(sr, "configured"), " by now")
 218             add("and the remainder of this cycle would probably recover: ",
 219                 self.format_recovered(esr, "configured"))
 220             add("and the whole cycle would probably recover: ",
 221                 self.format_recovered(ecr, "configured"))
 222
 223         add("if we were strictly using each lease's default 31-day lease lifetime "
 224             "(instead of our configured behavior), "
 225             "this cycle would be expected to recover: ",
 226             self.format_recovered(ecr, "original"))
 227
 228         if so_far["corrupt-shares"]:
 229             add("Corrupt shares:",
 230                 T.ul[ [T.li[ ["SI %s shnum %d" % corrupt_share
 231                               for corrupt_share in so_far["corrupt-shares"] ]
 232                              ]]])
 233
 234         return ctx.tag["Current cycle:", p]
 235
 236     def render_lease_last_cycle_results(self, ctx, data):
 237         lc = self.storage.lease_checker
 238         h = lc.get_state()["history"]
 239         if not h:
 240             return ""
 241         last = h[max(h.keys())]
 242
 243         start, end = last["cycle-start-finish-times"]
 244         ctx.tag["Last complete cycle (which took %s and finished %s ago)"
 245                 " recovered: " % (abbreviate_time(end-start),
 246                                   abbreviate_time(time.time() - end)),
 247                 self.format_recovered(last["space-recovered"], "actual")
 248                 ]
 249
 250         p = T.ul()
 251         def add(*pieces):
 252             p[T.li[pieces]]
 253
 254         saw = self.format_recovered(last["space-recovered"], "examined")
 255         add("and saw a total of ", saw)
 256
 257         if not last["expiration-enabled"]:
 258             rec = self.format_recovered(last["space-recovered"], "configured")
 259             add("but expiration was not enabled. If it had been, "
 260                 "it would have recovered: ", rec)
 261
 262         if last["corrupt-shares"]:
 263             add("Corrupt shares:",
 264                 T.ul[ [T.li[ ["SI %s shnum %d" % corrupt_share
 265                               for corrupt_share in last["corrupt-shares"] ]
 266                              ]]])
 267
 268         return ctx.tag[p]
 269