expirer: track mutable-vs-immutable sharecounts and sizes, report them on the web...

diff --git a/src/allmydata/storage/expirer.py b/src/allmydata/storage/expirer.py
index d41d097778ec97beac6170f29f89b328273a048c..d4fd9a01829f4d51ae7a29ed25cab6a0358fa18a 100644 (file)
--- a/src/allmydata/storage/expirer.py
+++ b/src/allmydata/storage/expirer.py
@@ -84,9 +84,7 @@ class LeaseCheckingCrawler(ShareCrawler):
 
     def create_empty_cycle_dict(self):
         recovered = self.create_empty_recovered_dict()
-        so_far = {"buckets-examined": 0,
-                  "shares-examined": 0,
-                  "corrupt-shares": [],
+        so_far = {"corrupt-shares": [],
                   "space-recovered": recovered,
                   "lease-age-histogram": {}, # (minage,maxage)->count
                   "leases-per-share-histogram": {}, # leasecount->numshares
@@ -95,9 +93,11 @@ class LeaseCheckingCrawler(ShareCrawler):
 
     def create_empty_recovered_dict(self):
         recovered = {}
-        for a in ("actual", "original-leasetimer", "configured-leasetimer"):
-            for b in ("numbuckets", "numshares", "sharebytes", "diskbytes"):
+        for a in ("actual", "original", "configured", "examined"):
+            for b in ("buckets", "shares", "sharebytes", "diskbytes"):
                 recovered[a+"-"+b] = 0
+                recovered[a+"-"+b+"-mutable"] = 0
+                recovered[a+"-"+b+"-immutable"] = 0
         return recovered
 
     def started_cycle(self, cycle):
@@ -128,27 +128,36 @@ class LeaseCheckingCrawler(ShareCrawler):
                 twlog.err()
                 which = (storage_index_b32, shnum)
                 self.state["cycle-to-date"]["corrupt-shares"].append(which)
-                wks = (1, 1, 1)
+                wks = (1, 1, 1, "unknown")
             would_keep_shares.append(wks)
-        recovered = self.state["cycle-to-date"]["space-recovered"]
+        sharetype = None
+        if wks:
+            sharetype = wks[3]
+        rec = self.state["cycle-to-date"]["space-recovered"]
+        self.increment(rec, "examined-buckets", 1)
+        if sharetype:
+            self.increment(rec, "examined-buckets-"+sharetype, 1)
+
         if sum([wks[0] for wks in would_keep_shares]) == 0:
-            self.increment(recovered,
-                           "original-leasetimer-diskbytes", bucket_diskbytes)
-            self.increment(recovered, "original-leasetimer-numbuckets", 1)
+            self.increment(rec, "original-diskbytes", bucket_diskbytes)
+            self.increment(rec, "original-diskbytes-"+sharetype, bucket_diskbytes)
+            self.increment(rec, "original-buckets", 1)
+            self.increment(rec, "original-buckets-"+sharetype, 1)
         if sum([wks[1] for wks in would_keep_shares]) == 0:
-            self.increment(recovered,
-                           "configured-leasetimer-diskbytes", bucket_diskbytes)
-            self.increment(recovered, "configured-leasetimer-numbuckets", 1)
+            self.increment(rec, "configured-diskbytes", bucket_diskbytes)
+            self.increment(rec, "configured-diskbytes-"+sharetype, bucket_diskbytes)
+            self.increment(rec, "configured-buckets", 1)
+            self.increment(rec, "configured-buckets-"+sharetype, 1)
         if sum([wks[2] for wks in would_keep_shares]) == 0:
-            self.increment(recovered,
-                           "actual-diskbytes", bucket_diskbytes)
-            self.increment(recovered, "actual-numbuckets", 1)
-        self.state["cycle-to-date"]["buckets-examined"] += 1
+            self.increment(rec, "actual-diskbytes", bucket_diskbytes)
+            self.increment(rec, "actual-diskbytes-"+sharetype, bucket_diskbytes)
+            self.increment(rec, "actual-buckets", 1)
+            self.increment(rec, "actual-buckets-"+sharetype, 1)
 
     def process_share(self, sharefilename):
         # first, find out what kind of a share it is
         sf = get_share_file(sharefilename)
-        sftype = sf.sharetype
+        sharetype = sf.sharetype
         now = time.time()
         s = self.stat(sharefilename)
 
@@ -179,7 +188,7 @@ class LeaseCheckingCrawler(ShareCrawler):
                 date_cutoff = self.mode[1]
                 if grant_renew_time < date_cutoff:
                     expired = True
-            if sftype not in self.sharetypes_to_expire:
+            if sharetype not in self.sharetypes_to_expire:
                 expired = False
 
             if expired:
@@ -189,11 +198,9 @@ class LeaseCheckingCrawler(ShareCrawler):
 
         so_far = self.state["cycle-to-date"]
         self.increment(so_far["leases-per-share-histogram"], num_leases, 1)
-        so_far["shares-examined"] += 1
-        # TODO: accumulate share-sizes too, so we can display "the whole
-        # cycle would probably recover x GB out of y GB total"
+        self.increment_space("examined", s, sharetype)
 
-        would_keep_share = [1, 1, 1]
+        would_keep_share = [1, 1, 1, sharetype]
 
         if self.expiration_enabled:
             for li in expired_leases_configured:
@@ -201,18 +208,18 @@ class LeaseCheckingCrawler(ShareCrawler):
 
         if num_valid_leases_original == 0:
             would_keep_share[0] = 0
-            self.increment_space("original-leasetimer", s)
+            self.increment_space("original", s, sharetype)
 
         if num_valid_leases_configured == 0:
             would_keep_share[1] = 0
-            self.increment_space("configured-leasetimer", s)
+            self.increment_space("configured", s, sharetype)
             if self.expiration_enabled:
                 would_keep_share[2] = 0
-                self.increment_space("actual", s)
+                self.increment_space("actual", s, sharetype)
 
         return would_keep_share
 
-    def increment_space(self, a, s):
+    def increment_space(self, a, s, sharetype):
         sharebytes = s.st_size
         try:
             # note that stat(2) says that st_blocks is 512 bytes, and that
@@ -224,9 +231,12 @@ class LeaseCheckingCrawler(ShareCrawler):
             # MacOS. But it isn't available on windows.
             diskbytes = sharebytes
         so_far_sr = self.state["cycle-to-date"]["space-recovered"]
-        self.increment(so_far_sr, a+"-numshares", 1)
+        self.increment(so_far_sr, a+"-shares", 1)
+        self.increment(so_far_sr, a+"-shares-"+sharetype, 1)
         self.increment(so_far_sr, a+"-sharebytes", sharebytes)
+        self.increment(so_far_sr, a+"-sharebytes-"+sharetype, sharebytes)
         self.increment(so_far_sr, a+"-diskbytes", diskbytes)
+        self.increment(so_far_sr, a+"-diskbytes-"+sharetype, diskbytes)
 
     def increment(self, d, k, delta=1):
         if k not in d:
@@ -272,8 +282,6 @@ class LeaseCheckingCrawler(ShareCrawler):
         lah = self.convert_lease_age_histogram(s["lease-age-histogram"])
         h["lease-age-histogram"] = lah
         h["leases-per-share-histogram"] = s["leases-per-share-histogram"].copy()
-        h["buckets-examined"] = s["buckets-examined"]
-        h["shares-examined"] = s["shares-examined"]
         h["corrupt-shares"] = s["corrupt-shares"][:]
         # note: if ["shares-recovered"] ever acquires an internal dict, this
         # copy() needs to become a deepcopy
@@ -304,22 +312,16 @@ class LeaseCheckingCrawler(ShareCrawler):
           lease-age-histogram (list of (minage,maxage,sharecount) tuples)
           leases-per-share-histogram
           corrupt-shares (list of (si_b32,shnum) tuples, minimal verification)
-          buckets-examined
-          shares-examined
           space-recovered
 
          estimated-remaining-cycle:
           # Values may be None if not enough data has been gathered to
           # produce an estimate.
-          buckets-examined
-          shares-examined
           space-recovered
 
          estimated-current-cycle:
           # cycle-to-date plus estimated-remaining. Values may be None if
           # not enough data has been gathered to produce an estimate.
-          buckets-examined
-          shares-examined
           space-recovered
 
          history: maps cyclenum to a dict with the following keys:
@@ -329,27 +331,33 @@ class LeaseCheckingCrawler(ShareCrawler):
           lease-age-histogram
           leases-per-share-histogram
           corrupt-shares
-          buckets-examined
-          shares-examined
           space-recovered
 
          The 'space-recovered' structure is a dictionary with the following
          keys:
+          # 'examined' is what was looked at
+          examined-buckets, examined-buckets-mutable, examined-buckets-immutable
+          examined-shares, -mutable, -immutable
+          examined-sharebytes, -mutable, -immutable
+          examined-diskbytes, -mutable, -immutable
+
           # 'actual' is what was actually deleted
-          actual-numbuckets
-          actual-numshares
-          actual-sharebytes
-          actual-diskbytes
+          actual-buckets, -mutable, -immutable
+          actual-shares, -mutable, -immutable
+          actual-sharebytes, -mutable, -immutable
+          actual-diskbytes, -mutable, -immutable
+
           # would have been deleted, if the original lease timer was used
-          original-leasetimer-numbuckets
-          original-leasetimer-numshares
-          original-leasetimer-sharebytes
-          original-leasetimer-diskbytes
+          original-buckets, -mutable, -immutable
+          original-shares, -mutable, -immutable
+          original-sharebytes, -mutable, -immutable
+          original-diskbytes, -mutable, -immutable
+
           # would have been deleted, if our configured max_age was used
-          configured-leasetimer-numbuckets
-          configured-leasetimer-numshares
-          configured-leasetimer-sharebytes
-          configured-leasetimer-diskbytes
+          configured-buckets, -mutable, -immutable
+          configured-shares, -mutable, -immutable
+          configured-sharebytes, -mutable, -immutable
+          configured-diskbytes, -mutable, -immutable
 
         """
         progress = self.get_progress()
@@ -379,27 +387,19 @@ class LeaseCheckingCrawler(ShareCrawler):
         if progress["cycle-complete-percentage"] > 0.0:
             pc = progress["cycle-complete-percentage"] / 100.0
             m = (1-pc)/pc
-            for a in ("actual", "original-leasetimer", "configured-leasetimer"):
-                for b in ("numbuckets", "numshares", "sharebytes", "diskbytes"):
-                    k = a+"-"+b
-                    remaining_sr[k] = m * so_far_sr[k]
-                    cycle_sr[k] = so_far_sr[k] + remaining_sr[k]
-            predshares = m * so_far["shares-examined"]
-            remaining["shares-examined"] = predshares
-            cycle["shares-examined"] = so_far["shares-examined"] + predshares
-            predbuckets = m * so_far["buckets-examined"]
-            remaining["buckets-examined"] = predbuckets
-            cycle["buckets-examined"] = so_far["buckets-examined"] + predbuckets
+            for a in ("actual", "original", "configured", "examined"):
+                for b in ("buckets", "shares", "sharebytes", "diskbytes"):
+                    for c in ("", "-mutable", "-immutable"):
+                        k = a+"-"+b+c
+                        remaining_sr[k] = m * so_far_sr[k]
+                        cycle_sr[k] = so_far_sr[k] + remaining_sr[k]
         else:
-            for a in ("actual", "original-leasetimer", "configured-leasetimer"):
-                for b in ("numbuckets", "numshares", "sharebytes", "diskbytes"):
-                    k = a+"-"+b
-                    remaining_sr[k] = None
-                    cycle_sr[k] = None
-            remaining["shares-examined"] = None
-            cycle["shares-examined"] = None
-            remaining["buckets-examined"] = None
-            cycle["buckets-examined"] = None
+            for a in ("actual", "original", "configured", "examined"):
+                for b in ("buckets", "shares", "sharebytes", "diskbytes"):
+                    for c in ("", "-mutable", "-immutable"):
+                        k = a+"-"+b+c
+                        remaining_sr[k] = None
+                        cycle_sr[k] = None
 
         state["estimated-remaining-cycle"] = remaining
         state["estimated-current-cycle"] = cycle

diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py
index ef308a24a5855e92c901ff2644fb8d3cb225deee..494330287491887f1907aa2851486864a47c6674 100644 (file)
--- a/src/allmydata/test/test_storage.py
+++ b/src/allmydata/test/test_storage.py
@@ -1634,38 +1634,40 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
             self.failUnlessEqual(len(lah), 1)
             self.failUnlessEqual(lah, [ (0.0, DAY, 1) ] )
             self.failUnlessEqual(so_far["leases-per-share-histogram"], {1: 1})
-            self.failUnlessEqual(so_far["buckets-examined"], 1)
-            self.failUnlessEqual(so_far["shares-examined"], 1)
             self.failUnlessEqual(so_far["corrupt-shares"], [])
             sr1 = so_far["space-recovered"]
-            self.failUnlessEqual(sr1["actual-numshares"], 0)
-            self.failUnlessEqual(sr1["configured-leasetimer-diskbytes"], 0)
-            self.failUnlessEqual(sr1["original-leasetimer-sharebytes"], 0)
+            self.failUnlessEqual(sr1["examined-buckets"], 1)
+            self.failUnlessEqual(sr1["examined-shares"], 1)
+            self.failUnlessEqual(sr1["actual-shares"], 0)
+            self.failUnlessEqual(sr1["configured-diskbytes"], 0)
+            self.failUnlessEqual(sr1["original-sharebytes"], 0)
             left = initial_state["estimated-remaining-cycle"]
-            self.failUnless(left["buckets-examined"] > 0,
-                            left["buckets-examined"])
-            self.failUnless(left["shares-examined"] > 0,
-                            left["shares-examined"])
             sr2 = left["space-recovered"]
-            self.failIfEqual(sr2["actual-numshares"], None)
-            self.failIfEqual(sr2["configured-leasetimer-diskbytes"], None)
-            self.failIfEqual(sr2["original-leasetimer-sharebytes"], None)
+            self.failUnless(sr2["examined-buckets"] > 0, sr2["examined-buckets"])
+            self.failUnless(sr2["examined-shares"] > 0, sr2["examined-shares"])
+            self.failIfEqual(sr2["actual-shares"], None)
+            self.failIfEqual(sr2["configured-diskbytes"], None)
+            self.failIfEqual(sr2["original-sharebytes"], None)
         d.addCallback(_after_first_bucket)
         d.addCallback(lambda ign: self.render1(webstatus))
         def _check_html_in_cycle(html):
             s = remove_tags(html)
             self.failUnlessIn("So far, this cycle has examined "
-                              "1 shares in 1 buckets "
-                              "and has recovered: "
-                              "0 shares, 0 buckets, 0 B ", s)
+                              "1 shares in 1 buckets (0 mutable / 1 immutable) ", s)
+            self.failUnlessIn("and has recovered: "
+                              "0 shares, 0 buckets (0 mutable / 0 immutable), "
+                              "0 B (0 B / 0 B)", s)
             self.failUnlessIn("If expiration were enabled, "
                               "we would have recovered: "
-                              "0 shares, 0 buckets, 0 B by now", s)
+                              "0 shares, 0 buckets (0 mutable / 0 immutable),"
+                              " 0 B (0 B / 0 B) by now", s)
             self.failUnlessIn("and the remainder of this cycle "
                               "would probably recover: "
-                              "0 shares, 0 buckets, 0 B ", s)
+                              "0 shares, 0 buckets (0 mutable / 0 immutable),"
+                              " 0 B (0 B / 0 B)", s)
             self.failUnlessIn("and the whole cycle would probably recover: "
-                              "0 shares, 0 buckets, 0 B ", s)
+                              "0 shares, 0 buckets (0 mutable / 0 immutable),"
+                              " 0 B (0 B / 0 B)", s)
             self.failUnlessIn("if we were using each lease's default "
                               "31-day lease lifetime", s)
             self.failUnlessIn("this cycle would be expected to recover: ", s)
@@ -1694,25 +1696,24 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
             self.failUnlessEqual(len(lah), 1)
             self.failUnlessEqual(lah, [ (0.0, DAY, 6) ] )
 
-            self.failUnlessEqual(last["leases-per-share-histogram"],
-                                 {1: 2, 2: 2})
-            self.failUnlessEqual(last["buckets-examined"], 4)
-            self.failUnlessEqual(last["shares-examined"], 4)
+            self.failUnlessEqual(last["leases-per-share-histogram"], {1: 2, 2: 2})
             self.failUnlessEqual(last["corrupt-shares"], [])
 
             rec = last["space-recovered"]
-            self.failUnlessEqual(rec["actual-numbuckets"], 0)
-            self.failUnlessEqual(rec["original-leasetimer-numbuckets"], 0)
-            self.failUnlessEqual(rec["configured-leasetimer-numbuckets"], 0)
-            self.failUnlessEqual(rec["actual-numshares"], 0)
-            self.failUnlessEqual(rec["original-leasetimer-numshares"], 0)
-            self.failUnlessEqual(rec["configured-leasetimer-numshares"], 0)
+            self.failUnlessEqual(rec["examined-buckets"], 4)
+            self.failUnlessEqual(rec["examined-shares"], 4)
+            self.failUnlessEqual(rec["actual-buckets"], 0)
+            self.failUnlessEqual(rec["original-buckets"], 0)
+            self.failUnlessEqual(rec["configured-buckets"], 0)
+            self.failUnlessEqual(rec["actual-shares"], 0)
+            self.failUnlessEqual(rec["original-shares"], 0)
+            self.failUnlessEqual(rec["configured-shares"], 0)
             self.failUnlessEqual(rec["actual-diskbytes"], 0)
-            self.failUnlessEqual(rec["original-leasetimer-diskbytes"], 0)
-            self.failUnlessEqual(rec["configured-leasetimer-diskbytes"], 0)
+            self.failUnlessEqual(rec["original-diskbytes"], 0)
+            self.failUnlessEqual(rec["configured-diskbytes"], 0)
             self.failUnlessEqual(rec["actual-sharebytes"], 0)
-            self.failUnlessEqual(rec["original-leasetimer-sharebytes"], 0)
-            self.failUnlessEqual(rec["configured-leasetimer-sharebytes"], 0)
+            self.failUnlessEqual(rec["original-sharebytes"], 0)
+            self.failUnlessEqual(rec["configured-sharebytes"], 0)
 
             def _get_sharefile(si):
                 return list(ss._iter_share_files(si))[0]
@@ -1726,7 +1727,8 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
         d.addCallback(lambda ign: self.render1(webstatus))
         def _check_html(html):
             s = remove_tags(html)
-            self.failUnlessIn("recovered: 0 shares, 0 buckets, 0 B "
+            self.failUnlessIn("recovered: 0 shares, 0 buckets "
+                              "(0 mutable / 0 immutable), 0 B (0 B / 0 B) "
                               "but expiration was not enabled", s)
         d.addCallback(_check_html)
         return d
@@ -1843,39 +1845,37 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
             last = s["history"][0]
 
             self.failUnlessEqual(last["expiration-enabled"], True)
-            self.failUnlessEqual(last["configured-expiration-mode"],
-                                 ("age",2000))
-            self.failUnlessEqual(last["buckets-examined"], 4)
-            self.failUnlessEqual(last["shares-examined"], 4)
-            self.failUnlessEqual(last["leases-per-share-histogram"],
-                                 {1: 2, 2: 2})
+            self.failUnlessEqual(last["configured-expiration-mode"], ("age",2000))
+            self.failUnlessEqual(last["leases-per-share-histogram"], {1: 2, 2: 2})
 
             rec = last["space-recovered"]
-            self.failUnlessEqual(rec["actual-numbuckets"], 2)
-            self.failUnlessEqual(rec["original-leasetimer-numbuckets"], 2)
-            self.failUnlessEqual(rec["configured-leasetimer-numbuckets"], 2)
-            self.failUnlessEqual(rec["actual-numshares"], 2)
-            self.failUnlessEqual(rec["original-leasetimer-numshares"], 2)
-            self.failUnlessEqual(rec["configured-leasetimer-numshares"], 2)
+            self.failUnlessEqual(rec["examined-buckets"], 4)
+            self.failUnlessEqual(rec["examined-shares"], 4)
+            self.failUnlessEqual(rec["actual-buckets"], 2)
+            self.failUnlessEqual(rec["original-buckets"], 2)
+            self.failUnlessEqual(rec["configured-buckets"], 2)
+            self.failUnlessEqual(rec["actual-shares"], 2)
+            self.failUnlessEqual(rec["original-shares"], 2)
+            self.failUnlessEqual(rec["configured-shares"], 2)
             size = sf0_size + sf2_size
             self.failUnlessEqual(rec["actual-sharebytes"], size)
-            self.failUnlessEqual(rec["original-leasetimer-sharebytes"], size)
-            self.failUnlessEqual(rec["configured-leasetimer-sharebytes"], size)
+            self.failUnlessEqual(rec["original-sharebytes"], size)
+            self.failUnlessEqual(rec["configured-sharebytes"], size)
             # different platforms have different notions of "blocks used by
             # this file", so merely assert that it's a number
             self.failUnless(rec["actual-diskbytes"] >= 0,
                             rec["actual-diskbytes"])
-            self.failUnless(rec["original-leasetimer-diskbytes"] >= 0,
-                            rec["original-leasetimer-diskbytes"])
-            self.failUnless(rec["configured-leasetimer-diskbytes"] >= 0,
-                            rec["configured-leasetimer-diskbytes"])
+            self.failUnless(rec["original-diskbytes"] >= 0,
+                            rec["original-diskbytes"])
+            self.failUnless(rec["configured-diskbytes"] >= 0,
+                            rec["configured-diskbytes"])
         d.addCallback(_after_first_cycle)
         d.addCallback(lambda ign: self.render1(webstatus))
         def _check_html(html):
             s = remove_tags(html)
             self.failUnlessIn("Expiration Enabled: expired leases will be removed", s)
             self.failUnlessIn("leases created or last renewed more than 33 minutes ago will be considered expired", s)
-            self.failUnlessIn(" recovered: 2 shares, 2 buckets, ", s)
+            self.failUnlessIn(" recovered: 2 shares, 2 buckets (1 mutable / 1 immutable), ", s)
         d.addCallback(_check_html)
         return d
 
@@ -1986,30 +1986,30 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
             self.failUnlessEqual(last["expiration-enabled"], True)
             self.failUnlessEqual(last["configured-expiration-mode"],
                                  ("date-cutoff",then))
-            self.failUnlessEqual(last["buckets-examined"], 4)
-            self.failUnlessEqual(last["shares-examined"], 4)
             self.failUnlessEqual(last["leases-per-share-histogram"],
                                  {1: 2, 2: 2})
 
             rec = last["space-recovered"]
-            self.failUnlessEqual(rec["actual-numbuckets"], 2)
-            self.failUnlessEqual(rec["original-leasetimer-numbuckets"], 0)
-            self.failUnlessEqual(rec["configured-leasetimer-numbuckets"], 2)
-            self.failUnlessEqual(rec["actual-numshares"], 2)
-            self.failUnlessEqual(rec["original-leasetimer-numshares"], 0)
-            self.failUnlessEqual(rec["configured-leasetimer-numshares"], 2)
+            self.failUnlessEqual(rec["examined-buckets"], 4)
+            self.failUnlessEqual(rec["examined-shares"], 4)
+            self.failUnlessEqual(rec["actual-buckets"], 2)
+            self.failUnlessEqual(rec["original-buckets"], 0)
+            self.failUnlessEqual(rec["configured-buckets"], 2)
+            self.failUnlessEqual(rec["actual-shares"], 2)
+            self.failUnlessEqual(rec["original-shares"], 0)
+            self.failUnlessEqual(rec["configured-shares"], 2)
             size = sf0_size + sf2_size
             self.failUnlessEqual(rec["actual-sharebytes"], size)
-            self.failUnlessEqual(rec["original-leasetimer-sharebytes"], 0)
-            self.failUnlessEqual(rec["configured-leasetimer-sharebytes"], size)
+            self.failUnlessEqual(rec["original-sharebytes"], 0)
+            self.failUnlessEqual(rec["configured-sharebytes"], size)
             # different platforms have different notions of "blocks used by
             # this file", so merely assert that it's a number
             self.failUnless(rec["actual-diskbytes"] >= 0,
                             rec["actual-diskbytes"])
-            self.failUnless(rec["original-leasetimer-diskbytes"] >= 0,
-                            rec["original-leasetimer-diskbytes"])
-            self.failUnless(rec["configured-leasetimer-diskbytes"] >= 0,
-                            rec["configured-leasetimer-diskbytes"])
+            self.failUnless(rec["original-diskbytes"] >= 0,
+                            rec["original-diskbytes"])
+            self.failUnless(rec["configured-diskbytes"] >= 0,
+                            rec["configured-diskbytes"])
         d.addCallback(_after_first_cycle)
         d.addCallback(lambda ign: self.render1(webstatus))
         def _check_html(html):
@@ -2019,7 +2019,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
             date = time.strftime("%d-%b-%Y", time.gmtime(then))
             self.failUnlessIn("leases created or last renewed before %s"
                               " will be considered expired" % date, s)
-            self.failUnlessIn(" recovered: 2 shares, 2 buckets, ", s)
+            self.failUnlessIn(" recovered: 2 shares, 2 buckets (1 mutable / 1 immutable), ", s)
         d.addCallback(_check_html)
         return d
 
@@ -2220,32 +2220,32 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
             self.failUnless("estimated-current-cycle" in s)
 
             left = s["estimated-remaining-cycle"]["space-recovered"]
-            self.failUnlessEqual(left["actual-numbuckets"], None)
-            self.failUnlessEqual(left["original-leasetimer-numbuckets"], None)
-            self.failUnlessEqual(left["configured-leasetimer-numbuckets"], None)
-            self.failUnlessEqual(left["actual-numshares"], None)
-            self.failUnlessEqual(left["original-leasetimer-numshares"], None)
-            self.failUnlessEqual(left["configured-leasetimer-numshares"], None)
+            self.failUnlessEqual(left["actual-buckets"], None)
+            self.failUnlessEqual(left["original-buckets"], None)
+            self.failUnlessEqual(left["configured-buckets"], None)
+            self.failUnlessEqual(left["actual-shares"], None)
+            self.failUnlessEqual(left["original-shares"], None)
+            self.failUnlessEqual(left["configured-shares"], None)
             self.failUnlessEqual(left["actual-diskbytes"], None)
-            self.failUnlessEqual(left["original-leasetimer-diskbytes"], None)
-            self.failUnlessEqual(left["configured-leasetimer-diskbytes"], None)
+            self.failUnlessEqual(left["original-diskbytes"], None)
+            self.failUnlessEqual(left["configured-diskbytes"], None)
             self.failUnlessEqual(left["actual-sharebytes"], None)
-            self.failUnlessEqual(left["original-leasetimer-sharebytes"], None)
-            self.failUnlessEqual(left["configured-leasetimer-sharebytes"], None)
+            self.failUnlessEqual(left["original-sharebytes"], None)
+            self.failUnlessEqual(left["configured-sharebytes"], None)
 
             full = s["estimated-remaining-cycle"]["space-recovered"]
-            self.failUnlessEqual(full["actual-numbuckets"], None)
-            self.failUnlessEqual(full["original-leasetimer-numbuckets"], None)
-            self.failUnlessEqual(full["configured-leasetimer-numbuckets"], None)
-            self.failUnlessEqual(full["actual-numshares"], None)
-            self.failUnlessEqual(full["original-leasetimer-numshares"], None)
-            self.failUnlessEqual(full["configured-leasetimer-numshares"], None)
+            self.failUnlessEqual(full["actual-buckets"], None)
+            self.failUnlessEqual(full["original-buckets"], None)
+            self.failUnlessEqual(full["configured-buckets"], None)
+            self.failUnlessEqual(full["actual-shares"], None)
+            self.failUnlessEqual(full["original-shares"], None)
+            self.failUnlessEqual(full["configured-shares"], None)
             self.failUnlessEqual(full["actual-diskbytes"], None)
-            self.failUnlessEqual(full["original-leasetimer-diskbytes"], None)
-            self.failUnlessEqual(full["configured-leasetimer-diskbytes"], None)
+            self.failUnlessEqual(full["original-diskbytes"], None)
+            self.failUnlessEqual(full["configured-diskbytes"], None)
             self.failUnlessEqual(full["actual-sharebytes"], None)
-            self.failUnlessEqual(full["original-leasetimer-sharebytes"], None)
-            self.failUnlessEqual(full["configured-leasetimer-sharebytes"], None)
+            self.failUnlessEqual(full["original-sharebytes"], None)
+            self.failUnlessEqual(full["configured-sharebytes"], None)
 
         d.addCallback(_check)
         return d
@@ -2255,7 +2255,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
         fileutil.make_dirs(basedir)
         ss = No_ST_BLOCKS_StorageServer(basedir, "\x00" * 20,
                                         expiration_mode=("age",-1000))
-        # a negative expiration_time= means the "configured-leasetimer-"
+        # a negative expiration_time= means the "configured-"
         # space-recovered counts will be non-zero, since all shares will have
         # expired by then
 
@@ -2273,14 +2273,14 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
             s = lc.get_state()
             last = s["history"][0]
             rec = last["space-recovered"]
-            self.failUnlessEqual(rec["configured-leasetimer-numbuckets"], 4)
-            self.failUnlessEqual(rec["configured-leasetimer-numshares"], 4)
-            self.failUnless(rec["configured-leasetimer-sharebytes"] > 0,
-                            rec["configured-leasetimer-sharebytes"])
+            self.failUnlessEqual(rec["configured-buckets"], 4)
+            self.failUnlessEqual(rec["configured-shares"], 4)
+            self.failUnless(rec["configured-sharebytes"] > 0,
+                            rec["configured-sharebytes"])
             # without the .st_blocks field in os.stat() results, we should be
             # reporting diskbytes==sharebytes
-            self.failUnlessEqual(rec["configured-leasetimer-sharebytes"],
-                                 rec["configured-leasetimer-diskbytes"])
+            self.failUnlessEqual(rec["configured-sharebytes"],
+                                 rec["configured-diskbytes"])
         d.addCallback(_check)
         return d
 
@@ -2321,8 +2321,9 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
         # processed.
         def _after_first_bucket(ignored):
             so_far = lc.get_state()["cycle-to-date"]
-            self.failUnlessEqual(so_far["buckets-examined"], 1)
-            self.failUnlessEqual(so_far["shares-examined"], 0)
+            rec = so_far["space-recovered"]
+            self.failUnlessEqual(rec["examined-buckets"], 1)
+            self.failUnlessEqual(rec["examined-shares"], 0)
             self.failUnlessEqual(so_far["corrupt-shares"], [(first_b32, 0)])
         d.addCallback(_after_first_bucket)
 
@@ -2348,8 +2349,9 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
         def _after_first_cycle(ignored):
             s = lc.get_state()
             last = s["history"][0]
-            self.failUnlessEqual(last["buckets-examined"], 4)
-            self.failUnlessEqual(last["shares-examined"], 3)
+            rec = last["space-recovered"]
+            self.failUnlessEqual(rec["examined-buckets"], 4)
+            self.failUnlessEqual(rec["examined-shares"], 3)
             self.failUnlessEqual(last["corrupt-shares"], [(first_b32, 0)])
         d.addCallback(_after_first_cycle)
         d.addCallback(lambda ign: self.render_json(w))

diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py
index 581d821b32887359be8078801377f15d453b577b..3491ea63bba5b998ac7e6ff0f61a3d13ce0f6abc 100644 (file)
--- a/src/allmydata/web/storage.py
+++ b/src/allmydata/web/storage.py
@@ -148,10 +148,15 @@ class StorageStatus(rend.Page):
             if d is None:
                 return "?"
             return "%d" % d
-        space = abbreviate_space(sr["%s-diskbytes" % a])
-        return "%s shares, %s buckets, %s" % (maybe(sr["%s-numshares" % a]),
-                                              maybe(sr["%s-numbuckets" % a]),
-                                              space)
+        return "%s shares, %s buckets (%s mutable / %s immutable), %s (%s / %s)" % \
+               (maybe(sr["%s-shares" % a]),
+                maybe(sr["%s-buckets" % a]),
+                maybe(sr["%s-buckets-mutable" % a]),
+                maybe(sr["%s-buckets-immutable" % a]),
+                abbreviate_space(sr["%s-diskbytes" % a]),
+                abbreviate_space(sr["%s-diskbytes-mutable" % a]),
+                abbreviate_space(sr["%s-diskbytes-immutable" % a]),
+                )
 
     def render_lease_current_cycle_progress(self, ctx, data):
         lc = self.storage.lease_checker
@@ -181,27 +186,32 @@ class StorageStatus(rend.Page):
                 return "?"
             return "%d" % d
         add("So far, this cycle has examined %d shares in %d buckets"
-            % (so_far["shares-examined"], so_far["buckets-examined"]))
+            % (sr["examined-shares"], sr["examined-buckets"]),
+            " (%d mutable / %d immutable)"
+            % (sr["examined-buckets-mutable"], sr["examined-buckets-immutable"]),
+            " (%s / %s)" % (abbreviate_space(sr["examined-diskbytes-mutable"]),
+                            abbreviate_space(sr["examined-diskbytes-immutable"])),
+            )
         add("and has recovered: ", self.format_recovered(sr, "actual"))
         if so_far["expiration-enabled"]:
             add("The remainder of this cycle is expected to recover: ",
                 self.format_recovered(esr, "actual"))
             add("The whole cycle is expected to examine %s shares in %s buckets"
-                % (maybe(ec["shares-examined"]), maybe(ec["buckets-examined"])))
+                % (maybe(ecr["examined-shares"]), maybe(ecr["examined-buckets"])))
             add("and to recover: ", self.format_recovered(ecr, "actual"))
 
         else:
             add("If expiration were enabled, we would have recovered: ",
-                self.format_recovered(sr, "configured-leasetimer"), " by now")
+                self.format_recovered(sr, "configured"), " by now")
             add("and the remainder of this cycle would probably recover: ",
-                self.format_recovered(esr, "configured-leasetimer"))
+                self.format_recovered(esr, "configured"))
             add("and the whole cycle would probably recover: ",
-                self.format_recovered(ecr, "configured-leasetimer"))
+                self.format_recovered(ecr, "configured"))
 
         add("if we were using each lease's default 31-day lease lifetime "
             "(instead of our configured node), "
             "this cycle would be expected to recover: ",
-            self.format_recovered(ecr, "original-leasetimer"))
+            self.format_recovered(ecr, "original"))
 
         if so_far["corrupt-shares"]:
             add("Corrupt shares:",
@@ -231,8 +241,7 @@ class StorageStatus(rend.Page):
             p[T.li[pieces]]
 
         if not last["expiration-enabled"]:
-            rec = self.format_recovered(last["space-recovered"],
-                                        "configured-leasetimer")
+            rec = self.format_recovered(last["space-recovered"], "configured")
             add("but expiration was not enabled. If it had been, "
                 "it would have recovered: ", rec)