From fffab0d7249cbf3934982b5465d65c1edd47bc47 Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Wed, 18 Mar 2009 13:25:04 -0700 Subject: [PATCH] expirer: track mutable-vs-immutable sharecounts and sizes, report them on the web status page for comparison --- src/allmydata/storage/expirer.py | 140 ++++++++++---------- src/allmydata/test/test_storage.py | 198 +++++++++++++++-------------- src/allmydata/web/storage.py | 33 +++-- 3 files changed, 191 insertions(+), 180 deletions(-) diff --git a/src/allmydata/storage/expirer.py b/src/allmydata/storage/expirer.py index d41d0977..d4fd9a01 100644 --- a/src/allmydata/storage/expirer.py +++ b/src/allmydata/storage/expirer.py @@ -84,9 +84,7 @@ class LeaseCheckingCrawler(ShareCrawler): def create_empty_cycle_dict(self): recovered = self.create_empty_recovered_dict() - so_far = {"buckets-examined": 0, - "shares-examined": 0, - "corrupt-shares": [], + so_far = {"corrupt-shares": [], "space-recovered": recovered, "lease-age-histogram": {}, # (minage,maxage)->count "leases-per-share-histogram": {}, # leasecount->numshares @@ -95,9 +93,11 @@ class LeaseCheckingCrawler(ShareCrawler): def create_empty_recovered_dict(self): recovered = {} - for a in ("actual", "original-leasetimer", "configured-leasetimer"): - for b in ("numbuckets", "numshares", "sharebytes", "diskbytes"): + for a in ("actual", "original", "configured", "examined"): + for b in ("buckets", "shares", "sharebytes", "diskbytes"): recovered[a+"-"+b] = 0 + recovered[a+"-"+b+"-mutable"] = 0 + recovered[a+"-"+b+"-immutable"] = 0 return recovered def started_cycle(self, cycle): @@ -128,27 +128,36 @@ class LeaseCheckingCrawler(ShareCrawler): twlog.err() which = (storage_index_b32, shnum) self.state["cycle-to-date"]["corrupt-shares"].append(which) - wks = (1, 1, 1) + wks = (1, 1, 1, "unknown") would_keep_shares.append(wks) - recovered = self.state["cycle-to-date"]["space-recovered"] + sharetype = None + if wks: + sharetype = wks[3] + rec = self.state["cycle-to-date"]["space-recovered"] + self.increment(rec, "examined-buckets", 1) + if sharetype: + self.increment(rec, "examined-buckets-"+sharetype, 1) + if sum([wks[0] for wks in would_keep_shares]) == 0: - self.increment(recovered, - "original-leasetimer-diskbytes", bucket_diskbytes) - self.increment(recovered, "original-leasetimer-numbuckets", 1) + self.increment(rec, "original-diskbytes", bucket_diskbytes) + self.increment(rec, "original-diskbytes-"+sharetype, bucket_diskbytes) + self.increment(rec, "original-buckets", 1) + self.increment(rec, "original-buckets-"+sharetype, 1) if sum([wks[1] for wks in would_keep_shares]) == 0: - self.increment(recovered, - "configured-leasetimer-diskbytes", bucket_diskbytes) - self.increment(recovered, "configured-leasetimer-numbuckets", 1) + self.increment(rec, "configured-diskbytes", bucket_diskbytes) + self.increment(rec, "configured-diskbytes-"+sharetype, bucket_diskbytes) + self.increment(rec, "configured-buckets", 1) + self.increment(rec, "configured-buckets-"+sharetype, 1) if sum([wks[2] for wks in would_keep_shares]) == 0: - self.increment(recovered, - "actual-diskbytes", bucket_diskbytes) - self.increment(recovered, "actual-numbuckets", 1) - self.state["cycle-to-date"]["buckets-examined"] += 1 + self.increment(rec, "actual-diskbytes", bucket_diskbytes) + self.increment(rec, "actual-diskbytes-"+sharetype, bucket_diskbytes) + self.increment(rec, "actual-buckets", 1) + self.increment(rec, "actual-buckets-"+sharetype, 1) def process_share(self, sharefilename): # first, find out what kind of a share it is sf = get_share_file(sharefilename) - sftype = sf.sharetype + sharetype = sf.sharetype now = time.time() s = self.stat(sharefilename) @@ -179,7 +188,7 @@ class LeaseCheckingCrawler(ShareCrawler): date_cutoff = self.mode[1] if grant_renew_time < date_cutoff: expired = True - if sftype not in self.sharetypes_to_expire: + if sharetype not in self.sharetypes_to_expire: expired = False if expired: @@ -189,11 +198,9 @@ class LeaseCheckingCrawler(ShareCrawler): so_far = self.state["cycle-to-date"] self.increment(so_far["leases-per-share-histogram"], num_leases, 1) - so_far["shares-examined"] += 1 - # TODO: accumulate share-sizes too, so we can display "the whole - # cycle would probably recover x GB out of y GB total" + self.increment_space("examined", s, sharetype) - would_keep_share = [1, 1, 1] + would_keep_share = [1, 1, 1, sharetype] if self.expiration_enabled: for li in expired_leases_configured: @@ -201,18 +208,18 @@ class LeaseCheckingCrawler(ShareCrawler): if num_valid_leases_original == 0: would_keep_share[0] = 0 - self.increment_space("original-leasetimer", s) + self.increment_space("original", s, sharetype) if num_valid_leases_configured == 0: would_keep_share[1] = 0 - self.increment_space("configured-leasetimer", s) + self.increment_space("configured", s, sharetype) if self.expiration_enabled: would_keep_share[2] = 0 - self.increment_space("actual", s) + self.increment_space("actual", s, sharetype) return would_keep_share - def increment_space(self, a, s): + def increment_space(self, a, s, sharetype): sharebytes = s.st_size try: # note that stat(2) says that st_blocks is 512 bytes, and that @@ -224,9 +231,12 @@ class LeaseCheckingCrawler(ShareCrawler): # MacOS. But it isn't available on windows. diskbytes = sharebytes so_far_sr = self.state["cycle-to-date"]["space-recovered"] - self.increment(so_far_sr, a+"-numshares", 1) + self.increment(so_far_sr, a+"-shares", 1) + self.increment(so_far_sr, a+"-shares-"+sharetype, 1) self.increment(so_far_sr, a+"-sharebytes", sharebytes) + self.increment(so_far_sr, a+"-sharebytes-"+sharetype, sharebytes) self.increment(so_far_sr, a+"-diskbytes", diskbytes) + self.increment(so_far_sr, a+"-diskbytes-"+sharetype, diskbytes) def increment(self, d, k, delta=1): if k not in d: @@ -272,8 +282,6 @@ class LeaseCheckingCrawler(ShareCrawler): lah = self.convert_lease_age_histogram(s["lease-age-histogram"]) h["lease-age-histogram"] = lah h["leases-per-share-histogram"] = s["leases-per-share-histogram"].copy() - h["buckets-examined"] = s["buckets-examined"] - h["shares-examined"] = s["shares-examined"] h["corrupt-shares"] = s["corrupt-shares"][:] # note: if ["shares-recovered"] ever acquires an internal dict, this # copy() needs to become a deepcopy @@ -304,22 +312,16 @@ class LeaseCheckingCrawler(ShareCrawler): lease-age-histogram (list of (minage,maxage,sharecount) tuples) leases-per-share-histogram corrupt-shares (list of (si_b32,shnum) tuples, minimal verification) - buckets-examined - shares-examined space-recovered estimated-remaining-cycle: # Values may be None if not enough data has been gathered to # produce an estimate. - buckets-examined - shares-examined space-recovered estimated-current-cycle: # cycle-to-date plus estimated-remaining. Values may be None if # not enough data has been gathered to produce an estimate. - buckets-examined - shares-examined space-recovered history: maps cyclenum to a dict with the following keys: @@ -329,27 +331,33 @@ class LeaseCheckingCrawler(ShareCrawler): lease-age-histogram leases-per-share-histogram corrupt-shares - buckets-examined - shares-examined space-recovered The 'space-recovered' structure is a dictionary with the following keys: + # 'examined' is what was looked at + examined-buckets, examined-buckets-mutable, examined-buckets-immutable + examined-shares, -mutable, -immutable + examined-sharebytes, -mutable, -immutable + examined-diskbytes, -mutable, -immutable + # 'actual' is what was actually deleted - actual-numbuckets - actual-numshares - actual-sharebytes - actual-diskbytes + actual-buckets, -mutable, -immutable + actual-shares, -mutable, -immutable + actual-sharebytes, -mutable, -immutable + actual-diskbytes, -mutable, -immutable + # would have been deleted, if the original lease timer was used - original-leasetimer-numbuckets - original-leasetimer-numshares - original-leasetimer-sharebytes - original-leasetimer-diskbytes + original-buckets, -mutable, -immutable + original-shares, -mutable, -immutable + original-sharebytes, -mutable, -immutable + original-diskbytes, -mutable, -immutable + # would have been deleted, if our configured max_age was used - configured-leasetimer-numbuckets - configured-leasetimer-numshares - configured-leasetimer-sharebytes - configured-leasetimer-diskbytes + configured-buckets, -mutable, -immutable + configured-shares, -mutable, -immutable + configured-sharebytes, -mutable, -immutable + configured-diskbytes, -mutable, -immutable """ progress = self.get_progress() @@ -379,27 +387,19 @@ class LeaseCheckingCrawler(ShareCrawler): if progress["cycle-complete-percentage"] > 0.0: pc = progress["cycle-complete-percentage"] / 100.0 m = (1-pc)/pc - for a in ("actual", "original-leasetimer", "configured-leasetimer"): - for b in ("numbuckets", "numshares", "sharebytes", "diskbytes"): - k = a+"-"+b - remaining_sr[k] = m * so_far_sr[k] - cycle_sr[k] = so_far_sr[k] + remaining_sr[k] - predshares = m * so_far["shares-examined"] - remaining["shares-examined"] = predshares - cycle["shares-examined"] = so_far["shares-examined"] + predshares - predbuckets = m * so_far["buckets-examined"] - remaining["buckets-examined"] = predbuckets - cycle["buckets-examined"] = so_far["buckets-examined"] + predbuckets + for a in ("actual", "original", "configured", "examined"): + for b in ("buckets", "shares", "sharebytes", "diskbytes"): + for c in ("", "-mutable", "-immutable"): + k = a+"-"+b+c + remaining_sr[k] = m * so_far_sr[k] + cycle_sr[k] = so_far_sr[k] + remaining_sr[k] else: - for a in ("actual", "original-leasetimer", "configured-leasetimer"): - for b in ("numbuckets", "numshares", "sharebytes", "diskbytes"): - k = a+"-"+b - remaining_sr[k] = None - cycle_sr[k] = None - remaining["shares-examined"] = None - cycle["shares-examined"] = None - remaining["buckets-examined"] = None - cycle["buckets-examined"] = None + for a in ("actual", "original", "configured", "examined"): + for b in ("buckets", "shares", "sharebytes", "diskbytes"): + for c in ("", "-mutable", "-immutable"): + k = a+"-"+b+c + remaining_sr[k] = None + cycle_sr[k] = None state["estimated-remaining-cycle"] = remaining state["estimated-current-cycle"] = cycle diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index ef308a24..49433028 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -1634,38 +1634,40 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): self.failUnlessEqual(len(lah), 1) self.failUnlessEqual(lah, [ (0.0, DAY, 1) ] ) self.failUnlessEqual(so_far["leases-per-share-histogram"], {1: 1}) - self.failUnlessEqual(so_far["buckets-examined"], 1) - self.failUnlessEqual(so_far["shares-examined"], 1) self.failUnlessEqual(so_far["corrupt-shares"], []) sr1 = so_far["space-recovered"] - self.failUnlessEqual(sr1["actual-numshares"], 0) - self.failUnlessEqual(sr1["configured-leasetimer-diskbytes"], 0) - self.failUnlessEqual(sr1["original-leasetimer-sharebytes"], 0) + self.failUnlessEqual(sr1["examined-buckets"], 1) + self.failUnlessEqual(sr1["examined-shares"], 1) + self.failUnlessEqual(sr1["actual-shares"], 0) + self.failUnlessEqual(sr1["configured-diskbytes"], 0) + self.failUnlessEqual(sr1["original-sharebytes"], 0) left = initial_state["estimated-remaining-cycle"] - self.failUnless(left["buckets-examined"] > 0, - left["buckets-examined"]) - self.failUnless(left["shares-examined"] > 0, - left["shares-examined"]) sr2 = left["space-recovered"] - self.failIfEqual(sr2["actual-numshares"], None) - self.failIfEqual(sr2["configured-leasetimer-diskbytes"], None) - self.failIfEqual(sr2["original-leasetimer-sharebytes"], None) + self.failUnless(sr2["examined-buckets"] > 0, sr2["examined-buckets"]) + self.failUnless(sr2["examined-shares"] > 0, sr2["examined-shares"]) + self.failIfEqual(sr2["actual-shares"], None) + self.failIfEqual(sr2["configured-diskbytes"], None) + self.failIfEqual(sr2["original-sharebytes"], None) d.addCallback(_after_first_bucket) d.addCallback(lambda ign: self.render1(webstatus)) def _check_html_in_cycle(html): s = remove_tags(html) self.failUnlessIn("So far, this cycle has examined " - "1 shares in 1 buckets " - "and has recovered: " - "0 shares, 0 buckets, 0 B ", s) + "1 shares in 1 buckets (0 mutable / 1 immutable) ", s) + self.failUnlessIn("and has recovered: " + "0 shares, 0 buckets (0 mutable / 0 immutable), " + "0 B (0 B / 0 B)", s) self.failUnlessIn("If expiration were enabled, " "we would have recovered: " - "0 shares, 0 buckets, 0 B by now", s) + "0 shares, 0 buckets (0 mutable / 0 immutable)," + " 0 B (0 B / 0 B) by now", s) self.failUnlessIn("and the remainder of this cycle " "would probably recover: " - "0 shares, 0 buckets, 0 B ", s) + "0 shares, 0 buckets (0 mutable / 0 immutable)," + " 0 B (0 B / 0 B)", s) self.failUnlessIn("and the whole cycle would probably recover: " - "0 shares, 0 buckets, 0 B ", s) + "0 shares, 0 buckets (0 mutable / 0 immutable)," + " 0 B (0 B / 0 B)", s) self.failUnlessIn("if we were using each lease's default " "31-day lease lifetime", s) self.failUnlessIn("this cycle would be expected to recover: ", s) @@ -1694,25 +1696,24 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): self.failUnlessEqual(len(lah), 1) self.failUnlessEqual(lah, [ (0.0, DAY, 6) ] ) - self.failUnlessEqual(last["leases-per-share-histogram"], - {1: 2, 2: 2}) - self.failUnlessEqual(last["buckets-examined"], 4) - self.failUnlessEqual(last["shares-examined"], 4) + self.failUnlessEqual(last["leases-per-share-histogram"], {1: 2, 2: 2}) self.failUnlessEqual(last["corrupt-shares"], []) rec = last["space-recovered"] - self.failUnlessEqual(rec["actual-numbuckets"], 0) - self.failUnlessEqual(rec["original-leasetimer-numbuckets"], 0) - self.failUnlessEqual(rec["configured-leasetimer-numbuckets"], 0) - self.failUnlessEqual(rec["actual-numshares"], 0) - self.failUnlessEqual(rec["original-leasetimer-numshares"], 0) - self.failUnlessEqual(rec["configured-leasetimer-numshares"], 0) + self.failUnlessEqual(rec["examined-buckets"], 4) + self.failUnlessEqual(rec["examined-shares"], 4) + self.failUnlessEqual(rec["actual-buckets"], 0) + self.failUnlessEqual(rec["original-buckets"], 0) + self.failUnlessEqual(rec["configured-buckets"], 0) + self.failUnlessEqual(rec["actual-shares"], 0) + self.failUnlessEqual(rec["original-shares"], 0) + self.failUnlessEqual(rec["configured-shares"], 0) self.failUnlessEqual(rec["actual-diskbytes"], 0) - self.failUnlessEqual(rec["original-leasetimer-diskbytes"], 0) - self.failUnlessEqual(rec["configured-leasetimer-diskbytes"], 0) + self.failUnlessEqual(rec["original-diskbytes"], 0) + self.failUnlessEqual(rec["configured-diskbytes"], 0) self.failUnlessEqual(rec["actual-sharebytes"], 0) - self.failUnlessEqual(rec["original-leasetimer-sharebytes"], 0) - self.failUnlessEqual(rec["configured-leasetimer-sharebytes"], 0) + self.failUnlessEqual(rec["original-sharebytes"], 0) + self.failUnlessEqual(rec["configured-sharebytes"], 0) def _get_sharefile(si): return list(ss._iter_share_files(si))[0] @@ -1726,7 +1727,8 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): d.addCallback(lambda ign: self.render1(webstatus)) def _check_html(html): s = remove_tags(html) - self.failUnlessIn("recovered: 0 shares, 0 buckets, 0 B " + self.failUnlessIn("recovered: 0 shares, 0 buckets " + "(0 mutable / 0 immutable), 0 B (0 B / 0 B) " "but expiration was not enabled", s) d.addCallback(_check_html) return d @@ -1843,39 +1845,37 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): last = s["history"][0] self.failUnlessEqual(last["expiration-enabled"], True) - self.failUnlessEqual(last["configured-expiration-mode"], - ("age",2000)) - self.failUnlessEqual(last["buckets-examined"], 4) - self.failUnlessEqual(last["shares-examined"], 4) - self.failUnlessEqual(last["leases-per-share-histogram"], - {1: 2, 2: 2}) + self.failUnlessEqual(last["configured-expiration-mode"], ("age",2000)) + self.failUnlessEqual(last["leases-per-share-histogram"], {1: 2, 2: 2}) rec = last["space-recovered"] - self.failUnlessEqual(rec["actual-numbuckets"], 2) - self.failUnlessEqual(rec["original-leasetimer-numbuckets"], 2) - self.failUnlessEqual(rec["configured-leasetimer-numbuckets"], 2) - self.failUnlessEqual(rec["actual-numshares"], 2) - self.failUnlessEqual(rec["original-leasetimer-numshares"], 2) - self.failUnlessEqual(rec["configured-leasetimer-numshares"], 2) + self.failUnlessEqual(rec["examined-buckets"], 4) + self.failUnlessEqual(rec["examined-shares"], 4) + self.failUnlessEqual(rec["actual-buckets"], 2) + self.failUnlessEqual(rec["original-buckets"], 2) + self.failUnlessEqual(rec["configured-buckets"], 2) + self.failUnlessEqual(rec["actual-shares"], 2) + self.failUnlessEqual(rec["original-shares"], 2) + self.failUnlessEqual(rec["configured-shares"], 2) size = sf0_size + sf2_size self.failUnlessEqual(rec["actual-sharebytes"], size) - self.failUnlessEqual(rec["original-leasetimer-sharebytes"], size) - self.failUnlessEqual(rec["configured-leasetimer-sharebytes"], size) + self.failUnlessEqual(rec["original-sharebytes"], size) + self.failUnlessEqual(rec["configured-sharebytes"], size) # different platforms have different notions of "blocks used by # this file", so merely assert that it's a number self.failUnless(rec["actual-diskbytes"] >= 0, rec["actual-diskbytes"]) - self.failUnless(rec["original-leasetimer-diskbytes"] >= 0, - rec["original-leasetimer-diskbytes"]) - self.failUnless(rec["configured-leasetimer-diskbytes"] >= 0, - rec["configured-leasetimer-diskbytes"]) + self.failUnless(rec["original-diskbytes"] >= 0, + rec["original-diskbytes"]) + self.failUnless(rec["configured-diskbytes"] >= 0, + rec["configured-diskbytes"]) d.addCallback(_after_first_cycle) d.addCallback(lambda ign: self.render1(webstatus)) def _check_html(html): s = remove_tags(html) self.failUnlessIn("Expiration Enabled: expired leases will be removed", s) self.failUnlessIn("leases created or last renewed more than 33 minutes ago will be considered expired", s) - self.failUnlessIn(" recovered: 2 shares, 2 buckets, ", s) + self.failUnlessIn(" recovered: 2 shares, 2 buckets (1 mutable / 1 immutable), ", s) d.addCallback(_check_html) return d @@ -1986,30 +1986,30 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): self.failUnlessEqual(last["expiration-enabled"], True) self.failUnlessEqual(last["configured-expiration-mode"], ("date-cutoff",then)) - self.failUnlessEqual(last["buckets-examined"], 4) - self.failUnlessEqual(last["shares-examined"], 4) self.failUnlessEqual(last["leases-per-share-histogram"], {1: 2, 2: 2}) rec = last["space-recovered"] - self.failUnlessEqual(rec["actual-numbuckets"], 2) - self.failUnlessEqual(rec["original-leasetimer-numbuckets"], 0) - self.failUnlessEqual(rec["configured-leasetimer-numbuckets"], 2) - self.failUnlessEqual(rec["actual-numshares"], 2) - self.failUnlessEqual(rec["original-leasetimer-numshares"], 0) - self.failUnlessEqual(rec["configured-leasetimer-numshares"], 2) + self.failUnlessEqual(rec["examined-buckets"], 4) + self.failUnlessEqual(rec["examined-shares"], 4) + self.failUnlessEqual(rec["actual-buckets"], 2) + self.failUnlessEqual(rec["original-buckets"], 0) + self.failUnlessEqual(rec["configured-buckets"], 2) + self.failUnlessEqual(rec["actual-shares"], 2) + self.failUnlessEqual(rec["original-shares"], 0) + self.failUnlessEqual(rec["configured-shares"], 2) size = sf0_size + sf2_size self.failUnlessEqual(rec["actual-sharebytes"], size) - self.failUnlessEqual(rec["original-leasetimer-sharebytes"], 0) - self.failUnlessEqual(rec["configured-leasetimer-sharebytes"], size) + self.failUnlessEqual(rec["original-sharebytes"], 0) + self.failUnlessEqual(rec["configured-sharebytes"], size) # different platforms have different notions of "blocks used by # this file", so merely assert that it's a number self.failUnless(rec["actual-diskbytes"] >= 0, rec["actual-diskbytes"]) - self.failUnless(rec["original-leasetimer-diskbytes"] >= 0, - rec["original-leasetimer-diskbytes"]) - self.failUnless(rec["configured-leasetimer-diskbytes"] >= 0, - rec["configured-leasetimer-diskbytes"]) + self.failUnless(rec["original-diskbytes"] >= 0, + rec["original-diskbytes"]) + self.failUnless(rec["configured-diskbytes"] >= 0, + rec["configured-diskbytes"]) d.addCallback(_after_first_cycle) d.addCallback(lambda ign: self.render1(webstatus)) def _check_html(html): @@ -2019,7 +2019,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): date = time.strftime("%d-%b-%Y", time.gmtime(then)) self.failUnlessIn("leases created or last renewed before %s" " will be considered expired" % date, s) - self.failUnlessIn(" recovered: 2 shares, 2 buckets, ", s) + self.failUnlessIn(" recovered: 2 shares, 2 buckets (1 mutable / 1 immutable), ", s) d.addCallback(_check_html) return d @@ -2220,32 +2220,32 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): self.failUnless("estimated-current-cycle" in s) left = s["estimated-remaining-cycle"]["space-recovered"] - self.failUnlessEqual(left["actual-numbuckets"], None) - self.failUnlessEqual(left["original-leasetimer-numbuckets"], None) - self.failUnlessEqual(left["configured-leasetimer-numbuckets"], None) - self.failUnlessEqual(left["actual-numshares"], None) - self.failUnlessEqual(left["original-leasetimer-numshares"], None) - self.failUnlessEqual(left["configured-leasetimer-numshares"], None) + self.failUnlessEqual(left["actual-buckets"], None) + self.failUnlessEqual(left["original-buckets"], None) + self.failUnlessEqual(left["configured-buckets"], None) + self.failUnlessEqual(left["actual-shares"], None) + self.failUnlessEqual(left["original-shares"], None) + self.failUnlessEqual(left["configured-shares"], None) self.failUnlessEqual(left["actual-diskbytes"], None) - self.failUnlessEqual(left["original-leasetimer-diskbytes"], None) - self.failUnlessEqual(left["configured-leasetimer-diskbytes"], None) + self.failUnlessEqual(left["original-diskbytes"], None) + self.failUnlessEqual(left["configured-diskbytes"], None) self.failUnlessEqual(left["actual-sharebytes"], None) - self.failUnlessEqual(left["original-leasetimer-sharebytes"], None) - self.failUnlessEqual(left["configured-leasetimer-sharebytes"], None) + self.failUnlessEqual(left["original-sharebytes"], None) + self.failUnlessEqual(left["configured-sharebytes"], None) full = s["estimated-remaining-cycle"]["space-recovered"] - self.failUnlessEqual(full["actual-numbuckets"], None) - self.failUnlessEqual(full["original-leasetimer-numbuckets"], None) - self.failUnlessEqual(full["configured-leasetimer-numbuckets"], None) - self.failUnlessEqual(full["actual-numshares"], None) - self.failUnlessEqual(full["original-leasetimer-numshares"], None) - self.failUnlessEqual(full["configured-leasetimer-numshares"], None) + self.failUnlessEqual(full["actual-buckets"], None) + self.failUnlessEqual(full["original-buckets"], None) + self.failUnlessEqual(full["configured-buckets"], None) + self.failUnlessEqual(full["actual-shares"], None) + self.failUnlessEqual(full["original-shares"], None) + self.failUnlessEqual(full["configured-shares"], None) self.failUnlessEqual(full["actual-diskbytes"], None) - self.failUnlessEqual(full["original-leasetimer-diskbytes"], None) - self.failUnlessEqual(full["configured-leasetimer-diskbytes"], None) + self.failUnlessEqual(full["original-diskbytes"], None) + self.failUnlessEqual(full["configured-diskbytes"], None) self.failUnlessEqual(full["actual-sharebytes"], None) - self.failUnlessEqual(full["original-leasetimer-sharebytes"], None) - self.failUnlessEqual(full["configured-leasetimer-sharebytes"], None) + self.failUnlessEqual(full["original-sharebytes"], None) + self.failUnlessEqual(full["configured-sharebytes"], None) d.addCallback(_check) return d @@ -2255,7 +2255,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): fileutil.make_dirs(basedir) ss = No_ST_BLOCKS_StorageServer(basedir, "\x00" * 20, expiration_mode=("age",-1000)) - # a negative expiration_time= means the "configured-leasetimer-" + # a negative expiration_time= means the "configured-" # space-recovered counts will be non-zero, since all shares will have # expired by then @@ -2273,14 +2273,14 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): s = lc.get_state() last = s["history"][0] rec = last["space-recovered"] - self.failUnlessEqual(rec["configured-leasetimer-numbuckets"], 4) - self.failUnlessEqual(rec["configured-leasetimer-numshares"], 4) - self.failUnless(rec["configured-leasetimer-sharebytes"] > 0, - rec["configured-leasetimer-sharebytes"]) + self.failUnlessEqual(rec["configured-buckets"], 4) + self.failUnlessEqual(rec["configured-shares"], 4) + self.failUnless(rec["configured-sharebytes"] > 0, + rec["configured-sharebytes"]) # without the .st_blocks field in os.stat() results, we should be # reporting diskbytes==sharebytes - self.failUnlessEqual(rec["configured-leasetimer-sharebytes"], - rec["configured-leasetimer-diskbytes"]) + self.failUnlessEqual(rec["configured-sharebytes"], + rec["configured-diskbytes"]) d.addCallback(_check) return d @@ -2321,8 +2321,9 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): # processed. def _after_first_bucket(ignored): so_far = lc.get_state()["cycle-to-date"] - self.failUnlessEqual(so_far["buckets-examined"], 1) - self.failUnlessEqual(so_far["shares-examined"], 0) + rec = so_far["space-recovered"] + self.failUnlessEqual(rec["examined-buckets"], 1) + self.failUnlessEqual(rec["examined-shares"], 0) self.failUnlessEqual(so_far["corrupt-shares"], [(first_b32, 0)]) d.addCallback(_after_first_bucket) @@ -2348,8 +2349,9 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): def _after_first_cycle(ignored): s = lc.get_state() last = s["history"][0] - self.failUnlessEqual(last["buckets-examined"], 4) - self.failUnlessEqual(last["shares-examined"], 3) + rec = last["space-recovered"] + self.failUnlessEqual(rec["examined-buckets"], 4) + self.failUnlessEqual(rec["examined-shares"], 3) self.failUnlessEqual(last["corrupt-shares"], [(first_b32, 0)]) d.addCallback(_after_first_cycle) d.addCallback(lambda ign: self.render_json(w)) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index 581d821b..3491ea63 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -148,10 +148,15 @@ class StorageStatus(rend.Page): if d is None: return "?" return "%d" % d - space = abbreviate_space(sr["%s-diskbytes" % a]) - return "%s shares, %s buckets, %s" % (maybe(sr["%s-numshares" % a]), - maybe(sr["%s-numbuckets" % a]), - space) + return "%s shares, %s buckets (%s mutable / %s immutable), %s (%s / %s)" % \ + (maybe(sr["%s-shares" % a]), + maybe(sr["%s-buckets" % a]), + maybe(sr["%s-buckets-mutable" % a]), + maybe(sr["%s-buckets-immutable" % a]), + abbreviate_space(sr["%s-diskbytes" % a]), + abbreviate_space(sr["%s-diskbytes-mutable" % a]), + abbreviate_space(sr["%s-diskbytes-immutable" % a]), + ) def render_lease_current_cycle_progress(self, ctx, data): lc = self.storage.lease_checker @@ -181,27 +186,32 @@ class StorageStatus(rend.Page): return "?" return "%d" % d add("So far, this cycle has examined %d shares in %d buckets" - % (so_far["shares-examined"], so_far["buckets-examined"])) + % (sr["examined-shares"], sr["examined-buckets"]), + " (%d mutable / %d immutable)" + % (sr["examined-buckets-mutable"], sr["examined-buckets-immutable"]), + " (%s / %s)" % (abbreviate_space(sr["examined-diskbytes-mutable"]), + abbreviate_space(sr["examined-diskbytes-immutable"])), + ) add("and has recovered: ", self.format_recovered(sr, "actual")) if so_far["expiration-enabled"]: add("The remainder of this cycle is expected to recover: ", self.format_recovered(esr, "actual")) add("The whole cycle is expected to examine %s shares in %s buckets" - % (maybe(ec["shares-examined"]), maybe(ec["buckets-examined"]))) + % (maybe(ecr["examined-shares"]), maybe(ecr["examined-buckets"]))) add("and to recover: ", self.format_recovered(ecr, "actual")) else: add("If expiration were enabled, we would have recovered: ", - self.format_recovered(sr, "configured-leasetimer"), " by now") + self.format_recovered(sr, "configured"), " by now") add("and the remainder of this cycle would probably recover: ", - self.format_recovered(esr, "configured-leasetimer")) + self.format_recovered(esr, "configured")) add("and the whole cycle would probably recover: ", - self.format_recovered(ecr, "configured-leasetimer")) + self.format_recovered(ecr, "configured")) add("if we were using each lease's default 31-day lease lifetime " "(instead of our configured node), " "this cycle would be expected to recover: ", - self.format_recovered(ecr, "original-leasetimer")) + self.format_recovered(ecr, "original")) if so_far["corrupt-shares"]: add("Corrupt shares:", @@ -231,8 +241,7 @@ class StorageStatus(rend.Page): p[T.li[pieces]] if not last["expiration-enabled"]: - rec = self.format_recovered(last["space-recovered"], - "configured-leasetimer") + rec = self.format_recovered(last["space-recovered"], "configured") add("but expiration was not enabled. If it had been, " "it would have recovered: ", rec) -- 2.45.2