From 8eaee2855047ecd1ad7841c040bd76509062d093 Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Wed, 18 Mar 2009 17:21:38 -0700 Subject: [PATCH] expirer: change setup, config options, in preparation for adding tahoe.cfg controls --- src/allmydata/storage/expirer.py | 49 +++++++++++++++++++----------- src/allmydata/storage/server.py | 23 ++++++++------ src/allmydata/test/test_storage.py | 40 +++++++++++++----------- src/allmydata/web/storage.py | 28 +++++++++-------- 4 files changed, 83 insertions(+), 57 deletions(-) diff --git a/src/allmydata/storage/expirer.py b/src/allmydata/storage/expirer.py index d4fd9a01..84f1649e 100644 --- a/src/allmydata/storage/expirer.py +++ b/src/allmydata/storage/expirer.py @@ -49,19 +49,25 @@ class LeaseCheckingCrawler(ShareCrawler): minimum_cycle_time = 12*60*60 # not more than twice per day def __init__(self, server, statefile, historyfile, - expiration_enabled, expiration_mode): + expiration_enabled, mode, + override_lease_duration, # used if expiration_mode=="age" + date_cutoff, # used if expiration_mode=="date-cutoff" + sharetypes): self.historyfile = historyfile self.expiration_enabled = expiration_enabled - self.mode = expiration_mode - if self.mode[0] not in ("age", "date-cutoff"): - raise ValueError("garbage-collection mode '%s' must be 'age' or 'date-cutoff'" % self.mode[0]) - if self.mode[0] == "age": - assert isinstance(expiration_mode[1], int) # seconds - elif self.mode[0] == "date-cutoff": - assert isinstance(expiration_mode[1], int) # seconds-since-epoch - self.sharetypes_to_expire = ("mutable", "immutable") - if len(self.mode) > 2: - self.sharetypes_to_expire = self.mode[2] + self.mode = mode + self.override_lease_duration = None + self.date_cutoff = None + if self.mode == "age": + assert isinstance(override_lease_duration, (int, type(None))) + self.override_lease_duration = override_lease_duration # seconds + elif self.mode == "date-cutoff": + assert isinstance(date_cutoff, int) # seconds-since-epoch + assert date_cutoff is not None + self.date_cutoff = date_cutoff + else: + raise ValueError("GC mode '%s' must be 'age' or 'date-cutoff'" % mode) + self.sharetypes_to_expire = sharetypes ShareCrawler.__init__(self, server, statefile) def add_initial_state(self): @@ -179,14 +185,15 @@ class LeaseCheckingCrawler(ShareCrawler): # expired-or-not according to our configured age limit expired = False - if self.mode[0] == "age": - age_limit = self.mode[1] + if self.mode == "age": + age_limit = original_expiration_time + if self.override_lease_duration is not None: + age_limit = self.override_lease_duration if age > age_limit: expired = True else: - assert self.mode[0] == "date-cutoff" - date_cutoff = self.mode[1] - if grant_renew_time < date_cutoff: + assert self.mode == "date-cutoff" + if grant_renew_time < self.date_cutoff: expired = True if sharetype not in self.sharetypes_to_expire: expired = False @@ -269,7 +276,10 @@ class LeaseCheckingCrawler(ShareCrawler): now = time.time() h["cycle-start-finish-times"] = (start, now) h["expiration-enabled"] = self.expiration_enabled - h["configured-expiration-mode"] = self.mode + h["configured-expiration-mode"] = (self.mode, + self.override_lease_duration, + self.date_cutoff, + self.sharetypes_to_expire) s = self.state["cycle-to-date"] @@ -376,7 +386,10 @@ class LeaseCheckingCrawler(ShareCrawler): lah = so_far["lease-age-histogram"] so_far["lease-age-histogram"] = self.convert_lease_age_histogram(lah) so_far["expiration-enabled"] = self.expiration_enabled - so_far["configured-expiration-mode"] = self.mode + so_far["configured-expiration-mode"] = (self.mode, + self.override_lease_duration, + self.date_cutoff, + self.sharetypes_to_expire) so_far_sr = so_far["space-recovered"] remaining_sr = {} diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 1f07ed7e..08279708 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -41,7 +41,10 @@ class StorageServer(service.MultiService, Referenceable): discard_storage=False, readonly_storage=False, stats_provider=None, expiration_enabled=False, - expiration_mode=("age", 31*24*60*60)): + expiration_mode="age", + expiration_override_lease_duration=None, + expiration_date_cutoff=None, + expiration_sharetypes=("mutable", "immutable")): service.MultiService.__init__(self) assert isinstance(nodeid, str) assert len(nodeid) == 20 @@ -82,22 +85,22 @@ class StorageServer(service.MultiService, Referenceable): "cancel": [], } self.add_bucket_counter() - self.add_lease_checker(expiration_enabled, expiration_mode) - def add_bucket_counter(self): - statefile = os.path.join(self.storedir, "bucket_counter.state") - self.bucket_counter = BucketCountingCrawler(self, statefile) - self.bucket_counter.setServiceParent(self) - - def add_lease_checker(self, expiration_enabled, expiration_mode): statefile = os.path.join(self.storedir, "lease_checker.state") historyfile = os.path.join(self.storedir, "lease_checker.history") klass = self.LeaseCheckerClass self.lease_checker = klass(self, statefile, historyfile, - expiration_enabled=expiration_enabled, - expiration_mode=expiration_mode) + expiration_enabled, expiration_mode, + expiration_override_lease_duration, + expiration_date_cutoff, + expiration_sharetypes) self.lease_checker.setServiceParent(self) + def add_bucket_counter(self): + statefile = os.path.join(self.storedir, "bucket_counter.state") + self.bucket_counter = BucketCountingCrawler(self, statefile) + self.bucket_counter.setServiceParent(self) + def count(self, name, delta=1): if self.stats_provider: self.stats_provider.count("storage_server." + name, delta) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 49433028..f3452b43 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -1753,7 +1753,8 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): # than 2000s old will be expired. ss = InstrumentedStorageServer(basedir, "\x00" * 20, expiration_enabled=True, - expiration_mode=("age",2000)) + expiration_mode="age", + expiration_override_lease_duration=2000) # make it start sooner than usual. lc = ss.lease_checker lc.slow_start = 0 @@ -1845,7 +1846,8 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): last = s["history"][0] self.failUnlessEqual(last["expiration-enabled"], True) - self.failUnlessEqual(last["configured-expiration-mode"], ("age",2000)) + self.failUnlessEqual(last["configured-expiration-mode"], + ("age", 2000, None, ("mutable", "immutable"))) self.failUnlessEqual(last["leases-per-share-histogram"], {1: 2, 2: 2}) rec = last["space-recovered"] @@ -1874,7 +1876,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): def _check_html(html): s = remove_tags(html) self.failUnlessIn("Expiration Enabled: expired leases will be removed", s) - self.failUnlessIn("leases created or last renewed more than 33 minutes ago will be considered expired", s) + self.failUnlessIn("Leases created or last renewed more than 33 minutes ago will be considered expired.", s) self.failUnlessIn(" recovered: 2 shares, 2 buckets (1 mutable / 1 immutable), ", s) d.addCallback(_check_html) return d @@ -1888,7 +1890,8 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): then = int(now - 2000) ss = InstrumentedStorageServer(basedir, "\x00" * 20, expiration_enabled=True, - expiration_mode=("date-cutoff",then)) + expiration_mode="date-cutoff", + expiration_date_cutoff=then) # make it start sooner than usual. lc = ss.lease_checker lc.slow_start = 0 @@ -1985,7 +1988,8 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): self.failUnlessEqual(last["expiration-enabled"], True) self.failUnlessEqual(last["configured-expiration-mode"], - ("date-cutoff",then)) + ("date-cutoff", None, then, + ("mutable", "immutable"))) self.failUnlessEqual(last["leases-per-share-histogram"], {1: 2, 2: 2}) @@ -2017,8 +2021,8 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): self.failUnlessIn("Expiration Enabled:" " expired leases will be removed", s) date = time.strftime("%d-%b-%Y", time.gmtime(then)) - self.failUnlessIn("leases created or last renewed before %s" - " will be considered expired" % date, s) + self.failUnlessIn("Leases created or last renewed before %s" + " will be considered expired." % date, s) self.failUnlessIn(" recovered: 2 shares, 2 buckets (1 mutable / 1 immutable), ", s) d.addCallback(_check_html) return d @@ -2030,8 +2034,9 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): then = int(now - 2000) ss = StorageServer(basedir, "\x00" * 20, expiration_enabled=True, - expiration_mode=("date-cutoff", - then, ("immutable",))) + expiration_mode="date-cutoff", + expiration_date_cutoff=then, + expiration_sharetypes=("immutable",)) lc = ss.lease_checker lc.slow_start = 0 webstatus = StorageStatus(ss) @@ -2075,7 +2080,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): d.addCallback(lambda ign: self.render1(webstatus)) def _check_html(html): s = remove_tags(html) - self.failUnlessIn("only the following sharetypes will be expired: immutable Next crawl", s) + self.failUnlessIn("The following sharetypes will be expired: immutable.", s) d.addCallback(_check_html) return d @@ -2086,8 +2091,9 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): then = int(now - 2000) ss = StorageServer(basedir, "\x00" * 20, expiration_enabled=True, - expiration_mode=("date-cutoff", - then, ("mutable",))) + expiration_mode="date-cutoff", + expiration_date_cutoff=then, + expiration_sharetypes=("mutable",)) lc = ss.lease_checker lc.slow_start = 0 webstatus = StorageStatus(ss) @@ -2131,7 +2137,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): d.addCallback(lambda ign: self.render1(webstatus)) def _check_html(html): s = remove_tags(html) - self.failUnlessIn("only the following sharetypes will be expired: mutable Next crawl", s) + self.failUnlessIn("The following sharetypes will be expired: mutable.", s) d.addCallback(_check_html) return d @@ -2140,9 +2146,8 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): fileutil.make_dirs(basedir) e = self.failUnlessRaises(ValueError, StorageServer, basedir, "\x00" * 20, - expiration_mode=("bogus", 0)) - self.failUnless("garbage-collection mode 'bogus'" - " must be 'age' or 'date-cutoff'" in str(e), str(e)) + expiration_mode="bogus") + self.failUnless("GC mode 'bogus' must be 'age' or 'date-cutoff'" in str(e), str(e)) def test_parse_duration(self): DAY = 24*60*60 @@ -2254,7 +2259,8 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): basedir = "storage/LeaseCrawler/no_st_blocks" fileutil.make_dirs(basedir) ss = No_ST_BLOCKS_StorageServer(basedir, "\x00" * 20, - expiration_mode=("age",-1000)) + expiration_mode="age", + expiration_override_lease_duration=-1000) # a negative expiration_time= means the "configured-" # space-recovered counts will be non-zero, since all shares will have # expired by then diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index 3491ea63..56db9026 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -128,19 +128,23 @@ class StorageStatus(rend.Page): return ctx.tag["Disabled: scan-only mode, no leases will be removed"] def render_lease_expiration_mode(self, ctx, data): - mode = self.storage.lease_checker.mode - if mode[0] == "age": - ctx.tag["leases created or last renewed more than %s ago " - "will be considered expired" - % abbreviate_time(mode[1])] + lc = self.storage.lease_checker + if lc.mode == "age": + if lc.override_lease_duration is None: + ctx.tag["Leases will expire naturally, probably 31 days after " + "creation or renewal."] + else: + ctx.tag["Leases created or last renewed more than %s ago " + "will be considered expired." + % abbreviate_time(lc.override_lease_duration)] else: - assert mode[0] == "date-cutoff" - date = time.strftime("%d-%b-%Y", time.gmtime(mode[1])) - ctx.tag["leases created or last renewed before %s " - "will be considered expired" % date] - if len(mode) > 2: - ctx.tag[", and only the following sharetypes will be expired: ", - sorted(mode[2])] + assert lc.mode == "date-cutoff" + date = time.strftime("%d-%b-%Y", time.gmtime(lc.date_cutoff)) + ctx.tag["Leases created or last renewed before %s " + "will be considered expired." % date] + if len(lc.mode) > 2: + ctx.tag[" The following sharetypes will be expired: ", + sorted(lc.sharetypes_to_expire), "."] return ctx.tag def format_recovered(self, sr, a): -- 2.45.2