From f0071c2571599cd52e23a29df870b9386fea23b6 Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Wed, 18 Mar 2009 18:00:09 -0700 Subject: [PATCH] expirer: clean up constructor args, add tahoe.cfg controls, use cutoff_date instead of date_cutoff --- docs/proposed/garbage-collection.txt | 12 +++++------ src/allmydata/client.py | 32 +++++++++++++++++++++++++++- src/allmydata/storage/expirer.py | 22 +++++++++---------- src/allmydata/storage/server.py | 4 ++-- src/allmydata/test/test_storage.py | 22 +++++++++---------- 5 files changed, 61 insertions(+), 31 deletions(-) diff --git a/docs/proposed/garbage-collection.txt b/docs/proposed/garbage-collection.txt index 758e1919..4cacadfb 100644 --- a/docs/proposed/garbage-collection.txt +++ b/docs/proposed/garbage-collection.txt @@ -114,11 +114,11 @@ expire.enabled = (boolean, optional) have expired. Other controls dictate when leases are considered to have expired. The default is False. -expire.mode = (string, "age" or "date-cutoff", required if expiration enabled) +expire.mode = (string, "age" or "cutoff-date", required if expiration enabled) If this string is "age", the age-based expiration scheme is used, and the "expire.override_lease_duration" setting can be provided to influence the - lease ages. If it is "date-cutoff", the absolute-date-cutoff mode is used, + lease ages. If it is "cutoff-date", the absolute-date-cutoff mode is used, and the "expire.cutoff_date" setting must be provided to specify the cutoff date. The mode setting currently has no default: you must provide a value. @@ -160,12 +160,12 @@ expire.override_lease_duration = (duration string, optional) been passed. This key is only valid when age-based expiration is in use (i.e. when - "expire.mode = age" is used). It will be rejected if date-cutoff expiration + "expire.mode = age" is used). It will be rejected if cutoff-date expiration is in use. -expire.cutoff_date = (date string, required if mode=date-cutoff) +expire.cutoff_date = (date string, required if mode=cutoff-date) - When date-cutoff expiration is in use, a lease will be expired if its + When cutoff-date expiration is in use, a lease will be expired if its create/renew timestamp is older than the cutoff date. This string will be a date in the following format: @@ -179,7 +179,7 @@ expire.cutoff_date = (date string, required if mode=date-cutoff) last renewal time and the cutoff date. This key is only valid when cutoff-based expiration is in use (i.e. when - "expire.mode = date-cutoff"). It will be rejected if age-based expiration is + "expire.mode = cutoff-date"). It will be rejected if age-based expiration is in use. expire.immutable = (boolean, optional) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index b997f306..04401a3b 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -19,6 +19,7 @@ from allmydata.control import ControlServer from allmydata.introducer.client import IntroducerClient from allmydata.util import hashutil, base32, pollmixin, cachedir from allmydata.util.abbreviate import parse_abbreviated_size +from allmydata.util.time_format import parse_duration, parse_date from allmydata.uri import LiteralFileURI from allmydata.dirnode import NewDirectoryNode from allmydata.mutable.filenode import MutableFileNode @@ -164,12 +165,41 @@ class Client(node.Node, pollmixin.PollMixin): reserved = 0 discard = self.get_config("storage", "debug_discard", False, boolean=True) + + expire = self.get_config("storage", "expire.enabled", False, boolean=True) + if expire: + mode = self.get_config("storage", "expire.mode") # require a mode + else: + mode = self.get_config("storage", "expire.mode", "age") + + o_l_d = self.get_config("storage", "expire.override_lease_duration", None) + if o_l_d is not None: + o_l_d = parse_duration(o_l_d) + + cutoff_date = None + if mode == "cutoff-date": + cutoff_date = self.get_config("storage", "expire.cutoff_date") + cutoff_date = parse_date(cutoff_date) + + sharetypes = [] + if self.get_config("storage", "expire.immutable", True, boolean=True): + sharetypes.append("immutable") + if self.get_config("storage", "expire.mutable", True, boolean=True): + sharetypes.append("mutable") + expiration_sharetypes = tuple(sharetypes) + ss = StorageServer(storedir, self.nodeid, reserved_space=reserved, discard_storage=discard, readonly_storage=readonly, - stats_provider=self.stats_provider) + stats_provider=self.stats_provider, + expiration_enabled=expire, + expiration_mode=mode, + expiration_override_lease_duration=o_l_d, + expiration_cutoff_date=cutoff_date, + expiration_sharetypes=expiration_sharetypes) self.add_service(ss) + d = self.when_tub_ready() # we can't do registerReference until the Tub is ready def _publish(res): diff --git a/src/allmydata/storage/expirer.py b/src/allmydata/storage/expirer.py index 84f1649e..1c638fa2 100644 --- a/src/allmydata/storage/expirer.py +++ b/src/allmydata/storage/expirer.py @@ -51,22 +51,22 @@ class LeaseCheckingCrawler(ShareCrawler): def __init__(self, server, statefile, historyfile, expiration_enabled, mode, override_lease_duration, # used if expiration_mode=="age" - date_cutoff, # used if expiration_mode=="date-cutoff" + cutoff_date, # used if expiration_mode=="cutoff-date" sharetypes): self.historyfile = historyfile self.expiration_enabled = expiration_enabled self.mode = mode self.override_lease_duration = None - self.date_cutoff = None + self.cutoff_date = None if self.mode == "age": assert isinstance(override_lease_duration, (int, type(None))) self.override_lease_duration = override_lease_duration # seconds - elif self.mode == "date-cutoff": - assert isinstance(date_cutoff, int) # seconds-since-epoch - assert date_cutoff is not None - self.date_cutoff = date_cutoff + elif self.mode == "cutoff-date": + assert isinstance(cutoff_date, int) # seconds-since-epoch + assert cutoff_date is not None + self.cutoff_date = cutoff_date else: - raise ValueError("GC mode '%s' must be 'age' or 'date-cutoff'" % mode) + raise ValueError("GC mode '%s' must be 'age' or 'cutoff-date'" % mode) self.sharetypes_to_expire = sharetypes ShareCrawler.__init__(self, server, statefile) @@ -192,8 +192,8 @@ class LeaseCheckingCrawler(ShareCrawler): if age > age_limit: expired = True else: - assert self.mode == "date-cutoff" - if grant_renew_time < self.date_cutoff: + assert self.mode == "cutoff-date" + if grant_renew_time < self.cutoff_date: expired = True if sharetype not in self.sharetypes_to_expire: expired = False @@ -278,7 +278,7 @@ class LeaseCheckingCrawler(ShareCrawler): h["expiration-enabled"] = self.expiration_enabled h["configured-expiration-mode"] = (self.mode, self.override_lease_duration, - self.date_cutoff, + self.cutoff_date, self.sharetypes_to_expire) s = self.state["cycle-to-date"] @@ -388,7 +388,7 @@ class LeaseCheckingCrawler(ShareCrawler): so_far["expiration-enabled"] = self.expiration_enabled so_far["configured-expiration-mode"] = (self.mode, self.override_lease_duration, - self.date_cutoff, + self.cutoff_date, self.sharetypes_to_expire) so_far_sr = so_far["space-recovered"] diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 08279708..1ea8e1ed 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -43,7 +43,7 @@ class StorageServer(service.MultiService, Referenceable): expiration_enabled=False, expiration_mode="age", expiration_override_lease_duration=None, - expiration_date_cutoff=None, + expiration_cutoff_date=None, expiration_sharetypes=("mutable", "immutable")): service.MultiService.__init__(self) assert isinstance(nodeid, str) @@ -92,7 +92,7 @@ class StorageServer(service.MultiService, Referenceable): self.lease_checker = klass(self, statefile, historyfile, expiration_enabled, expiration_mode, expiration_override_lease_duration, - expiration_date_cutoff, + expiration_cutoff_date, expiration_sharetypes) self.lease_checker.setServiceParent(self) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 39e8806b..23898df3 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -1881,17 +1881,17 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): d.addCallback(_check_html) return d - def test_expire_date_cutoff(self): - basedir = "storage/LeaseCrawler/expire_date_cutoff" + def test_expire_cutoff_date(self): + basedir = "storage/LeaseCrawler/expire_cutoff_date" fileutil.make_dirs(basedir) - # setting date-cutoff to 2000 seconds ago means that any lease which + # setting cutoff-date to 2000 seconds ago means that any lease which # is more than 2000s old will be expired. now = time.time() then = int(now - 2000) ss = InstrumentedStorageServer(basedir, "\x00" * 20, expiration_enabled=True, - expiration_mode="date-cutoff", - expiration_date_cutoff=then) + expiration_mode="cutoff-date", + expiration_cutoff_date=then) # make it start sooner than usual. lc = ss.lease_checker lc.slow_start = 0 @@ -1988,7 +1988,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): self.failUnlessEqual(last["expiration-enabled"], True) self.failUnlessEqual(last["configured-expiration-mode"], - ("date-cutoff", None, then, + ("cutoff-date", None, then, ("mutable", "immutable"))) self.failUnlessEqual(last["leases-per-share-histogram"], {1: 2, 2: 2}) @@ -2034,8 +2034,8 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): then = int(now - 2000) ss = StorageServer(basedir, "\x00" * 20, expiration_enabled=True, - expiration_mode="date-cutoff", - expiration_date_cutoff=then, + expiration_mode="cutoff-date", + expiration_cutoff_date=then, expiration_sharetypes=("immutable",)) lc = ss.lease_checker lc.slow_start = 0 @@ -2091,8 +2091,8 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): then = int(now - 2000) ss = StorageServer(basedir, "\x00" * 20, expiration_enabled=True, - expiration_mode="date-cutoff", - expiration_date_cutoff=then, + expiration_mode="cutoff-date", + expiration_cutoff_date=then, expiration_sharetypes=("mutable",)) lc = ss.lease_checker lc.slow_start = 0 @@ -2147,7 +2147,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): e = self.failUnlessRaises(ValueError, StorageServer, basedir, "\x00" * 20, expiration_mode="bogus") - self.failUnless("GC mode 'bogus' must be 'age' or 'date-cutoff'" in str(e), str(e)) + self.failUnless("GC mode 'bogus' must be 'age' or 'cutoff-date'" in str(e), str(e)) def test_parse_duration(self): DAY = 24*60*60 -- 2.45.2