minimum_cycle_time = 12*60*60 # not more than twice per day
def __init__(self, server, statefile, historyfile,
- expiration_enabled, expiration_mode):
+ expiration_enabled, mode,
+ override_lease_duration, # used if expiration_mode=="age"
+ date_cutoff, # used if expiration_mode=="date-cutoff"
+ sharetypes):
self.historyfile = historyfile
self.expiration_enabled = expiration_enabled
- self.mode = expiration_mode
- if self.mode[0] not in ("age", "date-cutoff"):
- raise ValueError("garbage-collection mode '%s' must be 'age' or 'date-cutoff'" % self.mode[0])
- if self.mode[0] == "age":
- assert isinstance(expiration_mode[1], int) # seconds
- elif self.mode[0] == "date-cutoff":
- assert isinstance(expiration_mode[1], int) # seconds-since-epoch
- self.sharetypes_to_expire = ("mutable", "immutable")
- if len(self.mode) > 2:
- self.sharetypes_to_expire = self.mode[2]
+ self.mode = mode
+ self.override_lease_duration = None
+ self.date_cutoff = None
+ if self.mode == "age":
+ assert isinstance(override_lease_duration, (int, type(None)))
+ self.override_lease_duration = override_lease_duration # seconds
+ elif self.mode == "date-cutoff":
+ assert isinstance(date_cutoff, int) # seconds-since-epoch
+ assert date_cutoff is not None
+ self.date_cutoff = date_cutoff
+ else:
+ raise ValueError("GC mode '%s' must be 'age' or 'date-cutoff'" % mode)
+ self.sharetypes_to_expire = sharetypes
ShareCrawler.__init__(self, server, statefile)
def add_initial_state(self):
# expired-or-not according to our configured age limit
expired = False
- if self.mode[0] == "age":
- age_limit = self.mode[1]
+ if self.mode == "age":
+ age_limit = original_expiration_time
+ if self.override_lease_duration is not None:
+ age_limit = self.override_lease_duration
if age > age_limit:
expired = True
else:
- assert self.mode[0] == "date-cutoff"
- date_cutoff = self.mode[1]
- if grant_renew_time < date_cutoff:
+ assert self.mode == "date-cutoff"
+ if grant_renew_time < self.date_cutoff:
expired = True
if sharetype not in self.sharetypes_to_expire:
expired = False
now = time.time()
h["cycle-start-finish-times"] = (start, now)
h["expiration-enabled"] = self.expiration_enabled
- h["configured-expiration-mode"] = self.mode
+ h["configured-expiration-mode"] = (self.mode,
+ self.override_lease_duration,
+ self.date_cutoff,
+ self.sharetypes_to_expire)
s = self.state["cycle-to-date"]
lah = so_far["lease-age-histogram"]
so_far["lease-age-histogram"] = self.convert_lease_age_histogram(lah)
so_far["expiration-enabled"] = self.expiration_enabled
- so_far["configured-expiration-mode"] = self.mode
+ so_far["configured-expiration-mode"] = (self.mode,
+ self.override_lease_duration,
+ self.date_cutoff,
+ self.sharetypes_to_expire)
so_far_sr = so_far["space-recovered"]
remaining_sr = {}
discard_storage=False, readonly_storage=False,
stats_provider=None,
expiration_enabled=False,
- expiration_mode=("age", 31*24*60*60)):
+ expiration_mode="age",
+ expiration_override_lease_duration=None,
+ expiration_date_cutoff=None,
+ expiration_sharetypes=("mutable", "immutable")):
service.MultiService.__init__(self)
assert isinstance(nodeid, str)
assert len(nodeid) == 20
"cancel": [],
}
self.add_bucket_counter()
- self.add_lease_checker(expiration_enabled, expiration_mode)
- def add_bucket_counter(self):
- statefile = os.path.join(self.storedir, "bucket_counter.state")
- self.bucket_counter = BucketCountingCrawler(self, statefile)
- self.bucket_counter.setServiceParent(self)
-
- def add_lease_checker(self, expiration_enabled, expiration_mode):
statefile = os.path.join(self.storedir, "lease_checker.state")
historyfile = os.path.join(self.storedir, "lease_checker.history")
klass = self.LeaseCheckerClass
self.lease_checker = klass(self, statefile, historyfile,
- expiration_enabled=expiration_enabled,
- expiration_mode=expiration_mode)
+ expiration_enabled, expiration_mode,
+ expiration_override_lease_duration,
+ expiration_date_cutoff,
+ expiration_sharetypes)
self.lease_checker.setServiceParent(self)
+ def add_bucket_counter(self):
+ statefile = os.path.join(self.storedir, "bucket_counter.state")
+ self.bucket_counter = BucketCountingCrawler(self, statefile)
+ self.bucket_counter.setServiceParent(self)
+
def count(self, name, delta=1):
if self.stats_provider:
self.stats_provider.count("storage_server." + name, delta)
# than 2000s old will be expired.
ss = InstrumentedStorageServer(basedir, "\x00" * 20,
expiration_enabled=True,
- expiration_mode=("age",2000))
+ expiration_mode="age",
+ expiration_override_lease_duration=2000)
# make it start sooner than usual.
lc = ss.lease_checker
lc.slow_start = 0
last = s["history"][0]
self.failUnlessEqual(last["expiration-enabled"], True)
- self.failUnlessEqual(last["configured-expiration-mode"], ("age",2000))
+ self.failUnlessEqual(last["configured-expiration-mode"],
+ ("age", 2000, None, ("mutable", "immutable")))
self.failUnlessEqual(last["leases-per-share-histogram"], {1: 2, 2: 2})
rec = last["space-recovered"]
def _check_html(html):
s = remove_tags(html)
self.failUnlessIn("Expiration Enabled: expired leases will be removed", s)
- self.failUnlessIn("leases created or last renewed more than 33 minutes ago will be considered expired", s)
+ self.failUnlessIn("Leases created or last renewed more than 33 minutes ago will be considered expired.", s)
self.failUnlessIn(" recovered: 2 shares, 2 buckets (1 mutable / 1 immutable), ", s)
d.addCallback(_check_html)
return d
then = int(now - 2000)
ss = InstrumentedStorageServer(basedir, "\x00" * 20,
expiration_enabled=True,
- expiration_mode=("date-cutoff",then))
+ expiration_mode="date-cutoff",
+ expiration_date_cutoff=then)
# make it start sooner than usual.
lc = ss.lease_checker
lc.slow_start = 0
self.failUnlessEqual(last["expiration-enabled"], True)
self.failUnlessEqual(last["configured-expiration-mode"],
- ("date-cutoff",then))
+ ("date-cutoff", None, then,
+ ("mutable", "immutable")))
self.failUnlessEqual(last["leases-per-share-histogram"],
{1: 2, 2: 2})
self.failUnlessIn("Expiration Enabled:"
" expired leases will be removed", s)
date = time.strftime("%d-%b-%Y", time.gmtime(then))
- self.failUnlessIn("leases created or last renewed before %s"
- " will be considered expired" % date, s)
+ self.failUnlessIn("Leases created or last renewed before %s"
+ " will be considered expired." % date, s)
self.failUnlessIn(" recovered: 2 shares, 2 buckets (1 mutable / 1 immutable), ", s)
d.addCallback(_check_html)
return d
then = int(now - 2000)
ss = StorageServer(basedir, "\x00" * 20,
expiration_enabled=True,
- expiration_mode=("date-cutoff",
- then, ("immutable",)))
+ expiration_mode="date-cutoff",
+ expiration_date_cutoff=then,
+ expiration_sharetypes=("immutable",))
lc = ss.lease_checker
lc.slow_start = 0
webstatus = StorageStatus(ss)
d.addCallback(lambda ign: self.render1(webstatus))
def _check_html(html):
s = remove_tags(html)
- self.failUnlessIn("only the following sharetypes will be expired: immutable Next crawl", s)
+ self.failUnlessIn("The following sharetypes will be expired: immutable.", s)
d.addCallback(_check_html)
return d
then = int(now - 2000)
ss = StorageServer(basedir, "\x00" * 20,
expiration_enabled=True,
- expiration_mode=("date-cutoff",
- then, ("mutable",)))
+ expiration_mode="date-cutoff",
+ expiration_date_cutoff=then,
+ expiration_sharetypes=("mutable",))
lc = ss.lease_checker
lc.slow_start = 0
webstatus = StorageStatus(ss)
d.addCallback(lambda ign: self.render1(webstatus))
def _check_html(html):
s = remove_tags(html)
- self.failUnlessIn("only the following sharetypes will be expired: mutable Next crawl", s)
+ self.failUnlessIn("The following sharetypes will be expired: mutable.", s)
d.addCallback(_check_html)
return d
fileutil.make_dirs(basedir)
e = self.failUnlessRaises(ValueError,
StorageServer, basedir, "\x00" * 20,
- expiration_mode=("bogus", 0))
- self.failUnless("garbage-collection mode 'bogus'"
- " must be 'age' or 'date-cutoff'" in str(e), str(e))
+ expiration_mode="bogus")
+ self.failUnless("GC mode 'bogus' must be 'age' or 'date-cutoff'" in str(e), str(e))
def test_parse_duration(self):
DAY = 24*60*60
basedir = "storage/LeaseCrawler/no_st_blocks"
fileutil.make_dirs(basedir)
ss = No_ST_BLOCKS_StorageServer(basedir, "\x00" * 20,
- expiration_mode=("age",-1000))
+ expiration_mode="age",
+ expiration_override_lease_duration=-1000)
# a negative expiration_time= means the "configured-"
# space-recovered counts will be non-zero, since all shares will have
# expired by then
return ctx.tag["Disabled: scan-only mode, no leases will be removed"]
def render_lease_expiration_mode(self, ctx, data):
- mode = self.storage.lease_checker.mode
- if mode[0] == "age":
- ctx.tag["leases created or last renewed more than %s ago "
- "will be considered expired"
- % abbreviate_time(mode[1])]
+ lc = self.storage.lease_checker
+ if lc.mode == "age":
+ if lc.override_lease_duration is None:
+ ctx.tag["Leases will expire naturally, probably 31 days after "
+ "creation or renewal."]
+ else:
+ ctx.tag["Leases created or last renewed more than %s ago "
+ "will be considered expired."
+ % abbreviate_time(lc.override_lease_duration)]
else:
- assert mode[0] == "date-cutoff"
- date = time.strftime("%d-%b-%Y", time.gmtime(mode[1]))
- ctx.tag["leases created or last renewed before %s "
- "will be considered expired" % date]
- if len(mode) > 2:
- ctx.tag[", and only the following sharetypes will be expired: ",
- sorted(mode[2])]
+ assert lc.mode == "date-cutoff"
+ date = time.strftime("%d-%b-%Y", time.gmtime(lc.date_cutoff))
+ ctx.tag["Leases created or last renewed before %s "
+ "will be considered expired." % date]
+ if len(lc.mode) > 2:
+ ctx.tag[" The following sharetypes will be expired: ",
+ sorted(lc.sharetypes_to_expire), "."]
return ctx.tag
def format_recovered(self, sr, a):
projects / tahoe-lafs / tahoe-lafs.git / commitdiff