From 24ab5ec26fab61e0e18d7a0d2b00f101db6d61be Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Mon, 16 Mar 2009 23:51:18 -0700 Subject: [PATCH] expirer: add mode to expire only-mutable or only-immutable shares --- src/allmydata/storage/expirer.py | 24 ++++-- src/allmydata/storage/immutable.py | 1 + src/allmydata/storage/mutable.py | 1 + src/allmydata/test/test_storage.py | 114 ++++++++++++++++++++++++++++- src/allmydata/web/storage.py | 14 ++-- 5 files changed, 140 insertions(+), 14 deletions(-) diff --git a/src/allmydata/storage/expirer.py b/src/allmydata/storage/expirer.py index 714a63ac..d41d0977 100644 --- a/src/allmydata/storage/expirer.py +++ b/src/allmydata/storage/expirer.py @@ -59,6 +59,9 @@ class LeaseCheckingCrawler(ShareCrawler): assert isinstance(expiration_mode[1], int) # seconds elif self.mode[0] == "date-cutoff": assert isinstance(expiration_mode[1], int) # seconds-since-epoch + self.sharetypes_to_expire = ("mutable", "immutable") + if len(self.mode) > 2: + self.sharetypes_to_expire = self.mode[2] ShareCrawler.__init__(self, server, statefile) def add_initial_state(self): @@ -145,6 +148,7 @@ class LeaseCheckingCrawler(ShareCrawler): def process_share(self, sharefilename): # first, find out what kind of a share it is sf = get_share_file(sharefilename) + sftype = sf.sharetype now = time.time() s = self.stat(sharefilename) @@ -165,19 +169,23 @@ class LeaseCheckingCrawler(ShareCrawler): num_valid_leases_original += 1 # expired-or-not according to our configured age limit + expired = False if self.mode[0] == "age": age_limit = self.mode[1] - if age < age_limit: - num_valid_leases_configured += 1 - else: - expired_leases_configured.append(li) + if age > age_limit: + expired = True else: assert self.mode[0] == "date-cutoff" date_cutoff = self.mode[1] - if grant_renew_time > date_cutoff: - num_valid_leases_configured += 1 - else: - expired_leases_configured.append(li) + if grant_renew_time < date_cutoff: + expired = True + if sftype not in self.sharetypes_to_expire: + expired = False + + if expired: + expired_leases_configured.append(li) + else: + num_valid_leases_configured += 1 so_far = self.state["cycle-to-date"] self.increment(so_far["leases-per-share-histogram"], num_leases, 1) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 363dc0dc..61d277c6 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -37,6 +37,7 @@ from allmydata.storage.common import UnknownImmutableContainerVersionError, \ class ShareFile: LEASE_SIZE = struct.calcsize(">L32s32sL") + sharetype = "immutable" def __init__(self, filename, max_size=None, create=False): """ If max_size is not None then I won't allow more than max_size to be written to me. If create=True and max_size must not be None. """ diff --git a/src/allmydata/storage/mutable.py b/src/allmydata/storage/mutable.py index c7619530..59e662c3 100644 --- a/src/allmydata/storage/mutable.py +++ b/src/allmydata/storage/mutable.py @@ -32,6 +32,7 @@ assert struct.calcsize("Q"), 8 # The struct module doc says that Q's are 8 bytes class MutableShareFile: + sharetype = "mutable" DATA_LENGTH_OFFSET = struct.calcsize(">32s20s32s") EXTRA_LEASE_OFFSET = DATA_LENGTH_OFFSET + 8 HEADER_SIZE = struct.calcsize(">32s20s32sQQ") # doesn't include leases diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 85d39033..d2dd5724 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -1741,7 +1741,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): f = open(sf.home, 'rb+') sf._write_lease_record(f, i, lease) f.close() - return + return raise IndexError("unable to renew non-existent lease") def test_expire_age(self): @@ -2023,6 +2023,118 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): d.addCallback(_check_html) return d + def test_only_immutable(self): + basedir = "storage/LeaseCrawler/only_immutable" + fileutil.make_dirs(basedir) + now = time.time() + then = int(now - 2000) + ss = StorageServer(basedir, "\x00" * 20, + expiration_enabled=True, + expiration_mode=("date-cutoff", + then, ("immutable",))) + lc = ss.lease_checker + lc.slow_start = 0 + webstatus = StorageStatus(ss) + + self.make_shares(ss) + [immutable_si_0, immutable_si_1, mutable_si_2, mutable_si_3] = self.sis + # set all leases to be expirable + new_expiration_time = now - 3000 + 31*24*60*60 + + def count_shares(si): + return len(list(ss._iter_share_files(si))) + def _get_sharefile(si): + return list(ss._iter_share_files(si))[0] + def count_leases(si): + return len(list(_get_sharefile(si).get_leases())) + + sf0 = _get_sharefile(immutable_si_0) + self.backdate_lease(sf0, self.renew_secrets[0], new_expiration_time) + sf1 = _get_sharefile(immutable_si_1) + self.backdate_lease(sf1, self.renew_secrets[1], new_expiration_time) + self.backdate_lease(sf1, self.renew_secrets[2], new_expiration_time) + sf2 = _get_sharefile(mutable_si_2) + self.backdate_lease(sf2, self.renew_secrets[3], new_expiration_time) + sf3 = _get_sharefile(mutable_si_3) + self.backdate_lease(sf3, self.renew_secrets[4], new_expiration_time) + self.backdate_lease(sf3, self.renew_secrets[5], new_expiration_time) + + ss.setServiceParent(self.s) + def _wait(): + return bool(lc.get_state()["last-cycle-finished"] is not None) + d = self.poll(_wait) + + def _after_first_cycle(ignored): + self.failUnlessEqual(count_shares(immutable_si_0), 0) + self.failUnlessEqual(count_shares(immutable_si_1), 0) + self.failUnlessEqual(count_shares(mutable_si_2), 1) + self.failUnlessEqual(count_leases(mutable_si_2), 1) + self.failUnlessEqual(count_shares(mutable_si_3), 1) + self.failUnlessEqual(count_leases(mutable_si_3), 2) + d.addCallback(_after_first_cycle) + d.addCallback(lambda ign: self.render1(webstatus)) + def _check_html(html): + s = remove_tags(html) + self.failUnlessIn("only the following sharetypes will be expired: immutable Next crawl", s) + d.addCallback(_check_html) + return d + + def test_only_mutable(self): + basedir = "storage/LeaseCrawler/only_mutable" + fileutil.make_dirs(basedir) + now = time.time() + then = int(now - 2000) + ss = StorageServer(basedir, "\x00" * 20, + expiration_enabled=True, + expiration_mode=("date-cutoff", + then, ("mutable",))) + lc = ss.lease_checker + lc.slow_start = 0 + webstatus = StorageStatus(ss) + + self.make_shares(ss) + [immutable_si_0, immutable_si_1, mutable_si_2, mutable_si_3] = self.sis + # set all leases to be expirable + new_expiration_time = now - 3000 + 31*24*60*60 + + def count_shares(si): + return len(list(ss._iter_share_files(si))) + def _get_sharefile(si): + return list(ss._iter_share_files(si))[0] + def count_leases(si): + return len(list(_get_sharefile(si).get_leases())) + + sf0 = _get_sharefile(immutable_si_0) + self.backdate_lease(sf0, self.renew_secrets[0], new_expiration_time) + sf1 = _get_sharefile(immutable_si_1) + self.backdate_lease(sf1, self.renew_secrets[1], new_expiration_time) + self.backdate_lease(sf1, self.renew_secrets[2], new_expiration_time) + sf2 = _get_sharefile(mutable_si_2) + self.backdate_lease(sf2, self.renew_secrets[3], new_expiration_time) + sf3 = _get_sharefile(mutable_si_3) + self.backdate_lease(sf3, self.renew_secrets[4], new_expiration_time) + self.backdate_lease(sf3, self.renew_secrets[5], new_expiration_time) + + ss.setServiceParent(self.s) + def _wait(): + return bool(lc.get_state()["last-cycle-finished"] is not None) + d = self.poll(_wait) + + def _after_first_cycle(ignored): + self.failUnlessEqual(count_shares(immutable_si_0), 1) + self.failUnlessEqual(count_leases(immutable_si_0), 1) + self.failUnlessEqual(count_shares(immutable_si_1), 1) + self.failUnlessEqual(count_leases(immutable_si_1), 2) + self.failUnlessEqual(count_shares(mutable_si_2), 0) + self.failUnlessEqual(count_shares(mutable_si_3), 0) + d.addCallback(_after_first_cycle) + d.addCallback(lambda ign: self.render1(webstatus)) + def _check_html(html): + s = remove_tags(html) + self.failUnlessIn("only the following sharetypes will be expired: mutable Next crawl", s) + d.addCallback(_check_html) + return d + def test_bad_mode(self): basedir = "storage/LeaseCrawler/bad_mode" fileutil.make_dirs(basedir) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index dec972ba..581d821b 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -130,14 +130,18 @@ class StorageStatus(rend.Page): def render_lease_expiration_mode(self, ctx, data): mode = self.storage.lease_checker.mode if mode[0] == "age": - return ctx.tag["leases created or last renewed more than %s ago " - "will be considered expired" - % abbreviate_time(mode[1])] + ctx.tag["leases created or last renewed more than %s ago " + "will be considered expired" + % abbreviate_time(mode[1])] else: assert mode[0] == "date-cutoff" date = time.strftime("%d-%b-%Y", time.gmtime(mode[1])) - return ctx.tag["leases created or last renewed before %s " - "will be considered expired" % date] + ctx.tag["leases created or last renewed before %s " + "will be considered expired" % date] + if len(mode) > 2: + ctx.tag[", and only the following sharetypes will be expired: ", + sorted(mode[2])] + return ctx.tag def format_recovered(self, sr, a): def maybe(d): -- 2.45.2