From: Brian Warner <warner@allmydata.com>
Date: Thu, 19 Mar 2009 00:21:38 +0000 (-0700)
Subject: expirer: change setup, config options, in preparation for adding tahoe.cfg controls
X-Git-Tag: allmydata-tahoe-1.4.0~46
X-Git-Url: https://git.rkrishnan.org/simplejson/...?a=commitdiff_plain;h=8eaee2855047ecd1ad7841c040bd76509062d093;p=tahoe-lafs%2Ftahoe-lafs.git

expirer: change setup, config options, in preparation for adding tahoe.cfg controls
---

diff --git a/src/allmydata/storage/expirer.py b/src/allmydata/storage/expirer.py
index d4fd9a01..84f1649e 100644
--- a/src/allmydata/storage/expirer.py
+++ b/src/allmydata/storage/expirer.py
@@ -49,19 +49,25 @@ class LeaseCheckingCrawler(ShareCrawler):
     minimum_cycle_time = 12*60*60 # not more than twice per day
 
     def __init__(self, server, statefile, historyfile,
-                 expiration_enabled, expiration_mode):
+                 expiration_enabled, mode,
+                 override_lease_duration, # used if expiration_mode=="age"
+                 date_cutoff, # used if expiration_mode=="date-cutoff"
+                 sharetypes):
         self.historyfile = historyfile
         self.expiration_enabled = expiration_enabled
-        self.mode = expiration_mode
-        if self.mode[0] not in ("age", "date-cutoff"):
-            raise ValueError("garbage-collection mode '%s' must be 'age' or 'date-cutoff'" % self.mode[0])
-        if self.mode[0] == "age":
-            assert isinstance(expiration_mode[1], int) # seconds
-        elif self.mode[0] == "date-cutoff":
-            assert isinstance(expiration_mode[1], int) # seconds-since-epoch
-        self.sharetypes_to_expire = ("mutable", "immutable")
-        if len(self.mode) > 2:
-            self.sharetypes_to_expire = self.mode[2]
+        self.mode = mode
+        self.override_lease_duration = None
+        self.date_cutoff = None
+        if self.mode == "age":
+            assert isinstance(override_lease_duration, (int, type(None)))
+            self.override_lease_duration = override_lease_duration # seconds
+        elif self.mode == "date-cutoff":
+            assert isinstance(date_cutoff, int) # seconds-since-epoch
+            assert date_cutoff is not None
+            self.date_cutoff = date_cutoff
+        else:
+            raise ValueError("GC mode '%s' must be 'age' or 'date-cutoff'" % mode)
+        self.sharetypes_to_expire = sharetypes
         ShareCrawler.__init__(self, server, statefile)
 
     def add_initial_state(self):
@@ -179,14 +185,15 @@ class LeaseCheckingCrawler(ShareCrawler):
 
             #  expired-or-not according to our configured age limit
             expired = False
-            if self.mode[0] == "age":
-                age_limit = self.mode[1]
+            if self.mode == "age":
+                age_limit = original_expiration_time
+                if self.override_lease_duration is not None:
+                    age_limit = self.override_lease_duration
                 if age > age_limit:
                     expired = True
             else:
-                assert self.mode[0] == "date-cutoff"
-                date_cutoff = self.mode[1]
-                if grant_renew_time < date_cutoff:
+                assert self.mode == "date-cutoff"
+                if grant_renew_time < self.date_cutoff:
                     expired = True
             if sharetype not in self.sharetypes_to_expire:
                 expired = False
@@ -269,7 +276,10 @@ class LeaseCheckingCrawler(ShareCrawler):
         now = time.time()
         h["cycle-start-finish-times"] = (start, now)
         h["expiration-enabled"] = self.expiration_enabled
-        h["configured-expiration-mode"] = self.mode
+        h["configured-expiration-mode"] = (self.mode,
+                                           self.override_lease_duration,
+                                           self.date_cutoff,
+                                           self.sharetypes_to_expire)
 
         s = self.state["cycle-to-date"]
 
@@ -376,7 +386,10 @@ class LeaseCheckingCrawler(ShareCrawler):
         lah = so_far["lease-age-histogram"]
         so_far["lease-age-histogram"] = self.convert_lease_age_histogram(lah)
         so_far["expiration-enabled"] = self.expiration_enabled
-        so_far["configured-expiration-mode"] = self.mode
+        so_far["configured-expiration-mode"] = (self.mode,
+                                                self.override_lease_duration,
+                                                self.date_cutoff,
+                                                self.sharetypes_to_expire)
 
         so_far_sr = so_far["space-recovered"]
         remaining_sr = {}
diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py
index 1f07ed7e..08279708 100644
--- a/src/allmydata/storage/server.py
+++ b/src/allmydata/storage/server.py
@@ -41,7 +41,10 @@ class StorageServer(service.MultiService, Referenceable):
                  discard_storage=False, readonly_storage=False,
                  stats_provider=None,
                  expiration_enabled=False,
-                 expiration_mode=("age", 31*24*60*60)):
+                 expiration_mode="age",
+                 expiration_override_lease_duration=None,
+                 expiration_date_cutoff=None,
+                 expiration_sharetypes=("mutable", "immutable")):
         service.MultiService.__init__(self)
         assert isinstance(nodeid, str)
         assert len(nodeid) == 20
@@ -82,22 +85,22 @@ class StorageServer(service.MultiService, Referenceable):
                           "cancel": [],
                           }
         self.add_bucket_counter()
-        self.add_lease_checker(expiration_enabled, expiration_mode)
 
-    def add_bucket_counter(self):
-        statefile = os.path.join(self.storedir, "bucket_counter.state")
-        self.bucket_counter = BucketCountingCrawler(self, statefile)
-        self.bucket_counter.setServiceParent(self)
-
-    def add_lease_checker(self, expiration_enabled, expiration_mode):
         statefile = os.path.join(self.storedir, "lease_checker.state")
         historyfile = os.path.join(self.storedir, "lease_checker.history")
         klass = self.LeaseCheckerClass
         self.lease_checker = klass(self, statefile, historyfile,
-                                   expiration_enabled=expiration_enabled,
-                                   expiration_mode=expiration_mode)
+                                   expiration_enabled, expiration_mode,
+                                   expiration_override_lease_duration,
+                                   expiration_date_cutoff,
+                                   expiration_sharetypes)
         self.lease_checker.setServiceParent(self)
 
+    def add_bucket_counter(self):
+        statefile = os.path.join(self.storedir, "bucket_counter.state")
+        self.bucket_counter = BucketCountingCrawler(self, statefile)
+        self.bucket_counter.setServiceParent(self)
+
     def count(self, name, delta=1):
         if self.stats_provider:
             self.stats_provider.count("storage_server." + name, delta)
diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py
index 49433028..f3452b43 100644
--- a/src/allmydata/test/test_storage.py
+++ b/src/allmydata/test/test_storage.py
@@ -1753,7 +1753,8 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
         # than 2000s old will be expired.
         ss = InstrumentedStorageServer(basedir, "\x00" * 20,
                                        expiration_enabled=True,
-                                       expiration_mode=("age",2000))
+                                       expiration_mode="age",
+                                       expiration_override_lease_duration=2000)
         # make it start sooner than usual.
         lc = ss.lease_checker
         lc.slow_start = 0
@@ -1845,7 +1846,8 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
             last = s["history"][0]
 
             self.failUnlessEqual(last["expiration-enabled"], True)
-            self.failUnlessEqual(last["configured-expiration-mode"], ("age",2000))
+            self.failUnlessEqual(last["configured-expiration-mode"],
+                                 ("age", 2000, None, ("mutable", "immutable")))
             self.failUnlessEqual(last["leases-per-share-histogram"], {1: 2, 2: 2})
 
             rec = last["space-recovered"]
@@ -1874,7 +1876,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
         def _check_html(html):
             s = remove_tags(html)
             self.failUnlessIn("Expiration Enabled: expired leases will be removed", s)
-            self.failUnlessIn("leases created or last renewed more than 33 minutes ago will be considered expired", s)
+            self.failUnlessIn("Leases created or last renewed more than 33 minutes ago will be considered expired.", s)
             self.failUnlessIn(" recovered: 2 shares, 2 buckets (1 mutable / 1 immutable), ", s)
         d.addCallback(_check_html)
         return d
@@ -1888,7 +1890,8 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
         then = int(now - 2000)
         ss = InstrumentedStorageServer(basedir, "\x00" * 20,
                                        expiration_enabled=True,
-                                       expiration_mode=("date-cutoff",then))
+                                       expiration_mode="date-cutoff",
+                                       expiration_date_cutoff=then)
         # make it start sooner than usual.
         lc = ss.lease_checker
         lc.slow_start = 0
@@ -1985,7 +1988,8 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
 
             self.failUnlessEqual(last["expiration-enabled"], True)
             self.failUnlessEqual(last["configured-expiration-mode"],
-                                 ("date-cutoff",then))
+                                 ("date-cutoff", None, then,
+                                  ("mutable", "immutable")))
             self.failUnlessEqual(last["leases-per-share-histogram"],
                                  {1: 2, 2: 2})
 
@@ -2017,8 +2021,8 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
             self.failUnlessIn("Expiration Enabled:"
                               " expired leases will be removed", s)
             date = time.strftime("%d-%b-%Y", time.gmtime(then))
-            self.failUnlessIn("leases created or last renewed before %s"
-                              " will be considered expired" % date, s)
+            self.failUnlessIn("Leases created or last renewed before %s"
+                              " will be considered expired." % date, s)
             self.failUnlessIn(" recovered: 2 shares, 2 buckets (1 mutable / 1 immutable), ", s)
         d.addCallback(_check_html)
         return d
@@ -2030,8 +2034,9 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
         then = int(now - 2000)
         ss = StorageServer(basedir, "\x00" * 20,
                            expiration_enabled=True,
-                           expiration_mode=("date-cutoff",
-                                            then, ("immutable",)))
+                           expiration_mode="date-cutoff",
+                           expiration_date_cutoff=then,
+                           expiration_sharetypes=("immutable",))
         lc = ss.lease_checker
         lc.slow_start = 0
         webstatus = StorageStatus(ss)
@@ -2075,7 +2080,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
         d.addCallback(lambda ign: self.render1(webstatus))
         def _check_html(html):
             s = remove_tags(html)
-            self.failUnlessIn("only the following sharetypes will be expired: immutable Next crawl", s)
+            self.failUnlessIn("The following sharetypes will be expired: immutable.", s)
         d.addCallback(_check_html)
         return d
 
@@ -2086,8 +2091,9 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
         then = int(now - 2000)
         ss = StorageServer(basedir, "\x00" * 20,
                            expiration_enabled=True,
-                           expiration_mode=("date-cutoff",
-                                            then, ("mutable",)))
+                           expiration_mode="date-cutoff",
+                           expiration_date_cutoff=then,
+                           expiration_sharetypes=("mutable",))
         lc = ss.lease_checker
         lc.slow_start = 0
         webstatus = StorageStatus(ss)
@@ -2131,7 +2137,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
         d.addCallback(lambda ign: self.render1(webstatus))
         def _check_html(html):
             s = remove_tags(html)
-            self.failUnlessIn("only the following sharetypes will be expired: mutable Next crawl", s)
+            self.failUnlessIn("The following sharetypes will be expired: mutable.", s)
         d.addCallback(_check_html)
         return d
 
@@ -2140,9 +2146,8 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
         fileutil.make_dirs(basedir)
         e = self.failUnlessRaises(ValueError,
                                   StorageServer, basedir, "\x00" * 20,
-                                  expiration_mode=("bogus", 0))
-        self.failUnless("garbage-collection mode 'bogus'"
-                        " must be 'age' or 'date-cutoff'" in str(e), str(e))
+                                  expiration_mode="bogus")
+        self.failUnless("GC mode 'bogus' must be 'age' or 'date-cutoff'" in str(e), str(e))
 
     def test_parse_duration(self):
         DAY = 24*60*60
@@ -2254,7 +2259,8 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
         basedir = "storage/LeaseCrawler/no_st_blocks"
         fileutil.make_dirs(basedir)
         ss = No_ST_BLOCKS_StorageServer(basedir, "\x00" * 20,
-                                        expiration_mode=("age",-1000))
+                                        expiration_mode="age",
+                                        expiration_override_lease_duration=-1000)
         # a negative expiration_time= means the "configured-"
         # space-recovered counts will be non-zero, since all shares will have
         # expired by then
diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py
index 3491ea63..56db9026 100644
--- a/src/allmydata/web/storage.py
+++ b/src/allmydata/web/storage.py
@@ -128,19 +128,23 @@ class StorageStatus(rend.Page):
             return ctx.tag["Disabled: scan-only mode, no leases will be removed"]
 
     def render_lease_expiration_mode(self, ctx, data):
-        mode = self.storage.lease_checker.mode
-        if mode[0] == "age":
-            ctx.tag["leases created or last renewed more than %s ago "
-                    "will be considered expired"
-                    % abbreviate_time(mode[1])]
+        lc = self.storage.lease_checker
+        if lc.mode == "age":
+            if lc.override_lease_duration is None:
+                ctx.tag["Leases will expire naturally, probably 31 days after "
+                        "creation or renewal."]
+            else:
+                ctx.tag["Leases created or last renewed more than %s ago "
+                        "will be considered expired."
+                        % abbreviate_time(lc.override_lease_duration)]
         else:
-            assert mode[0] == "date-cutoff"
-            date = time.strftime("%d-%b-%Y", time.gmtime(mode[1]))
-            ctx.tag["leases created or last renewed before %s "
-                    "will be considered expired" % date]
-        if len(mode) > 2:
-            ctx.tag[", and only the following sharetypes will be expired: ",
-                    sorted(mode[2])]
+            assert lc.mode == "date-cutoff"
+            date = time.strftime("%d-%b-%Y", time.gmtime(lc.date_cutoff))
+            ctx.tag["Leases created or last renewed before %s "
+                    "will be considered expired." % date]
+        if len(lc.mode) > 2:
+            ctx.tag[" The following sharetypes will be expired: ",
+                    sorted(lc.sharetypes_to_expire), "."]
         return ctx.tag
 
     def format_recovered(self, sr, a):