From: Brian Warner <warner@allmydata.com>
Date: Fri, 20 Mar 2009 19:18:16 +0000 (-0700)
Subject: expirer: tolerate empty buckets, refactor bucketsize-counting code a bit, don't incre... 
X-Git-Tag: allmydata-tahoe-1.4.0~40
X-Git-Url: https://git.rkrishnan.org/simplejson/%22news.html/running.html?a=commitdiff_plain;h=3b656079269cca967ed5c258affe13eef1262950;p=tahoe-lafs%2Ftahoe-lafs.git

expirer: tolerate empty buckets, refactor bucketsize-counting code a bit, don't increment -mutable/-immutable counters unless we actually know the sharetype
---

diff --git a/src/allmydata/storage/expirer.py b/src/allmydata/storage/expirer.py
index 1c638fa2..bb94e11b 100644
--- a/src/allmydata/storage/expirer.py
+++ b/src/allmydata/storage/expirer.py
@@ -114,11 +114,10 @@ class LeaseCheckingCrawler(ShareCrawler):
 
     def process_bucket(self, cycle, prefix, prefixdir, storage_index_b32):
         bucketdir = os.path.join(prefixdir, storage_index_b32)
-        try:
-            bucket_diskbytes = self.stat(bucketdir).st_blocks * 512
-        except AttributeError:
-            bucket_diskbytes = 0 # no stat().st_blocks on windows
+        s = self.stat(bucketdir)
         would_keep_shares = []
+        wks = None
+
         for fn in os.listdir(bucketdir):
             try:
                 shnum = int(fn)
@@ -136,29 +135,26 @@ class LeaseCheckingCrawler(ShareCrawler):
                 self.state["cycle-to-date"]["corrupt-shares"].append(which)
                 wks = (1, 1, 1, "unknown")
             would_keep_shares.append(wks)
+
         sharetype = None
         if wks:
+            # use the last share's sharetype as the buckettype
             sharetype = wks[3]
         rec = self.state["cycle-to-date"]["space-recovered"]
         self.increment(rec, "examined-buckets", 1)
         if sharetype:
             self.increment(rec, "examined-buckets-"+sharetype, 1)
 
+        try:
+            bucket_diskbytes = s.st_blocks * 512
+        except AttributeError:
+            bucket_diskbytes = 0 # no stat().st_blocks on windows
         if sum([wks[0] for wks in would_keep_shares]) == 0:
-            self.increment(rec, "original-diskbytes", bucket_diskbytes)
-            self.increment(rec, "original-diskbytes-"+sharetype, bucket_diskbytes)
-            self.increment(rec, "original-buckets", 1)
-            self.increment(rec, "original-buckets-"+sharetype, 1)
+            self.increment_bucketspace("original", bucket_diskbytes, sharetype)
         if sum([wks[1] for wks in would_keep_shares]) == 0:
-            self.increment(rec, "configured-diskbytes", bucket_diskbytes)
-            self.increment(rec, "configured-diskbytes-"+sharetype, bucket_diskbytes)
-            self.increment(rec, "configured-buckets", 1)
-            self.increment(rec, "configured-buckets-"+sharetype, 1)
+            self.increment_bucketspace("configured", bucket_diskbytes, sharetype)
         if sum([wks[2] for wks in would_keep_shares]) == 0:
-            self.increment(rec, "actual-diskbytes", bucket_diskbytes)
-            self.increment(rec, "actual-diskbytes-"+sharetype, bucket_diskbytes)
-            self.increment(rec, "actual-buckets", 1)
-            self.increment(rec, "actual-buckets-"+sharetype, 1)
+            self.increment_bucketspace("actual", bucket_diskbytes, sharetype)
 
     def process_share(self, sharefilename):
         # first, find out what kind of a share it is
@@ -239,11 +235,20 @@ class LeaseCheckingCrawler(ShareCrawler):
             diskbytes = sharebytes
         so_far_sr = self.state["cycle-to-date"]["space-recovered"]
         self.increment(so_far_sr, a+"-shares", 1)
-        self.increment(so_far_sr, a+"-shares-"+sharetype, 1)
         self.increment(so_far_sr, a+"-sharebytes", sharebytes)
-        self.increment(so_far_sr, a+"-sharebytes-"+sharetype, sharebytes)
         self.increment(so_far_sr, a+"-diskbytes", diskbytes)
-        self.increment(so_far_sr, a+"-diskbytes-"+sharetype, diskbytes)
+        if sharetype:
+            self.increment(so_far_sr, a+"-shares-"+sharetype, 1)
+            self.increment(so_far_sr, a+"-sharebytes-"+sharetype, sharebytes)
+            self.increment(so_far_sr, a+"-diskbytes-"+sharetype, diskbytes)
+
+    def increment_bucketspace(self, a, bucket_diskbytes, sharetype):
+        rec = self.state["cycle-to-date"]["space-recovered"]
+        self.increment(rec, a+"-diskbytes", bucket_diskbytes)
+        self.increment(rec, a+"-buckets", 1)
+        if sharetype:
+            self.increment(rec, a+"-diskbytes-"+sharetype, bucket_diskbytes)
+            self.increment(rec, a+"-buckets-"+sharetype, 1)
 
     def increment(self, d, k, delta=1):
         if k not in d:
diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py
index b1e1ed12..0d09678c 100644
--- a/src/allmydata/test/test_storage.py
+++ b/src/allmydata/test/test_storage.py
@@ -2324,6 +2324,12 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
         # of share we corrupted, this will trigger an
         # UnknownImmutableContainerVersionError.
 
+        # also create an empty bucket
+        empty_si = base32.b2a("\x04"*16)
+        empty_bucket_dir = os.path.join(ss.sharedir,
+                                        storage_index_to_dir(empty_si))
+        fileutil.make_dirs(empty_bucket_dir)
+
         ss.setServiceParent(self.s)
 
         d = eventual.fireEventually()
@@ -2361,7 +2367,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
             s = lc.get_state()
             last = s["history"][0]
             rec = last["space-recovered"]
-            self.failUnlessEqual(rec["examined-buckets"], 4)
+            self.failUnlessEqual(rec["examined-buckets"], 5)
             self.failUnlessEqual(rec["examined-shares"], 3)
             self.failUnlessEqual(last["corrupt-shares"], [(first_b32, 0)])
         d.addCallback(_after_first_cycle)