From c205a54965236fc370873977186b9ef452600606 Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Thu, 30 Oct 2008 13:01:20 -0700 Subject: [PATCH] util/cachedir.py: add a cache-directory manager class, which expires+deletes unused files after a while --- src/allmydata/test/test_util.py | 65 ++++++++++++++++++++++++++++++++- src/allmydata/util/cachedir.py | 43 ++++++++++++++++++++++ 2 files changed, 107 insertions(+), 1 deletion(-) create mode 100644 src/allmydata/util/cachedir.py diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 1dbce56b..ba74c34f 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -8,7 +8,7 @@ from twisted.python import failure from allmydata.util import base32, idlib, humanreadable, mathutil, hashutil from allmydata.util import assertutil, fileutil, deferredutil -from allmydata.util import limiter, time_format, pollmixin +from allmydata.util import limiter, time_format, pollmixin, cachedir class Base32(unittest.TestCase): def test_b2a_matches_Pythons(self): @@ -541,3 +541,66 @@ class TimeFormat(unittest.TestCase): self.failUnless("not a complete ISO8601 timestamp" in str(e)) s = time_format.iso_utc_time_to_localseconds("1970-01-01_00:00:01.500") self.failUnlessEqual(s, 1.5) + +class CacheDir(unittest.TestCase): + def test_basic(self): + basedir = "test_util/CacheDir/test_basic" + + def _failIfExists(name): + absfn = os.path.join(basedir, name) + self.failIf(os.path.exists(absfn), + "%s exists but it shouldn't" % absfn) + + def _failUnlessExists(name): + absfn = os.path.join(basedir, name) + self.failUnless(os.path.exists(absfn), + "%s doesn't exist but it should" % absfn) + + cdm = cachedir.CacheDirectoryManager(basedir) + a = cdm.get_file("a") + b = cdm.get_file("b") + c = cdm.get_file("c") + f = open(a.get_filename(), "wb"); f.write("hi"); f.close(); del f + f = open(b.get_filename(), "wb"); f.write("hi"); f.close(); del f + f = open(c.get_filename(), "wb"); f.write("hi"); f.close(); del f + + _failUnlessExists("a") + _failUnlessExists("b") + _failUnlessExists("c") + + cdm.check() + + _failUnlessExists("a") + _failUnlessExists("b") + _failUnlessExists("c") + + del a + # this file won't be deleted yet, because it isn't old enough + cdm.check() + _failUnlessExists("a") + _failUnlessExists("b") + _failUnlessExists("c") + + # we change the definition of "old" to make everything old + cdm.old = -10 + + cdm.check() + _failIfExists("a") + _failUnlessExists("b") + _failUnlessExists("c") + + cdm.old = 60*60 + + del b + + cdm.check() + _failIfExists("a") + _failUnlessExists("b") + _failUnlessExists("c") + + b2 = cdm.get_file("b") + + cdm.check() + _failIfExists("a") + _failUnlessExists("b") + _failUnlessExists("c") diff --git a/src/allmydata/util/cachedir.py b/src/allmydata/util/cachedir.py new file mode 100644 index 00000000..c4902c3e --- /dev/null +++ b/src/allmydata/util/cachedir.py @@ -0,0 +1,43 @@ + +import os.path, stat, weakref, time +from twisted.application import service, internet +from allmydata.util import fileutil + +HOUR = 60*60 + +class CacheDirectoryManager(service.MultiService): + def __init__(self, basedir, pollinterval=1*HOUR, old=1*HOUR): + service.MultiService.__init__(self) + self.basedir = basedir + fileutil.make_dirs(basedir) + self.old = old + self.files = weakref.WeakValueDictionary() + + t = internet.TimerService(pollinterval, self.check) + t.setServiceParent(self) + + def get_file(self, key): + assert isinstance(key, str) # used as filename + absfn = os.path.join(self.basedir, key) + if os.path.exists(absfn): + os.utime(absfn, None) + cf = CacheFile(absfn) + self.files[key] = cf + return cf + + def check(self): + now = time.time() + for fn in os.listdir(self.basedir): + if fn in self.files: + continue + absfn = os.path.join(self.basedir, fn) + mtime = os.stat(absfn)[stat.ST_MTIME] + if now - mtime > self.old: + os.remove(absfn) + +class CacheFile: + def __init__(self, absfn): + self.filename = absfn + + def get_filename(self): + return self.filename -- 2.45.2