directories: make initialization of the download cache lazy

If you open up a directory containing thousands of files, it currently computes the cache filename and checks for the cache file on disk immediately for each immutble file in that directory.  With this patch, it delays those steps until you try to do something with an immutable file that could use the cache.

diff --git a/src/allmydata/client.py b/src/allmydata/client.py
index 5df1acbc83ba40c0f0371d6624e650779c1335db..df35c89aba18cfa551509714778faa6f119effab 100644 (file)
--- a/src/allmydata/client.py
+++ b/src/allmydata/client.py
@@ -428,9 +428,7 @@ class Client(node.Node, pollmixin.PollMixin):
                 if isinstance(u, LiteralFileURI):
                     node = LiteralFileNode(u, self) # LIT
                 else:
-                    key = base32.b2a(u.storage_index)
-                    cachefile = self.download_cache_dirman.get_file(key)
-                    node = FileNode(u, self, cachefile) # CHK
+                    node = FileNode(u, self, self.download_cache_dirman) # CHK
             else:
                 assert IMutableFileURI.providedBy(u), u
                 node = MutableFileNode(self).init_from_uri(u)

diff --git a/src/allmydata/immutable/filenode.py b/src/allmydata/immutable/filenode.py
index 7ff2aacadf40fa5b8644be3a72b883aa669238d2..1e83f344839c5a38b6cd4e7c6bd3e52ca57fa26b 100644 (file)
--- a/src/allmydata/immutable/filenode.py
+++ b/src/allmydata/immutable/filenode.py
@@ -68,12 +68,13 @@ class PortionOfFile:
 class DownloadCache:
     implements(IDownloadTarget)
 
-    def __init__(self, node, cachefile):
+    def __init__(self, node, cachedirectorymanager):
         self._downloader = node._client.getServiceNamed("downloader")
         self._uri = node.get_uri()
         self._storage_index = node.get_storage_index()
         self.milestones = set() # of (offset,size,Deferred)
-        self.cachefile = cachefile
+        self.cachedirectorymanager = cachedirectorymanager
+        self.cachefile = None
         self.download_in_progress = False
         # five states:
         #  new FileNode, no downloads ever performed
@@ -103,6 +104,8 @@ class DownloadCache:
 
     def read(self, consumer, offset, size):
         assert offset+size <= self.get_filesize()
+        if not self.cachefile:
+            self.cachefile = self.cachedirectorymanager.get_file(self._storage_index)
         f = PortionOfFile(self.cachefile.get_filename(), offset, size)
         d = basic.FileSender().beginFileTransfer(f, consumer)
         d.addCallback(lambda lastSent: consumer)
@@ -142,6 +145,8 @@ class DownloadCache:
                         umid="8PKOhg", level=log.NOISY)
 
     def get_filesize(self):
+        if not self.cachefile:
+            self.cachefile = self.cachedirectorymanager.get_file(self._storage_index)
         try:
             filesize = os.stat(self.cachefile.get_filename())[stat.ST_SIZE]
         except OSError:
@@ -150,6 +155,8 @@ class DownloadCache:
 
 
     def open(self, size):
+        if not self.cachefile:
+            self.cachefile = self.cachedirectorymanager.get_file(self._storage_index)
         self.f = open(self.cachefile.get_filename(), "wb")
 
     def write(self, data):
@@ -176,9 +183,9 @@ class DownloadCache:
 
 
 class FileNode(_ImmutableFileNodeBase, log.PrefixingLogMixin):
-    def __init__(self, uri, client, cachefile):
+    def __init__(self, uri, client, cachedirectorymanager):
         _ImmutableFileNodeBase.__init__(self, uri, client)
-        self.download_cache = DownloadCache(self, cachefile)
+        self.download_cache = DownloadCache(self, cachedirectorymanager)
         prefix = uri.get_verify_cap().to_string()
         log.PrefixingLogMixin.__init__(self, "allmydata.immutable.filenode", prefix=prefix)
         self.log("starting", level=log.OPERATIONAL)