From c0d1e7deaec145d6eeb24358417967a901c3841e Mon Sep 17 00:00:00 2001 From: Zooko O'Whielacronx Date: Tue, 7 Jul 2009 17:40:40 -0700 Subject: [PATCH] directories: make initialization of the download cache lazy If you open up a directory containing thousands of files, it currently computes the cache filename and checks for the cache file on disk immediately for each immutble file in that directory. With this patch, it delays those steps until you try to do something with an immutable file that could use the cache. --- src/allmydata/client.py | 4 +--- src/allmydata/immutable/filenode.py | 15 +++++++++++---- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 5df1acbc..df35c89a 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -428,9 +428,7 @@ class Client(node.Node, pollmixin.PollMixin): if isinstance(u, LiteralFileURI): node = LiteralFileNode(u, self) # LIT else: - key = base32.b2a(u.storage_index) - cachefile = self.download_cache_dirman.get_file(key) - node = FileNode(u, self, cachefile) # CHK + node = FileNode(u, self, self.download_cache_dirman) # CHK else: assert IMutableFileURI.providedBy(u), u node = MutableFileNode(self).init_from_uri(u) diff --git a/src/allmydata/immutable/filenode.py b/src/allmydata/immutable/filenode.py index 7ff2aaca..1e83f344 100644 --- a/src/allmydata/immutable/filenode.py +++ b/src/allmydata/immutable/filenode.py @@ -68,12 +68,13 @@ class PortionOfFile: class DownloadCache: implements(IDownloadTarget) - def __init__(self, node, cachefile): + def __init__(self, node, cachedirectorymanager): self._downloader = node._client.getServiceNamed("downloader") self._uri = node.get_uri() self._storage_index = node.get_storage_index() self.milestones = set() # of (offset,size,Deferred) - self.cachefile = cachefile + self.cachedirectorymanager = cachedirectorymanager + self.cachefile = None self.download_in_progress = False # five states: # new FileNode, no downloads ever performed @@ -103,6 +104,8 @@ class DownloadCache: def read(self, consumer, offset, size): assert offset+size <= self.get_filesize() + if not self.cachefile: + self.cachefile = self.cachedirectorymanager.get_file(self._storage_index) f = PortionOfFile(self.cachefile.get_filename(), offset, size) d = basic.FileSender().beginFileTransfer(f, consumer) d.addCallback(lambda lastSent: consumer) @@ -142,6 +145,8 @@ class DownloadCache: umid="8PKOhg", level=log.NOISY) def get_filesize(self): + if not self.cachefile: + self.cachefile = self.cachedirectorymanager.get_file(self._storage_index) try: filesize = os.stat(self.cachefile.get_filename())[stat.ST_SIZE] except OSError: @@ -150,6 +155,8 @@ class DownloadCache: def open(self, size): + if not self.cachefile: + self.cachefile = self.cachedirectorymanager.get_file(self._storage_index) self.f = open(self.cachefile.get_filename(), "wb") def write(self, data): @@ -176,9 +183,9 @@ class DownloadCache: class FileNode(_ImmutableFileNodeBase, log.PrefixingLogMixin): - def __init__(self, uri, client, cachefile): + def __init__(self, uri, client, cachedirectorymanager): _ImmutableFileNodeBase.__init__(self, uri, client) - self.download_cache = DownloadCache(self, cachefile) + self.download_cache = DownloadCache(self, cachedirectorymanager) prefix = uri.get_verify_cap().to_string() log.PrefixingLogMixin.__init__(self, "allmydata.immutable.filenode", prefix=prefix) self.log("starting", level=log.OPERATIONAL) -- 2.45.2