From: Brian Warner Date: Wed, 18 Nov 2009 19:16:24 +0000 (-0800) Subject: make get_size/get_current_size consistent for all IFilesystemNode classes X-Git-Url: https://git.rkrishnan.org/%5B/%5D%20//%22?a=commitdiff_plain;h=e046744f40d59e7014c9ba7b8d9edbb1ddc79132;p=tahoe-lafs%2Ftahoe-lafs.git make get_size/get_current_size consistent for all IFilesystemNode classes * stop caching most_recent_size in dirnode, rely upon backing filenode for it * start caching most_recent_size in MutableFileNode * return None when you don't know, not "?" * only render None as "?" in the web "more info" page * add get_size/get_current_size to UnknownNode --- diff --git a/src/allmydata/dirnode.py b/src/allmydata/dirnode.py index ddab1cc2..fb3cd2da 100644 --- a/src/allmydata/dirnode.py +++ b/src/allmydata/dirnode.py @@ -195,7 +195,6 @@ class DirectoryNode: self._uri = wrap_dirnode_cap(filenode_cap) self._nodemaker = nodemaker self._uploader = uploader - self._most_recent_size = None def __repr__(self): return "<%s %s-%s %s>" % (self.__class__.__name__, @@ -204,22 +203,19 @@ class DirectoryNode: hasattr(self, '_uri') and self._uri.abbrev()) def get_size(self): - # return the size of our backing mutable file, in bytes, if we've - # fetched it. - if not self._node.is_mutable(): - # TODO?: consider using IMutableFileNode.providedBy(self._node) - return self._node.get_size() - return self._most_recent_size + """Return the size of our backing mutable file, in bytes, if we've + fetched it. Otherwise return None. This returns synchronously.""" + return self._node.get_size() - def _set_size(self, data): - self._most_recent_size = len(data) - return data + def get_current_size(self): + """Calculate the size of our backing mutable file, in bytes. Returns + a Deferred that fires with the result.""" + return self._node.get_current_size() def _read(self): if self._node.is_mutable(): # use the IMutableFileNode API. d = self._node.download_best_version() - d.addCallback(self._set_size) else: d = self._node.download_to_data() d.addCallback(self._unpack_contents) @@ -706,9 +702,10 @@ class DeepStats: def enter_directory(self, parent, children): dirsize_bytes = parent.get_size() + if dirsize_bytes is not None: + self.add("size-directories", dirsize_bytes) + self.max("largest-directory", dirsize_bytes) dirsize_children = len(children) - self.add("size-directories", dirsize_bytes) - self.max("largest-directory", dirsize_bytes) self.max("largest-directory-children", dirsize_children) def add(self, key, value=1): diff --git a/src/allmydata/immutable/filenode.py b/src/allmydata/immutable/filenode.py index 0e8aabd1..fbc4adb7 100644 --- a/src/allmydata/immutable/filenode.py +++ b/src/allmydata/immutable/filenode.py @@ -196,6 +196,8 @@ class FileNode(_ImmutableFileNodeBase, log.PrefixingLogMixin): def get_size(self): return self.u.get_size() + def get_current_size(self): + return defer.succeed(self.get_size()) def get_cap(self): return self.u @@ -323,6 +325,8 @@ class LiteralFileNode(_ImmutableFileNodeBase): def get_size(self): return len(self.u.data) + def get_current_size(self): + return defer.succeed(self.get_size()) def get_cap(self): return self.u diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 769cc028..2366d864 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -550,6 +550,19 @@ class IFilesystemNode(Interface): file. """ + def get_size(): + """Return the length (in bytes) of the data this node represents. For + directory nodes, I return the size of the backing store. I return + synchronously and do not consult the network, so for mutable objects, + I will return the most recently observed size for the object, or None + if I don't remember a size. Use get_current_size, which returns a + Deferred, if you want more up-to-date information.""" + + def get_current_size(): + """I return a Deferred that fires with the length (in bytes) of the + data this node represents. + """ + class IMutableFilesystemNode(IFilesystemNode): pass @@ -561,9 +574,6 @@ class IFileNode(IFilesystemNode): """Download the file's contents. Return a Deferred that fires with those contents.""" - def get_size(): - """Return the length (in bytes) of the data this node represents.""" - def read(consumer, offset=0, size=None): """Download a portion (possibly all) of the file's contents, making them available to the given IConsumer. Return a Deferred that fires diff --git a/src/allmydata/mutable/filenode.py b/src/allmydata/mutable/filenode.py index 4192a8fc..4e531b96 100644 --- a/src/allmydata/mutable/filenode.py +++ b/src/allmydata/mutable/filenode.py @@ -63,6 +63,7 @@ class MutableFileNode: self._total_shares = default_encoding_parameters["n"] self._sharemap = {} # known shares, shnum-to-[nodeids] self._cache = ResponseCache() + self._most_recent_size = None # all users of this MutableFileNode go through the serializer. This # takes advantage of the fact that Deferreds discard the callbacks @@ -186,7 +187,14 @@ class MutableFileNode: # IFilesystemNode def get_size(self): - return "?" # TODO: this is likely to cause problems, not being an int + return self._most_recent_size + def get_current_size(self): + d = self.get_size_of_best_version() + d.addCallback(self._stash_size) + return d + def _stash_size(self, size): + self._most_recent_size = size + return size def get_cap(self): return self._uri @@ -427,7 +435,12 @@ class MutableFileNode: r = Retrieve(self, servermap, version, fetch_privkey) if self._history: self._history.notify_retrieve(r.get_status()) - return r.download() + d = r.download() + d.addCallback(self._downloaded_version) + return d + def _downloaded_version(self, data): + self._most_recent_size = len(data) + return data def upload(self, new_contents, servermap): return self._do_serialized(self._upload, new_contents, servermap) @@ -436,4 +449,9 @@ class MutableFileNode: p = Publish(self, self._storage_broker, servermap) if self._history: self._history.notify_publish(p.get_status(), len(new_contents)) - return p.publish(new_contents) + d = p.publish(new_contents) + d.addCallback(self._did_upload, len(new_contents)) + return d + def _did_upload(self, res, size): + self._most_recent_size = size + return res diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index d0580750..311dc873 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -201,7 +201,9 @@ class FakeMutableFileNode: def get_writekey(self): return "\x00"*16 def get_size(self): - return "?" # TODO: see mutable.MutableFileNode.get_size + return len(self.all_contents[self.storage_index]) + def get_current_size(self): + return self.get_size_of_best_version() def get_size_of_best_version(self): return defer.succeed(len(self.all_contents[self.storage_index])) diff --git a/src/allmydata/unknown.py b/src/allmydata/unknown.py index c64e6368..55c84c5b 100644 --- a/src/allmydata/unknown.py +++ b/src/allmydata/unknown.py @@ -19,6 +19,10 @@ class UnknownNode: return None def get_repair_cap(self): return None + def get_size(self): + return None + def get_current_size(self): + return defer.succeed(None) def check(self, monitor, verify, add_lease): return defer.succeed(None) def check_and_repair(self, monitor, verify, add_lease): diff --git a/src/allmydata/web/info.py b/src/allmydata/web/info.py index edaab44b..fa81f6c2 100644 --- a/src/allmydata/web/info.py +++ b/src/allmydata/web/info.py @@ -55,17 +55,12 @@ class MoreInfo(rend.Page): def render_size(self, ctx, data): node = self.original si = node.get_storage_index() - if IDirectoryNode.providedBy(node): - d = node._node.get_size_of_best_version() - elif IFileNode.providedBy(node): - if node.is_mutable(): - d = node.get_size_of_best_version() - else: - # for immutable files and LIT files, we get the size from the - # URI - d = defer.succeed(node.get_size()) - else: - d = defer.succeed("?") + d = node.get_current_size() + def _no_size(size): + if size is None: + return "?" + return size + d.addCallback(_no_size) def _handle_unrecoverable(f): f.trap(UnrecoverableFileError) return "?"