From 815e0673e65ce938b41d59d1ee5e4cb9c7058ad7 Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Wed, 19 Nov 2008 15:03:47 -0700 Subject: [PATCH] manifest: include stats in results. webapi is unchanged. --- docs/frontends/webapi.txt | 4 +++- src/allmydata/dirnode.py | 29 ++++++++++++++--------------- src/allmydata/interfaces.py | 18 ++++++++++++------ src/allmydata/test/test_dirnode.py | 15 +++++++++------ src/allmydata/test/test_system.py | 8 ++++---- src/allmydata/web/directory.py | 7 ++++--- 6 files changed, 46 insertions(+), 35 deletions(-) diff --git a/docs/frontends/webapi.txt b/docs/frontends/webapi.txt index 2ecba410..cfa96ea7 100644 --- a/docs/frontends/webapi.txt +++ b/docs/frontends/webapi.txt @@ -964,11 +964,13 @@ POST $DIRURL?t=start-manifest (must add &ophandle=XYZ) by a space. If output=JSON is added to the queryargs, then the results will be a - JSON-formatted dictionary with three keys: + JSON-formatted dictionary with four keys: finished (bool): if False then you must reload the page until True origin_si (str): the storage index of the starting point manifest: list of (path, cap) tuples, where path is a list of strings. + stats: a dictionary with the same keys as the t=deep-stats command + (described below) POST $DIRURL?t=start-deep-size (must add &ophandle=XYZ) diff --git a/src/allmydata/dirnode.py b/src/allmydata/dirnode.py index 86d23d08..42db6ab2 100644 --- a/src/allmydata/dirnode.py +++ b/src/allmydata/dirnode.py @@ -536,21 +536,6 @@ class NewDirectoryNode: return self.deep_traverse(DeepChecker(self, verify, repair=True)) -class ManifestWalker: - def __init__(self, origin): - self.manifest = [] - self.origin = origin - def set_monitor(self, monitor): - self.monitor = monitor - monitor.origin_si = self.origin.get_storage_index() - monitor.set_status(self.manifest) - def add_node(self, node, path): - self.manifest.append( (tuple(path), node.get_uri()) ) - def enter_directory(self, parent, children): - pass - def finish(self): - return self.manifest - class DeepStats: def __init__(self, origin): @@ -651,6 +636,20 @@ class DeepStats: def finish(self): return self.get_results() +class ManifestWalker(DeepStats): + def __init__(self, origin): + DeepStats.__init__(self, origin) + self.manifest = [] + + def add_node(self, node, path): + self.manifest.append( (tuple(path), node.get_uri()) ) + return DeepStats.add_node(self, node, path) + + def finish(self): + return {"manifest": self.manifest, + "stats": self.get_results(), + } + class DeepChecker: def __init__(self, root, verify, repair): diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index c678b07f..885b29cb 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -871,12 +871,18 @@ class IDirectoryNode(IMutableFilesystemNode): NoSuchChildError if I do not have a child by that name.""" def build_manifest(): - """Return a Monitor. The Monitor's results will be a list of (path, - cap) tuples for nodes (directories and files) reachable from this - one. 'path' will be a tuple of unicode strings. The origin dirnode - will be represented by an empty path tuple. The Monitor will also - have an .origin_si attribute with the (binary) storage index of the - starting point. + """I generate a table of everything reachable from this directory. + I also compute deep-stats as described below. + + I return a Monitor. The Monitor's results will be a dictionary with + two elements. The 'manifest' element is a list of (path, cap) tuples + for nodes (directories and files) reachable from this one. 'path' + will be a tuple of unicode strings. The origin dirnode will be + represented by an empty path tuple. The 'stats' element is a + dictionary, the same that is generated by start_deep_stats() below. + + The Monitor will also have an .origin_si attribute with the (binary) + storage index of the starting point. """ def start_deep_stats(): diff --git a/src/allmydata/test/test_dirnode.py b/src/allmydata/test/test_dirnode.py index 3cf94087..c838d6dd 100644 --- a/src/allmydata/test/test_dirnode.py +++ b/src/allmydata/test/test_dirnode.py @@ -335,12 +335,6 @@ class Dirnode(unittest.TestCase, testutil.ShouldFailMixin, testutil.StallMixin): self.failUnlessEqual(sorted(children.keys()), sorted([u"child", u"subdir"]))) - d.addCallback(lambda res: n.build_manifest().when_done()) - def _check_manifest(manifest): - self.failUnlessEqual(sorted(manifest), - sorted(self.expected_manifest)) - d.addCallback(_check_manifest) - d.addCallback(lambda res: n.start_deep_stats().when_done()) def _check_deepstats(stats): self.failUnless(isinstance(stats, dict)) @@ -367,6 +361,15 @@ class Dirnode(unittest.TestCase, testutil.ShouldFailMixin, testutil.StallMixin): self.failUnlessEqual(stats["size-files-histogram"], []) d.addCallback(_check_deepstats) + d.addCallback(lambda res: n.build_manifest().when_done()) + def _check_manifest(res): + manifest = res["manifest"] + self.failUnlessEqual(sorted(manifest), + sorted(self.expected_manifest)) + stats = res["stats"] + _check_deepstats(stats) + d.addCallback(_check_manifest) + def _add_subsubdir(res): return self.subdir.create_empty_directory(u"subsubdir") d.addCallback(_add_subsubdir) diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index e3047f1f..5cbf36c3 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -684,8 +684,8 @@ class SystemTest(SystemTestMixin, unittest.TestCase): d1.addCallback(lambda res: dnode.has_child(u"see recursive")) d1.addCallback(lambda answer: self.failUnlessEqual(answer, True)) d1.addCallback(lambda res: dnode.build_manifest().when_done()) - d1.addCallback(lambda manifest: - self.failUnlessEqual(len(manifest), 1)) + d1.addCallback(lambda res: + self.failUnlessEqual(len(res["manifest"]), 1)) return d1 d.addCallback(_created_dirnode) @@ -975,8 +975,8 @@ class SystemTest(SystemTestMixin, unittest.TestCase): # P/personal/sekrit data # P/s2-rw (same as P/s2-ro) # P/s2-rw/mydata992 (same as P/s2-rw/mydata992) - d1.addCallback(lambda manifest: - self.failUnlessEqual(len(manifest), 5)) + d1.addCallback(lambda res: + self.failUnlessEqual(len(res["manifest"]), 5)) d1.addCallback(lambda res: home.start_deep_stats().when_done()) def _check_stats(stats): expected = {"count-immutable-files": 1, diff --git a/src/allmydata/web/directory.py b/src/allmydata/web/directory.py index 04f3ae74..96835630 100644 --- a/src/allmydata/web/directory.py +++ b/src/allmydata/web/directory.py @@ -724,14 +724,15 @@ class ManifestResults(rend.Page, ReloadMixin): lines = [] is_finished = self.monitor.is_finished() lines.append("finished: " + {True: "yes", False: "no"}[is_finished]) - for (path, cap) in self.monitor.get_status(): + for (path, cap) in self.monitor.get_status()["manifest"]: lines.append(self.slashify_path(path) + " " + cap) return "\n".join(lines) + "\n" def json(self, ctx): inevow.IRequest(ctx).setHeader("content-type", "text/plain") m = self.monitor - status = {"manifest": m.get_status(), + status = {"manifest": m.get_status()["manifest"], + "stats": m.get_status()["stats"], "finished": m.is_finished(), "origin": base32.b2a(m.origin_si), } @@ -747,7 +748,7 @@ class ManifestResults(rend.Page, ReloadMixin): return T.p["Manifest of SI=%s" % self._si_abbrev()] def data_items(self, ctx, data): - return self.monitor.get_status() + return self.monitor.get_status()["manifest"] def render_row(self, ctx, (path, cap)): ctx.fillSlots("path", self.slashify_path(path)) -- 2.45.2