From b84c2c6541793cf53c586d21fe2149e292250fbc Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Wed, 19 Nov 2008 16:00:27 -0700 Subject: [PATCH] manifest: add storage-index strings to the json results --- docs/frontends/webapi.txt | 5 +++-- src/allmydata/dirnode.py | 12 +++++++++--- src/allmydata/interfaces.py | 17 ++++++++++++----- src/allmydata/scripts/cli.py | 1 + src/allmydata/test/test_dirnode.py | 3 +++ src/allmydata/test/test_system.py | 1 + src/allmydata/test/test_web.py | 8 ++++++-- src/allmydata/web/directory.py | 1 + 8 files changed, 36 insertions(+), 12 deletions(-) diff --git a/docs/frontends/webapi.txt b/docs/frontends/webapi.txt index cfa96ea7..500fc3d2 100644 --- a/docs/frontends/webapi.txt +++ b/docs/frontends/webapi.txt @@ -964,11 +964,12 @@ POST $DIRURL?t=start-manifest (must add &ophandle=XYZ) by a space. If output=JSON is added to the queryargs, then the results will be a - JSON-formatted dictionary with four keys: + JSON-formatted dictionary with five keys: finished (bool): if False then you must reload the page until True - origin_si (str): the storage index of the starting point + origin_si (base32 str): the storage index of the starting point manifest: list of (path, cap) tuples, where path is a list of strings. + storage-index: list of (base32) storage index strings stats: a dictionary with the same keys as the t=deep-stats command (described below) diff --git a/src/allmydata/dirnode.py b/src/allmydata/dirnode.py index 42db6ab2..a485a24e 100644 --- a/src/allmydata/dirnode.py +++ b/src/allmydata/dirnode.py @@ -565,7 +565,7 @@ class DeepStats: def set_monitor(self, monitor): self.monitor = monitor monitor.origin_si = self.origin.get_storage_index() - monitor.set_status(self.stats) + monitor.set_status(self.get_results()) def add_node(self, node, childpath): if IDirectoryNode.providedBy(node): @@ -640,14 +640,20 @@ class ManifestWalker(DeepStats): def __init__(self, origin): DeepStats.__init__(self, origin) self.manifest = [] + self.storage_index_strings = set() def add_node(self, node, path): self.manifest.append( (tuple(path), node.get_uri()) ) + si = node.get_storage_index() + if si: + self.storage_index_strings.add(base32.b2a(si)) return DeepStats.add_node(self, node, path) - def finish(self): + def get_results(self): + stats = DeepStats.get_results(self) return {"manifest": self.manifest, - "stats": self.get_results(), + "storage-index": self.storage_index_strings, + "stats": stats, } diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 885b29cb..798b4baf 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -875,11 +875,18 @@ class IDirectoryNode(IMutableFilesystemNode): I also compute deep-stats as described below. I return a Monitor. The Monitor's results will be a dictionary with - two elements. The 'manifest' element is a list of (path, cap) tuples - for nodes (directories and files) reachable from this one. 'path' - will be a tuple of unicode strings. The origin dirnode will be - represented by an empty path tuple. The 'stats' element is a - dictionary, the same that is generated by start_deep_stats() below. + three elements: + + res['manifest']: a list of (path, cap) tuples for all nodes + (directories and files) reachable from this one. + 'path' will be a tuple of unicode strings. The + origin dirnode will be represented by an empty path + tuple. + res['storage-index']: a list of (base32) storage index strings, + one for each reachable node. This is a set: + duplicates have been removed. + res['stats']: a dictionary, the same that is generated by + start_deep_stats() below. The Monitor will also have an .origin_si attribute with the (binary) storage index of the starting point. diff --git a/src/allmydata/scripts/cli.py b/src/allmydata/scripts/cli.py index 6a0d6030..c7bdfab7 100644 --- a/src/allmydata/scripts/cli.py +++ b/src/allmydata/scripts/cli.py @@ -200,6 +200,7 @@ class WebopenOptions(VDriveOptions): class ManifestOptions(VDriveOptions): optFlags = [ ("storage-index", "s", "Only print storage index strings, not pathname+cap"), + ("raw", "r", "Display raw JSON data instead of parsed"), ] def parseArgs(self, where=''): self.where = where diff --git a/src/allmydata/test/test_dirnode.py b/src/allmydata/test/test_dirnode.py index c838d6dd..3e1c9729 100644 --- a/src/allmydata/test/test_dirnode.py +++ b/src/allmydata/test/test_dirnode.py @@ -27,6 +27,7 @@ class Marker: nodeuri = nodeuri.to_string() self.nodeuri = nodeuri si = hashutil.tagged_hash("tag1", nodeuri)[:16] + self.storage_index = si fp = hashutil.tagged_hash("tag2", nodeuri) self.verifieruri = uri.SSKVerifierURI(storage_index=si, fingerprint=fp) def get_uri(self): @@ -35,6 +36,8 @@ class Marker: return self.nodeuri def get_verifier(self): return self.verifieruri + def get_storage_index(self): + return self.storage_index def check(self, monitor, verify=False): r = CheckerResults("", None) diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index 5cbf36c3..c46b552d 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -2399,6 +2399,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): "--node-directory", basedir, self.root_uri]) def _check((out,err)): + self.failUnlessEqual(err, "") lines = [l for l in out.split("\n") if l] self.failUnlessEqual(len(lines), 5) caps = {} diff --git a/src/allmydata/test/test_web.py b/src/allmydata/test/test_web.py index 39e59d1b..019049bd 100644 --- a/src/allmydata/test/test_web.py +++ b/src/allmydata/test/test_web.py @@ -940,13 +940,17 @@ class Web(WebMixin, testutil.StallMixin, unittest.TestCase): self.failUnless("\nsub/baz.txt URI:CHK:" in manifest) d.addCallback(_got_text) d.addCallback(getman, "JSON") - def _got_json(manifest): - data = manifest["manifest"] + def _got_json(res): + data = res["manifest"] got = {} for (path_list, cap) in data: got[tuple(path_list)] = cap self.failUnlessEqual(got[(u"sub",)], self._sub_uri) self.failUnless((u"sub",u"baz.txt") in got) + self.failUnless("finished" in res) + self.failUnless("origin" in res) + self.failUnless("storage-index" in res) + self.failUnless("stats" in res) d.addCallback(_got_json) return d diff --git a/src/allmydata/web/directory.py b/src/allmydata/web/directory.py index 96835630..1e3920f8 100644 --- a/src/allmydata/web/directory.py +++ b/src/allmydata/web/directory.py @@ -732,6 +732,7 @@ class ManifestResults(rend.Page, ReloadMixin): inevow.IRequest(ctx).setHeader("content-type", "text/plain") m = self.monitor status = {"manifest": m.get_status()["manifest"], + "storage-index": list(m.get_status()["storage-index"]), "stats": m.get_status()["stats"], "finished": m.is_finished(), "origin": base32.b2a(m.origin_si), -- 2.45.2