From bc53c240031bc77d6dc3040918fb0d70098191d9 Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Mon, 24 Nov 2008 14:40:46 -0700 Subject: [PATCH] dirnode manifest: add verifycaps, both to internal API and to webapi. This will give the manual-GC tools more to work with, so they can estimate how much space will be freed. --- docs/CLI.txt | 5 ++++- docs/frontends/webapi.txt | 3 ++- src/allmydata/dirnode.py | 5 +++++ src/allmydata/interfaces.py | 9 ++++++--- src/allmydata/test/test_dirnode.py | 18 ++++++++++++++++-- src/allmydata/test/test_system.py | 23 +++++++++++++++++++++++ src/allmydata/test/test_web.py | 1 + src/allmydata/web/directory.py | 8 +++++--- 8 files changed, 62 insertions(+), 10 deletions(-) diff --git a/docs/CLI.txt b/docs/CLI.txt index 93ab4f87..f88464c6 100644 --- a/docs/CLI.txt +++ b/docs/CLI.txt @@ -329,6 +329,7 @@ tahoe mv tahoe:uploaded.txt fun:uploaded.txt tahoe manifest tahoe: tahoe manifest --storage-index tahoe: +tahoe manifest --raw tahoe: This performs a recursive walk of the given directory, visiting every file and directory that can be reached from that point. It then emits one line to @@ -339,7 +340,9 @@ tahoe manifest --storage-index tahoe: If --storage-index is added, each line will instead contain the object's storage index. This (string) value is useful to determine which share files - (on the server) are associated with this directory tree. + (on the server) are associated with this directory tree. If --raw is + provided instead, the output will be a JSON-encoded dictionary that includes + keys for storage index strings, verifycaps, and deep-stats. tahoe stats tahoe: diff --git a/docs/frontends/webapi.txt b/docs/frontends/webapi.txt index 500fc3d2..5ec59067 100644 --- a/docs/frontends/webapi.txt +++ b/docs/frontends/webapi.txt @@ -964,11 +964,12 @@ POST $DIRURL?t=start-manifest (must add &ophandle=XYZ) by a space. If output=JSON is added to the queryargs, then the results will be a - JSON-formatted dictionary with five keys: + JSON-formatted dictionary with six keys: finished (bool): if False then you must reload the page until True origin_si (base32 str): the storage index of the starting point manifest: list of (path, cap) tuples, where path is a list of strings. + verifycaps: list of (printable) verify cap strings storage-index: list of (base32) storage index strings stats: a dictionary with the same keys as the t=deep-stats command (described below) diff --git a/src/allmydata/dirnode.py b/src/allmydata/dirnode.py index a485a24e..b976dbff 100644 --- a/src/allmydata/dirnode.py +++ b/src/allmydata/dirnode.py @@ -641,17 +641,22 @@ class ManifestWalker(DeepStats): DeepStats.__init__(self, origin) self.manifest = [] self.storage_index_strings = set() + self.verifycaps = set() def add_node(self, node, path): self.manifest.append( (tuple(path), node.get_uri()) ) si = node.get_storage_index() if si: self.storage_index_strings.add(base32.b2a(si)) + v = node.get_verifier() + if v: + self.verifycaps.add(v.to_string()) return DeepStats.add_node(self, node, path) def get_results(self): stats = DeepStats.get_results(self) return {"manifest": self.manifest, + "verifycaps": self.verifycaps, "storage-index": self.storage_index_strings, "stats": stats, } diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 47f7afba..5b935be4 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -863,16 +863,19 @@ class IDirectoryNode(IMutableFilesystemNode): I also compute deep-stats as described below. I return a Monitor. The Monitor's results will be a dictionary with - three elements: + four elements: res['manifest']: a list of (path, cap) tuples for all nodes (directories and files) reachable from this one. 'path' will be a tuple of unicode strings. The origin dirnode will be represented by an empty path tuple. + res['verifycaps']: a list of (printable) verifycap strings, one for + each reachable non-LIT node. This is a set: + it will contain no duplicates. res['storage-index']: a list of (base32) storage index strings, - one for each reachable node. This is a set: - duplicates have been removed. + one for each reachable non-LIT node. This is + a set: it will contain no duplicates. res['stats']: a dictionary, the same that is generated by start_deep_stats() below. diff --git a/src/allmydata/test/test_dirnode.py b/src/allmydata/test/test_dirnode.py index cc2be476..b1dd67f0 100644 --- a/src/allmydata/test/test_dirnode.py +++ b/src/allmydata/test/test_dirnode.py @@ -9,7 +9,7 @@ from allmydata.interfaces import IURI, IClient, IMutableFileNode, \ INewDirectoryURI, IReadonlyNewDirectoryURI, IFileNode, \ ExistingChildError, NoSuchChildError, \ IDeepCheckResults, IDeepCheckAndRepairResults -from allmydata.util import hashutil +from allmydata.util import hashutil, base32 from allmydata.monitor import Monitor from allmydata.test.common import make_chk_file_uri, make_mutable_file_uri, \ FakeDirectoryNode, create_chk_filenode, ErrorMixin @@ -281,6 +281,8 @@ class Dirnode(unittest.TestCase, def test_create(self): self.expected_manifest = [] + self.expected_verifycaps = set() + self.expected_storage_indexes = set() d = self.client.create_empty_dirnode() def _then(n): @@ -294,8 +296,11 @@ class Dirnode(unittest.TestCase, u_v = n.get_verifier().to_string() self.failUnless(u_v.startswith("URI:DIR2-Verifier:"), u_v) self.expected_manifest.append( ((), u) ) + self.expected_verifycaps.add(u_v) + si = n.get_storage_index() + self.expected_storage_indexes.add(base32.b2a(si)) expected_si = n._uri._filenode_uri.storage_index - self.failUnlessEqual(n.get_storage_index(), expected_si) + self.failUnlessEqual(si, expected_si) d = n.list() d.addCallback(lambda res: self.failUnlessEqual(res, {})) @@ -306,6 +311,8 @@ class Dirnode(unittest.TestCase, m = Marker(fake_file_uri) ffu_v = m.get_verifier().to_string() self.expected_manifest.append( ((u"child",) , m.get_uri()) ) + self.expected_verifycaps.add(ffu_v) + self.expected_storage_indexes.add(base32.b2a(m.get_storage_index())) d.addCallback(lambda res: n.set_uri(u"child", fake_file_uri)) d.addCallback(lambda res: self.shouldFail(ExistingChildError, "set_uri-no", @@ -326,6 +333,9 @@ class Dirnode(unittest.TestCase, new_v = subdir.get_verifier().to_string() assert isinstance(new_v, str) self.expected_manifest.append( ((u"subdir",), subdir.get_uri()) ) + self.expected_verifycaps.add(new_v) + si = subdir.get_storage_index() + self.expected_storage_indexes.add(base32.b2a(si)) d.addCallback(_created) d.addCallback(lambda res: @@ -372,6 +382,10 @@ class Dirnode(unittest.TestCase, sorted(self.expected_manifest)) stats = res["stats"] _check_deepstats(stats) + self.failUnlessEqual(self.expected_verifycaps, + res["verifycaps"]) + self.failUnlessEqual(self.expected_storage_indexes, + res["storage-index"]) d.addCallback(_check_manifest) def _add_subsubdir(res): diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index c46b552d..a3c564b4 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -2423,6 +2423,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): "--node-directory", basedir, "--storage-index", self.root_uri])) def _check2((out,err)): + self.failUnlessEqual(err, "") lines = [l for l in out.split("\n") if l] self.failUnlessEqual(len(lines), 3) self.failUnless(base32.b2a(self.root.get_storage_index()) in lines) @@ -2430,6 +2431,28 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): self.failUnless(base32.b2a(self.large.get_storage_index()) in lines) d.addCallback(_check2) + d.addCallback(lambda res: + self._run_cli(["manifest", + "--node-directory", basedir, + "--raw", self.root_uri])) + def _check2r((out,err)): + self.failUnlessEqual(err, "") + data = simplejson.loads(out) + sis = data["storage-index"] + self.failUnlessEqual(len(sis), 3) + self.failUnless(base32.b2a(self.root.get_storage_index()) in sis) + self.failUnless(base32.b2a(self.mutable.get_storage_index()) in sis) + self.failUnless(base32.b2a(self.large.get_storage_index()) in sis) + self.failUnlessEqual(data["stats"]["count-files"], 4) + self.failUnlessEqual(data["origin"], + base32.b2a(self.root.get_storage_index())) + verifycaps = data["verifycaps"] + self.failUnlessEqual(len(verifycaps), 3) + self.failUnless(self.root.get_verifier().to_string() in verifycaps) + self.failUnless(self.mutable.get_verifier().to_string() in verifycaps) + self.failUnless(self.large.get_verifier().to_string() in verifycaps) + d.addCallback(_check2r) + d.addCallback(lambda res: self._run_cli(["stats", "--node-directory", basedir, diff --git a/src/allmydata/test/test_web.py b/src/allmydata/test/test_web.py index 019049bd..2a9d770d 100644 --- a/src/allmydata/test/test_web.py +++ b/src/allmydata/test/test_web.py @@ -950,6 +950,7 @@ class Web(WebMixin, testutil.StallMixin, unittest.TestCase): self.failUnless("finished" in res) self.failUnless("origin" in res) self.failUnless("storage-index" in res) + self.failUnless("verifycaps" in res) self.failUnless("stats" in res) d.addCallback(_got_json) return d diff --git a/src/allmydata/web/directory.py b/src/allmydata/web/directory.py index 1e3920f8..648484c4 100644 --- a/src/allmydata/web/directory.py +++ b/src/allmydata/web/directory.py @@ -731,9 +731,11 @@ class ManifestResults(rend.Page, ReloadMixin): def json(self, ctx): inevow.IRequest(ctx).setHeader("content-type", "text/plain") m = self.monitor - status = {"manifest": m.get_status()["manifest"], - "storage-index": list(m.get_status()["storage-index"]), - "stats": m.get_status()["stats"], + s = m.get_status() + status = {"manifest": s["manifest"], + "verifycaps": list(s["verifycaps"]), + "storage-index": list(s["storage-index"]), + "stats": s["stats"], "finished": m.is_finished(), "origin": base32.b2a(m.origin_si), } -- 2.45.2