From ef1b6ae8e312af2177733b8fb162d9356e1d3b1a Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Thu, 2 Jul 2009 18:07:49 -0700 Subject: [PATCH] Tolerate unknown URI types in directory structures. Part of #683. The idea is that future versions of Tahoe will add new URI types that this version won't recognize, but might store them in directories that we *can* read. We should handle these "objects from the future" as best we can. Previous releases of Tahoe would just explode. With this change, we'll continue to be able to work with everything else in the directory. The code change is to wrap anything we don't recognize as an UnknownNode instance (as opposed to a FileNode or DirectoryNode). Then webapi knows how to render these (mostly by leaving fields blank), deep-check knows to skip over them, deep-stats counts them in "count-unknown". You can rename and delete these things, but you can't add new ones (because we wouldn't know how to generate a readcap to put into the dirnode's rocap slot, and because this lets us catch typos better). --- docs/frontends/webapi.txt | 1 + src/allmydata/client.py | 20 +++-- src/allmydata/dirnode.py | 37 ++++++++-- src/allmydata/interfaces.py | 3 + src/allmydata/test/test_client.py | 39 +++++++++- src/allmydata/test/test_dirnode.py | 81 ++++++++++++++++++++- src/allmydata/test/test_uri.py | 12 +-- src/allmydata/test/test_web.py | 113 +++++++++++++++++++++++++++-- src/allmydata/unknown.py | 25 +++++++ src/allmydata/uri.py | 13 +++- src/allmydata/web/directory.py | 82 ++++++++++++++------- src/allmydata/web/filenode.py | 12 ++- src/allmydata/web/info.py | 50 +++++++++---- src/allmydata/web/info.xhtml | 2 +- 14 files changed, 414 insertions(+), 76 deletions(-) create mode 100644 src/allmydata/unknown.py diff --git a/docs/frontends/webapi.txt b/docs/frontends/webapi.txt index 6abbd42a..1353f233 100644 --- a/docs/frontends/webapi.txt +++ b/docs/frontends/webapi.txt @@ -1232,6 +1232,7 @@ POST $DIRURL?t=start-deep-stats (must add &ophandle=XYZ) count-literal-files: same, for LIT files (data contained inside the URI) count-files: sum of the above three count-directories: count of directories + count-unknown: count of unrecognized objects (perhaps from the future) size-immutable-files: total bytes for all CHK files in the set, =deep-size size-mutable-files (TODO): same, for current version of all mutable files size-literal-files: same, for LIT files diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 08d559e5..8c1dc21f 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -20,9 +20,10 @@ from allmydata.introducer.client import IntroducerClient from allmydata.util import hashutil, base32, pollmixin, cachedir, log from allmydata.util.abbreviate import parse_abbreviated_size from allmydata.util.time_format import parse_duration, parse_date -from allmydata.uri import LiteralFileURI +from allmydata.uri import LiteralFileURI, UnknownURI from allmydata.dirnode import NewDirectoryNode from allmydata.mutable.filenode import MutableFileNode +from allmydata.unknown import UnknownNode from allmydata.stats import StatsProvider from allmydata.history import History from allmydata.interfaces import IURI, INewDirectoryURI, IStatsProducer, \ @@ -404,11 +405,17 @@ class Client(node.Node, pollmixin.PollMixin): # dirnodes. The first takes a URI and produces a filenode or (new-style) # dirnode. The other three create brand-new filenodes/dirnodes. - def create_node_from_uri(self, u, readcap=None): + def create_node_from_uri(self, writecap, readcap=None): # this returns synchronously. + u = writecap or readcap if not u: - u = readcap + # maybe the writecap was hidden because we're in a readonly + # directory, and the future cap format doesn't have a readcap, or + # something. + return UnknownNode(writecap, readcap) u = IURI(u) + if isinstance(u, UnknownURI): + return UnknownNode(writecap, readcap) u_s = u.to_string() if u_s not in self._node_cache: if IReadonlyNewDirectoryURI.providedBy(u): @@ -427,13 +434,12 @@ class Client(node.Node, pollmixin.PollMixin): else: assert IMutableFileURI.providedBy(u), u node = MutableFileNode(self).init_from_uri(u) - self._node_cache[u_s] = node + self._node_cache[u_s] = node # note: WeakValueDictionary return self._node_cache[u_s] def create_empty_dirnode(self): - n = NewDirectoryNode(self) - d = n.create(self._generate_pubprivkeys, self.DEFAULT_MUTABLE_KEYSIZE) - d.addCallback(lambda res: n) + d = self.create_mutable_file() + d.addCallback(NewDirectoryNode.create_with_mutablefile, self) return d def create_mutable_file(self, contents="", keysize=None): diff --git a/src/allmydata/dirnode.py b/src/allmydata/dirnode.py index b8113b3a..73143199 100644 --- a/src/allmydata/dirnode.py +++ b/src/allmydata/dirnode.py @@ -7,9 +7,11 @@ from foolscap.api import fireEventually import simplejson from allmydata.mutable.common import NotMutableError from allmydata.mutable.filenode import MutableFileNode +from allmydata.unknown import UnknownNode from allmydata.interfaces import IMutableFileNode, IDirectoryNode,\ IURI, IFileNode, IMutableFileURI, IFilesystemNode, \ - ExistingChildError, NoSuchChildError, ICheckable, IDeepCheckable + ExistingChildError, NoSuchChildError, ICheckable, IDeepCheckable, \ + CannotPackUnknownNodeError from allmydata.check_results import DeepCheckResults, \ DeepCheckAndRepairResults from allmydata.monitor import Monitor @@ -137,6 +139,12 @@ class NewDirectoryNode: self._node.init_from_uri(self._uri.get_filenode_uri()) return self + @classmethod + def create_with_mutablefile(cls, filenode, client): + self = cls(client) + self._node = filenode + return self._filenode_created(filenode) + def create(self, keypair_generator=None, keysize=None): """ Returns a deferred that eventually fires with self once the directory @@ -226,9 +234,7 @@ class NewDirectoryNode: for name in sorted(children.keys()): child, metadata = children[name] assert isinstance(name, unicode) - assert (IFileNode.providedBy(child) - or IMutableFileNode.providedBy(child) - or IDirectoryNode.providedBy(child)), (name,child) + assert IFilesystemNode.providedBy(child), (name,child) assert isinstance(metadata, dict) rwcap = child.get_uri() # might be RO if the child is not writeable if rwcap is None: @@ -381,6 +387,13 @@ class NewDirectoryNode: precondition(isinstance(name, unicode), name) precondition(isinstance(child_uri, str), child_uri) child_node = self._create_node(child_uri, None) + if isinstance(child_node, UnknownNode): + # don't be willing to pack unknown nodes: we might accidentally + # put some write-authority into the rocap slot because we don't + # know how to diminish the URI they gave us. We don't even know + # if they gave us a readcap or a writecap. + msg = "cannot pack unknown node as child %s" % str(name) + raise CannotPackUnknownNodeError(msg) d = self.set_node(name, child_node, metadata, overwrite) d.addCallback(lambda res: child_node) return d @@ -397,7 +410,11 @@ class NewDirectoryNode: assert len(e) == 3 name, child_uri, metadata = e assert isinstance(name, unicode) - a.set_node(name, self._create_node(child_uri, None), metadata) + child_node = self._create_node(child_uri, None) + if isinstance(child_node, UnknownNode): + msg = "cannot pack unknown node as child %s" % str(name) + raise CannotPackUnknownNodeError(msg) + a.set_node(name, child_node, metadata) return self._node.modify(a.modify) def set_node(self, name, child, metadata=None, overwrite=True): @@ -560,12 +577,15 @@ class NewDirectoryNode: dirkids = [] filekids = [] for name, (child, metadata) in sorted(children.iteritems()): + childpath = path + [name] + if isinstance(child, UnknownNode): + walker.add_node(child, childpath) + continue verifier = child.get_verify_cap() # allow LIT files (for which verifier==None) to be processed if (verifier is not None) and (verifier in found): continue found.add(verifier) - childpath = path + [name] if IDirectoryNode.providedBy(child): dirkids.append( (child, childpath) ) else: @@ -618,6 +638,7 @@ class DeepStats: "count-literal-files", "count-files", "count-directories", + "count-unknown", "size-immutable-files", #"size-mutable-files", "size-literal-files", @@ -640,7 +661,9 @@ class DeepStats: monitor.set_status(self.get_results()) def add_node(self, node, childpath): - if IDirectoryNode.providedBy(node): + if isinstance(node, UnknownNode): + self.add("count-unknown") + elif IDirectoryNode.providedBy(node): self.add("count-directories") elif IMutableFileNode.providedBy(node): self.add("count-files") diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 86e83124..42188e7b 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -478,6 +478,9 @@ class INewDirectoryURI(Interface): class IReadonlyNewDirectoryURI(Interface): pass +class CannotPackUnknownNodeError(Exception): + """UnknownNodes (using filecaps from the future that we don't understand) + cannot yet be copied safely, so I refuse to copy them.""" class IFilesystemNode(Interface): def get_uri(): diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py index b652fbab..38cbdc22 100644 --- a/src/allmydata/test/test_client.py +++ b/src/allmydata/test/test_client.py @@ -8,7 +8,8 @@ import allmydata from allmydata import client from allmydata.storage_client import StorageFarmBroker from allmydata.introducer.client import IntroducerClient -from allmydata.util import base32 +from allmydata.util import base32, fileutil +from allmydata.interfaces import IFilesystemNode, IFileNode, IDirectoryNode from foolscap.api import flushEventualQueue import common_util as testutil @@ -234,3 +235,39 @@ class Run(unittest.TestCase, testutil.StallMixin): d.addCallback(_restart) return d +class NodeMaker(unittest.TestCase): + def test_maker(self): + basedir = "client/NodeMaker/maker" + fileutil.make_dirs(basedir) + f = open(os.path.join(basedir, "tahoe.cfg"), "w") + f.write(BASECONFIG) + f.close() + c = client.Client(basedir) + + n = c.create_node_from_uri("URI:CHK:6nmrpsubgbe57udnexlkiwzmlu:bjt7j6hshrlmadjyr7otq3dc24end5meo5xcr5xe5r663po6itmq:3:10:7277") + self.failUnless(IFilesystemNode.providedBy(n)) + self.failUnless(IFileNode.providedBy(n)) + self.failIf(IDirectoryNode.providedBy(n)) + self.failUnless(n.is_readonly()) + self.failIf(n.is_mutable()) + + n = c.create_node_from_uri("URI:DIR2:n6x24zd3seu725yluj75q5boaa:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq") + self.failUnless(IFilesystemNode.providedBy(n)) + self.failIf(IFileNode.providedBy(n)) + self.failUnless(IDirectoryNode.providedBy(n)) + self.failIf(n.is_readonly()) + self.failUnless(n.is_mutable()) + + n = c.create_node_from_uri("URI:DIR2-RO:b7sr5qsifnicca7cbk3rhrhbvq:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq") + self.failUnless(IFilesystemNode.providedBy(n)) + self.failIf(IFileNode.providedBy(n)) + self.failUnless(IDirectoryNode.providedBy(n)) + self.failUnless(n.is_readonly()) + self.failUnless(n.is_mutable()) + + future = "x-tahoe-crazy://future_cap_format." + n = c.create_node_from_uri(future) + self.failUnless(IFilesystemNode.providedBy(n)) + self.failIf(IFileNode.providedBy(n)) + self.failIf(IDirectoryNode.providedBy(n)) + self.failUnlessEqual(n.get_uri(), future) diff --git a/src/allmydata/test/test_dirnode.py b/src/allmydata/test/test_dirnode.py index 0321726a..555d647e 100644 --- a/src/allmydata/test/test_dirnode.py +++ b/src/allmydata/test/test_dirnode.py @@ -4,11 +4,12 @@ from zope.interface import implements from twisted.trial import unittest from twisted.internet import defer from allmydata import uri, dirnode +from allmydata.client import Client from allmydata.immutable import upload from allmydata.interfaces import IURI, IClient, IMutableFileNode, \ INewDirectoryURI, IReadonlyNewDirectoryURI, IFileNode, \ ExistingChildError, NoSuchChildError, \ - IDeepCheckResults, IDeepCheckAndRepairResults + IDeepCheckResults, IDeepCheckAndRepairResults, CannotPackUnknownNodeError from allmydata.mutable.filenode import MutableFileNode from allmydata.mutable.common import UncoordinatedWriteError from allmydata.util import hashutil, base32 @@ -17,6 +18,7 @@ from allmydata.test.common import make_chk_file_uri, make_mutable_file_uri, \ FakeDirectoryNode, create_chk_filenode, ErrorMixin from allmydata.test.no_network import GridTestMixin from allmydata.check_results import CheckResults, CheckAndRepairResults +from allmydata.unknown import UnknownNode import common_util as testutil # to test dirnode.py, we want to construct a tree of real DirectoryNodes that @@ -715,6 +717,83 @@ class Dirnode(unittest.TestCase, d.addErrback(self.explain_error) return d +class FakeMutableFile: + counter = 0 + def __init__(self, initial_contents=""): + self.data = initial_contents + counter = FakeMutableFile.counter + FakeMutableFile.counter += 1 + writekey = hashutil.ssk_writekey_hash(str(counter)) + fingerprint = hashutil.ssk_pubkey_fingerprint_hash(str(counter)) + self.uri = uri.WriteableSSKFileURI(writekey, fingerprint) + def get_uri(self): + return self.uri.to_string() + def download_best_version(self): + return defer.succeed(self.data) + def get_writekey(self): + return "writekey" + def is_readonly(self): + return False + def is_mutable(self): + return True + def modify(self, modifier): + self.data = modifier(self.data, None, True) + return defer.succeed(None) + +class FakeClient2(Client): + def __init__(self): + pass + def create_mutable_file(self, initial_contents=""): + return defer.succeed(FakeMutableFile(initial_contents)) + +class Dirnode2(unittest.TestCase, testutil.ShouldFailMixin): + def setUp(self): + self.client = FakeClient2() + + def test_from_future(self): + # create a dirnode that contains unknown URI types, and make sure we + # tolerate them properly. Since dirnodes aren't allowed to add + # unknown node types, we have to be tricky. + d = self.client.create_empty_dirnode() + future_writecap = "x-tahoe-crazy://I_am_from_the_future." + future_readcap = "x-tahoe-crazy-readonly://I_am_from_the_future." + future_node = UnknownNode(future_writecap, future_readcap) + def _then(n): + self._node = n + return n.set_node(u"future", future_node) + d.addCallback(_then) + + # we should be prohibited from adding an unknown URI to a directory, + # since we don't know how to diminish the cap to a readcap (for the + # dirnode's rocap slot), and we don't want to accidentally grant + # write access to a holder of the dirnode's readcap. + d.addCallback(lambda ign: + self.shouldFail(CannotPackUnknownNodeError, + "copy unknown", + "cannot pack unknown node as child add", + self._node.set_uri, u"add", future_writecap)) + d.addCallback(lambda ign: self._node.list()) + def _check(children): + self.failUnlessEqual(len(children), 1) + (fn, metadata) = children[u"future"] + self.failUnless(isinstance(fn, UnknownNode), fn) + self.failUnlessEqual(fn.get_uri(), future_writecap) + self.failUnlessEqual(fn.get_readonly_uri(), future_readcap) + # but we *should* be allowed to copy this node, because the + # UnknownNode contains all the information that was in the + # original directory (readcap and writecap), so we're preserving + # everything. + return self._node.set_node(u"copy", fn) + d.addCallback(_check) + d.addCallback(lambda ign: self._node.list()) + def _check2(children): + self.failUnlessEqual(len(children), 2) + (fn, metadata) = children[u"copy"] + self.failUnless(isinstance(fn, UnknownNode), fn) + self.failUnlessEqual(fn.get_uri(), future_writecap) + self.failUnlessEqual(fn.get_readonly_uri(), future_readcap) + return d + class DeepStats(unittest.TestCase): timeout = 240 # It takes longer than 120 seconds on Francois's arm box. def test_stats(self): diff --git a/src/allmydata/test/test_uri.py b/src/allmydata/test/test_uri.py index e81d466d..7ac63469 100644 --- a/src/allmydata/test/test_uri.py +++ b/src/allmydata/test/test_uri.py @@ -61,7 +61,7 @@ class Compare(unittest.TestCase): def test_is_uri(self): lit1 = uri.LiteralFileURI("some data").to_string() self.failUnless(uri.is_uri(lit1)) - self.failIf(uri.is_uri("this is not a uri")) + self.failIf(uri.is_uri(None)) class CHKFile(unittest.TestCase): def test_pack(self): @@ -175,10 +175,12 @@ class Extension(unittest.TestCase): readable = uri.unpack_extension_readable(ext) class Invalid(unittest.TestCase): - def test_create_invalid(self): - not_uri = "I am not a URI" - self.failUnlessRaises(TypeError, uri.from_string, not_uri) - + def test_from_future(self): + # any URI type that we don't recognize should be treated as unknown + future_uri = "I am a URI from the future. Whatever you do, don't " + u = uri.from_string(future_uri) + self.failUnless(isinstance(u, uri.UnknownURI)) + self.failUnlessEqual(u.to_string(), future_uri) class Constraint(unittest.TestCase): def test_constraint(self): diff --git a/src/allmydata/test/test_web.py b/src/allmydata/test/test_web.py index c6638ce7..6a1a8638 100644 --- a/src/allmydata/test/test_web.py +++ b/src/allmydata/test/test_web.py @@ -11,6 +11,7 @@ from allmydata import interfaces, uri, webish from allmydata.storage.shares import get_share_file from allmydata.storage_client import StorageFarmBroker from allmydata.immutable import upload, download +from allmydata.unknown import UnknownNode from allmydata.web import status, common from allmydata.scripts.debug import CorruptShareOptions, corrupt_share from allmydata.util import fileutil, base32 @@ -2781,6 +2782,73 @@ class Grid(GridTestMixin, WebErrorMixin, unittest.TestCase, ShouldFailMixin): d.addErrback(self.explain_web_error) return d + def test_unknown(self): + self.basedir = "web/Grid/unknown" + self.set_up_grid() + c0 = self.g.clients[0] + self.uris = {} + self.fileurls = {} + + future_writecap = "x-tahoe-crazy://I_am_from_the_future." + future_readcap = "x-tahoe-crazy-readonly://I_am_from_the_future." + # the future cap format may contain slashes, which must be tolerated + expected_info_url = "uri/%s?t=info" % urllib.quote(future_writecap, + safe="") + future_node = UnknownNode(future_writecap, future_readcap) + + d = c0.create_empty_dirnode() + def _stash_root_and_create_file(n): + self.rootnode = n + self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/" + self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/" + return self.rootnode.set_node(u"future", future_node) + d.addCallback(_stash_root_and_create_file) + # make sure directory listing tolerates unknown nodes + d.addCallback(lambda ign: self.GET(self.rooturl)) + def _check_html(res): + self.failUnlessIn("future", res) + # find the More Info link for "future", should be relative + mo = re.search(r'More Info', res) + info_url = mo.group(1) + self.failUnlessEqual(info_url, "future?t=info") + + d.addCallback(_check_html) + d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json")) + def _check_json(res, expect_writecap): + data = simplejson.loads(res) + self.failUnlessEqual(data[0], "dirnode") + f = data[1]["children"]["future"] + self.failUnlessEqual(f[0], "unknown") + if expect_writecap: + self.failUnlessEqual(f[1]["rw_uri"], future_writecap) + else: + self.failIfIn("rw_uri", f[1]) + self.failUnlessEqual(f[1]["ro_uri"], future_readcap) + self.failUnless("metadata" in f[1]) + d.addCallback(_check_json, expect_writecap=True) + d.addCallback(lambda ign: self.GET(expected_info_url)) + def _check_info(res, expect_readcap): + self.failUnlessIn("Object Type: unknown", res) + self.failUnlessIn(future_writecap, res) + if expect_readcap: + self.failUnlessIn(future_readcap, res) + self.failIfIn("Raw data as", res) + self.failIfIn("Directory writecap", res) + self.failIfIn("Checker Operations", res) + self.failIfIn("Mutable File Operations", res) + self.failIfIn("Directory Operations", res) + d.addCallback(_check_info, expect_readcap=False) + d.addCallback(lambda ign: self.GET(self.rooturl+"future?t=info")) + d.addCallback(_check_info, expect_readcap=True) + + # and make sure that a read-only version of the directory can be + # rendered too. This version will not have future_writecap + d.addCallback(lambda ign: self.GET(self.rourl)) + d.addCallback(_check_html) + d.addCallback(lambda ign: self.GET(self.rourl+"?t=json")) + d.addCallback(_check_json, expect_writecap=False) + return d + def test_deep_check(self): self.basedir = "web/Grid/deep_check" self.set_up_grid() @@ -2809,6 +2877,13 @@ class Grid(GridTestMixin, WebErrorMixin, unittest.TestCase, ShouldFailMixin): convergence=""))) d.addCallback(_stash_uri, "sick") + # this tests that deep-check and stream-manifest will ignore + # UnknownNode instances. Hopefully this will also cover deep-stats. + future_writecap = "x-tahoe-crazy://I_am_from_the_future." + future_readcap = "x-tahoe-crazy-readonly://I_am_from_the_future." + future_node = UnknownNode(future_writecap, future_readcap) + d.addCallback(lambda ign: self.rootnode.set_node(u"future",future_node)) + def _clobber_shares(ignored): self.delete_shares_numbered(self.uris["sick"], [0,1]) d.addCallback(_clobber_shares) @@ -2817,13 +2892,19 @@ class Grid(GridTestMixin, WebErrorMixin, unittest.TestCase, ShouldFailMixin): # root/good # root/small # root/sick + # root/future d.addCallback(self.CHECK, "root", "t=stream-deep-check") def _done(res): - units = [simplejson.loads(line) - for line in res.splitlines() - if line] - self.failUnlessEqual(len(units), 4+1) + try: + units = [simplejson.loads(line) + for line in res.splitlines() + if line] + except ValueError: + print "response is:", res + print "undecodeable line was '%s'" % line + raise + self.failUnlessEqual(len(units), 5+1) # should be parent-first u0 = units[0] self.failUnlessEqual(u0["path"], []) @@ -2844,8 +2925,27 @@ class Grid(GridTestMixin, WebErrorMixin, unittest.TestCase, ShouldFailMixin): self.failUnlessEqual(s["count-immutable-files"], 2) self.failUnlessEqual(s["count-literal-files"], 1) self.failUnlessEqual(s["count-directories"], 1) + self.failUnlessEqual(s["count-unknown"], 1) d.addCallback(_done) + d.addCallback(self.CHECK, "root", "t=stream-manifest") + def _check_manifest(res): + self.failUnless(res.endswith("\n")) + units = [simplejson.loads(t) for t in res[:-1].split("\n")] + self.failUnlessEqual(len(units), 5+1) + self.failUnlessEqual(units[-1]["type"], "stats") + first = units[0] + self.failUnlessEqual(first["path"], []) + self.failUnlessEqual(first["cap"], self.rootnode.get_uri()) + self.failUnlessEqual(first["type"], "directory") + stats = units[-1]["stats"] + self.failUnlessEqual(stats["count-immutable-files"], 2) + self.failUnlessEqual(stats["count-literal-files"], 1) + self.failUnlessEqual(stats["count-mutable-files"], 0) + self.failUnlessEqual(stats["count-immutable-files"], 2) + self.failUnlessEqual(stats["count-unknown"], 1) + d.addCallback(_check_manifest) + # now add root/subdir and root/subdir/grandchild, then make subdir # unrecoverable, then see what happens @@ -2866,6 +2966,7 @@ class Grid(GridTestMixin, WebErrorMixin, unittest.TestCase, ShouldFailMixin): # root/good # root/small # root/sick + # root/future # root/subdir [unrecoverable] # root/subdir/grandchild @@ -2888,7 +2989,7 @@ class Grid(GridTestMixin, WebErrorMixin, unittest.TestCase, ShouldFailMixin): error_line) self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback units = [simplejson.loads(line) for line in lines[:first_error]] - self.failUnlessEqual(len(units), 5) # includes subdir + self.failUnlessEqual(len(units), 6) # includes subdir last_unit = units[-1] self.failUnlessEqual(last_unit["path"], ["subdir"]) d.addCallback(_check_broken_manifest) @@ -2909,7 +3010,7 @@ class Grid(GridTestMixin, WebErrorMixin, unittest.TestCase, ShouldFailMixin): error_line) self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback units = [simplejson.loads(line) for line in lines[:first_error]] - self.failUnlessEqual(len(units), 5) # includes subdir + self.failUnlessEqual(len(units), 6) # includes subdir last_unit = units[-1] self.failUnlessEqual(last_unit["path"], ["subdir"]) r = last_unit["check-results"]["results"] diff --git a/src/allmydata/unknown.py b/src/allmydata/unknown.py new file mode 100644 index 00000000..c64e6368 --- /dev/null +++ b/src/allmydata/unknown.py @@ -0,0 +1,25 @@ +from zope.interface import implements +from twisted.internet import defer +from allmydata.interfaces import IFilesystemNode + +class UnknownNode: + implements(IFilesystemNode) + def __init__(self, writecap, readcap): + assert writecap is None or isinstance(writecap, str) + self.writecap = writecap + assert readcap is None or isinstance(readcap, str) + self.readcap = readcap + def get_uri(self): + return self.writecap + def get_readonly_uri(self): + return self.readcap + def get_storage_index(self): + return None + def get_verify_cap(self): + return None + def get_repair_cap(self): + return None + def check(self, monitor, verify, add_lease): + return defer.succeed(None) + def check_and_repair(self, monitor, verify, add_lease): + return defer.succeed(None) diff --git a/src/allmydata/uri.py b/src/allmydata/uri.py index 297fe93c..9be92a41 100644 --- a/src/allmydata/uri.py +++ b/src/allmydata/uri.py @@ -428,10 +428,16 @@ class NewDirectoryURIVerifier(_NewDirectoryBaseURI): def get_filenode_uri(self): return self._filenode_uri - +class UnknownURI: + def __init__(self, uri): + self._uri = uri + def to_string(self): + return self._uri def from_string(s): - if s.startswith('URI:CHK:'): + if not isinstance(s, str): + raise TypeError("unknown URI type: %s.." % str(s)[:100]) + elif s.startswith('URI:CHK:'): return CHKFileURI.init_from_string(s) elif s.startswith('URI:CHK-Verifier:'): return CHKFileVerifierURI.init_from_string(s) @@ -449,8 +455,7 @@ def from_string(s): return ReadonlyNewDirectoryURI.init_from_string(s) elif s.startswith('URI:DIR2-Verifier:'): return NewDirectoryURIVerifier.init_from_string(s) - else: - raise TypeError("unknown URI type: %s.." % s[:12]) + return UnknownURI(s) registerAdapter(from_string, str, IURI) diff --git a/src/allmydata/web/directory.py b/src/allmydata/web/directory.py index 5c83b51b..1bfd3826 100644 --- a/src/allmydata/web/directory.py +++ b/src/allmydata/web/directory.py @@ -15,7 +15,7 @@ from foolscap.api import fireEventually from allmydata.util import base32, time_format from allmydata.uri import from_string_dirnode from allmydata.interfaces import IDirectoryNode, IFileNode, IMutableFileNode, \ - ExistingChildError, NoSuchChildError + IFilesystemNode, ExistingChildError, NoSuchChildError from allmydata.monitor import Monitor, OperationCancelledError from allmydata import dirnode from allmydata.web.common import text_plain, WebError, \ @@ -46,7 +46,7 @@ def make_handler_for(node, client, parentnode=None, name=None): return FileNodeHandler(client, node, parentnode, name) if IDirectoryNode.providedBy(node): return DirectoryNodeHandler(client, node, parentnode, name) - raise WebError("Cannot provide handler for '%s'" % node) + return UnknownNodeHandler(client, node, parentnode, name) class DirectoryNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin): addSlash = True @@ -617,11 +617,9 @@ class DirectoryAsHTML(rend.Page): times.append("m: " + mtime) ctx.fillSlots("times", times) - assert (IFileNode.providedBy(target) - or IDirectoryNode.providedBy(target) - or IMutableFileNode.providedBy(target)), target - - quoted_uri = urllib.quote(target.get_uri()) + assert IFilesystemNode.providedBy(target), target + writecap = target.get_uri() or "" + quoted_uri = urllib.quote(writecap, safe="") # escape slashes too if IMutableFileNode.providedBy(target): # to prevent javascript in displayed .html files from stealing a @@ -650,7 +648,7 @@ class DirectoryAsHTML(rend.Page): elif IDirectoryNode.providedBy(target): # directory - uri_link = "%s/uri/%s/" % (root, urllib.quote(target.get_uri())) + uri_link = "%s/uri/%s/" % (root, urllib.quote(writecap)) ctx.fillSlots("filename", T.a(href=uri_link)[html.escape(name)]) if target.is_readonly(): @@ -661,6 +659,15 @@ class DirectoryAsHTML(rend.Page): ctx.fillSlots("size", "-") info_link = "%s/uri/%s/?t=info" % (root, quoted_uri) + else: + # unknown + ctx.fillSlots("filename", html.escape(name)) + ctx.fillSlots("type", "?") + ctx.fillSlots("size", "-") + # use a directory-relative info link, so we can extract both the + # writecap and the readcap + info_link = "%s?t=info" % urllib.quote(name) + ctx.fillSlots("info", T.a(href=info_link)["More Info"]) return ctx.tag @@ -727,20 +734,23 @@ def DirectoryJSONMetadata(ctx, dirnode): def _got(children): kids = {} for name, (childnode, metadata) in children.iteritems(): - if childnode.is_readonly(): - rw_uri = None - ro_uri = childnode.get_uri() - else: - rw_uri = childnode.get_uri() - ro_uri = childnode.get_readonly_uri() + assert IFilesystemNode.providedBy(childnode), childnode + rw_uri = childnode.get_uri() + ro_uri = childnode.get_readonly_uri() + if (IDirectoryNode.providedBy(childnode) + or IFileNode.providedBy(childnode)): + if childnode.is_readonly(): + rw_uri = None if IFileNode.providedBy(childnode): kiddata = ("filenode", {'size': childnode.get_size(), - 'metadata': metadata, + 'mutable': childnode.is_mutable(), }) + elif IDirectoryNode.providedBy(childnode): + kiddata = ("dirnode", {'mutable': childnode.is_mutable(), + }) else: - assert IDirectoryNode.providedBy(childnode), (childnode, - children,) - kiddata = ("dirnode", {'metadata': metadata}) + kiddata = ("unknown", {}) + kiddata[1]["metadata"] = metadata if ro_uri: kiddata[1]["ro_uri"] = ro_uri if rw_uri: @@ -748,7 +758,6 @@ def DirectoryJSONMetadata(ctx, dirnode): verifycap = childnode.get_verify_cap() if verifycap: kiddata[1]['verify_uri'] = verifycap.to_string() - kiddata[1]['mutable'] = childnode.is_mutable() kids[name] = kiddata if dirnode.is_readonly(): drw_uri = None @@ -879,13 +888,16 @@ class ManifestResults(rend.Page, ReloadMixin): ctx.fillSlots("path", self.slashify_path(path)) root = get_root(ctx) # TODO: we need a clean consistent way to get the type of a cap string - if cap.startswith("URI:CHK") or cap.startswith("URI:SSK"): - nameurl = urllib.quote(path[-1].encode("utf-8")) - uri_link = "%s/file/%s/@@named=/%s" % (root, urllib.quote(cap), - nameurl) + if cap: + if cap.startswith("URI:CHK") or cap.startswith("URI:SSK"): + nameurl = urllib.quote(path[-1].encode("utf-8")) + uri_link = "%s/file/%s/@@named=/%s" % (root, urllib.quote(cap), + nameurl) + else: + uri_link = "%s/uri/%s" % (root, urllib.quote(cap, safe="")) + ctx.fillSlots("cap", T.a(href=uri_link)[cap]) else: - uri_link = "%s/uri/%s" % (root, urllib.quote(cap)) - ctx.fillSlots("cap", T.a(href=uri_link)[cap]) + ctx.fillSlots("cap", "") return ctx.tag class DeepSizeResults(rend.Page): @@ -951,8 +963,10 @@ class ManifestStreamer(dirnode.DeepStats): if IDirectoryNode.providedBy(node): d["type"] = "directory" - else: + elif IFileNode.providedBy(node): d["type"] = "file" + else: + d["type"] = "unknown" v = node.get_verify_cap() if v: @@ -1058,3 +1072,19 @@ class DeepCheckStreamer(dirnode.DeepStats): assert "\n" not in j self.req.write(j+"\n") return "" + +class UnknownNodeHandler(RenderMixin, rend.Page): + + def __init__(self, client, node, parentnode=None, name=None): + rend.Page.__init__(self) + assert node + self.node = node + + def render_GET(self, ctx): + req = IRequest(ctx) + t = get_arg(req, "t", "").strip() + if t == "info": + return MoreInfo(self.node) + raise WebError("GET unknown: can only do t=info, not t=%s" % t) + + diff --git a/src/allmydata/web/filenode.py b/src/allmydata/web/filenode.py index 727d9b68..05c12279 100644 --- a/src/allmydata/web/filenode.py +++ b/src/allmydata/web/filenode.py @@ -6,10 +6,11 @@ from twisted.internet import defer from nevow import url, rend from nevow.inevow import IRequest -from allmydata.interfaces import ExistingChildError +from allmydata.interfaces import ExistingChildError, CannotPackUnknownNodeError from allmydata.monitor import Monitor from allmydata.immutable.upload import FileHandle from allmydata.immutable.filenode import LiteralFileNode +from allmydata.unknown import UnknownNode from allmydata.util import log, base32 from allmydata.web.common import text_plain, WebError, RenderMixin, \ @@ -55,7 +56,14 @@ class ReplaceMeMixin: def replace_me_with_a_childcap(self, req, client, replace): req.content.seek(0) childcap = req.content.read() - childnode = client.create_node_from_uri(childcap) + childnode = client.create_node_from_uri(childcap, childcap+"readonly") + if isinstance(childnode, UnknownNode): + # don't be willing to pack unknown nodes: we might accidentally + # put some write-authority into the rocap slot because we don't + # know how to diminish the URI they gave us. We don't even know + # if they gave us a readcap or a writecap. + msg = "cannot attach unknown node as child %s" % str(self.name) + raise CannotPackUnknownNodeError(msg) d = self.parentnode.set_node(self.name, childnode, overwrite=replace) d.addCallback(lambda res: childnode.get_uri()) return d diff --git a/src/allmydata/web/info.py b/src/allmydata/web/info.py index e7c42e60..edaab44b 100644 --- a/src/allmydata/web/info.py +++ b/src/allmydata/web/info.py @@ -6,7 +6,7 @@ from nevow import rend, tags as T from nevow.inevow import IRequest from allmydata.util import base32 -from allmydata.interfaces import IDirectoryNode +from allmydata.interfaces import IDirectoryNode, IFileNode from allmydata.web.common import getxmlfile from allmydata.mutable.common import UnrecoverableFileError # TODO: move @@ -21,14 +21,16 @@ class MoreInfo(rend.Page): def get_type(self): node = self.original - si = node.get_storage_index() if IDirectoryNode.providedBy(node): return "directory" - if si: - if node.is_mutable(): - return "mutable file" - return "immutable file" - return "LIT file" + if IFileNode.providedBy(node): + si = node.get_storage_index() + if si: + if node.is_mutable(): + return "mutable file" + return "immutable file" + return "LIT file" + return "unknown" def render_title(self, ctx, data): node = self.original @@ -55,11 +57,15 @@ class MoreInfo(rend.Page): si = node.get_storage_index() if IDirectoryNode.providedBy(node): d = node._node.get_size_of_best_version() - elif node.is_mutable(): - d = node.get_size_of_best_version() + elif IFileNode.providedBy(node): + if node.is_mutable(): + d = node.get_size_of_best_version() + else: + # for immutable files and LIT files, we get the size from the + # URI + d = defer.succeed(node.get_size()) else: - # for immutable files and LIT files, we get the size from the URI - d = defer.succeed(node.get_size()) + d = defer.succeed("?") def _handle_unrecoverable(f): f.trap(UnrecoverableFileError) return "?" @@ -92,15 +98,22 @@ class MoreInfo(rend.Page): node = self.original if IDirectoryNode.providedBy(node): node = node._node - if node.is_readonly(): + if ((IDirectoryNode.providedBy(node) or IFileNode.providedBy(node)) + and node.is_readonly()): return "" - return ctx.tag[node.get_uri()] + writecap = node.get_uri() + if not writecap: + return "" + return ctx.tag[writecap] def render_file_readcap(self, ctx, data): node = self.original if IDirectoryNode.providedBy(node): node = node._node - return ctx.tag[node.get_readonly_uri()] + readcap = node.get_readonly_uri() + if not readcap: + return "" + return ctx.tag[readcap] def render_file_verifycap(self, ctx, data): node = self.original @@ -122,10 +135,14 @@ class MoreInfo(rend.Page): node = self.original if IDirectoryNode.providedBy(node): node = node._node + elif IFileNode.providedBy(node): + pass + else: + return "" root = self.get_root(ctx) quoted_uri = urllib.quote(node.get_uri()) text_plain_url = "%s/file/%s/@@named=/raw.txt" % (root, quoted_uri) - return ctx.tag[text_plain_url] + return T.li["Raw data as ", T.a(href=text_plain_url)["text/plain"]] def render_is_checkable(self, ctx, data): node = self.original @@ -167,7 +184,8 @@ class MoreInfo(rend.Page): node = self.original if IDirectoryNode.providedBy(node): return "" - if node.is_mutable() and not node.is_readonly(): + if (IFileNode.providedBy(node) + and node.is_mutable() and not node.is_readonly()): return ctx.tag return "" diff --git a/src/allmydata/web/info.xhtml b/src/allmydata/web/info.xhtml index 5e0a09c1..9237f38f 100644 --- a/src/allmydata/web/info.xhtml +++ b/src/allmydata/web/info.xhtml @@ -45,7 +45,7 @@
  • JSON
    • -
  • Raw data as text/plain
    • +
  • -- 2.45.2