From dad354b2753d2ffa115b6060ab836fe424221ee3 Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Thu, 13 Oct 2011 09:29:51 -0700 Subject: [PATCH] webapi: handle format=, remove mutable-type= * fix CLI commands (put, mkdir) to send format=, not mutable-type= * fix tests * test_cli: fix tests that observe t=json output, don't ignore failures in 'tahoe put' * fix handling of version= to make it easier to use the default * interpret ?mutable=true&format=MDMF as MDMF, not SDMF --- src/allmydata/client.py | 2 +- src/allmydata/nodemaker.py | 8 ++--- src/allmydata/scripts/tahoe_mkdir.py | 4 +-- src/allmydata/scripts/tahoe_put.py | 8 +++-- src/allmydata/test/test_cli.py | 17 ++++++----- src/allmydata/test/test_web.py | 23 ++++++++++---- src/allmydata/web/common.py | 28 ++++++++++++----- src/allmydata/web/directory.py | 45 +++++++++------------------- src/allmydata/web/filenode.py | 36 ++++++++++------------ src/allmydata/web/root.py | 22 +++++--------- src/allmydata/web/unlinked.py | 36 +++++++++++----------- 11 files changed, 113 insertions(+), 116 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index ed0ff9a0..aa55da7a 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -499,7 +499,7 @@ class Client(node.Node, pollmixin.PollMixin): # may get an opaque node if there were any problems. return self.nodemaker.create_from_cap(write_uri, read_uri, deep_immutable=deep_immutable, name=name) - def create_dirnode(self, initial_children={}, version=SDMF_VERSION): + def create_dirnode(self, initial_children={}, version=None): d = self.nodemaker.create_new_mutable_directory(initial_children, version=version) return d diff --git a/src/allmydata/nodemaker.py b/src/allmydata/nodemaker.py index 5d46d893..6fdaf479 100644 --- a/src/allmydata/nodemaker.py +++ b/src/allmydata/nodemaker.py @@ -109,8 +109,9 @@ class NodeMaker: return self._create_dirnode(filenode) return None - def create_mutable_file(self, contents=None, keysize=None, - version=SDMF_VERSION): + def create_mutable_file(self, contents=None, keysize=None, version=None): + if version is None: + version = SDMF_VERSION n = MutableFileNode(self.storage_broker, self.secret_holder, self.default_encoding_parameters, self.history) d = self.key_generator.generate(keysize) @@ -118,8 +119,7 @@ class NodeMaker: d.addCallback(lambda res: n) return d - def create_new_mutable_directory(self, initial_children={}, - version=SDMF_VERSION): + def create_new_mutable_directory(self, initial_children={}, version=None): # initial_children must have metadata (i.e. {} instead of None) for (name, (node, metadata)) in initial_children.iteritems(): precondition(isinstance(metadata, dict), diff --git a/src/allmydata/scripts/tahoe_mkdir.py b/src/allmydata/scripts/tahoe_mkdir.py index 50ed0672..cc0374bf 100644 --- a/src/allmydata/scripts/tahoe_mkdir.py +++ b/src/allmydata/scripts/tahoe_mkdir.py @@ -23,7 +23,7 @@ def mkdir(options): # create a new unlinked directory url = nodeurl + "uri?t=mkdir" if options["mutable-type"]: - url += "&mutable-type=%s" % urllib.quote(options['mutable-type']) + url += "&format=%s" % urllib.quote(options['mutable-type']) resp = do_http("POST", url) rc = check_http_error(resp, stderr) if rc: @@ -40,7 +40,7 @@ def mkdir(options): url = nodeurl + "uri/%s/%s?t=mkdir" % (urllib.quote(rootcap), urllib.quote(path)) if options['mutable-type']: - url += "&mutable-type=%s" % urllib.quote(options['mutable-type']) + url += "&format=%s" % urllib.quote(options['mutable-type']) resp = do_http("POST", url) check_http_error(resp, stderr) diff --git a/src/allmydata/scripts/tahoe_put.py b/src/allmydata/scripts/tahoe_put.py index a92cfc53..3a3b3945 100644 --- a/src/allmydata/scripts/tahoe_put.py +++ b/src/allmydata/scripts/tahoe_put.py @@ -63,11 +63,15 @@ def put(options): else: # unlinked upload url = nodeurl + "uri" + + file_format = None if mutable: - url += "?mutable=true" + file_format = "SDMF" if mutable_type: assert mutable - url += "&mutable-type=%s" % mutable_type + file_format = mutable_type.upper() + if file_format: + url += "?format=%s" % file_format if from_file: infileobj = open(os.path.expanduser(from_file), "rb") diff --git a/src/allmydata/test/test_cli.py b/src/allmydata/test/test_cli.py index fb6f143a..123df201 100644 --- a/src/allmydata/test/test_cli.py +++ b/src/allmydata/test/test_cli.py @@ -1145,7 +1145,7 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): def _check_mdmf_json(self, (rc, json, err)): self.failUnlessEqual(rc, 0) self.failUnlessEqual(err, "") - self.failUnlessIn('"mutable-type": "mdmf"', json) + self.failUnlessIn('"format": "mdmf"', json) # We also want a valid MDMF cap to be in the json. self.failUnlessIn("URI:MDMF", json) self.failUnlessIn("URI:MDMF-RO", json) @@ -1154,7 +1154,7 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): def _check_sdmf_json(self, (rc, json, err)): self.failUnlessEqual(rc, 0) self.failUnlessEqual(err, "") - self.failUnlessIn('"mutable-type": "sdmf"', json) + self.failUnlessIn('"format": "sdmf"', json) # We also want to see the appropriate SDMF caps. self.failUnlessIn("URI:SSK", json) self.failUnlessIn("URI:SSK-RO", json) @@ -1171,6 +1171,9 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): def _put_and_ls(ign, mutable_type, filename): d2 = self.do_cli("put", "--mutable", "--mutable-type="+mutable_type, fn1, filename) + def _dont_fail((rc, out, err)): + self.failUnlessEqual(rc, 0) + d2.addCallback(_dont_fail) d2.addCallback(lambda ign: self.do_cli("ls", "--json", filename)) return d2 @@ -1615,8 +1618,8 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnlessIn(self._sdmf_uri, out) self.failUnlessIn(self._sdmf_readonly_uri, out) self.failUnlessIn(self._imm_uri, out) - self.failUnlessIn('"mutable-type": "sdmf"', out) - self.failUnlessIn('"mutable-type": "mdmf"', out) + self.failUnlessIn('"format": "sdmf"', out) + self.failUnlessIn('"format": "mdmf"', out) d.addCallback(_got_json) return d @@ -3315,7 +3318,7 @@ class Mkdir(GridTestMixin, CLITestMixin, unittest.TestCase): d2.addCallback(lambda ign: self.do_cli("ls", "--json", dirname)) d2.addCallback(_check, uri_prefix) d2.addCallback(lambda ign: self.do_cli("ls", "--json", self._filecap)) - d2.addCallback(_check, '"mutable-type": "%s"' % (mutable_type.lower(),)) + d2.addCallback(_check, '"format": "%s"' % (mutable_type.lower(),)) return d2 d.addCallback(_mkdir, "sdmf", "URI:DIR2", "tahoe:foo") @@ -3345,13 +3348,13 @@ class Mkdir(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(_stash_dircap) d.addCallback(lambda res: self.do_cli("ls", "--json", self._filecap)) - d.addCallback(_check, '"mutable-type": "sdmf"') + d.addCallback(_check, '"format": "sdmf"') d.addCallback(lambda res: self.do_cli("mkdir", "--mutable-type=mdmf")) d.addCallback(_check, "URI:DIR2-MDMF") d.addCallback(_stash_dircap) d.addCallback(lambda res: self.do_cli("ls", "--json", self._filecap)) - d.addCallback(_check, '"mutable-type": "mdmf"') + d.addCallback(_check, '"format": "mdmf"') return d def test_mkdir_bad_mutable_type(self): diff --git a/src/allmydata/test/test_web.py b/src/allmydata/test/test_web.py index c2c168f5..ceb468b9 100644 --- a/src/allmydata/test/test_web.py +++ b/src/allmydata/test/test_web.py @@ -2123,17 +2123,27 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi filename = format + ".txt" d = self.POST("/uri?t=upload&format=" + format, file=(filename, self.NEWFILE_CONTENTS * 300000)) - def _got_filecap(filecap): - self.failUnless(filecap.startswith(uri_prefix)) + def _got_results(results): + if format.upper() in ("SDMF", "MDMF"): + # webapi.rst says this returns a filecap + filecap = results + else: + # for immutable, it returns an "upload results page", and + # the filecap is buried inside + line = [l for l in results.split("\n") if "URI: " in l][0] + mo = re.search(r'([^<]+)', line) + filecap = mo.group(1) + self.failUnless(filecap.startswith(uri_prefix), + (uri_prefix, filecap)) return self.GET("/uri/%s?t=json" % filecap) - d.addCallback(_got_filecap) + d.addCallback(_got_results) def _got_json(json): data = simplejson.loads(json) data = data[1] self.failUnlessIn("format", data) - self.failUnlessEqual(data["format"], format) + self.failUnlessEqual(data["format"], format.lower()) d.addCallback(_got_json) - + return d d = defer.succeed(None) d.addCallback(_check_upload_unlinked, "chk", "URI:CHK") d.addCallback(_check_upload_unlinked, "CHK", "URI:CHK") @@ -2165,8 +2175,9 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi data = simplejson.loads(json) data = data[1] self.failUnlessIn("format", data) - self.failUnlessEqual(data["format"], format) + self.failUnlessEqual(data["format"], format.lower()) d.addCallback(_got_json) + return d d = defer.succeed(None) d.addCallback(_check_upload, "chk", "URI:CHK") diff --git a/src/allmydata/web/common.py b/src/allmydata/web/common.py index 0a14bf30..9bfed072 100644 --- a/src/allmydata/web/common.py +++ b/src/allmydata/web/common.py @@ -34,17 +34,29 @@ def parse_replace_arg(replace): return boolean_of_arg(replace) -def parse_mutable_type_arg(arg): +def get_format(req, default="CHK"): + arg = get_arg(req, "format", None) if not arg: - return None # interpreted by the caller as "let the nodemaker decide" + if boolean_of_arg(get_arg(req, "mutable", "false")): + return "SDMF" + return default + if arg.upper() == "CHK": + return "CHK" + elif arg.upper() == "SDMF": + return "SDMF" + elif arg.upper() == "MDMF": + return "MDMF" + else: + raise WebError("Unknown format: %s, I know CHK, SDMF, MDMF" % arg, + http.BAD_REQUEST) - arg = arg.lower() - if arg == "mdmf": - return MDMF_VERSION - elif arg == "sdmf": +def get_mutable_type(file_format): # accepts result of get_format() + if file_format == "SDMF": return SDMF_VERSION - - return "invalid" + elif file_format == "MDMF": + return MDMF_VERSION + else: + return None def parse_offset_arg(offset): diff --git a/src/allmydata/web/directory.py b/src/allmydata/web/directory.py index c1ecec6c..5c58c495 100644 --- a/src/allmydata/web/directory.py +++ b/src/allmydata/web/directory.py @@ -25,7 +25,7 @@ from allmydata.web.common import text_plain, WebError, \ boolean_of_arg, get_arg, get_root, parse_replace_arg, \ should_create_intermediate_directories, \ getxmlfile, RenderMixin, humanize_failure, convert_children_json, \ - parse_mutable_type_arg + get_format, get_mutable_type from allmydata.web.filenode import ReplaceMeMixin, \ FileNodeHandler, PlaceHolderNodeHandler from allmydata.web.check_results import CheckResults, \ @@ -107,17 +107,12 @@ class DirectoryNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin): kids_json = req.content.read() kids = convert_children_json(self.client.nodemaker, kids_json) + file_format = get_format(req, None) mutable = True + mt = get_mutable_type(file_format) if t == "mkdir-immutable": mutable = False - mt = None - if mutable: - arg = get_arg(req, "mutable-type", None) - mt = parse_mutable_type_arg(arg) - if mt is "invalid": - raise WebError("Unknown type: %s" % arg, - http.BAD_REQUEST) d = self.node.create_subdirectory(name, kids, mutable=mutable, mutable_version=mt) @@ -251,15 +246,9 @@ class DirectoryNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin): name = name.decode("utf-8") replace = boolean_of_arg(get_arg(req, "replace", "true")) kids = {} - arg = get_arg(req, "mutable-type", None) - mt = parse_mutable_type_arg(arg) - if mt is not None and mt is not "invalid": - d = self.node.create_subdirectory(name, kids, overwrite=replace, + mt = get_mutable_type(get_format(req, None)) + d = self.node.create_subdirectory(name, kids, overwrite=replace, mutable_version=mt) - elif mt is "invalid": - raise WebError("Unknown type: %s" % arg, http.BAD_REQUEST) - else: - d = self.node.create_subdirectory(name, kids, overwrite=replace) d.addCallback(lambda child: child.get_uri()) # TODO: urlencode return d @@ -275,15 +264,9 @@ class DirectoryNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin): req.content.seek(0) kids_json = req.content.read() kids = convert_children_json(self.client.nodemaker, kids_json) - arg = get_arg(req, "mutable-type", None) - mt = parse_mutable_type_arg(arg) - if mt is not None and mt is not "invalid": - d = self.node.create_subdirectory(name, kids, overwrite=False, - mutable_version=mt) - elif mt is "invalid": - raise WebError("Unknown type: %s" % arg) - else: - d = self.node.create_subdirectory(name, kids, overwrite=False) + mt = get_mutable_type(get_format(req, None)) + d = self.node.create_subdirectory(name, kids, overwrite=False, + mutable_version=mt) d.addCallback(lambda child: child.get_uri()) # TODO: urlencode return d @@ -882,16 +865,16 @@ def DirectoryJSONMetadata(ctx, dirnode): kiddata = ("filenode", {'size': childnode.get_size(), 'mutable': childnode.is_mutable(), }) - if childnode.is_mutable() and \ - childnode.get_version() is not None: + if childnode.is_mutable(): mutable_type = childnode.get_version() assert mutable_type in (SDMF_VERSION, MDMF_VERSION) - if mutable_type == MDMF_VERSION: - mutable_type = "mdmf" + file_format = "mdmf" else: - mutable_type = "sdmf" - kiddata[1]['mutable-type'] = mutable_type + file_format = "sdmf" + else: + file_format = "chk" + kiddata[1]['format'] = file_format elif IDirectoryNode.providedBy(childnode): kiddata = ("dirnode", {'mutable': childnode.is_mutable()}) diff --git a/src/allmydata/web/filenode.py b/src/allmydata/web/filenode.py index ef8d3bae..89db41bf 100644 --- a/src/allmydata/web/filenode.py +++ b/src/allmydata/web/filenode.py @@ -18,7 +18,7 @@ from allmydata.blacklist import FileProhibited, ProhibitedNode from allmydata.web.common import text_plain, WebError, RenderMixin, \ boolean_of_arg, get_arg, should_create_intermediate_directories, \ MyExceptionHandler, parse_replace_arg, parse_offset_arg, \ - parse_mutable_type_arg + get_format, get_mutable_type from allmydata.web.check_results import CheckResults, \ CheckAndRepairResults, LiteralCheckResults from allmydata.web.info import MoreInfo @@ -26,13 +26,9 @@ from allmydata.web.info import MoreInfo class ReplaceMeMixin: def replace_me_with_a_child(self, req, client, replace): # a new file is being uploaded in our place. - mutable = boolean_of_arg(get_arg(req, "mutable", "false")) - if mutable: - arg = get_arg(req, "mutable-type", None) - mutable_type = parse_mutable_type_arg(arg) - if mutable_type is "invalid": - raise WebError("Unknown type: %s" % arg, http.BAD_REQUEST) - + file_format = get_format(req, "CHK") + if file_format in ("SDMF", "MDMF"): + mutable_type = get_mutable_type(file_format) data = MutableFileHandle(req.content) d = client.create_mutable_file(data, version=mutable_type) def _uploaded(newnode): @@ -42,6 +38,7 @@ class ReplaceMeMixin: return d2 d.addCallback(_uploaded) else: + assert file_format == "CHK" uploadable = FileHandle(req.content, convergence=client.convergence) d = self.parentnode.add_file(self.name, uploadable, overwrite=replace) @@ -70,15 +67,10 @@ class ReplaceMeMixin: def replace_me_with_a_formpost(self, req, client, replace): # create a new file, maybe mutable, maybe immutable - mutable = boolean_of_arg(get_arg(req, "mutable", "false")) - - # create an immutable file + file_format = get_format(req, "CHK") contents = req.fields["file"] - if mutable: - arg = get_arg(req, "mutable-type", None) - mutable_type = parse_mutable_type_arg(arg) - if mutable_type is "invalid": - raise WebError("Unknown type: %s" % arg, http.BAD_REQUEST) + if file_format in ("SDMF", "MDMF"): + mutable_type = get_mutable_type(file_format) uploadable = MutableFileHandle(contents.file) d = client.create_mutable_file(uploadable, version=mutable_type) def _uploaded(newnode): @@ -518,14 +510,16 @@ def FileJSONMetadata(ctx, filenode, edge_metadata): if edge_metadata is not None: data[1]['metadata'] = edge_metadata - if filenode.is_mutable() and filenode.get_version() is not None: + if filenode.is_mutable(): mutable_type = filenode.get_version() - assert mutable_type in (MDMF_VERSION, SDMF_VERSION) + assert mutable_type in (SDMF_VERSION, MDMF_VERSION) if mutable_type == MDMF_VERSION: - mutable_type = "mdmf" + file_format = "mdmf" else: - mutable_type = "sdmf" - data[1]['mutable-type'] = mutable_type + file_format = "sdmf" + else: + file_format = "chk" + data[1]['format'] = file_format return text_plain(simplejson.dumps(data, indent=1) + "\n", ctx) diff --git a/src/allmydata/web/root.py b/src/allmydata/web/root.py index b669d50e..47a6e9bf 100644 --- a/src/allmydata/web/root.py +++ b/src/allmydata/web/root.py @@ -16,7 +16,7 @@ from allmydata.interfaces import IFileNode from allmydata.web import filenode, directory, unlinked, status, operations from allmydata.web import reliability, storage from allmydata.web.common import abbreviate_size, getxmlfile, WebError, \ - get_arg, RenderMixin, boolean_of_arg, parse_mutable_type_arg + get_arg, RenderMixin, get_format, get_mutable_type class URIHandler(RenderMixin, rend.Page): @@ -45,14 +45,9 @@ class URIHandler(RenderMixin, rend.Page): # "PUT /uri?t=mkdir" to create an unlinked directory t = get_arg(req, "t", "").strip() if t == "": - mutable = boolean_of_arg(get_arg(req, "mutable", "false").strip()) - if mutable: - arg = get_arg(req, "mutable-type", None) - version = parse_mutable_type_arg(arg) - if version == "invalid": - errmsg = "Unknown type: %s" % arg - raise WebError(errmsg, http.BAD_REQUEST) - + file_format = get_format(req, "CHK") + if file_format in ("SDMF", "MDMF"): + version = get_mutable_type(file_format) return unlinked.PUTUnlinkedSSK(req, self.client, version) else: return unlinked.PUTUnlinkedCHK(req, self.client) @@ -69,12 +64,9 @@ class URIHandler(RenderMixin, rend.Page): req = IRequest(ctx) t = get_arg(req, "t", "").strip() if t in ("", "upload"): - mutable = bool(get_arg(req, "mutable", "").strip()) - if mutable: - arg = get_arg(req, "mutable-type", None) - version = parse_mutable_type_arg(arg) - if version is "invalid": - raise WebError("Unknown type: %s" % arg, http.BAD_REQUEST) + file_format = get_format(req) + if file_format in ("SDMF", "MDMF"): + version = get_mutable_type(file_format) return unlinked.POSTUnlinkedSSK(req, self.client, version) else: return unlinked.POSTUnlinkedCHK(req, self.client) diff --git a/src/allmydata/web/unlinked.py b/src/allmydata/web/unlinked.py index 482bd285..94dd0621 100644 --- a/src/allmydata/web/unlinked.py +++ b/src/allmydata/web/unlinked.py @@ -6,7 +6,7 @@ from nevow import rend, url, tags as T from allmydata.immutable.upload import FileHandle from allmydata.mutable.publish import MutableFileHandle from allmydata.web.common import getxmlfile, get_arg, boolean_of_arg, \ - convert_children_json, WebError, parse_mutable_type_arg + convert_children_json, WebError, get_format, get_mutable_type from allmydata.web import status def PUTUnlinkedCHK(req, client): @@ -27,15 +27,14 @@ def PUTUnlinkedSSK(req, client, version): def PUTUnlinkedCreateDirectory(req, client): # "PUT /uri?t=mkdir", to create an unlinked directory. - arg = get_arg(req, "mutable-type", None) - mt = parse_mutable_type_arg(arg) - if mt is not None and mt is not "invalid": - d = client.create_dirnode(version=mt) - elif mt is "invalid": - msg = "Unknown type: %s" % arg - raise WebError(msg, http.BAD_REQUEST) - else: - d = client.create_dirnode() + file_format = get_format(req, None) + if file_format == "CHK": + raise WebError("format=CHK not currently accepted for PUT /uri?t=mkdir", + http.BAD_REQUEST) + mt = None + if file_format: + mt = get_mutable_type(file_format) + d = client.create_dirnode(version=mt) d.addCallback(lambda dirnode: dirnode.get_uri()) # XXX add redirect_to_result return d @@ -112,15 +111,14 @@ def POSTUnlinkedCreateDirectory(req, client): raise WebError("t=mkdir does not accept children=, " "try t=mkdir-with-children instead", http.BAD_REQUEST) - arg = get_arg(req, "mutable-type", None) - mt = parse_mutable_type_arg(arg) - if mt is not None and mt is not "invalid": - d = client.create_dirnode(version=mt) - elif mt is "invalid": - msg = "Unknown type: %s" % arg - raise WebError(msg, http.BAD_REQUEST) - else: - d = client.create_dirnode() + file_format = get_format(req, None) + if file_format == "CHK": + raise WebError("format=CHK not currently accepted for POST /uri?t=mkdir", + http.BAD_REQUEST) + mt = None + if file_format: + mt = get_mutable_type(file_format) + d = client.create_dirnode(version=mt) redirect = get_arg(req, "redirect_to_result", "false") if boolean_of_arg(redirect): def _then_redir(res): -- 2.45.2