From b30041c5ecf3e2b6d1b6d0b489daf626abd84fd2 Mon Sep 17 00:00:00 2001 From: Brian Warner <warner@lothar.com> Date: Mon, 12 Oct 2009 19:34:44 -0700 Subject: [PATCH] webapi: t=mkdir now accepts initial children, using the same JSON that t=json emits. client.create_dirnode(initial_children=) now works. --- docs/frontends/webapi.txt | 57 ++++++++++++-- src/allmydata/client.py | 1 - src/allmydata/test/test_web.py | 134 ++++++++++++++++++++++++++++++++- src/allmydata/web/common.py | 17 +++++ src/allmydata/web/directory.py | 15 +++- src/allmydata/web/unlinked.py | 12 ++- 6 files changed, 223 insertions(+), 13 deletions(-) diff --git a/docs/frontends/webapi.txt b/docs/frontends/webapi.txt index 4bbd7541..28b4fd44 100644 --- a/docs/frontends/webapi.txt +++ b/docs/frontends/webapi.txt @@ -345,10 +345,46 @@ PUT /uri POST /uri?t=mkdir PUT /uri?t=mkdir - Create a new empty directory and return its write-cap as the HTTP response - body. This does not make the newly created directory visible from the - virtual drive. The "PUT" operation is provided for backwards compatibility: - new code should use POST. + Create a new directory (either empty or with some initial children) and + return its write-cap as the HTTP response body. This does not make the newly + created directory visible from the virtual drive. The "PUT" operation is + provided for backwards compatibility: new code should use POST. + + Initial children are provided in the "children" field of the POST form, or + as the request body of the PUT request. This is more efficient than doing + separate mkdir and add-children operations. If this value is empty, the new + directory will be empty. + + If not empty, it will be interpreted as a JSON-encoded dictionary of + children with which the new directory should be populated, using the same + format as would be returned in the 'children' value of the t=json GET + request, described below. Each dictionary key should be a child name, and + each value should be a list of [TYPE, PROPDICT], where PROPDICT contains + "rw_uri", "ro_uri", and "metadata" keys (all others are ignored). For + example, the PUT request body could be: + + { + "Fran\u00e7ais": [ "filenode", { + "ro_uri": "URI:CHK:...", + "size": bytes, + "metadata": { + "ctime": 1202777696.7564139, + "mtime": 1202777696.7564139, + "tahoe": { + "linkcrtime": 1202777696.7564139, + "linkmotime": 1202777696.7564139, + } } } ], + "subdir": [ "dirnode", { + "rw_uri": "URI:DIR2:...", + "ro_uri": "URI:DIR2-RO:...", + "metadata": { + "ctime": 1202778102.7589991, + "mtime": 1202778111.2160511, + "tahoe": { + "linkcrtime": 1202777696.7564139, + "linkmotime": 1202777696.7564139, + } } } ] + } POST /uri/$DIRCAP/[SUBDIRS../]SUBDIR?t=mkdir PUT /uri/$DIRCAP/[SUBDIRS../]SUBDIR?t=mkdir @@ -358,6 +394,9 @@ PUT /uri/$DIRCAP/[SUBDIRS../]SUBDIR?t=mkdir intermediate directories as necessary. If the named target directory already exists, this will make no changes to it. + If a directory is created, it will be populated with initial children via + the PUT request body or POST 'children' form field, as described above. + This will return an error if a blocking file is present at any of the parent names, preventing the server from creating the necessary parent directory. @@ -367,7 +406,9 @@ PUT /uri/$DIRCAP/[SUBDIRS../]SUBDIR?t=mkdir POST /uri/$DIRCAP/[SUBDIRS../]?t=mkdir&name=NAME Create a new empty directory and attach it to the given existing directory. - This will create additional intermediate directories as necessary. + This will create additional intermediate directories as necessary. The new + directory will be populated with initial children via the PUT request body + or POST 'children' form field, as described above. The URL of this form points to the parent of the bottom-most new directory, whereas the previous form has a URL that points directly to the bottom-most @@ -742,6 +783,9 @@ POST /uri?t=mkdir "false"), then the HTTP response body will simply be the write-cap of the new directory. + It accepts the same initial-children arguments as described in earlier + t=mkdir sections, but these are unlikely to be useful from a browser form. + POST /uri/$DIRCAP/[SUBDIRS../]?t=mkdir&name=CHILDNAME This creates a new directory as a child of the designated SUBDIR. This will @@ -753,6 +797,9 @@ POST /uri/$DIRCAP/[SUBDIRS../]?t=mkdir&name=CHILDNAME when_done= argument, the HTTP response will simply contain the write-cap of the directory that was just created. + It accepts the same initial-children arguments as described in earlier + t=mkdir sections, but these are unlikely to be useful from a browser form. + === Uploading a File === diff --git a/src/allmydata/client.py b/src/allmydata/client.py index f4e16df7..1187291a 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -459,7 +459,6 @@ class Client(node.Node, pollmixin.PollMixin): def create_dirnode(self, initial_children={}): d = self.nodemaker.create_new_mutable_directory() - assert not initial_children, "not ready yet: %s" % (initial_children,) if initial_children: d.addCallback(lambda n: n.set_children(initial_children)) return d diff --git a/src/allmydata/test/test_web.py b/src/allmydata/test/test_web.py index 3ac6c660..967de384 100644 --- a/src/allmydata/test/test_web.py +++ b/src/allmydata/test/test_web.py @@ -11,13 +11,14 @@ from allmydata import interfaces, uri, webish from allmydata.storage.shares import get_share_file from allmydata.storage_client import StorageFarmBroker from allmydata.immutable import upload, download +from allmydata.dirnode import DirectoryNode from allmydata.nodemaker import NodeMaker from allmydata.unknown import UnknownNode from allmydata.web import status, common from allmydata.scripts.debug import CorruptShareOptions, corrupt_share from allmydata.util import fileutil, base32 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \ - create_chk_filenode, WebErrorMixin, ShouldFailMixin + create_chk_filenode, WebErrorMixin, ShouldFailMixin, make_mutable_file_uri from allmydata.interfaces import IMutableFileNode from allmydata.mutable import servermap, publish, retrieve import common_util as testutil @@ -1121,6 +1122,66 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, unittest.TestCase): d.addCallback(self.failUnlessNodeKeysAre, []) return d + def test_PUT_NEWDIRURL_initial_children(self): + (newkids, filecap1, filecap2, filecap3, + dircap) = self._create_initial_children() + d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", + simplejson.dumps(newkids)) + def _check(uri): + n = self.s.create_node_from_uri(uri.strip()) + d2 = self.failUnlessNodeKeysAre(n, newkids.keys()) + d2.addCallback(lambda ign: + self.failUnlessChildURIIs(n, u"child-imm", filecap1)) + d2.addCallback(lambda ign: + n.get_child_and_metadata_at_path(u"child-imm")) + d2.addCallback(lambda (c1, md1): + self.failUnlessEqual(md1["metakey1"], "metavalue1")) + d2.addCallback(lambda ign: + self.failUnlessChildURIIs(n, u"child-mutable", + filecap2)) + d2.addCallback(lambda ign: + self.failUnlessChildURIIs(n, u"child-mutable-ro", + filecap3)) + d2.addCallback(lambda ign: + self.failUnlessChildURIIs(n, u"dirchild", dircap)) + return d2 + d.addCallback(_check) + d.addCallback(lambda res: + self.failUnlessNodeHasChild(self._foo_node, u"newdir")) + d.addCallback(lambda res: self._foo_node.get(u"newdir")) + d.addCallback(self.failUnlessNodeKeysAre, newkids.keys()) + d.addCallback(lambda res: self._foo_node.get(u"newdir")) + d.addCallback(self.failUnlessChildURIIs, u"child-imm", filecap1) + return d + + def test_POST_NEWDIRURL_initial_children(self): + (newkids, filecap1, filecap2, filecap3, + dircap) = self._create_initial_children() + d = self.POST(self.public_url + "/foo/newdir?t=mkdir", + children=simplejson.dumps(newkids)) + def _check(uri): + n = self.s.create_node_from_uri(uri.strip()) + d2 = self.failUnlessNodeKeysAre(n, newkids.keys()) + d2.addCallback(lambda ign: + self.failUnlessChildURIIs(n, u"child-imm", filecap1)) + d2.addCallback(lambda ign: + self.failUnlessChildURIIs(n, u"child-mutable", + filecap2)) + d2.addCallback(lambda ign: + self.failUnlessChildURIIs(n, u"child-mutable-ro", + filecap3)) + d2.addCallback(lambda ign: + self.failUnlessChildURIIs(n, u"dirchild", dircap)) + return d2 + d.addCallback(_check) + d.addCallback(lambda res: + self.failUnlessNodeHasChild(self._foo_node, u"newdir")) + d.addCallback(lambda res: self._foo_node.get(u"newdir")) + d.addCallback(self.failUnlessNodeKeysAre, newkids.keys()) + d.addCallback(lambda res: self._foo_node.get(u"newdir")) + d.addCallback(self.failUnlessChildURIIs, u"child-imm", filecap1) + return d + def test_PUT_NEWDIRURL_exists(self): d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "") d.addCallback(lambda res: @@ -1867,6 +1928,18 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, unittest.TestCase): d.addCallback(self.failUnlessNodeKeysAre, []) return d + def test_POST_mkdir_initial_children(self): + newkids, filecap1, ign, ign, ign = self._create_initial_children() + d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir", + children=simplejson.dumps(newkids)) + d.addCallback(lambda res: + self.failUnlessNodeHasChild(self._foo_node, u"newdir")) + d.addCallback(lambda res: self._foo_node.get(u"newdir")) + d.addCallback(self.failUnlessNodeKeysAre, newkids.keys()) + d.addCallback(lambda res: self._foo_node.get(u"newdir")) + d.addCallback(self.failUnlessChildURIIs, u"child-imm", filecap1) + return d + def test_POST_mkdir_2(self): d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "") d.addCallback(lambda res: @@ -1900,6 +1973,44 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, unittest.TestCase): d.addCallback(_check_target) return d + def _create_initial_children(self): + contents, n, filecap1 = self.makefile(12) + md1 = {"metakey1": "metavalue1"} + filecap2 = make_mutable_file_uri() + node3 = self.s.create_node_from_uri(make_mutable_file_uri()) + filecap3 = node3.get_readonly_uri() + node4 = self.s.create_node_from_uri(make_mutable_file_uri()) + dircap = DirectoryNode(node4, None, None).get_uri() + newkids = {u"child-imm": ["filenode", {"ro_uri": filecap1, + "metadata": md1, }], + u"child-mutable": ["filenode", {"rw_uri": filecap2}], + u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}], + u"dirchild": ["dirnode", {"rw_uri": dircap}], + } + return newkids, filecap1, filecap2, filecap3, dircap + + def test_POST_mkdir_no_parentdir_initial_children(self): + (newkids, filecap1, filecap2, filecap3, + dircap) = self._create_initial_children() + d = self.POST("/uri?t=mkdir", children=simplejson.dumps(newkids)) + def _after_mkdir(res): + self.failUnless(res.startswith("URI:DIR"), res) + n = self.s.create_node_from_uri(res) + d2 = self.failUnlessNodeKeysAre(n, newkids.keys()) + d2.addCallback(lambda ign: + self.failUnlessChildURIIs(n, u"child-imm", filecap1)) + d2.addCallback(lambda ign: + self.failUnlessChildURIIs(n, u"child-mutable", + filecap2)) + d2.addCallback(lambda ign: + self.failUnlessChildURIIs(n, u"child-mutable-ro", + filecap3)) + d2.addCallback(lambda ign: + self.failUnlessChildURIIs(n, u"dirchild", dircap)) + return d2 + d.addCallback(_after_mkdir) + return d + def test_POST_noparent_bad(self): d = self.shouldHTTPError("POST /uri?t=bogus", 400, "Bad Request", "/uri accepts only PUT, PUT?t=mkdir, " @@ -2392,6 +2503,27 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, unittest.TestCase): d.addCallback(self.failUnlessIsEmptyJSON) return d + def test_PUT_mkdir_initial_children(self): + (newkids, filecap1, filecap2, filecap3, + dircap) = self._create_initial_children() + d = self.PUT("/uri?t=mkdir", simplejson.dumps(newkids)) + def _check(uri): + n = self.s.create_node_from_uri(uri.strip()) + d2 = self.failUnlessNodeKeysAre(n, newkids.keys()) + d2.addCallback(lambda ign: + self.failUnlessChildURIIs(n, u"child-imm", filecap1)) + d2.addCallback(lambda ign: + self.failUnlessChildURIIs(n, u"child-mutable", + filecap2)) + d2.addCallback(lambda ign: + self.failUnlessChildURIIs(n, u"child-mutable-ro", + filecap3)) + d2.addCallback(lambda ign: + self.failUnlessChildURIIs(n, u"dirchild", dircap)) + return d2 + d.addCallback(_check) + return d + def test_POST_check(self): d = self.POST(self.public_url + "/foo", t="check", name="bar.txt") def _done(res): diff --git a/src/allmydata/web/common.py b/src/allmydata/web/common.py index 15462cff..2c97e9f8 100644 --- a/src/allmydata/web/common.py +++ b/src/allmydata/web/common.py @@ -1,4 +1,5 @@ +import simplejson from twisted.web import http, server from twisted.python import log from zope.interface import Interface @@ -53,6 +54,22 @@ def get_arg(ctx_or_req, argname, default=None, multiple=False): return results[0] return default +def convert_initial_children_json(initial_children_json): + initial_children = {} + if initial_children_json: + data = simplejson.loads(initial_children_json) + for (name, (ctype, propdict)) in data.iteritems(): + name = unicode(name) + writecap = propdict.get("rw_uri") + if writecap is not None: + writecap = str(writecap) + readcap = propdict.get("ro_uri") + if readcap is not None: + readcap = str(readcap) + metadata = propdict.get("metadata", {}) + initial_children[name] = (writecap, readcap, metadata) + return initial_children + def abbreviate_time(data): # 1.23s, 790ms, 132us if data is None: diff --git a/src/allmydata/web/directory.py b/src/allmydata/web/directory.py index 034b1ac7..d3febe95 100644 --- a/src/allmydata/web/directory.py +++ b/src/allmydata/web/directory.py @@ -22,7 +22,7 @@ from allmydata.web.common import text_plain, WebError, \ IOpHandleTable, NeedOperationHandleError, \ boolean_of_arg, get_arg, get_root, parse_replace_arg, \ should_create_intermediate_directories, \ - getxmlfile, RenderMixin, humanize_failure + getxmlfile, RenderMixin, humanize_failure, convert_initial_children_json from allmydata.web.filenode import ReplaceMeMixin, \ FileNodeHandler, PlaceHolderNodeHandler from allmydata.web.check_results import CheckResults, \ @@ -96,7 +96,13 @@ class DirectoryNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin): if (method,t) in [ ("POST","mkdir"), ("PUT","mkdir") ]: if DEBUG: print " making final directory" # final directory - d = self.node.create_subdirectory(name) + if method == "POST": + kids_json = get_arg(req, "children", "") + else: + req.content.seek(0) + kids_json = req.content.read() + initial_children = convert_initial_children_json(kids_json) + d = self.node.create_subdirectory(name, initial_children) d.addCallback(make_handler_for, self.client, self.node, name) return d @@ -221,7 +227,10 @@ class DirectoryNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin): return defer.succeed(self.node.get_uri()) # TODO: urlencode name = name.decode("utf-8") replace = boolean_of_arg(get_arg(req, "replace", "true")) - d = self.node.create_subdirectory(name, overwrite=replace) + children_json = get_arg(req, "children", "") + initial_children = convert_initial_children_json(children_json) + d = self.node.create_subdirectory(name, initial_children, + overwrite=replace) d.addCallback(lambda child: child.get_uri()) # TODO: urlencode return d diff --git a/src/allmydata/web/unlinked.py b/src/allmydata/web/unlinked.py index cc316f74..963b6452 100644 --- a/src/allmydata/web/unlinked.py +++ b/src/allmydata/web/unlinked.py @@ -4,7 +4,8 @@ from twisted.web import http from twisted.internet import defer from nevow import rend, url, tags as T from allmydata.immutable.upload import FileHandle -from allmydata.web.common import getxmlfile, get_arg, boolean_of_arg +from allmydata.web.common import getxmlfile, get_arg, boolean_of_arg, \ + convert_initial_children_json from allmydata.web import status def PUTUnlinkedCHK(req, client): @@ -25,7 +26,10 @@ def PUTUnlinkedSSK(req, client): def PUTUnlinkedCreateDirectory(req, client): # "PUT /uri?t=mkdir", to create an unlinked directory. - d = client.create_dirnode() + req.content.seek(0) + initial_children_json = req.content.read() + initial_children = convert_initial_children_json(initial_children_json) + d = client.create_dirnode(initial_children=initial_children) d.addCallback(lambda dirnode: dirnode.get_uri()) # XXX add redirect_to_result return d @@ -90,7 +94,9 @@ def POSTUnlinkedSSK(req, client): def POSTUnlinkedCreateDirectory(req, client): # "POST /uri?t=mkdir", to create an unlinked directory. - d = client.create_dirnode() + initial_json = get_arg(req, "children", "") + initial_children = convert_initial_children_json(initial_json) + d = client.create_dirnode(initial_children=initial_children) redirect = get_arg(req, "redirect_to_result", "false") if boolean_of_arg(redirect): def _then_redir(res): -- 2.45.2