webapi changes for MDMF
authorKevan Carstensen <kevan@isnotajoke.com>
Sun, 7 Aug 2011 00:43:48 +0000 (17:43 -0700)
committerKevan Carstensen <kevan@isnotajoke.com>
Sun, 7 Aug 2011 00:43:48 +0000 (17:43 -0700)
    - Learn how to create MDMF files and directories through the
      mutable-type argument.
    - Operate with the interface changes associated with MDMF and #993.
    - Learn how to do partial updates of mutable files.

src/allmydata/test/test_web.py
src/allmydata/web/common.py
src/allmydata/web/directory.py
src/allmydata/web/filenode.py
src/allmydata/web/info.py
src/allmydata/web/root.py
src/allmydata/web/unlinked.py

index 28c1323bc80d4aed49259253cbf6e0707e5e64f6..69d8211d7c53ce399a86aa808868b919fc31bb73 100644 (file)
@@ -24,8 +24,9 @@ from allmydata.util.consumer import download_to_data
 from allmydata.util.netstring import split_netstring
 from allmydata.util.encodingutil import to_str
 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
-     create_chk_filenode, WebErrorMixin, ShouldFailMixin, make_mutable_file_uri
-from allmydata.interfaces import IMutableFileNode
+     create_chk_filenode, WebErrorMixin, ShouldFailMixin, \
+     make_mutable_file_uri, create_mutable_filenode
+from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
 from allmydata.mutable import servermap, publish, retrieve
 import allmydata.test.common_util as testutil
 from allmydata.test.no_network import GridTestMixin
@@ -48,15 +49,24 @@ class FakeStatsProvider:
         return stats
 
 class FakeNodeMaker(NodeMaker):
+    encoding_params = {
+        'k': 3,
+        'n': 10,
+        'happy': 7,
+        'max_segment_size':128*1024 # 1024=KiB
+    }
     def _create_lit(self, cap):
         return FakeCHKFileNode(cap)
     def _create_immutable(self, cap):
         return FakeCHKFileNode(cap)
     def _create_mutable(self, cap):
-        return FakeMutableFileNode(None, None, None, None).init_from_cap(cap)
-    def create_mutable_file(self, contents="", keysize=None):
-        n = FakeMutableFileNode(None, None, None, None)
-        return n.create(contents)
+        return FakeMutableFileNode(None,
+                                   None,
+                                   self.encoding_params, None).init_from_cap(cap)
+    def create_mutable_file(self, contents="", keysize=None,
+                            version=SDMF_VERSION):
+        n = FakeMutableFileNode(None, None, self.encoding_params, None)
+        return n.create(contents, version=version)
 
 class FakeUploader(service.Service):
     name = "uploader"
@@ -164,6 +174,7 @@ class FakeClient(Client):
         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
                                        self.uploader, None,
                                        None, None)
+        self.mutable_file_default = SDMF_VERSION
 
     def startService(self):
         return service.MultiService.startService(self)
@@ -208,6 +219,17 @@ class WebMixin(object):
             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 
+            # sdmf
+            # XXX: Do we ever use this?
+            self.BAZ_CONTENTS, n, self._baz_txt_uri, self._baz_txt_readonly_uri = self.makefile_mutable(0)
+
+            foo.set_uri(u"baz.txt", self._baz_txt_uri, self._baz_txt_readonly_uri)
+
+            # mdmf
+            self.QUUX_CONTENTS, n, self._quux_txt_uri, self._quux_txt_readonly_uri = self.makefile_mutable(0, mdmf=True)
+            assert self._quux_txt_uri.startswith("URI:MDMF")
+            foo.set_uri(u"quux.txt", self._quux_txt_uri, self._quux_txt_readonly_uri)
+
             foo.set_uri(u"empty", res[3][1].get_uri(),
                         res[3][1].get_readonly_uri())
             sub_uri = res[4][1].get_uri()
@@ -239,6 +261,8 @@ class WebMixin(object):
             # public/
             # public/foo/
             # public/foo/bar.txt
+            # public/foo/baz.txt
+            # public/foo/quux.txt
             # public/foo/blockingfile
             # public/foo/empty/
             # public/foo/sub/
@@ -259,12 +283,23 @@ class WebMixin(object):
         n = create_chk_filenode(contents)
         return contents, n, n.get_uri()
 
+    def makefile_mutable(self, number, mdmf=False):
+        contents = "contents of mutable file %s\n" % number
+        n = create_mutable_filenode(contents, mdmf)
+        return contents, n, n.get_uri(), n.get_readonly_uri()
+
     def tearDown(self):
         return self.s.stopService()
 
     def failUnlessIsBarDotTxt(self, res):
         self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
 
+    def failUnlessIsQuuxDotTxt(self, res):
+        self.failUnlessReallyEqual(res, self.QUUX_CONTENTS, res)
+
+    def failUnlessIsBazDotTxt(self, res):
+        self.failUnlessReallyEqual(res, self.BAZ_CONTENTS, res)
+
     def failUnlessIsBarJSON(self, res):
         data = simplejson.loads(res)
         self.failUnless(isinstance(data, list))
@@ -276,6 +311,25 @@ class WebMixin(object):
         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
         self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
 
+    def failUnlessIsQuuxJSON(self, res, readonly=False):
+        data = simplejson.loads(res)
+        self.failUnless(isinstance(data, list))
+        self.failUnlessEqual(data[0], "filenode")
+        self.failUnless(isinstance(data[1], dict))
+        metadata = data[1]
+        return self.failUnlessIsQuuxDotTxtMetadata(metadata, readonly)
+
+    def failUnlessIsQuuxDotTxtMetadata(self, metadata, readonly):
+        self.failUnless(metadata['mutable'])
+        if readonly:
+            self.failIf("rw_uri" in metadata)
+        else:
+            self.failUnless("rw_uri" in metadata)
+            self.failUnlessEqual(metadata['rw_uri'], self._quux_txt_uri)
+        self.failUnless("ro_uri" in metadata)
+        self.failUnlessEqual(metadata['ro_uri'], self._quux_txt_readonly_uri)
+        self.failUnlessReallyEqual(metadata['size'], len(self.QUUX_CONTENTS))
+
     def failUnlessIsFooJSON(self, res):
         data = simplejson.loads(res)
         self.failUnless(isinstance(data, list))
@@ -289,8 +343,8 @@ class WebMixin(object):
 
         kidnames = sorted([unicode(n) for n in data[1]["children"]])
         self.failUnlessEqual(kidnames,
-                             [u"bar.txt", u"blockingfile", u"empty",
-                              u"n\u00fc.txt", u"sub"])
+                             [u"bar.txt", u"baz.txt", u"blockingfile",
+                              u"empty", u"n\u00fc.txt", u"quux.txt", u"sub"])
         kids = dict( [(unicode(name),value)
                       for (name,value)
                       in data[1]["children"].iteritems()] )
@@ -311,6 +365,11 @@ class WebMixin(object):
                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
         self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
                                    self._bar_txt_uri)
+        self.failUnlessIn("quux.txt", kids)
+        self.failUnlessReallyEqual(kids[u"quux.txt"][1]["rw_uri"],
+                                   self._quux_txt_uri)
+        self.failUnlessReallyEqual(kids[u"quux.txt"][1]["ro_uri"],
+                                   self._quux_txt_readonly_uri)
 
     def GET(self, urlpath, followRedirect=False, return_response=False,
             **kwargs):
@@ -783,6 +842,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                              self.PUT, base + "/@@name=/blah.txt", "")
         return d
 
+
     def test_GET_DIRURL_named_bad(self):
         base = "/file/%s" % urllib.quote(self._foo_uri)
         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
@@ -825,6 +885,39 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(self.failUnlessIsBarDotTxt)
         return d
 
+    def test_GET_FILE_URI_mdmf(self):
+        base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
+        d = self.GET(base)
+        d.addCallback(self.failUnlessIsQuuxDotTxt)
+        return d
+
+    def test_GET_FILE_URI_mdmf_extensions(self):
+        base = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
+        d = self.GET(base)
+        d.addCallback(self.failUnlessIsQuuxDotTxt)
+        return d
+
+    def test_GET_FILE_URI_mdmf_bare_cap(self):
+        cap_elements = self._quux_txt_uri.split(":")
+        # 6 == expected cap length with two extensions.
+        self.failUnlessEqual(len(cap_elements), 6)
+
+        # Now lop off the extension parameters and stitch everything
+        # back together
+        quux_uri = ":".join(cap_elements[:len(cap_elements) - 2])
+
+        # Now GET that. We should get back quux.
+        base = "/uri/%s" % urllib.quote(quux_uri)
+        d = self.GET(base)
+        d.addCallback(self.failUnlessIsQuuxDotTxt)
+        return d
+
+    def test_GET_FILE_URI_mdmf_readonly(self):
+        base = "/uri/%s" % urllib.quote(self._quux_txt_readonly_uri)
+        d = self.GET(base)
+        d.addCallback(self.failUnlessIsQuuxDotTxt)
+        return d
+
     def test_GET_FILE_URI_badchild(self):
         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
         errmsg = "Files have no children, certainly not named 'boguschild'"
@@ -841,6 +934,72 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                              self.PUT, base, "")
         return d
 
+    def test_PUT_FILE_URI_mdmf(self):
+        base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
+        self._quux_new_contents = "new_contents"
+        d = self.GET(base)
+        d.addCallback(lambda res:
+            self.failUnlessIsQuuxDotTxt(res))
+        d.addCallback(lambda ignored:
+            self.PUT(base, self._quux_new_contents))
+        d.addCallback(lambda ignored:
+            self.GET(base))
+        d.addCallback(lambda res:
+            self.failUnlessReallyEqual(res, self._quux_new_contents))
+        return d
+
+    def test_PUT_FILE_URI_mdmf_extensions(self):
+        base = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
+        self._quux_new_contents = "new_contents"
+        d = self.GET(base)
+        d.addCallback(lambda res: self.failUnlessIsQuuxDotTxt(res))
+        d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
+        d.addCallback(lambda ignored: self.GET(base))
+        d.addCallback(lambda res: self.failUnlessEqual(self._quux_new_contents,
+                                                       res))
+        return d
+
+    def test_PUT_FILE_URI_mdmf_bare_cap(self):
+        elements = self._quux_txt_uri.split(":")
+        self.failUnlessEqual(len(elements), 6)
+
+        quux_uri = ":".join(elements[:len(elements) - 2])
+        base = "/uri/%s" % urllib.quote(quux_uri)
+        self._quux_new_contents = "new_contents" * 50000
+
+        d = self.GET(base)
+        d.addCallback(self.failUnlessIsQuuxDotTxt)
+        d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
+        d.addCallback(lambda ignored: self.GET(base))
+        d.addCallback(lambda res:
+            self.failUnlessEqual(res, self._quux_new_contents))
+        return d
+
+    def test_PUT_FILE_URI_mdmf_readonly(self):
+        # We're not allowed to PUT things to a readonly cap.
+        base = "/uri/%s" % self._quux_txt_readonly_uri
+        d = self.GET(base)
+        d.addCallback(lambda res:
+            self.failUnlessIsQuuxDotTxt(res))
+        # What should we get here? We get a 500 error now; that's not right.
+        d.addCallback(lambda ignored:
+            self.shouldFail2(error.Error, "test_PUT_FILE_URI_mdmf_readonly",
+                             "400 Bad Request", "read-only cap",
+                             self.PUT, base, "new data"))
+        return d
+
+    def test_PUT_FILE_URI_sdmf_readonly(self):
+        # We're not allowed to put things to a readonly cap.
+        base = "/uri/%s" % self._baz_txt_readonly_uri
+        d = self.GET(base)
+        d.addCallback(lambda res:
+            self.failUnlessIsBazDotTxt(res))
+        d.addCallback(lambda ignored:
+            self.shouldFail2(error.Error, "test_PUT_FILE_URI_sdmf_readonly",
+                             "400 Bad Request", "read-only cap",
+                             self.PUT, base, "new_data"))
+        return d
+
     # TODO: version of this with a Unicode filename
     def test_GET_FILEURL_save(self):
         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
@@ -857,6 +1016,54 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addBoth(self.should404, "test_GET_FILEURL_missing")
         return d
 
+    def test_GET_FILEURL_info_mdmf(self):
+        d = self.GET("/uri/%s?t=info" % self._quux_txt_uri)
+        def _got(res):
+            self.failUnlessIn("mutable file (mdmf)", res)
+            self.failUnlessIn(self._quux_txt_uri, res)
+            self.failUnlessIn(self._quux_txt_readonly_uri, res)
+        d.addCallback(_got)
+        return d
+
+    def test_GET_FILEURL_info_mdmf_readonly(self):
+        d = self.GET("/uri/%s?t=info" % self._quux_txt_readonly_uri)
+        def _got(res):
+            self.failUnlessIn("mutable file (mdmf)", res)
+            self.failIfIn(self._quux_txt_uri, res)
+            self.failUnlessIn(self._quux_txt_readonly_uri, res)
+        d.addCallback(_got)
+        return d
+
+    def test_GET_FILEURL_info_sdmf(self):
+        d = self.GET("/uri/%s?t=info" % self._baz_txt_uri)
+        def _got(res):
+            self.failUnlessIn("mutable file (sdmf)", res)
+            self.failUnlessIn(self._baz_txt_uri, res)
+        d.addCallback(_got)
+        return d
+
+    def test_GET_FILEURL_info_mdmf_extensions(self):
+        d = self.GET("/uri/%s:3:131073?t=info" % self._quux_txt_uri)
+        def _got(res):
+            self.failUnlessIn("mutable file (mdmf)", res)
+            self.failUnlessIn(self._quux_txt_uri, res)
+            self.failUnlessIn(self._quux_txt_readonly_uri, res)
+        d.addCallback(_got)
+        return d
+
+    def test_GET_FILEURL_info_mdmf_bare_cap(self):
+        elements = self._quux_txt_uri.split(":")
+        self.failUnlessEqual(len(elements), 6)
+
+        quux_uri = ":".join(elements[:len(elements) - 2])
+        base = "/uri/%s?t=info" % urllib.quote(quux_uri)
+        d = self.GET(base)
+        def _got(res):
+            self.failUnlessIn("mutable file (mdmf)", res)
+            self.failUnlessIn(quux_uri, res)
+        d.addCallback(_got)
+        return d
+
     def test_PUT_overwrite_only_files(self):
         # create a directory, put a file in that directory.
         contents, n, filecap = self.makefile(8)
@@ -898,6 +1105,35 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                                                       self.NEWFILE_CONTENTS))
         return d
 
+    def test_PUT_NEWFILEURL_unlinked_mdmf(self):
+        # this should get us a few segments of an MDMF mutable file,
+        # which we can then test for.
+        contents = self.NEWFILE_CONTENTS * 300000
+        d = self.PUT("/uri?mutable=true&mutable-type=mdmf",
+                     contents)
+        def _got_filecap(filecap):
+            self.failUnless(filecap.startswith("URI:MDMF"))
+            return filecap
+        d.addCallback(_got_filecap)
+        d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
+        d.addCallback(lambda json: self.failUnlessIn("mdmf", json))
+        return d
+
+    def test_PUT_NEWFILEURL_unlinked_sdmf(self):
+        contents = self.NEWFILE_CONTENTS * 300000
+        d = self.PUT("/uri?mutable=true&mutable-type=sdmf",
+                     contents)
+        d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
+        d.addCallback(lambda json: self.failUnlessIn("sdmf", json))
+        return d
+
+    def test_PUT_NEWFILEURL_unlinked_bad_mutable_type(self):
+        contents = self.NEWFILE_CONTENTS * 300000
+        return self.shouldHTTPError("test bad mutable type",
+                                    400, "Bad Request", "Unknown type: foo",
+                                    self.PUT, "/uri?mutable=true&mutable-type=foo",
+                                    contents)
+
     def test_PUT_NEWFILEURL_range_bad(self):
         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
         target = self.public_url + "/foo/new.txt"
@@ -930,12 +1166,10 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         return d
 
     def test_PUT_NEWFILEURL_mutable_toobig(self):
-        d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_mutable_toobig",
-                             "413 Request Entity Too Large",
-                             "SDMF is limited to one segment, and 10001 > 10000",
-                             self.PUT,
-                             self.public_url + "/foo/new.txt?mutable=true",
-                             "b" * (self.s.MUTABLE_SIZELIMIT+1))
+        # It is okay to upload large mutable files, so we should be able
+        # to do that.
+        d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
+                     "b" * (self.s.MUTABLE_SIZELIMIT + 1))
         return d
 
     def test_PUT_NEWFILEURL_replace(self):
@@ -1030,6 +1264,80 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(_check1)
         return d
 
+    def test_GET_FILEURL_json_mutable_type(self):
+        # The JSON should include mutable-type, which says whether the
+        # file is SDMF or MDMF
+        d = self.PUT("/uri?mutable=true&mutable-type=mdmf",
+                     self.NEWFILE_CONTENTS * 300000)
+        d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
+        def _got_json(json, version):
+            data = simplejson.loads(json)
+            assert "filenode" == data[0]
+            data = data[1]
+            assert isinstance(data, dict)
+
+            self.failUnlessIn("mutable-type", data)
+            self.failUnlessEqual(data['mutable-type'], version)
+
+        d.addCallback(_got_json, "mdmf")
+        # Now make an SDMF file and check that it is reported correctly.
+        d.addCallback(lambda ignored:
+            self.PUT("/uri?mutable=true&mutable-type=sdmf",
+                      self.NEWFILE_CONTENTS * 300000))
+        d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
+        d.addCallback(_got_json, "sdmf")
+        return d
+
+    def test_GET_FILEURL_json_mdmf_extensions(self):
+        # A GET invoked against a URL that includes an MDMF cap with
+        # extensions should fetch the same JSON information as a GET
+        # invoked against a bare cap.
+        self._quux_txt_uri = "%s:3:131073" % self._quux_txt_uri
+        self._quux_txt_readonly_uri = "%s:3:131073" % self._quux_txt_readonly_uri
+        d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
+        d.addCallback(self.failUnlessIsQuuxJSON)
+        return d
+
+    def test_GET_FILEURL_json_mdmf_bare_cap(self):
+        elements = self._quux_txt_uri.split(":")
+        self.failUnlessEqual(len(elements), 6)
+
+        quux_uri = ":".join(elements[:len(elements) - 2])
+        # so failUnlessIsQuuxJSON will work.
+        self._quux_txt_uri = quux_uri
+
+        # we need to alter the readonly URI in the same way, again so
+        # failUnlessIsQuuxJSON will work
+        elements = self._quux_txt_readonly_uri.split(":")
+        self.failUnlessEqual(len(elements), 6)
+        quux_ro_uri = ":".join(elements[:len(elements) - 2])
+        self._quux_txt_readonly_uri = quux_ro_uri
+
+        base = "/uri/%s?t=json" % urllib.quote(quux_uri)
+        d = self.GET(base)
+        d.addCallback(self.failUnlessIsQuuxJSON)
+        return d
+
+    def test_GET_FILEURL_json_mdmf_bare_readonly_cap(self):
+        elements = self._quux_txt_readonly_uri.split(":")
+        self.failUnlessEqual(len(elements), 6)
+
+        quux_readonly_uri = ":".join(elements[:len(elements) - 2])
+        # so failUnlessIsQuuxJSON will work
+        self._quux_txt_readonly_uri = quux_readonly_uri
+        base = "/uri/%s?t=json" % quux_readonly_uri
+        d = self.GET(base)
+        # XXX: We may need to make a method that knows how to check for
+        # readonly JSON, or else alter that one so that it knows how to
+        # do that.
+        d.addCallback(self.failUnlessIsQuuxJSON, readonly=True)
+        return d
+
+    def test_GET_FILEURL_json_mdmf(self):
+        d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
+        d.addCallback(self.failUnlessIsQuuxJSON)
+        return d
+
     def test_GET_FILEURL_json_missing(self):
         d = self.GET(self.public_url + "/foo/missing?json")
         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
@@ -1062,19 +1370,75 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
             self.failUnless(CSS_STYLE.search(res), res)
         d.addCallback(_check)
         return d
-
+    
     def test_GET_FILEURL_uri_missing(self):
         d = self.GET(self.public_url + "/foo/missing?t=uri")
         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
         return d
 
-    def test_GET_DIRECTORY_html_banner(self):
+    def test_GET_DIRECTORY_html(self):
         d = self.GET(self.public_url + "/foo", followRedirect=True)
         def _check(res):
             self.failUnlessIn('<div class="toolbar-item"><a href="../../..">Return to Welcome page</a></div>',res)
+            # These are radio buttons that allow a user to toggle
+            # whether a particular mutable file is SDMF or MDMF.
+            self.failUnlessIn("mutable-type-mdmf", res)
+            self.failUnlessIn("mutable-type-sdmf", res)
+            # Similarly, these toggle whether a particular directory
+            # should be MDMF or SDMF.
+            self.failUnlessIn("mutable-directory-mdmf", res)
+            self.failUnlessIn("mutable-directory-sdmf", res)
+            self.failUnlessIn("quux", res)
         d.addCallback(_check)
         return d
 
+    def test_GET_root_html(self):
+        # make sure that we have the option to upload an unlinked
+        # mutable file in SDMF and MDMF formats.
+        d = self.GET("/")
+        def _got_html(html):
+            # These are radio buttons that allow the user to toggle
+            # whether a particular mutable file is MDMF or SDMF.
+            self.failUnlessIn("mutable-type-mdmf", html)
+            self.failUnlessIn("mutable-type-sdmf", html)
+            # We should also have the ability to create a mutable directory.
+            self.failUnlessIn("mkdir", html)
+            # ...and we should have the ability to say whether that's an
+            # MDMF or SDMF directory
+            self.failUnlessIn("mutable-directory-mdmf", html)
+            self.failUnlessIn("mutable-directory-sdmf", html)
+        d.addCallback(_got_html)
+        return d
+
+    def test_mutable_type_defaults(self):
+        # The checked="checked" attribute of the inputs corresponding to
+        # the mutable-type parameter should change as expected with the
+        # value configured in tahoe.cfg.
+        #
+        # By default, the value configured with the client is
+        # SDMF_VERSION, so that should be checked.
+        assert self.s.mutable_file_default == SDMF_VERSION
+
+        d = self.GET("/")
+        def _got_html(html, value):
+            i = 'input checked="checked" type="radio" id="mutable-type-%s"'
+            self.failUnlessIn(i % value, html)
+        d.addCallback(_got_html, "sdmf")
+        d.addCallback(lambda ignored:
+            self.GET(self.public_url + "/foo", followRedirect=True))
+        d.addCallback(_got_html, "sdmf")
+        # Now switch the configuration value to MDMF. The MDMF radio
+        # buttons should now be checked on these pages.
+        def _swap_values(ignored):
+            self.s.mutable_file_default = MDMF_VERSION
+        d.addCallback(_swap_values)
+        d.addCallback(lambda ignored: self.GET("/"))
+        d.addCallback(_got_html, "mdmf")
+        d.addCallback(lambda ignored:
+            self.GET(self.public_url + "/foo", followRedirect=True))
+        d.addCallback(_got_html, "mdmf")
+        return d
+
     def test_GET_DIRURL(self):
         # the addSlash means we get a redirect here
         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
@@ -1168,6 +1532,35 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(self.failUnlessIsFooJSON)
         return d
 
+    def test_GET_DIRURL_json_mutable_type(self):
+        d = self.PUT(self.public_url + \
+                     "/foo/sdmf.txt?mutable=true&mutable-type=sdmf",
+                     self.NEWFILE_CONTENTS * 300000)
+        d.addCallback(lambda ignored:
+            self.PUT(self.public_url + \
+                     "/foo/mdmf.txt?mutable=true&mutable-type=mdmf",
+                     self.NEWFILE_CONTENTS * 300000))
+        # Now we have an MDMF and SDMF file in the directory. If we GET
+        # its JSON, we should see their encodings.
+        d.addCallback(lambda ignored:
+            self.GET(self.public_url + "/foo?t=json"))
+        def _got_json(json):
+            data = simplejson.loads(json)
+            assert data[0] == "dirnode"
+
+            data = data[1]
+            kids = data['children']
+
+            mdmf_data = kids['mdmf.txt'][1]
+            self.failUnlessIn("mutable-type", mdmf_data)
+            self.failUnlessEqual(mdmf_data['mutable-type'], "mdmf")
+
+            sdmf_data = kids['sdmf.txt'][1]
+            self.failUnlessIn("mutable-type", sdmf_data)
+            self.failUnlessEqual(sdmf_data['mutable-type'], "sdmf")
+        d.addCallback(_got_json)
+        return d
+
 
     def test_POST_DIRURL_manifest_no_ophandle(self):
         d = self.shouldFail2(error.Error,
@@ -1263,15 +1656,15 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(self.get_operation_results, "127", "json")
         def _got_json(stats):
             expected = {"count-immutable-files": 3,
-                        "count-mutable-files": 0,
+                        "count-mutable-files": 2,
                         "count-literal-files": 0,
-                        "count-files": 3,
+                        "count-files": 5,
                         "count-directories": 3,
                         "size-immutable-files": 57,
                         "size-literal-files": 0,
                         #"size-directories": 1912, # varies
                         #"largest-directory": 1590,
-                        "largest-directory-children": 5,
+                        "largest-directory-children": 7,
                         "largest-immutable-file": 19,
                         }
             for k,v in expected.iteritems():
@@ -1288,7 +1681,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         def _check(res):
             self.failUnless(res.endswith("\n"))
             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
-            self.failUnlessReallyEqual(len(units), 7)
+            self.failUnlessReallyEqual(len(units), 9)
             self.failUnlessEqual(units[-1]["type"], "stats")
             first = units[0]
             self.failUnlessEqual(first["path"], [])
@@ -1299,6 +1692,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
             self.failIfEqual(baz["storage-index"], None)
             self.failIfEqual(baz["verifycap"], None)
             self.failIfEqual(baz["repaircap"], None)
+            # XXX: Add quux and baz to this test.
             return
         d.addCallback(_check)
         return d
@@ -1325,6 +1719,31 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(self.failUnlessNodeKeysAre, [])
         return d
 
+    def test_PUT_NEWDIRURL_mdmf(self):
+        d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&mutable-type=mdmf", "")
+        d.addCallback(lambda res:
+                      self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
+        d.addCallback(lambda res: self._foo_node.get(u"newdir"))
+        d.addCallback(lambda node:
+            self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
+        return d
+
+    def test_PUT_NEWDIRURL_sdmf(self):
+        d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&mutable-type=sdmf",
+                     "")
+        d.addCallback(lambda res:
+                      self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
+        d.addCallback(lambda res: self._foo_node.get(u"newdir"))
+        d.addCallback(lambda node:
+            self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
+        return d
+
+    def test_PUT_NEWDIRURL_bad_mutable_type(self):
+        return self.shouldHTTPError("test bad mutable type",
+                             400, "Bad Request", "Unknown type: foo",
+                             self.PUT, self.public_url + \
+                             "/foo/newdir=?t=mkdir&mutable-type=foo", "")
+
     def test_POST_NEWDIRURL(self):
         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
         d.addCallback(lambda res:
@@ -1333,6 +1752,30 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(self.failUnlessNodeKeysAre, [])
         return d
 
+    def test_POST_NEWDIRURL_mdmf(self):
+        d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&mutable-type=mdmf", "")
+        d.addCallback(lambda res:
+                      self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
+        d.addCallback(lambda res: self._foo_node.get(u"newdir"))
+        d.addCallback(lambda node:
+            self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
+        return d
+
+    def test_POST_NEWDIRURL_sdmf(self):
+        d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&mutable-type=sdmf", "")
+        d.addCallback(lambda res:
+            self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
+        d.addCallback(lambda res: self._foo_node.get(u"newdir"))
+        d.addCallback(lambda node:
+            self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
+        return d
+
+    def test_POST_NEWDIRURL_bad_mutable_type(self):
+        return self.shouldHTTPError("test bad mutable type",
+                                    400, "Bad Request", "Unknown type: foo",
+                                    self.POST2, self.public_url + \
+                                    "/foo/newdir?t=mkdir&mutable-type=foo", "")
+
     def test_POST_NEWDIRURL_emptyname(self):
         # an empty pathname component (i.e. a double-slash) is disallowed
         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_emptyname",
@@ -1341,13 +1784,21 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                              self.POST, self.public_url + "//?t=mkdir")
         return d
 
-    def test_POST_NEWDIRURL_initial_children(self):
+    def _do_POST_NEWDIRURL_initial_children_test(self, version=None):
         (newkids, caps) = self._create_initial_children()
-        d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-with-children",
+        query = "/foo/newdir?t=mkdir-with-children"
+        if version == MDMF_VERSION:
+            query += "&mutable-type=mdmf"
+        elif version == SDMF_VERSION:
+            query += "&mutable-type=sdmf"
+        else:
+            version = SDMF_VERSION # for later
+        d = self.POST2(self.public_url + query,
                        simplejson.dumps(newkids))
         def _check(uri):
             n = self.s.create_node_from_uri(uri.strip())
             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
+            self.failUnlessEqual(n._node.get_version(), version)
             d2.addCallback(lambda ign:
                            self.failUnlessROChildURIIs(n, u"child-imm",
                                                        caps['filecap1']))
@@ -1385,6 +1836,23 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
         return d
 
+    def test_POST_NEWDIRURL_initial_children(self):
+        return self._do_POST_NEWDIRURL_initial_children_test()
+
+    def test_POST_NEWDIRURL_initial_children_mdmf(self):
+        return self._do_POST_NEWDIRURL_initial_children_test(MDMF_VERSION)
+
+    def test_POST_NEWDIRURL_initial_children_sdmf(self):
+        return self._do_POST_NEWDIRURL_initial_children_test(SDMF_VERSION)
+
+    def test_POST_NEWDIRURL_initial_children_bad_mutable_type(self):
+        (newkids, caps) = self._create_initial_children()
+        return self.shouldHTTPError("test bad mutable type",
+                                    400, "Bad Request", "Unknown type: foo",
+                                    self.POST2, self.public_url + \
+                                    "/foo/newdir?t=mkdir-with-children&mutable-type=foo",
+                                    simplejson.dumps(newkids))
+
     def test_POST_NEWDIRURL_immutable(self):
         (newkids, caps) = self._create_immutable_children()
         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
@@ -1485,6 +1953,49 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(self.failUnlessNodeKeysAre, [])
         return d
 
+    def test_PUT_NEWDIRURL_mkdirs_mdmf(self):
+        d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&mutable-type=mdmf", "")
+        d.addCallback(lambda ignored:
+            self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
+        d.addCallback(lambda ignored:
+            self.failIfNodeHasChild(self._foo_node, u"newdir"))
+        d.addCallback(lambda ignored:
+            self._foo_node.get_child_at_path(u"subdir"))
+        def _got_subdir(subdir):
+            # XXX: What we want?
+            #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
+            self.failUnlessNodeHasChild(subdir, u"newdir")
+            return subdir.get_child_at_path(u"newdir")
+        d.addCallback(_got_subdir)
+        d.addCallback(lambda newdir:
+            self.failUnlessEqual(newdir._node.get_version(), MDMF_VERSION))
+        return d
+
+    def test_PUT_NEWDIRURL_mkdirs_sdmf(self):
+        d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&mutable-type=sdmf", "")
+        d.addCallback(lambda ignored:
+            self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
+        d.addCallback(lambda ignored:
+            self.failIfNodeHasChild(self._foo_node, u"newdir"))
+        d.addCallback(lambda ignored:
+            self._foo_node.get_child_at_path(u"subdir"))
+        def _got_subdir(subdir):
+            # XXX: What we want?
+            #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
+            self.failUnlessNodeHasChild(subdir, u"newdir")
+            return subdir.get_child_at_path(u"newdir")
+        d.addCallback(_got_subdir)
+        d.addCallback(lambda newdir:
+            self.failUnlessEqual(newdir._node.get_version(), SDMF_VERSION))
+        return d
+
+    def test_PUT_NEWDIRURL_mkdirs_bad_mutable_type(self):
+        return self.shouldHTTPError("test bad mutable type",
+                                    400, "Bad Request", "Unknown type: foo",
+                                    self.PUT, self.public_url + \
+                                    "/foo/subdir/newdir?t=mkdir&mutable-type=foo",
+                                    "")
+
     def test_DELETE_DIRURL(self):
         d = self.DELETE(self.public_url + "/foo")
         d.addCallback(lambda res:
@@ -1722,16 +2233,79 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         return d
 
     def test_POST_upload_no_link_mutable_toobig(self):
-        d = self.shouldFail2(error.Error,
-                             "test_POST_upload_no_link_mutable_toobig",
-                             "413 Request Entity Too Large",
-                             "SDMF is limited to one segment, and 10001 > 10000",
-                             self.POST,
-                             "/uri", t="upload", mutable="true",
-                             file=("new.txt",
-                                   "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
+        # The SDMF size limit is no longer in place, so we should be
+        # able to upload mutable files that are as large as we want them
+        # to be.
+        d = self.POST("/uri", t="upload", mutable="true",
+                      file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
+        return d
+
+
+    def test_POST_upload_mutable_type_unlinked(self):
+        d = self.POST("/uri?t=upload&mutable=true&mutable-type=sdmf",
+                      file=("sdmf.txt", self.NEWFILE_CONTENTS * 300000))
+        d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
+        def _got_json(json, version):
+            data = simplejson.loads(json)
+            data = data[1]
+
+            self.failUnlessIn("mutable-type", data)
+            self.failUnlessEqual(data['mutable-type'], version)
+        d.addCallback(_got_json, "sdmf")
+        d.addCallback(lambda ignored:
+            self.POST("/uri?t=upload&mutable=true&mutable-type=mdmf",
+                      file=('mdmf.txt', self.NEWFILE_CONTENTS * 300000)))
+        def _got_filecap(filecap):
+            self.failUnless(filecap.startswith("URI:MDMF"))
+            return filecap
+        d.addCallback(_got_filecap)
+        d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
+        d.addCallback(_got_json, "mdmf")
+        return d
+
+    def test_POST_upload_mutable_type_unlinked_bad_mutable_type(self):
+        return self.shouldHTTPError("test bad mutable type",
+                                    400, "Bad Request", "Unknown type: foo",
+                                    self.POST,
+                                    "/uri?5=upload&mutable=true&mutable-type=foo",
+                                    file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
+
+    def test_POST_upload_mutable_type(self):
+        d = self.POST(self.public_url + \
+                      "/foo?t=upload&mutable=true&mutable-type=sdmf",
+                      file=("sdmf.txt", self.NEWFILE_CONTENTS * 300000))
+        fn = self._foo_node
+        def _got_cap(filecap, filename):
+            filenameu = unicode(filename)
+            self.failUnlessURIMatchesRWChild(filecap, fn, filenameu)
+            return self.GET(self.public_url + "/foo/%s?t=json" % filename)
+        def _got_mdmf_cap(filecap):
+            self.failUnless(filecap.startswith("URI:MDMF"))
+            return filecap
+        d.addCallback(_got_cap, "sdmf.txt")
+        def _got_json(json, version):
+            data = simplejson.loads(json)
+            data = data[1]
+
+            self.failUnlessIn("mutable-type", data)
+            self.failUnlessEqual(data['mutable-type'], version)
+        d.addCallback(_got_json, "sdmf")
+        d.addCallback(lambda ignored:
+            self.POST(self.public_url + \
+                      "/foo?t=upload&mutable=true&mutable-type=mdmf",
+                      file=("mdmf.txt", self.NEWFILE_CONTENTS * 300000)))
+        d.addCallback(_got_mdmf_cap)
+        d.addCallback(_got_cap, "mdmf.txt")
+        d.addCallback(_got_json, "mdmf")
         return d
 
+    def test_POST_upload_bad_mutable_type(self):
+        return self.shouldHTTPError("test bad mutable type",
+                                    400, "Bad Request", "Unknown type: foo",
+                                    self.POST, self.public_url + \
+                                    "/foo?t=upload&mutable=true&mutable-type=foo",
+                                    file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
+
     def test_POST_upload_mutable(self):
         # this creates a mutable file
         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
@@ -1856,32 +2430,24 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
         d.addCallback(_got_headers)
 
-        # make sure that size errors are displayed correctly for overwrite
-        d.addCallback(lambda res:
-                      self.shouldFail2(error.Error,
-                                       "test_POST_upload_mutable-toobig",
-                                       "413 Request Entity Too Large",
-                                       "SDMF is limited to one segment, and 10001 > 10000",
-                                       self.POST,
-                                       self.public_url + "/foo", t="upload",
-                                       mutable="true",
-                                       file=("new.txt",
-                                             "b" * (self.s.MUTABLE_SIZELIMIT+1)),
-                                       ))
-
+        # make sure that outdated size limits aren't enforced anymore.
+        d.addCallback(lambda ignored:
+            self.POST(self.public_url + "/foo", t="upload",
+                      mutable="true",
+                      file=("new.txt",
+                            "b" * (self.s.MUTABLE_SIZELIMIT+1))))
         d.addErrback(self.dump_error)
         return d
 
     def test_POST_upload_mutable_toobig(self):
-        d = self.shouldFail2(error.Error,
-                             "test_POST_upload_mutable_toobig",
-                             "413 Request Entity Too Large",
-                             "SDMF is limited to one segment, and 10001 > 10000",
-                             self.POST,
-                             self.public_url + "/foo",
-                             t="upload", mutable="true",
-                             file=("new.txt",
-                                   "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
+        # SDMF had a size limti that was removed a while ago. MDMF has
+        # never had a size limit. Test to make sure that we do not
+        # encounter errors when trying to upload large mutable files,
+        # since there should be no coded prohibitions regarding large
+        # mutable files.
+        d = self.POST(self.public_url + "/foo",
+                      t="upload", mutable="true",
+                      file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
         return d
 
     def dump_error(self, f):
@@ -1969,8 +2535,8 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         # make sure that nothing was added
         d.addCallback(lambda res:
                       self.failUnlessNodeKeysAre(self._foo_node,
-                                                 [u"bar.txt", u"blockingfile",
-                                                  u"empty", u"n\u00fc.txt",
+                                                 [u"bar.txt", u"baz.txt", u"blockingfile",
+                                                  u"empty", u"n\u00fc.txt", u"quux.txt",
                                                   u"sub"]))
         return d
 
@@ -2092,6 +2658,31 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(_check3)
         return d
 
+    def test_POST_FILEURL_mdmf_check(self):
+        quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
+        d = self.POST(quux_url, t="check")
+        def _check(res):
+            self.failUnlessIn("Healthy", res)
+        d.addCallback(_check)
+        quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
+        d.addCallback(lambda ignored:
+            self.POST(quux_extension_url, t="check"))
+        d.addCallback(_check)
+        return d
+
+    def test_POST_FILEURL_mdmf_check_and_repair(self):
+        quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
+        d = self.POST(quux_url, t="check", repair="true")
+        def _check(res):
+            self.failUnlessIn("Healthy", res)
+        d.addCallback(_check)
+        quux_extension_url = "/uri/%s" %\
+            urllib.quote("%s:3:131073" % self._quux_txt_uri)
+        d.addCallback(lambda ignored:
+            self.POST(quux_extension_url, t="check", repair="true"))
+        d.addCallback(_check)
+        return d
+
     def wait_for_operation(self, ignored, ophandle):
         url = "/operations/" + ophandle
         url += "?t=status&output=JSON"
@@ -2137,13 +2728,13 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(self.wait_for_operation, "123")
         def _check_json(data):
             self.failUnlessReallyEqual(data["finished"], True)
-            self.failUnlessReallyEqual(data["count-objects-checked"], 8)
-            self.failUnlessReallyEqual(data["count-objects-healthy"], 8)
+            self.failUnlessReallyEqual(data["count-objects-checked"], 10)
+            self.failUnlessReallyEqual(data["count-objects-healthy"], 10)
         d.addCallback(_check_json)
         d.addCallback(self.get_operation_results, "123", "html")
         def _check_html(res):
-            self.failUnless("Objects Checked: <span>8</span>" in res)
-            self.failUnless("Objects Healthy: <span>8</span>" in res)
+            self.failUnless("Objects Checked: <span>10</span>" in res)
+            self.failUnless("Objects Healthy: <span>10</span>" in res)
         d.addCallback(_check_html)
 
         d.addCallback(lambda res:
@@ -2172,22 +2763,22 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(self.wait_for_operation, "124")
         def _check_json(data):
             self.failUnlessReallyEqual(data["finished"], True)
-            self.failUnlessReallyEqual(data["count-objects-checked"], 8)
-            self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 8)
+            self.failUnlessReallyEqual(data["count-objects-checked"], 10)
+            self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 10)
             self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
             self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
             self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
             self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
             self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
-            self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 8)
+            self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 10)
             self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
             self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
         d.addCallback(_check_json)
         d.addCallback(self.get_operation_results, "124", "html")
         def _check_html(res):
-            self.failUnless("Objects Checked: <span>8</span>" in res)
+            self.failUnless("Objects Checked: <span>10</span>" in res)
 
-            self.failUnless("Objects Healthy (before repair): <span>8</span>" in res)
+            self.failUnless("Objects Healthy (before repair): <span>10</span>" in res)
             self.failUnless("Objects Unhealthy (before repair): <span>0</span>" in res)
             self.failUnless("Corrupt Shares (before repair): <span>0</span>" in res)
 
@@ -2195,7 +2786,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
             self.failUnless("Repairs Successful: <span>0</span>" in res)
             self.failUnless("Repairs Unsuccessful: <span>0</span>" in res)
 
-            self.failUnless("Objects Healthy (after repair): <span>8</span>" in res)
+            self.failUnless("Objects Healthy (after repair): <span>10</span>" in res)
             self.failUnless("Objects Unhealthy (after repair): <span>0</span>" in res)
             self.failUnless("Corrupt Shares (after repair): <span>0</span>" in res)
         d.addCallback(_check_html)
@@ -2214,6 +2805,26 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(self.failUnlessNodeKeysAre, [])
         return d
 
+    def test_POST_mkdir_mdmf(self):
+        d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&mutable-type=mdmf")
+        d.addCallback(lambda res: self._foo_node.get(u"newdir"))
+        d.addCallback(lambda node:
+            self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
+        return d
+
+    def test_POST_mkdir_sdmf(self):
+        d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&mutable-type=sdmf")
+        d.addCallback(lambda res: self._foo_node.get(u"newdir"))
+        d.addCallback(lambda node:
+            self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
+        return d
+
+    def test_POST_mkdir_bad_mutable_type(self):
+        return self.shouldHTTPError("test bad mutable type",
+                                    400, "Bad Request", "Unknown type: foo",
+                                    self.POST, self.public_url + \
+                                    "/foo?t=mkdir&name=newdir&mutable-type=foo")
+
     def test_POST_mkdir_initial_children(self):
         (newkids, caps) = self._create_initial_children()
         d = self.POST2(self.public_url +
@@ -2227,6 +2838,45 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
         return d
 
+    def test_POST_mkdir_initial_children_mdmf(self):
+        (newkids, caps) = self._create_initial_children()
+        d = self.POST2(self.public_url +
+                       "/foo?t=mkdir-with-children&name=newdir&mutable-type=mdmf",
+                       simplejson.dumps(newkids))
+        d.addCallback(lambda res:
+                      self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
+        d.addCallback(lambda res: self._foo_node.get(u"newdir"))
+        d.addCallback(lambda node:
+            self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
+        d.addCallback(lambda res: self._foo_node.get(u"newdir"))
+        d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
+                       caps['filecap1'])
+        return d
+
+    # XXX: Duplication.
+    def test_POST_mkdir_initial_children_sdmf(self):
+        (newkids, caps) = self._create_initial_children()
+        d = self.POST2(self.public_url +
+                       "/foo?t=mkdir-with-children&name=newdir&mutable-type=sdmf",
+                       simplejson.dumps(newkids))
+        d.addCallback(lambda res:
+                      self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
+        d.addCallback(lambda res: self._foo_node.get(u"newdir"))
+        d.addCallback(lambda node:
+            self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
+        d.addCallback(lambda res: self._foo_node.get(u"newdir"))
+        d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
+                       caps['filecap1'])
+        return d
+
+    def test_POST_mkdir_initial_children_bad_mutable_type(self):
+        (newkids, caps) = self._create_initial_children()
+        return self.shouldHTTPError("test bad mutable type",
+                                    400, "Bad Request", "Unknown type: foo",
+                                    self.POST, self.public_url + \
+                                    "/foo?t=mkdir-with-children&name=newdir&mutable-type=foo",
+                                    simplejson.dumps(newkids))
+
     def test_POST_mkdir_immutable(self):
         (newkids, caps) = self._create_immutable_children()
         d = self.POST2(self.public_url +
@@ -2283,6 +2933,29 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(_after_mkdir)
         return d
 
+    def test_POST_mkdir_no_parentdir_noredirect_mdmf(self):
+        d = self.POST("/uri?t=mkdir&mutable-type=mdmf")
+        def _after_mkdir(res):
+            u = uri.from_string(res)
+            # Check that this is an MDMF writecap
+            self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
+        d.addCallback(_after_mkdir)
+        return d
+
+    def test_POST_mkdir_no_parentdir_noredirect_sdmf(self):
+        d = self.POST("/uri?t=mkdir&mutable-type=sdmf")
+        def _after_mkdir(res):
+            u = uri.from_string(res)
+            self.failUnlessIsInstance(u, uri.DirectoryURI)
+        d.addCallback(_after_mkdir)
+        return d
+
+    def test_POST_mkdir_no_parentdir_noredirect_bad_mutable_type(self):
+        return self.shouldHTTPError("test bad mutable type",
+                                    400, "Bad Request", "Unknown type: foo",
+                                    self.POST, self.public_url + \
+                                    "/uri?t=mkdir&mutable-type=foo")
+
     def test_POST_mkdir_no_parentdir_noredirect2(self):
         # make sure form-based arguments (as on the welcome page) still work
         d = self.POST("/uri", t="mkdir")
@@ -2325,6 +2998,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         filecap3 = node3.get_readonly_uri()
         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
         dircap = DirectoryNode(node4, None, None).get_uri()
+        mdmfcap = make_mutable_file_uri(mdmf=True)
         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
         emptydircap = "URI:DIR2-LIT:"
         newkids = {u"child-imm":        ["filenode", {"rw_uri": filecap1,
@@ -2341,6 +3015,8 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                                                       "ro_uri": self._make_readonly(dircap)}],
                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
+                   u"child-mutable-mdmf": ["filenode", {"rw_uri": mdmfcap,
+                                                        "ro_uri": self._make_readonly(mdmfcap)}],
                    }
         return newkids, {'filecap1': filecap1,
                          'filecap2': filecap2,
@@ -2350,7 +3026,8 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                          'unknown_immcap': unknown_immcap,
                          'dircap': dircap,
                          'litdircap': litdircap,
-                         'emptydircap': emptydircap}
+                         'emptydircap': emptydircap,
+                         'mdmfcap': mdmfcap}
 
     def _create_immutable_children(self):
         contents, n, filecap1 = self.makefile(12)
@@ -2891,6 +3568,46 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                                                       contents))
         return d
 
+    def test_PUT_NEWFILEURL_mdmf(self):
+        new_contents = self.NEWFILE_CONTENTS * 300000
+        d = self.PUT(self.public_url + \
+                     "/foo/mdmf.txt?mutable=true&mutable-type=mdmf",
+                     new_contents)
+        d.addCallback(lambda ignored:
+            self.GET(self.public_url + "/foo/mdmf.txt?t=json"))
+        def _got_json(json):
+            data = simplejson.loads(json)
+            data = data[1]
+            self.failUnlessIn("mutable-type", data)
+            self.failUnlessEqual(data['mutable-type'], "mdmf")
+            self.failUnless(data['rw_uri'].startswith("URI:MDMF"))
+            self.failUnless(data['ro_uri'].startswith("URI:MDMF"))
+        d.addCallback(_got_json)
+        return d
+
+    def test_PUT_NEWFILEURL_sdmf(self):
+        new_contents = self.NEWFILE_CONTENTS * 300000
+        d = self.PUT(self.public_url + \
+                     "/foo/sdmf.txt?mutable=true&mutable-type=sdmf",
+                     new_contents)
+        d.addCallback(lambda ignored:
+            self.GET(self.public_url + "/foo/sdmf.txt?t=json"))
+        def _got_json(json):
+            data = simplejson.loads(json)
+            data = data[1]
+            self.failUnlessIn("mutable-type", data)
+            self.failUnlessEqual(data['mutable-type'], "sdmf")
+        d.addCallback(_got_json)
+        return d
+
+    def test_PUT_NEWFILEURL_bad_mutable_type(self):
+       new_contents = self.NEWFILE_CONTENTS * 300000
+       return self.shouldHTTPError("test bad mutable type",
+                                   400, "Bad Request", "Unknown type: foo",
+                                   self.PUT, self.public_url + \
+                                   "/foo/foo.txt?mutable=true&mutable-type=foo",
+                                   new_contents)
+
     def test_PUT_NEWFILEURL_uri_replace(self):
         contents, n, new_uri = self.makefile(8)
         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
@@ -3000,6 +3717,29 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(self.failUnlessIsEmptyJSON)
         return d
 
+    def test_PUT_mkdir_mdmf(self):
+        d = self.PUT("/uri?t=mkdir&mutable-type=mdmf", "")
+        def _got(res):
+            u = uri.from_string(res)
+            # Check that this is an MDMF writecap
+            self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
+        d.addCallback(_got)
+        return d
+
+    def test_PUT_mkdir_sdmf(self):
+        d = self.PUT("/uri?t=mkdir&mutable-type=sdmf", "")
+        def _got(res):
+            u = uri.from_string(res)
+            self.failUnlessIsInstance(u, uri.DirectoryURI)
+        d.addCallback(_got)
+        return d
+
+    def test_PUT_mkdir_bad_mutable_type(self):
+        return self.shouldHTTPError("bad mutable type",
+                                    400, "Bad Request", "Unknown type: foo",
+                                    self.PUT, "/uri?t=mkdir&mutable-type=foo",
+                                    "")
+
     def test_POST_check(self):
         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
         def _done(res):
@@ -3012,6 +3752,75 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(_done)
         return d
 
+
+    def test_PUT_update_at_offset(self):
+        file_contents = "test file" * 100000 # about 900 KiB
+        d = self.PUT("/uri?mutable=true", file_contents)
+        def _then(filecap):
+            self.filecap = filecap
+            new_data = file_contents[:100]
+            new = "replaced and so on"
+            new_data += new
+            new_data += file_contents[len(new_data):]
+            assert len(new_data) == len(file_contents)
+            self.new_data = new_data
+        d.addCallback(_then)
+        d.addCallback(lambda ignored:
+            self.PUT("/uri/%s?replace=True&offset=100" % self.filecap,
+                     "replaced and so on"))
+        def _get_data(filecap):
+            n = self.s.create_node_from_uri(filecap)
+            return n.download_best_version()
+        d.addCallback(_get_data)
+        d.addCallback(lambda results:
+            self.failUnlessEqual(results, self.new_data))
+        # Now try appending things to the file
+        d.addCallback(lambda ignored:
+            self.PUT("/uri/%s?offset=%d" % (self.filecap, len(self.new_data)),
+                     "puppies" * 100))
+        d.addCallback(_get_data)
+        d.addCallback(lambda results:
+            self.failUnlessEqual(results, self.new_data + ("puppies" * 100)))
+        # and try replacing the beginning of the file
+        d.addCallback(lambda ignored:
+            self.PUT("/uri/%s?offset=0" % self.filecap, "begin"))
+        d.addCallback(_get_data)
+        d.addCallback(lambda results:
+            self.failUnlessEqual(results, "begin"+self.new_data[len("begin"):]+("puppies"*100)))
+        return d
+
+    def test_PUT_update_at_invalid_offset(self):
+        file_contents = "test file" * 100000 # about 900 KiB
+        d = self.PUT("/uri?mutable=true", file_contents)
+        def _then(filecap):
+            self.filecap = filecap
+        d.addCallback(_then)
+        # Negative offsets should cause an error.
+        d.addCallback(lambda ignored:
+            self.shouldHTTPError("test mutable invalid offset negative",
+                                 400, "Bad Request",
+                                 "Invalid offset",
+                                 self.PUT,
+                                 "/uri/%s?offset=-1" % self.filecap,
+                                 "foo"))
+        return d
+
+    def test_PUT_update_at_offset_immutable(self):
+        file_contents = "Test file" * 100000
+        d = self.PUT("/uri", file_contents)
+        def _then(filecap):
+            self.filecap = filecap
+        d.addCallback(_then)
+        d.addCallback(lambda ignored:
+            self.shouldHTTPError("test immutable update",
+                                 400, "Bad Request",
+                                 "immutable",
+                                 self.PUT,
+                                 "/uri/%s?offset=50" % self.filecap,
+                                 "foo"))
+        return d
+
+
     def test_bad_method(self):
         url = self.webish_url + self.public_url + "/foo/bar.txt"
         d = self.shouldHTTPError("test_bad_method",
@@ -3281,7 +4090,8 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
         def _stash_mutable_uri(n, which):
             self.uris[which] = n.get_uri()
             assert isinstance(self.uris[which], str)
-        d.addCallback(lambda ign: c0.create_mutable_file(DATA+"3"))
+        d.addCallback(lambda ign:
+            c0.create_mutable_file(publish.MutableData(DATA+"3")))
         d.addCallback(_stash_mutable_uri, "corrupt")
         d.addCallback(lambda ign:
                       c0.upload(upload.Data("literal", convergence="")))
@@ -3427,7 +4237,8 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
         def _stash_mutable_uri(n, which):
             self.uris[which] = n.get_uri()
             assert isinstance(self.uris[which], str)
-        d.addCallback(lambda ign: c0.create_mutable_file(DATA+"3"))
+        d.addCallback(lambda ign:
+            c0.create_mutable_file(publish.MutableData(DATA+"3")))
         d.addCallback(_stash_mutable_uri, "corrupt")
 
         def _compute_fileurls(ignored):
@@ -4089,7 +4900,8 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
         def _stash_mutable_uri(n, which):
             self.uris[which] = n.get_uri()
             assert isinstance(self.uris[which], str)
-        d.addCallback(lambda ign: c0.create_mutable_file(DATA+"2"))
+        d.addCallback(lambda ign:
+            c0.create_mutable_file(publish.MutableData(DATA+"2")))
         d.addCallback(_stash_mutable_uri, "mutable")
 
         def _compute_fileurls(ignored):
@@ -4188,7 +5000,8 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
                                                         convergence="")))
         d.addCallback(_stash_uri, "small")
 
-        d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
+        d.addCallback(lambda ign:
+            c0.create_mutable_file(publish.MutableData("mutable")))
         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
         d.addCallback(_stash_uri, "mutable")
 
index 22083a587ebb37aae4680417cf7cd32d3d7a0443..6e90554488bd81cafc140797e628305eaa9149af 100644 (file)
@@ -9,7 +9,7 @@ from nevow.util import resource_filename
 from allmydata.interfaces import ExistingChildError, NoSuchChildError, \
      FileTooLargeError, NotEnoughSharesError, NoSharesError, \
      EmptyPathnameComponentError, MustBeDeepImmutableError, \
-     MustBeReadonlyError, MustNotBeUnknownRWError
+     MustBeReadonlyError, MustNotBeUnknownRWError, SDMF_VERSION, MDMF_VERSION
 from allmydata.mutable.common import UnrecoverableFileError
 from allmydata.util import abbreviate
 from allmydata.util.encodingutil import to_str, quote_output
@@ -32,6 +32,32 @@ def parse_replace_arg(replace):
     else:
         return boolean_of_arg(replace)
 
+
+def parse_mutable_type_arg(arg):
+    if not arg:
+        return None # interpreted by the caller as "let the nodemaker decide"
+
+    arg = arg.lower()
+    if arg == "mdmf":
+        return MDMF_VERSION
+    elif arg == "sdmf":
+        return SDMF_VERSION
+
+    return "invalid"
+
+
+def parse_offset_arg(offset):
+    # XXX: This will raise a ValueError when invoked on something that
+    # is not an integer. Is that okay? Or do we want a better error
+    # message? Since this call is going to be used by programmers and
+    # their tools rather than users (through the wui), it is not
+    # inconsistent to return that, I guess.
+    if offset is not None:
+        offset = int(offset)
+
+    return offset
+
+
 def get_root(ctx_or_req):
     req = IRequest(ctx_or_req)
     # the addSlash=True gives us one extra (empty) segment
index 216a539d772cb17958fdc3b830e48de403fc9476..d5d202942b4031c76b3a8eebac6334c0dea104f5 100644 (file)
@@ -16,14 +16,15 @@ from allmydata.util import base32, time_format
 from allmydata.uri import from_string_dirnode
 from allmydata.interfaces import IDirectoryNode, IFileNode, IFilesystemNode, \
      IImmutableFileNode, IMutableFileNode, ExistingChildError, \
-     NoSuchChildError, EmptyPathnameComponentError
+     NoSuchChildError, EmptyPathnameComponentError, SDMF_VERSION, MDMF_VERSION
 from allmydata.monitor import Monitor, OperationCancelledError
 from allmydata import dirnode
 from allmydata.web.common import text_plain, WebError, \
      IOpHandleTable, NeedOperationHandleError, \
      boolean_of_arg, get_arg, get_root, parse_replace_arg, \
      should_create_intermediate_directories, \
-     getxmlfile, RenderMixin, humanize_failure, convert_children_json
+     getxmlfile, RenderMixin, humanize_failure, convert_children_json, \
+     parse_mutable_type_arg
 from allmydata.web.filenode import ReplaceMeMixin, \
      FileNodeHandler, PlaceHolderNodeHandler
 from allmydata.web.check_results import CheckResults, \
@@ -108,8 +109,17 @@ class DirectoryNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin):
                     mutable = True
                     if t == "mkdir-immutable":
                         mutable = False
+
+                    mt = None
+                    if mutable:
+                        arg = get_arg(req, "mutable-type", None)
+                        mt = parse_mutable_type_arg(arg)
+                        if mt is "invalid":
+                            raise WebError("Unknown type: %s" % arg,
+                                           http.BAD_REQUEST)
                     d = self.node.create_subdirectory(name, kids,
-                                                      mutable=mutable)
+                                                      mutable=mutable,
+                                                      mutable_version=mt)
                     d.addCallback(make_handler_for,
                                   self.client, self.node, name)
                     return d
@@ -150,7 +160,8 @@ class DirectoryNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin):
         if not t:
             # render the directory as HTML, using the docFactory and Nevow's
             # whole templating thing.
-            return DirectoryAsHTML(self.node)
+            return DirectoryAsHTML(self.node,
+                                   self.client.mutable_file_default)
 
         if t == "json":
             return DirectoryJSONMetadata(ctx, self.node)
@@ -239,7 +250,15 @@ class DirectoryNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin):
         name = name.decode("utf-8")
         replace = boolean_of_arg(get_arg(req, "replace", "true"))
         kids = {}
-        d = self.node.create_subdirectory(name, kids, overwrite=replace)
+        arg = get_arg(req, "mutable-type", None)
+        mt = parse_mutable_type_arg(arg)
+        if mt is not None and mt is not "invalid":
+            d = self.node.create_subdirectory(name, kids, overwrite=replace,
+                                          mutable_version=mt)
+        elif mt is "invalid":
+            raise WebError("Unknown type: %s" % arg, http.BAD_REQUEST)
+        else:
+            d = self.node.create_subdirectory(name, kids, overwrite=replace)
         d.addCallback(lambda child: child.get_uri()) # TODO: urlencode
         return d
 
@@ -255,7 +274,15 @@ class DirectoryNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin):
         req.content.seek(0)
         kids_json = req.content.read()
         kids = convert_children_json(self.client.nodemaker, kids_json)
-        d = self.node.create_subdirectory(name, kids, overwrite=False)
+        arg = get_arg(req, "mutable-type", None)
+        mt = parse_mutable_type_arg(arg)
+        if mt is not None and mt is not "invalid":
+            d = self.node.create_subdirectory(name, kids, overwrite=False,
+                                              mutable_version=mt)
+        elif mt is "invalid":
+            raise WebError("Unknown type: %s" % arg)
+        else:
+            d = self.node.create_subdirectory(name, kids, overwrite=False)
         d.addCallback(lambda child: child.get_uri()) # TODO: urlencode
         return d
 
@@ -552,10 +579,13 @@ class DirectoryAsHTML(rend.Page):
     docFactory = getxmlfile("directory.xhtml")
     addSlash = True
 
-    def __init__(self, node):
+    def __init__(self, node, default_mutable_format):
         rend.Page.__init__(self)
         self.node = node
 
+        assert default_mutable_format in (MDMF_VERSION, SDMF_VERSION)
+        self.default_mutable_format = default_mutable_format
+
     def beforeRender(self, ctx):
         # attempt to get the dirnode's children, stashing them (or the
         # failure that results) for later use
@@ -753,6 +783,7 @@ class DirectoryAsHTML(rend.Page):
 
         return ctx.tag
 
+    # XXX: Duplicated from root.py.
     def render_forms(self, ctx, data):
         forms = []
 
@@ -761,6 +792,12 @@ class DirectoryAsHTML(rend.Page):
         if self.dirnode_children is None:
             return T.div["No upload forms: directory is unreadable"]
 
+        mdmf_directory_input = T.input(type='radio', name='mutable-type',
+                                       id='mutable-directory-mdmf',
+                                       value='mdmf')
+        sdmf_directory_input = T.input(type='radio', name='mutable-type',
+                                       id='mutable-directory-sdmf',
+                                       value='sdmf', checked='checked')
         mkdir = T.form(action=".", method="post",
                        enctype="multipart/form-data")[
             T.fieldset[
@@ -769,10 +806,34 @@ class DirectoryAsHTML(rend.Page):
             T.legend(class_="freeform-form-label")["Create a new directory in this directory"],
             "New directory name: ",
             T.input(type="text", name="name"), " ",
+            T.label(for_='mutable-directory-sdmf')["SDMF"],
+            sdmf_directory_input,
+            T.label(for_='mutable-directory-mdmf')["MDMF"],
+            mdmf_directory_input,
             T.input(type="submit", value="Create"),
             ]]
         forms.append(T.div(class_="freeform-form")[mkdir])
 
+        # Build input elements for mutable file type. We do this outside
+        # of the list so we can check the appropriate format, based on
+        # the default configured in the client (which reflects the
+        # default configured in tahoe.cfg)
+        if self.default_mutable_format == MDMF_VERSION:
+            mdmf_input = T.input(type='radio', name='mutable-type',
+                                 id='mutable-type-mdmf', value='mdmf',
+                                 checked='checked')
+        else:
+            mdmf_input = T.input(type='radio', name='mutable-type',
+                                 id='mutable-type-mdmf', value='mdmf')
+
+        if self.default_mutable_format == SDMF_VERSION:
+            sdmf_input = T.input(type='radio', name='mutable-type',
+                                 id='mutable-type-sdmf', value='sdmf',
+                                 checked="checked")
+        else:
+            sdmf_input = T.input(type='radio', name='mutable-type',
+                                 id='mutable-type-sdmf', value='sdmf')
+
         upload = T.form(action=".", method="post",
                         enctype="multipart/form-data")[
             T.fieldset[
@@ -785,6 +846,9 @@ class DirectoryAsHTML(rend.Page):
             T.input(type="submit", value="Upload"),
             " Mutable?:",
             T.input(type="checkbox", name="mutable"),
+            sdmf_input, T.label(for_="mutable-type-sdmf")["SDMF"],
+            mdmf_input,
+            T.label(for_="mutable-type-mdmf")["MDMF (experimental)"],
             ]]
         forms.append(T.div(class_="freeform-form")[upload])
 
@@ -820,6 +884,17 @@ def DirectoryJSONMetadata(ctx, dirnode):
                 kiddata = ("filenode", {'size': childnode.get_size(),
                                         'mutable': childnode.is_mutable(),
                                         })
+                if childnode.is_mutable() and \
+                    childnode.get_version() is not None:
+                    mutable_type = childnode.get_version()
+                    assert mutable_type in (SDMF_VERSION, MDMF_VERSION)
+
+                    if mutable_type == MDMF_VERSION:
+                        mutable_type = "mdmf"
+                    else:
+                        mutable_type = "sdmf"
+                    kiddata[1]['mutable-type'] = mutable_type
+
             elif IDirectoryNode.providedBy(childnode):
                 kiddata = ("dirnode", {'mutable': childnode.is_mutable()})
             else:
index 447bbe00f945659c8394bf8ba5a782edd1045b80..d5f40bc44776ccaf4af714985f9317abcb633a47 100644 (file)
@@ -6,14 +6,17 @@ from twisted.internet import defer
 from nevow import url, rend
 from nevow.inevow import IRequest
 
-from allmydata.interfaces import ExistingChildError
+from allmydata.interfaces import ExistingChildError, SDMF_VERSION, MDMF_VERSION
 from allmydata.monitor import Monitor
 from allmydata.immutable.upload import FileHandle
+from allmydata.mutable.publish import MutableFileHandle
+from allmydata.mutable.common import MODE_READ
 from allmydata.util import log, base32
 
 from allmydata.web.common import text_plain, WebError, RenderMixin, \
      boolean_of_arg, get_arg, should_create_intermediate_directories, \
-     MyExceptionHandler, parse_replace_arg
+     MyExceptionHandler, parse_replace_arg, parse_offset_arg, \
+     parse_mutable_type_arg
 from allmydata.web.check_results import CheckResults, \
      CheckAndRepairResults, LiteralCheckResults
 from allmydata.web.info import MoreInfo
@@ -23,9 +26,13 @@ class ReplaceMeMixin:
         # a new file is being uploaded in our place.
         mutable = boolean_of_arg(get_arg(req, "mutable", "false"))
         if mutable:
-            req.content.seek(0)
-            data = req.content.read()
-            d = client.create_mutable_file(data)
+            arg = get_arg(req, "mutable-type", None)
+            mutable_type = parse_mutable_type_arg(arg)
+            if mutable_type is "invalid":
+                raise WebError("Unknown type: %s" % arg, http.BAD_REQUEST)
+
+            data = MutableFileHandle(req.content)
+            d = client.create_mutable_file(data, version=mutable_type)
             def _uploaded(newnode):
                 d2 = self.parentnode.set_node(self.name, newnode,
                                               overwrite=replace)
@@ -58,21 +65,20 @@ class ReplaceMeMixin:
         d.addCallback(lambda res: childnode.get_uri())
         return d
 
-    def _read_data_from_formpost(self, req):
-        # SDMF: files are small, and we can only upload data, so we read
-        # the whole file into memory before uploading.
-        contents = req.fields["file"]
-        contents.file.seek(0)
-        data = contents.file.read()
-        return data
 
     def replace_me_with_a_formpost(self, req, client, replace):
         # create a new file, maybe mutable, maybe immutable
         mutable = boolean_of_arg(get_arg(req, "mutable", "false"))
 
+        # create an immutable file
+        contents = req.fields["file"]
         if mutable:
-            data = self._read_data_from_formpost(req)
-            d = client.create_mutable_file(data)
+            arg = get_arg(req, "mutable-type", None)
+            mutable_type = parse_mutable_type_arg(arg)
+            if mutable_type is "invalid":
+                raise WebError("Unknown type: %s" % arg, http.BAD_REQUEST)
+            uploadable = MutableFileHandle(contents.file)
+            d = client.create_mutable_file(uploadable, version=mutable_type)
             def _uploaded(newnode):
                 d2 = self.parentnode.set_node(self.name, newnode,
                                               overwrite=replace)
@@ -80,13 +86,13 @@ class ReplaceMeMixin:
                 return d2
             d.addCallback(_uploaded)
             return d
-        # create an immutable file
-        contents = req.fields["file"]
+
         uploadable = FileHandle(contents.file, convergence=client.convergence)
         d = self.parentnode.add_file(self.name, uploadable, overwrite=replace)
         d.addCallback(lambda newnode: newnode.get_uri())
         return d
 
+
 class PlaceHolderNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin):
     def __init__(self, client, parentnode, name):
         rend.Page.__init__(self)
@@ -169,18 +175,27 @@ class FileNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin):
             # properly. So we assume that at least the browser will agree
             # with itself, and echo back the same bytes that we were given.
             filename = get_arg(req, "filename", self.name) or "unknown"
-            if self.node.is_mutable():
-                # some day: d = self.node.get_best_version()
-                d = makeMutableDownloadable(self.node)
-            else:
-                d = defer.succeed(self.node)
+            d = self.node.get_best_readable_version()
             d.addCallback(lambda dn: FileDownloader(dn, filename))
             return d
         if t == "json":
-            if self.parentnode and self.name:
-                d = self.parentnode.get_metadata_for(self.name)
+            # We do this to make sure that fields like size and
+            # mutable-type (which depend on the file on the grid and not
+            # just on the cap) are filled in. The latter gets used in
+            # tests, in particular.
+            #
+            # TODO: Make it so that the servermap knows how to update in
+            # a mode specifically designed to fill in these fields, and
+            # then update it in that mode.
+            if self.node.is_mutable():
+                d = self.node.get_servermap(MODE_READ)
             else:
                 d = defer.succeed(None)
+            if self.parentnode and self.name:
+                d.addCallback(lambda ignored:
+                    self.parentnode.get_metadata_for(self.name))
+            else:
+                d.addCallback(lambda ignored: None)
             d.addCallback(lambda md: FileJSONMetadata(ctx, self.node, md))
             return d
         if t == "info":
@@ -197,11 +212,7 @@ class FileNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin):
         if t:
             raise WebError("GET file: bad t=%s" % t)
         filename = get_arg(req, "filename", self.name) or "unknown"
-        if self.node.is_mutable():
-            # some day: d = self.node.get_best_version()
-            d = makeMutableDownloadable(self.node)
-        else:
-            d = defer.succeed(self.node)
+        d = self.node.get_best_readable_version()
         d.addCallback(lambda dn: FileDownloader(dn, filename))
         return d
 
@@ -209,17 +220,37 @@ class FileNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin):
         req = IRequest(ctx)
         t = get_arg(req, "t", "").strip()
         replace = parse_replace_arg(get_arg(req, "replace", "true"))
+        offset = parse_offset_arg(get_arg(req, "offset", None))
 
         if not t:
-            if self.node.is_mutable():
-                return self.replace_my_contents(req)
             if not replace:
                 # this is the early trap: if someone else modifies the
                 # directory while we're uploading, the add_file(overwrite=)
                 # call in replace_me_with_a_child will do the late trap.
                 raise ExistingChildError()
-            assert self.parentnode and self.name
-            return self.replace_me_with_a_child(req, self.client, replace)
+
+            if self.node.is_mutable():
+                # Are we a readonly filenode? We shouldn't allow callers
+                # to try to replace us if we are.
+                if self.node.is_readonly():
+                    raise WebError("PUT to a mutable file: replace or update"
+                                   " requested with read-only cap")
+                if offset is None:
+                    return self.replace_my_contents(req)
+
+                if offset >= 0:
+                    return self.update_my_contents(req, offset)
+
+                raise WebError("PUT to a mutable file: Invalid offset")
+
+            else:
+                if offset is not None:
+                    raise WebError("PUT to a file: append operation invoked "
+                                   "on an immutable cap")
+
+                assert self.parentnode and self.name
+                return self.replace_me_with_a_child(req, self.client, replace)
+
         if t == "uri":
             if not replace:
                 raise ExistingChildError()
@@ -280,46 +311,34 @@ class FileNodeHandler(RenderMixin, rend.Page, ReplaceMeMixin):
 
     def replace_my_contents(self, req):
         req.content.seek(0)
-        new_contents = req.content.read()
+        new_contents = MutableFileHandle(req.content)
         d = self.node.overwrite(new_contents)
         d.addCallback(lambda res: self.node.get_uri())
         return d
 
+
+    def update_my_contents(self, req, offset):
+        req.content.seek(0)
+        added_contents = MutableFileHandle(req.content)
+
+        d = self.node.get_best_mutable_version()
+        d.addCallback(lambda mv:
+            mv.update(added_contents, offset))
+        d.addCallback(lambda ignored:
+            self.node.get_uri())
+        return d
+
+
     def replace_my_contents_with_a_formpost(self, req):
         # we have a mutable file. Get the data from the formpost, and replace
         # the mutable file's contents with it.
-        new_contents = self._read_data_from_formpost(req)
+        new_contents = req.fields['file']
+        new_contents = MutableFileHandle(new_contents.file)
+
         d = self.node.overwrite(new_contents)
         d.addCallback(lambda res: self.node.get_uri())
         return d
 
-class MutableDownloadable:
-    #implements(IDownloadable)
-    def __init__(self, size, node):
-        self.size = size
-        self.node = node
-    def get_size(self):
-        return self.size
-    def is_mutable(self):
-        return True
-    def read(self, consumer, offset=0, size=None):
-        d = self.node.download_best_version()
-        d.addCallback(self._got_data, consumer, offset, size)
-        return d
-    def _got_data(self, contents, consumer, offset, size):
-        start = offset
-        if size is not None:
-            end = offset+size
-        else:
-            end = self.size
-        # SDMF: we can write the whole file in one big chunk
-        consumer.write(contents[start:end])
-        return consumer
-
-def makeMutableDownloadable(n):
-    d = defer.maybeDeferred(n.get_size_of_best_version)
-    d.addCallback(MutableDownloadable, n)
-    return d
 
 class FileDownloader(rend.Page):
     def __init__(self, filenode, filename):
@@ -494,6 +513,16 @@ def FileJSONMetadata(ctx, filenode, edge_metadata):
     data[1]['mutable'] = filenode.is_mutable()
     if edge_metadata is not None:
         data[1]['metadata'] = edge_metadata
+
+    if filenode.is_mutable() and filenode.get_version() is not None:
+        mutable_type = filenode.get_version()
+        assert mutable_type in (MDMF_VERSION, SDMF_VERSION)
+        if mutable_type == MDMF_VERSION:
+            mutable_type = "mdmf"
+        else:
+            mutable_type = "sdmf"
+        data[1]['mutable-type'] = mutable_type
+
     return text_plain(simplejson.dumps(data, indent=1) + "\n", ctx)
 
 def FileURI(ctx, filenode):
index 4a765859f23d2bbe8043b16755f4644a13632434..ee1affaecb44a20ba3800b78e39e58dedcd89387 100644 (file)
@@ -5,7 +5,7 @@ from nevow import rend, tags as T
 from nevow.inevow import IRequest
 
 from allmydata.util import base32
-from allmydata.interfaces import IDirectoryNode, IFileNode
+from allmydata.interfaces import IDirectoryNode, IFileNode, MDMF_VERSION
 from allmydata.web.common import getxmlfile
 from allmydata.mutable.common import UnrecoverableFileError # TODO: move
 
@@ -28,7 +28,12 @@ class MoreInfo(rend.Page):
             si = node.get_storage_index()
             if si:
                 if node.is_mutable():
-                    return "mutable file"
+                    ret = "mutable file"
+                    if node.get_version() == MDMF_VERSION:
+                        ret += " (mdmf)"
+                    else:
+                        ret += " (sdmf)"
+                    return ret
                 return "immutable file"
             return "immutable LIT file"
         return "unknown"
index d7a832f52fe7a2228b33f5e37df5e160620c2dbc..00495d894987ec116eb058fde33d1a052c475b7c 100644 (file)
@@ -12,11 +12,11 @@ import allmydata # to display import path
 from allmydata import get_package_versions_string
 from allmydata import provisioning
 from allmydata.util import idlib, log
-from allmydata.interfaces import IFileNode
+from allmydata.interfaces import IFileNode, MDMF_VERSION, SDMF_VERSION
 from allmydata.web import filenode, directory, unlinked, status, operations
 from allmydata.web import reliability, storage
 from allmydata.web.common import abbreviate_size, getxmlfile, WebError, \
-     get_arg, RenderMixin, boolean_of_arg
+     get_arg, RenderMixin, boolean_of_arg, parse_mutable_type_arg
 
 
 class URIHandler(RenderMixin, rend.Page):
@@ -47,7 +47,13 @@ class URIHandler(RenderMixin, rend.Page):
         if t == "":
             mutable = boolean_of_arg(get_arg(req, "mutable", "false").strip())
             if mutable:
-                return unlinked.PUTUnlinkedSSK(req, self.client)
+                arg = get_arg(req, "mutable-type", None)
+                version = parse_mutable_type_arg(arg)
+                if version == "invalid":
+                    errmsg = "Unknown type: %s" % arg
+                    raise WebError(errmsg, http.BAD_REQUEST)
+
+                return unlinked.PUTUnlinkedSSK(req, self.client, version)
             else:
                 return unlinked.PUTUnlinkedCHK(req, self.client)
         if t == "mkdir":
@@ -65,7 +71,11 @@ class URIHandler(RenderMixin, rend.Page):
         if t in ("", "upload"):
             mutable = bool(get_arg(req, "mutable", "").strip())
             if mutable:
-                return unlinked.POSTUnlinkedSSK(req, self.client)
+                arg = get_arg(req, "mutable-type", None)
+                version = parse_mutable_type_arg(arg)
+                if version is "invalid":
+                    raise WebError("Unknown type: %s" % arg, http.BAD_REQUEST)
+                return unlinked.POSTUnlinkedSSK(req, self.client, version)
             else:
                 return unlinked.POSTUnlinkedCHK(req, self.client)
         if t == "mkdir":
@@ -322,6 +332,30 @@ class Root(rend.Page):
 
     def render_upload_form(self, ctx, data):
         # this is a form where users can upload unlinked files
+        #
+        # for mutable files, users can choose the format by selecting
+        # MDMF or SDMF from a radio button. They can also configure a
+        # default format in tahoe.cfg, which they rightly expect us to
+        # obey. we convey to them that we are obeying their choice by
+        # ensuring that the one that they've chosen is selected in the
+        # interface.
+        if self.client.mutable_file_default == MDMF_VERSION:
+            mdmf_input = T.input(type='radio', name='mutable-type',
+                                 value='mdmf', id='mutable-type-mdmf',
+                                 checked='checked')
+        else:
+            mdmf_input = T.input(type='radio', name='mutable-type',
+                                 value='mdmf', id='mutable-type-mdmf')
+
+        if self.client.mutable_file_default == SDMF_VERSION:
+            sdmf_input = T.input(type='radio', name='mutable-type',
+                                 value='sdmf', id='mutable-type-sdmf',
+                                 checked='checked')
+        else:
+            sdmf_input = T.input(type='radio', name='mutable-type',
+                                 value='sdmf', id='mutable-type-sdmf')
+
+
         form = T.form(action="uri", method="post",
                       enctype="multipart/form-data")[
             T.fieldset[
@@ -330,16 +364,28 @@ class Root(rend.Page):
                   T.input(type="file", name="file", class_="freeform-input-file")],
             T.input(type="hidden", name="t", value="upload"),
             T.div[T.input(type="checkbox", name="mutable"), T.label(for_="mutable")["Create mutable file"],
+                  sdmf_input, T.label(for_="mutable-type-sdmf")["SDMF"],
+                  mdmf_input,
+                  T.label(for_='mutable-type-mdmf')['MDMF (experimental)'],
                   " ", T.input(type="submit", value="Upload!")],
             ]]
         return T.div[form]
 
     def render_mkdir_form(self, ctx, data):
         # this is a form where users can create new directories
+        mdmf_input = T.input(type='radio', name='mutable-type',
+                             value='mdmf', id='mutable-directory-mdmf')
+        sdmf_input = T.input(type='radio', name='mutable-type',
+                             value='sdmf', id='mutable-directory-sdmf',
+                             checked='checked')
         form = T.form(action="uri", method="post",
                       enctype="multipart/form-data")[
             T.fieldset[
             T.legend(class_="freeform-form-label")["Create a directory"],
+            T.label(for_='mutable-directory-sdmf')["SDMF"],
+            sdmf_input,
+            T.label(for_='mutable-directory-mdmf')["MDMF"],
+            mdmf_input,
             T.input(type="hidden", name="t", value="mkdir"),
             T.input(type="hidden", name="redirect_to_result", value="true"),
             T.input(type="submit", value="Create a directory"),
index 670aff68520c15a3032b350e90cf6ae1ecf8109e..482bd285ae5260f7b02c552662277825ac83bbed 100644 (file)
@@ -4,8 +4,9 @@ from twisted.web import http
 from twisted.internet import defer
 from nevow import rend, url, tags as T
 from allmydata.immutable.upload import FileHandle
+from allmydata.mutable.publish import MutableFileHandle
 from allmydata.web.common import getxmlfile, get_arg, boolean_of_arg, \
-     convert_children_json, WebError
+     convert_children_json, WebError, parse_mutable_type_arg
 from allmydata.web import status
 
 def PUTUnlinkedCHK(req, client):
@@ -16,17 +17,25 @@ def PUTUnlinkedCHK(req, client):
     # that fires with the URI of the new file
     return d
 
-def PUTUnlinkedSSK(req, client):
+def PUTUnlinkedSSK(req, client, version):
     # SDMF: files are small, and we can only upload data
     req.content.seek(0)
-    data = req.content.read()
-    d = client.create_mutable_file(data)
+    data = MutableFileHandle(req.content)
+    d = client.create_mutable_file(data, version=version)
     d.addCallback(lambda n: n.get_uri())
     return d
 
 def PUTUnlinkedCreateDirectory(req, client):
     # "PUT /uri?t=mkdir", to create an unlinked directory.
-    d = client.create_dirnode()
+    arg = get_arg(req, "mutable-type", None)
+    mt = parse_mutable_type_arg(arg)
+    if mt is not None and mt is not "invalid":
+        d = client.create_dirnode(version=mt)
+    elif mt is "invalid":
+        msg = "Unknown type: %s" % arg
+        raise WebError(msg, http.BAD_REQUEST)
+    else:
+        d = client.create_dirnode()
     d.addCallback(lambda dirnode: dirnode.get_uri())
     # XXX add redirect_to_result
     return d
@@ -79,13 +88,12 @@ class UploadResultsPage(status.UploadResultsRendererMixin, rend.Page):
                       ["/uri/" + res.uri])
         return d
 
-def POSTUnlinkedSSK(req, client):
+def POSTUnlinkedSSK(req, client, version):
     # "POST /uri", to create an unlinked file.
     # SDMF: files are small, and we can only upload data
-    contents = req.fields["file"]
-    contents.file.seek(0)
-    data = contents.file.read()
-    d = client.create_mutable_file(data)
+    contents = req.fields["file"].file
+    data = MutableFileHandle(contents)
+    d = client.create_mutable_file(data, version=version)
     d.addCallback(lambda n: n.get_uri())
     return d
 
@@ -104,7 +112,15 @@ def POSTUnlinkedCreateDirectory(req, client):
             raise WebError("t=mkdir does not accept children=, "
                            "try t=mkdir-with-children instead",
                            http.BAD_REQUEST)
-    d = client.create_dirnode()
+    arg = get_arg(req, "mutable-type", None)
+    mt = parse_mutable_type_arg(arg)
+    if mt is not None and mt is not "invalid":
+        d = client.create_dirnode(version=mt)
+    elif mt is "invalid":
+        msg = "Unknown type: %s" % arg
+        raise WebError(msg, http.BAD_REQUEST)
+    else:
+        d = client.create_dirnode()
     redirect = get_arg(req, "redirect_to_result", "false")
     if boolean_of_arg(redirect):
         def _then_redir(res):