Fix tests on platforms without time.tzset (e.g. Windows). fixes ticket:2725

[tahoe-lafs/tahoe-lafs.git] / src / allmydata / test / test_web.py

diff --git a/src/allmydata/test/test_web.py b/src/allmydata/test/test_web.py
index 377086416b76dc264cb14272ad93c9cc0e837301..2a968f9aed4f5087b6dd74849e9a0e69568364e2 100644 (file)
--- a/src/allmydata/test/test_web.py
+++ b/src/allmydata/test/test_web.py
@@ -1,36 +1,44 @@
-
-import os.path, re, urllib
+import os.path, re, urllib, time, cgi

 import simplejson
 from StringIO import StringIO
 import simplejson
 from StringIO import StringIO

+

 from twisted.application import service
 from twisted.trial import unittest
 from twisted.internet import defer, reactor
 from twisted.internet.task import Clock
 from twisted.web import client, error, http
 from twisted.python import failure, log
 from twisted.application import service
 from twisted.trial import unittest
 from twisted.internet import defer, reactor
 from twisted.internet.task import Clock
 from twisted.web import client, error, http
 from twisted.python import failure, log

+
+from foolscap.api import fireEventually, flushEventualQueue
+
+from nevow.util import escapeToXML

 from nevow import rend
 from nevow import rend

+

 from allmydata import interfaces, uri, webish, dirnode
 from allmydata.storage.shares import get_share_file
 from allmydata import interfaces, uri, webish, dirnode
 from allmydata.storage.shares import get_share_file

-from allmydata.storage_client import StorageFarmBroker
-from allmydata.immutable import upload, download
+from allmydata.storage_client import StorageFarmBroker, StubServer
+from allmydata.immutable import upload
+from allmydata.immutable.downloader.status import DownloadStatus

 from allmydata.dirnode import DirectoryNode
 from allmydata.nodemaker import NodeMaker
 from allmydata.unknown import UnknownNode
 from allmydata.web import status, common
 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
 from allmydata.dirnode import DirectoryNode
 from allmydata.nodemaker import NodeMaker
 from allmydata.unknown import UnknownNode
 from allmydata.web import status, common
 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share

-from allmydata.util import fileutil, base32
+from allmydata.util import fileutil, base32, hashutil

 from allmydata.util.consumer import download_to_data
 from allmydata.util.netstring import split_netstring
 from allmydata.util.encodingutil import to_str
 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
 from allmydata.util.consumer import download_to_data
 from allmydata.util.netstring import split_netstring
 from allmydata.util.encodingutil import to_str
 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \

-     create_chk_filenode, WebErrorMixin, ShouldFailMixin, make_mutable_file_uri
-from allmydata.interfaces import IMutableFileNode
+     create_chk_filenode, WebErrorMixin, ShouldFailMixin, \
+     make_mutable_file_uri, create_mutable_filenode
+from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION

 from allmydata.mutable import servermap, publish, retrieve
 import allmydata.test.common_util as testutil
 from allmydata.test.no_network import GridTestMixin
 from allmydata.test.common_web import HTTPClientGETFactory, \
      HTTPClientHEADFactory
 from allmydata.client import Client, SecretHolder
 from allmydata.mutable import servermap, publish, retrieve
 import allmydata.test.common_util as testutil
 from allmydata.test.no_network import GridTestMixin
 from allmydata.test.common_web import HTTPClientGETFactory, \
      HTTPClientHEADFactory
 from allmydata.client import Client, SecretHolder

+from allmydata.introducer import IntroducerNode

 
 # create a fake uploader/downloader, and a couple of fake dirnodes, then
 # create a webserver that works against them
 
 # create a fake uploader/downloader, and a couple of fake dirnodes, then
 # create a webserver that works against them


@@ -41,41 +49,111 @@ unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
 
 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
 

+FAVICON_MARKUP = '<link href="/icon.png" rel="shortcut icon" />'
+DIR_HTML_TAG = '<html lang="en">'
+

 class FakeStatsProvider:
     def get_stats(self):
         stats = {'stats': {}, 'counters': {}}
         return stats
 
 class FakeNodeMaker(NodeMaker):
 class FakeStatsProvider:
     def get_stats(self):
         stats = {'stats': {}, 'counters': {}}
         return stats
 
 class FakeNodeMaker(NodeMaker):

+    encoding_params = {
+        'k': 3,
+        'n': 10,
+        'happy': 7,
+        'max_segment_size':128*1024 # 1024=KiB
+    }

     def _create_lit(self, cap):
     def _create_lit(self, cap):

-        return FakeCHKFileNode(cap)
+        return FakeCHKFileNode(cap, self.all_contents)

     def _create_immutable(self, cap):
     def _create_immutable(self, cap):

-        return FakeCHKFileNode(cap)
+        return FakeCHKFileNode(cap, self.all_contents)

     def _create_mutable(self, cap):
     def _create_mutable(self, cap):

-        return FakeMutableFileNode(None, None, None, None).init_from_cap(cap)
-    def create_mutable_file(self, contents="", keysize=None):
-        n = FakeMutableFileNode(None, None, None, None)
-        return n.create(contents)
+        return FakeMutableFileNode(None, None,
+                                   self.encoding_params, None,
+                                   self.all_contents).init_from_cap(cap)
+    def create_mutable_file(self, contents="", keysize=None,
+                            version=SDMF_VERSION):
+        n = FakeMutableFileNode(None, None, self.encoding_params, None,
+                                self.all_contents)
+        return n.create(contents, version=version)

 
 class FakeUploader(service.Service):
     name = "uploader"
 
 class FakeUploader(service.Service):
     name = "uploader"

-    def upload(self, uploadable, history=None):
+    helper_furl = None
+    helper_connected = False
+
+    def upload(self, uploadable):

         d = uploadable.get_size()
         d.addCallback(lambda size: uploadable.read(size))
         def _got_data(datav):
             data = "".join(datav)
         d = uploadable.get_size()
         d.addCallback(lambda size: uploadable.read(size))
         def _got_data(datav):
             data = "".join(datav)

-            n = create_chk_filenode(data)
-            results = upload.UploadResults()
-            results.uri = n.get_uri()
-            return results
+            n = create_chk_filenode(data, self.all_contents)
+            ur = upload.UploadResults(file_size=len(data),
+                                      ciphertext_fetched=0,
+                                      preexisting_shares=0,
+                                      pushed_shares=10,
+                                      sharemap={},
+                                      servermap={},
+                                      timings={},
+                                      uri_extension_data={},
+                                      uri_extension_hash="fake",
+                                      verifycapstr="fakevcap")
+            ur.set_uri(n.get_uri())
+            return ur

         d.addCallback(_got_data)
         return d
         d.addCallback(_got_data)
         return d

+

     def get_helper_info(self):
     def get_helper_info(self):

-        return (None, False)
+        return (self.helper_furl, self.helper_connected)
+
+
+def build_one_ds():
+    ds = DownloadStatus("storage_index", 1234)
+    now = time.time()
+
+    serverA = StubServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
+    serverB = StubServer(hashutil.tagged_hash("foo", "serverid_b")[:20])
+    storage_index = hashutil.storage_index_hash("SI")
+    e0 = ds.add_segment_request(0, now)
+    e0.activate(now+0.5)
+    e0.deliver(now+1, 0, 100, 0.5) # when, start,len, decodetime
+    e1 = ds.add_segment_request(1, now+2)
+    e1.error(now+3)
+    # two outstanding requests
+    e2 = ds.add_segment_request(2, now+4)
+    e3 = ds.add_segment_request(3, now+5)
+    del e2,e3 # hush pyflakes
+
+    # simulate a segment which gets delivered faster than a system clock tick (ticket #1166)
+    e = ds.add_segment_request(4, now)
+    e.activate(now)
+    e.deliver(now, 0, 140, 0.5)
+
+    e = ds.add_dyhb_request(serverA, now)
+    e.finished([1,2], now+1)
+    e = ds.add_dyhb_request(serverB, now+2) # left unfinished
+
+    e = ds.add_read_event(0, 120, now)
+    e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
+    e.finished(now+1)
+    e = ds.add_read_event(120, 30, now+2) # left unfinished
+
+    e = ds.add_block_request(serverA, 1, 100, 20, now)
+    e.finished(20, now+1)
+    e = ds.add_block_request(serverB, 1, 120, 30, now+1) # left unfinished
+
+    # make sure that add_read_event() can come first too
+    ds1 = DownloadStatus(storage_index, 1234)
+    e = ds1.add_read_event(0, 120, now)
+    e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
+    e.finished(now+1)
+
+    return ds

 
 class FakeHistory:
     _all_upload_status = [upload.UploadStatus()]
 
 class FakeHistory:
     _all_upload_status = [upload.UploadStatus()]

-    _all_download_status = [download.DownloadStatus()]
+    _all_download_status = [build_one_ds()]

     _all_mapupdate_statuses = [servermap.UpdateStatus()]
     _all_publish_statuses = [publish.PublishStatus()]
     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
     _all_mapupdate_statuses = [servermap.UpdateStatus()]
     _all_publish_statuses = [publish.PublishStatus()]
     _all_retrieve_statuses = [retrieve.RetrieveStatus()]


@@ -93,26 +171,108 @@ class FakeHistory:
     def list_all_helper_statuses(self):
         return []
 
     def list_all_helper_statuses(self):
         return []
 

+class FakeDisplayableServer(StubServer):
+    def __init__(self, serverid, nickname, connected,
+                 last_connect_time, last_loss_time, last_rx_time):
+        StubServer.__init__(self, serverid)
+        self.announcement = {"my-version": "allmydata-tahoe-fake",
+                             "service-name": "storage",
+                             "nickname": nickname}
+        self.connected = connected
+        self.last_loss_time = last_loss_time
+        self.last_rx_time = last_rx_time
+        self.last_connect_time = last_connect_time
+    def is_connected(self):
+        return self.connected
+    def get_permutation_seed(self):
+        return ""
+    def get_remote_host(self):
+        return ""
+    def get_last_loss_time(self):
+        return self.last_loss_time
+    def get_last_received_data_time(self):
+        return self.last_rx_time
+    def get_last_connect_time(self):
+        return self.last_connect_time
+    def get_announcement(self):
+        return self.announcement
+    def get_nickname(self):
+        return self.announcement["nickname"]
+    def get_available_space(self):
+        return 123456
+
+class FakeBucketCounter(object):
+    def get_state(self):
+        return {"last-complete-bucket-count": 0}
+    def get_progress(self):
+        return {"estimated-time-per-cycle": 0,
+                "cycle-in-progress": False,
+                "remaining-wait-time": 0}
+
+class FakeLeaseChecker(object):
+    def __init__(self):
+        self.expiration_enabled = False
+        self.mode = "age"
+        self.override_lease_duration = None
+        self.sharetypes_to_expire = {}
+    def get_state(self):
+        return {"history": None}
+    def get_progress(self):
+        return {"estimated-time-per-cycle": 0,
+                "cycle-in-progress": False,
+                "remaining-wait-time": 0}
+
+class FakeStorageServer(service.MultiService):
+    name = 'storage'
+    def __init__(self, nodeid, nickname):
+        service.MultiService.__init__(self)
+        self.my_nodeid = nodeid
+        self.nickname = nickname
+        self.bucket_counter = FakeBucketCounter()
+        self.lease_checker = FakeLeaseChecker()
+    def get_stats(self):
+        return {"storage_server.accepting_immutable_shares": False}
+

 class FakeClient(Client):
     def __init__(self):
         # don't upcall to Client.__init__, since we only want to initialize a
         # minimal subset
         service.MultiService.__init__(self)
 class FakeClient(Client):
     def __init__(self):
         # don't upcall to Client.__init__, since we only want to initialize a
         # minimal subset
         service.MultiService.__init__(self)

+        self.all_contents = {}

         self.nodeid = "fake_nodeid"
         self.nodeid = "fake_nodeid"

-        self.nickname = "fake_nickname"
+        self.nickname = u"fake_nickname \u263A"

         self.introducer_furl = "None"
         self.stats_provider = FakeStatsProvider()
         self._secret_holder = SecretHolder("lease secret", "convergence secret")
         self.helper = None
         self.convergence = "some random string"
         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
         self.introducer_furl = "None"
         self.stats_provider = FakeStatsProvider()
         self._secret_holder = SecretHolder("lease secret", "convergence secret")
         self.helper = None
         self.convergence = "some random string"
         self.storage_broker = StorageFarmBroker(None, permute_peers=True)

+        # fake knowledge of another server
+        self.storage_broker.test_add_server("other_nodeid",
+            FakeDisplayableServer(
+                serverid="other_nodeid", nickname=u"other_nickname \u263B", connected = True,
+                last_connect_time = 10, last_loss_time = 20, last_rx_time = 30))
+        self.storage_broker.test_add_server("disconnected_nodeid",
+            FakeDisplayableServer(
+                serverid="other_nodeid", nickname=u"disconnected_nickname \u263B", connected = False,
+                last_connect_time = 15, last_loss_time = 25, last_rx_time = 35))

         self.introducer_client = None
         self.history = FakeHistory()
         self.uploader = FakeUploader()
         self.introducer_client = None
         self.history = FakeHistory()
         self.uploader = FakeUploader()

+        self.uploader.all_contents = self.all_contents

         self.uploader.setServiceParent(self)
         self.uploader.setServiceParent(self)

+        self.blacklist = None

         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,

-                                       self.uploader, None, None,
-                                       None, None)
+                                       self.uploader, None,
+                                       None, None, None)
+        self.nodemaker.all_contents = self.all_contents
+        self.mutable_file_default = SDMF_VERSION
+        self.addService(FakeStorageServer(self.nodeid, self.nickname))
+
+    def get_long_nodeid(self):
+        return "v0-nodeid"
+    def get_long_tubid(self):
+        return "tubid"

 
     def startService(self):
         return service.MultiService.startService(self)
 
     def startService(self):
         return service.MultiService.startService(self)


@@ -121,17 +281,21 @@ class FakeClient(Client):
 
     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 
 
     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 

-class WebMixin(object):
+class WebMixin(testutil.TimezoneMixin):

     def setUp(self):
     def setUp(self):

+        self.setTimezone('UTC-13:00')

         self.s = FakeClient()
         self.s.startService()
         self.staticdir = self.mktemp()
         self.clock = Clock()
         self.s = FakeClient()
         self.s.startService()
         self.staticdir = self.mktemp()
         self.clock = Clock()

+        self.fakeTime = 86460 # 1d 0h 1m 0s

         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,

-                                      clock=self.clock)
+                                      clock=self.clock, now_fn=lambda:self.fakeTime)

         self.ws.setServiceParent(self.s)
         self.ws.setServiceParent(self.s)

-        self.webish_port = port = self.ws.listener._port.getHost().port
-        self.webish_url = "http://localhost:%d" % port
+        self.webish_port = self.ws.getPortnum()
+        self.webish_url = self.ws.getURL()
+        assert self.webish_url.endswith("/")
+        self.webish_url = self.webish_url[:-1] # these tests add their own /

 
         l = [ self.s.create_dirnode() for x in range(6) ]
         d = defer.DeferredList(l)
 
         l = [ self.s.create_dirnode() for x in range(6) ]
         d = defer.DeferredList(l)


@@ -155,28 +319,50 @@ class WebMixin(object):
             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 
             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 

+            # sdmf
+            # XXX: Do we ever use this?
+            self.BAZ_CONTENTS, n, self._baz_txt_uri, self._baz_txt_readonly_uri = self.makefile_mutable(0)
+
+            foo.set_uri(u"baz.txt", self._baz_txt_uri, self._baz_txt_readonly_uri)
+
+            # mdmf
+            self.QUUX_CONTENTS, n, self._quux_txt_uri, self._quux_txt_readonly_uri = self.makefile_mutable(0, mdmf=True)
+            assert self._quux_txt_uri.startswith("URI:MDMF")
+            foo.set_uri(u"quux.txt", self._quux_txt_uri, self._quux_txt_readonly_uri)
+

             foo.set_uri(u"empty", res[3][1].get_uri(),
                         res[3][1].get_readonly_uri())
             sub_uri = res[4][1].get_uri()
             self._sub_uri = sub_uri
             foo.set_uri(u"sub", sub_uri, sub_uri)
             sub = self.s.create_node_from_uri(sub_uri)
             foo.set_uri(u"empty", res[3][1].get_uri(),
                         res[3][1].get_readonly_uri())
             sub_uri = res[4][1].get_uri()
             self._sub_uri = sub_uri
             foo.set_uri(u"sub", sub_uri, sub_uri)
             sub = self.s.create_node_from_uri(sub_uri)

+            self._sub_node = sub

 
             _ign, n, blocking_uri = self.makefile(1)
             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 
 
             _ign, n, blocking_uri = self.makefile(1)
             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 

+            # filenode to test for html encoding issues
+            self._htmlname_unicode = u"<&weirdly'named\"file>>>_<iframe />.txt"
+            self._htmlname_raw = self._htmlname_unicode.encode('utf-8')
+            self._htmlname_urlencoded = urllib.quote(self._htmlname_raw, '')
+            self._htmlname_escaped = escapeToXML(self._htmlname_raw)
+            self._htmlname_escaped_attr = cgi.escape(self._htmlname_raw, quote=True)
+            self._htmlname_escaped_double = escapeToXML(cgi.escape(self._htmlname_raw, quote=True))
+            self.HTMLNAME_CONTENTS, n, self._htmlname_txt_uri = self.makefile(0)
+            foo.set_uri(self._htmlname_unicode, self._htmlname_txt_uri, self._htmlname_txt_uri)
+

             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
             # still think of it as an umlaut
             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 
             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
             # still think of it as an umlaut
             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 

-            _ign, n, baz_file = self.makefile(2)
+            self.SUBBAZ_CONTENTS, n, baz_file = self.makefile(2)

             self._baz_file_uri = baz_file
             sub.set_uri(u"baz.txt", baz_file, baz_file)
 
             _ign, n, self._bad_file_uri = self.makefile(3)
             # this uri should not be downloadable
             self._baz_file_uri = baz_file
             sub.set_uri(u"baz.txt", baz_file, baz_file)
 
             _ign, n, self._bad_file_uri = self.makefile(3)
             # this uri should not be downloadable

-            del FakeCHKFileNode.all_contents[self._bad_file_uri]
+            del self.s.all_contents[self._bad_file_uri]

 
             rodir = res[5][1]
             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 
             rodir = res[5][1]
             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),


@@ -186,7 +372,10 @@ class WebMixin(object):
             # public/
             # public/foo/
             # public/foo/bar.txt
             # public/
             # public/foo/
             # public/foo/bar.txt

+            # public/foo/baz.txt
+            # public/foo/quux.txt

             # public/foo/blockingfile
             # public/foo/blockingfile

+            # public/foo/<&weirdly'named\"file>>>_<iframe />.txt

             # public/foo/empty/
             # public/foo/sub/
             # public/foo/sub/baz.txt
             # public/foo/empty/
             # public/foo/sub/
             # public/foo/sub/baz.txt


@@ -201,43 +390,79 @@ class WebMixin(object):
         d.addCallback(_got_metadata)
         return d
 
         d.addCallback(_got_metadata)
         return d
 

+    def get_all_contents(self):
+        return self.s.all_contents
+

     def makefile(self, number):
         contents = "contents of file %s\n" % number
     def makefile(self, number):
         contents = "contents of file %s\n" % number

-        n = create_chk_filenode(contents)
+        n = create_chk_filenode(contents, self.get_all_contents())

         return contents, n, n.get_uri()
 
         return contents, n, n.get_uri()
 

+    def makefile_mutable(self, number, mdmf=False):
+        contents = "contents of mutable file %s\n" % number
+        n = create_mutable_filenode(contents, mdmf, self.s.all_contents)
+        return contents, n, n.get_uri(), n.get_readonly_uri()
+

     def tearDown(self):
         return self.s.stopService()
 
     def failUnlessIsBarDotTxt(self, res):
         self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
 
     def tearDown(self):
         return self.s.stopService()
 
     def failUnlessIsBarDotTxt(self, res):
         self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
 

+    def failUnlessIsQuuxDotTxt(self, res):
+        self.failUnlessReallyEqual(res, self.QUUX_CONTENTS, res)
+
+    def failUnlessIsBazDotTxt(self, res):
+        self.failUnlessReallyEqual(res, self.BAZ_CONTENTS, res)
+
+    def failUnlessIsSubBazDotTxt(self, res):
+        self.failUnlessReallyEqual(res, self.SUBBAZ_CONTENTS, res)
+

     def failUnlessIsBarJSON(self, res):
         data = simplejson.loads(res)
         self.failUnless(isinstance(data, list))
         self.failUnlessEqual(data[0], "filenode")
         self.failUnless(isinstance(data[1], dict))
         self.failIf(data[1]["mutable"])
     def failUnlessIsBarJSON(self, res):
         data = simplejson.loads(res)
         self.failUnless(isinstance(data, list))
         self.failUnlessEqual(data[0], "filenode")
         self.failUnless(isinstance(data[1], dict))
         self.failIf(data[1]["mutable"])

-        self.failIf("rw_uri" in data[1]) # immutable
+        self.failIfIn("rw_uri", data[1]) # immutable

         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
         self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
 
         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
         self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
 

+    def failUnlessIsQuuxJSON(self, res, readonly=False):
+        data = simplejson.loads(res)
+        self.failUnless(isinstance(data, list))
+        self.failUnlessEqual(data[0], "filenode")
+        self.failUnless(isinstance(data[1], dict))
+        metadata = data[1]
+        return self.failUnlessIsQuuxDotTxtMetadata(metadata, readonly)
+
+    def failUnlessIsQuuxDotTxtMetadata(self, metadata, readonly):
+        self.failUnless(metadata['mutable'])
+        if readonly:
+            self.failIfIn("rw_uri", metadata)
+        else:
+            self.failUnlessIn("rw_uri", metadata)
+            self.failUnlessEqual(metadata['rw_uri'], self._quux_txt_uri)
+        self.failUnlessIn("ro_uri", metadata)
+        self.failUnlessEqual(metadata['ro_uri'], self._quux_txt_readonly_uri)
+        self.failUnlessReallyEqual(metadata['size'], len(self.QUUX_CONTENTS))
+

     def failUnlessIsFooJSON(self, res):
         data = simplejson.loads(res)
         self.failUnless(isinstance(data, list))
         self.failUnlessEqual(data[0], "dirnode", res)
         self.failUnless(isinstance(data[1], dict))
         self.failUnless(data[1]["mutable"])
     def failUnlessIsFooJSON(self, res):
         data = simplejson.loads(res)
         self.failUnless(isinstance(data, list))
         self.failUnlessEqual(data[0], "dirnode", res)
         self.failUnless(isinstance(data[1], dict))
         self.failUnless(data[1]["mutable"])

-        self.failUnless("rw_uri" in data[1]) # mutable
+        self.failUnlessIn("rw_uri", data[1]) # mutable

         self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
 
         kidnames = sorted([unicode(n) for n in data[1]["children"]])
         self.failUnlessEqual(kidnames,
         self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
 
         kidnames = sorted([unicode(n) for n in data[1]["children"]])
         self.failUnlessEqual(kidnames,

-                             [u"bar.txt", u"blockingfile", u"empty",
-                              u"n\u00fc.txt", u"sub"])
+                             [self._htmlname_unicode, u"bar.txt", u"baz.txt",
+                              u"blockingfile", u"empty", u"n\u00fc.txt", u"quux.txt", u"sub"])

         kids = dict( [(unicode(name),value)
                       for (name,value)
                       in data[1]["children"].iteritems()] )
         kids = dict( [(unicode(name),value)
                       for (name,value)
                       in data[1]["children"].iteritems()] )


@@ -258,6 +483,11 @@ class WebMixin(object):
                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
         self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
                                    self._bar_txt_uri)
                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
         self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
                                    self._bar_txt_uri)

+        self.failUnlessIn("quux.txt", kids)
+        self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["rw_uri"]),
+                                   self._quux_txt_uri)
+        self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["ro_uri"]),
+                                   self._quux_txt_readonly_uri)

 
     def GET(self, urlpath, followRedirect=False, return_response=False,
             **kwargs):
 
     def GET(self, urlpath, followRedirect=False, return_response=False,
             **kwargs):


@@ -337,13 +567,9 @@ class WebMixin(object):
         if isinstance(res, failure.Failure):
             res.trap(expected_failure)
             if substring:
         if isinstance(res, failure.Failure):
             res.trap(expected_failure)
             if substring:

-                self.failUnless(substring in str(res),
-                                "substring '%s' not in '%s'"
-                                % (substring, str(res)))
+                self.failUnlessIn(substring, str(res), which)

             if response_substring:
             if response_substring:

-                self.failUnless(response_substring in res.value.response,
-                                "response substring '%s' not in '%s'"
-                                % (response_substring, res.value.response))
+                self.failUnlessIn(response_substring, res.value.response, which)

         else:
             self.fail("%s was supposed to raise %s, not get '%s'" %
                       (which, expected_failure, res))
         else:
             self.fail("%s was supposed to raise %s, not get '%s'" %
                       (which, expected_failure, res))


@@ -358,14 +584,16 @@ class WebMixin(object):
             if isinstance(res, failure.Failure):
                 res.trap(expected_failure)
                 if substring:
             if isinstance(res, failure.Failure):
                 res.trap(expected_failure)
                 if substring:

-                    self.failUnless(substring in str(res),
-                                    "%s: substring '%s' not in '%s'"
-                                    % (which, substring, str(res)))
+                    self.failUnlessIn(substring, str(res),
+                                      "'%s' not in '%s' (response is '%s') for test '%s'" % \
+                                      (substring, str(res),
+                                       getattr(res.value, "response", ""),
+                                       which))

                 if response_substring:
                 if response_substring:

-                    self.failUnless(response_substring in res.value.response,
-                                    "%s: response substring '%s' not in '%s'"
-                                    % (which,
-                                       response_substring, res.value.response))
+                    self.failUnlessIn(response_substring, res.value.response,
+                                      "'%s' not in '%s' for test '%s'" % \
+                                      (response_substring, res.value.response,
+                                       which))

             else:
                 self.fail("%s was supposed to raise %s, not get '%s'" %
                           (which, expected_failure, res))
             else:
                 self.fail("%s was supposed to raise %s, not get '%s'" %
                           (which, expected_failure, res))


@@ -386,8 +614,7 @@ class WebMixin(object):
             self.failUnlessReallyEqual(res.value.status, "302")
         else:
             self.fail("%s was supposed to Error(302), not get '%s'" %
             self.failUnlessReallyEqual(res.value.status, "302")
         else:
             self.fail("%s was supposed to Error(302), not get '%s'" %

-                        (which, res))
-
+                      (which, res))

 
 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
     def test_create(self):
 
 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
     def test_create(self):


@@ -396,7 +623,39 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
     def test_welcome(self):
         d = self.GET("/")
         def _check(res):
     def test_welcome(self):
         d = self.GET("/")
         def _check(res):

-            self.failUnless('Welcome To Tahoe-LAFS' in res, res)
+            self.failUnlessIn('<title>Tahoe-LAFS - Welcome</title>', res)
+            self.failUnlessIn(FAVICON_MARKUP, res)
+            self.failUnlessIn('<a href="status">Recent and Active Operations</a>', res)
+            self.failUnlessIn('<a href="statistics">Operational Statistics</a>', res)
+            self.failUnless(re.search('<input (type="hidden" |name="t" |value="report-incident" ){3}/>',res), res)
+            self.failUnlessIn('Page rendered at', res)
+            self.failUnlessIn('Tahoe-LAFS code imported from:', res)
+            res_u = res.decode('utf-8')
+            self.failUnlessIn(u'<td>fake_nickname \u263A</td>', res_u)
+            self.failUnlessIn(u'<div class="nickname">other_nickname \u263B</div>', res_u)
+            self.failUnlessIn(u'Connected to <span>1</span>\n              of <span>2</span> known storage servers', res_u)
+            def timestamp(t):
+                return (u'"%s"' % (t,)) if self.have_working_tzset() else u'"[^"]*"'
+            self.failUnless(re.search(
+                u'<div class="status-indicator"><img (src="img/connected-yes.png" |alt="Connected" ){2}/>'
+                u'</div>\n                <a( class="timestamp"| title=%s){2}>1d\u00A00h\u00A00m\u00A050s</a>'
+                % timestamp(u'1970-01-01 13:00:10'), res_u), repr(res_u))
+            self.failUnless(re.search(
+                u'<div class="status-indicator"><img (src="img/connected-no.png" |alt="Disconnected" ){2}/>'
+                u'</div>\n                <a( class="timestamp"| title=%s){2}>1d\u00A00h\u00A00m\u00A035s</a>'
+                % timestamp(u'1970-01-01 13:00:25'), res_u), repr(res_u))
+            self.failUnless(re.search(
+                u'<td class="service-last-received-data"><a( class="timestamp"| title=%s){2}>'
+                u'1d\u00A00h\u00A00m\u00A030s</a></td>'
+                % timestamp(u'1970-01-01 13:00:30'), res_u), repr(res_u))
+            self.failUnless(re.search(
+                u'<td class="service-last-received-data"><a( class="timestamp"| title=%s){2}>'
+                u'1d\u00A00h\u00A00m\u00A025s</a></td>'
+                % timestamp(u'1970-01-01 13:00:35'), res_u), repr(res_u))
+
+            self.failUnlessIn(u'\u00A9 <a href="https://tahoe-lafs.org/">Tahoe-LAFS Software Foundation', res_u)
+            self.failUnlessIn('<td><h3>Available</h3></td>', res)
+            self.failUnlessIn('123.5kB', res)

 
             self.s.basedir = 'web/test_welcome'
             fileutil.make_dirs("web/test_welcome")
 
             self.s.basedir = 'web/test_welcome'
             fileutil.make_dirs("web/test_welcome")


@@ -405,85 +664,102 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(_check)
         return d
 
         d.addCallback(_check)
         return d
 

-    def test_provisioning(self):
-        d = self.GET("/provisioning/")
-        def _check(res):
-            self.failUnless('Provisioning Tool' in res)
-            fields = {'filled': True,
-                      "num_users": int(50e3),
-                      "files_per_user": 1000,
-                      "space_per_user": int(1e9),
-                      "sharing_ratio": 1.0,
-                      "encoding_parameters": "3-of-10-5",
-                      "num_servers": 30,
-                      "ownership_mode": "A",
-                      "download_rate": 100,
-                      "upload_rate": 10,
-                      "delete_rate": 10,
-                      "lease_timer": 7,
-                      }
-            return self.POST("/provisioning/", **fields)
+    def test_introducer_status(self):
+        class MockIntroducerClient(object):
+            def __init__(self, connected):
+                self.connected = connected
+            def connected_to_introducer(self):
+                return self.connected

 
 

-        d.addCallback(_check)
-        def _check2(res):
-            self.failUnless('Provisioning Tool' in res)
-            self.failUnless("Share space consumed: 167.01TB" in res)
-
-            fields = {'filled': True,
-                      "num_users": int(50e6),
-                      "files_per_user": 1000,
-                      "space_per_user": int(5e9),
-                      "sharing_ratio": 1.0,
-                      "encoding_parameters": "25-of-100-50",
-                      "num_servers": 30000,
-                      "ownership_mode": "E",
-                      "drive_failure_model": "U",
-                      "drive_size": 1000,
-                      "download_rate": 1000,
-                      "upload_rate": 100,
-                      "delete_rate": 100,
-                      "lease_timer": 7,
-                      }
-            return self.POST("/provisioning/", **fields)
-        d.addCallback(_check2)
-        def _check3(res):
-            self.failUnless("Share space consumed: huge!" in res)
-            fields = {'filled': True}
-            return self.POST("/provisioning/", **fields)
-        d.addCallback(_check3)
-        def _check4(res):
-            self.failUnless("Share space consumed:" in res)
-        d.addCallback(_check4)
+        d = defer.succeed(None)
+
+        # introducer not connected, unguessable furl
+        def _set_introducer_not_connected_unguessable(ign):
+            self.s.introducer_furl = "pb://someIntroducer/secret"
+            self.s.introducer_client = MockIntroducerClient(False)
+            return self.GET("/")
+        d.addCallback(_set_introducer_not_connected_unguessable)
+        def _check_introducer_not_connected_unguessable(res):
+            html = res.replace('\n', ' ')
+            self.failUnlessIn('<div class="furl">pb://someIntroducer/[censored]</div>', html)
+            self.failIfIn('pb://someIntroducer/secret', html)
+            self.failUnless(re.search('<img (alt="Disconnected" |src="img/connected-no.png" ){2}/>', html), res)
+        d.addCallback(_check_introducer_not_connected_unguessable)
+
+        # introducer connected, unguessable furl
+        def _set_introducer_connected_unguessable(ign):
+            self.s.introducer_furl = "pb://someIntroducer/secret"
+            self.s.introducer_client = MockIntroducerClient(True)
+            return self.GET("/")
+        d.addCallback(_set_introducer_connected_unguessable)
+        def _check_introducer_connected_unguessable(res):
+            html = res.replace('\n', ' ')
+            self.failUnlessIn('<div class="furl">pb://someIntroducer/[censored]</div>', html)
+            self.failIfIn('pb://someIntroducer/secret', html)
+            self.failUnless(re.search('<img (src="img/connected-yes.png" |alt="Connected" ){2}/>', html), res)
+        d.addCallback(_check_introducer_connected_unguessable)
+
+        # introducer connected, guessable furl
+        def _set_introducer_connected_guessable(ign):
+            self.s.introducer_furl = "pb://someIntroducer/introducer"
+            self.s.introducer_client = MockIntroducerClient(True)
+            return self.GET("/")
+        d.addCallback(_set_introducer_connected_guessable)
+        def _check_introducer_connected_guessable(res):
+            html = res.replace('\n', ' ')
+            self.failUnlessIn('<div class="furl">pb://someIntroducer/introducer</div>', html)
+            self.failUnless(re.search('<img (src="img/connected-yes.png" |alt="Connected" ){2}/>', html), res)
+        d.addCallback(_check_introducer_connected_guessable)

         return d
 
         return d
 

-    def test_reliability_tool(self):
-        try:
-            from allmydata import reliability
-            _hush_pyflakes = reliability
-            del _hush_pyflakes
-        except:
-            raise unittest.SkipTest("reliability tool requires NumPy")
+    def test_helper_status(self):
+        d = defer.succeed(None)

 
 

-        d = self.GET("/reliability/")
-        def _check(res):
-            self.failUnless('Reliability Tool' in res)
-            fields = {'drive_lifetime': "8Y",
-                      "k": "3",
-                      "R": "7",
-                      "N": "10",
-                      "delta": "100000",
-                      "check_period": "1M",
-                      "report_period": "3M",
-                      "report_span": "5Y",
-                      }
-            return self.POST("/reliability/", **fields)
+        # set helper furl to None
+        def _set_no_helper(ign):
+            self.s.uploader.helper_furl = None
+            return self.GET("/")
+        d.addCallback(_set_no_helper)
+        def _check_no_helper(res):
+            html = res.replace('\n', ' ')
+            self.failUnless(re.search('<img (src="img/connected-not-configured.png" |alt="Not Configured" ){2}/>', html), res)
+        d.addCallback(_check_no_helper)
+
+        # enable helper, not connected
+        def _set_helper_not_connected(ign):
+            self.s.uploader.helper_furl = "pb://someHelper/secret"
+            self.s.uploader.helper_connected = False
+            return self.GET("/")
+        d.addCallback(_set_helper_not_connected)
+        def _check_helper_not_connected(res):
+            html = res.replace('\n', ' ')
+            self.failUnlessIn('<div class="furl">pb://someHelper/[censored]</div>', html)
+            self.failIfIn('pb://someHelper/secret', html)
+            self.failUnless(re.search('<img (src="img/connected-no.png" |alt="Disconnected" ){2}/>', html), res)
+        d.addCallback(_check_helper_not_connected)
+
+        # enable helper, connected
+        def _set_helper_connected(ign):
+            self.s.uploader.helper_furl = "pb://someHelper/secret"
+            self.s.uploader.helper_connected = True
+            return self.GET("/")
+        d.addCallback(_set_helper_connected)
+        def _check_helper_connected(res):
+            html = res.replace('\n', ' ')
+            self.failUnlessIn('<div class="furl">pb://someHelper/[censored]</div>', html)
+            self.failIfIn('pb://someHelper/secret', html)
+            self.failUnless(re.search('<img (src="img/connected-yes.png" |alt="Connected" ){2}/>', html), res)
+        d.addCallback(_check_helper_connected)
+        return d

 
 

+    def test_storage(self):
+        d = self.GET("/storage")
+        def _check(res):
+            self.failUnlessIn('Storage Server Status', res)
+            self.failUnlessIn(FAVICON_MARKUP, res)
+            res_u = res.decode('utf-8')
+            self.failUnlessIn(u'<li>Server Nickname: <span class="nickname mine">fake_nickname \u263A</span></li>', res_u)

         d.addCallback(_check)
         d.addCallback(_check)

-        def _check2(res):
-            self.failUnless('Reliability Tool' in res)
-            r = r'Probability of loss \(no maintenance\):\s+<span>0.033591'
-            self.failUnless(re.search(r, res), res)
-        d.addCallback(_check2)

         return d
 
     def test_status(self):
         return d
 
     def test_status(self):


@@ -495,12 +771,12 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
         d = self.GET("/status", followRedirect=True)
         def _check(res):
         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
         d = self.GET("/status", followRedirect=True)
         def _check(res):

-            self.failUnless('Upload and Download Status' in res, res)
-            self.failUnless('"down-%d"' % dl_num in res, res)
-            self.failUnless('"up-%d"' % ul_num in res, res)
-            self.failUnless('"mapupdate-%d"' % mu_num in res, res)
-            self.failUnless('"publish-%d"' % pub_num in res, res)
-            self.failUnless('"retrieve-%d"' % ret_num in res, res)
+            self.failUnlessIn('Recent and Active Operations', res)
+            self.failUnlessIn('"down-%d"' % dl_num, res)
+            self.failUnlessIn('"up-%d"' % ul_num, res)
+            self.failUnlessIn('"mapupdate-%d"' % mu_num, res)
+            self.failUnlessIn('"publish-%d"' % pub_num, res)
+            self.failUnlessIn('"retrieve-%d"' % ret_num, res)

         d.addCallback(_check)
         d.addCallback(lambda res: self.GET("/status/?t=json"))
         def _check_json(res):
         d.addCallback(_check)
         d.addCallback(lambda res: self.GET("/status/?t=json"))
         def _check_json(res):


@@ -513,23 +789,48 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
 
         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
         def _check_dl(res):
 
         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
         def _check_dl(res):

-            self.failUnless("File Download Status" in res, res)
+            self.failUnlessIn("File Download Status", res)

         d.addCallback(_check_dl)
         d.addCallback(_check_dl)

+        d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
+        def _check_dl_json(res):
+            data = simplejson.loads(res)
+            self.failUnless(isinstance(data, dict))
+            self.failUnlessIn("read", data)
+            self.failUnlessEqual(data["read"][0]["length"], 120)
+            self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
+            self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
+            self.failUnlessEqual(data["segment"][2]["finish_time"], None)
+            phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
+            cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
+            # serverids[] keys are strings, since that's what JSON does, but
+            # we'd really like them to be ints
+            self.failUnlessEqual(data["serverids"]["0"], "phwrsjte")
+            self.failUnless(data["serverids"].has_key("1"),
+                            str(data["serverids"]))
+            self.failUnlessEqual(data["serverids"]["1"], "cmpuvkjm",
+                                 str(data["serverids"]))
+            self.failUnlessEqual(data["server_info"][phwr_id]["short"],
+                                 "phwrsjte")
+            self.failUnlessEqual(data["server_info"][cmpu_id]["short"],
+                                 "cmpuvkjm")
+            self.failUnlessIn("dyhb", data)
+            self.failUnlessIn("misc", data)
+        d.addCallback(_check_dl_json)

         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
         def _check_ul(res):
         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
         def _check_ul(res):

-            self.failUnless("File Upload Status" in res, res)
+            self.failUnlessIn("File Upload Status", res)

         d.addCallback(_check_ul)
         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
         def _check_mapupdate(res):
         d.addCallback(_check_ul)
         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
         def _check_mapupdate(res):

-            self.failUnless("Mutable File Servermap Update Status" in res, res)
+            self.failUnlessIn("Mutable File Servermap Update Status", res)

         d.addCallback(_check_mapupdate)
         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
         def _check_publish(res):
         d.addCallback(_check_mapupdate)
         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
         def _check_publish(res):

-            self.failUnless("Mutable File Publish Status" in res, res)
+            self.failUnlessIn("Mutable File Publish Status", res)

         d.addCallback(_check_publish)
         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
         def _check_retrieve(res):
         d.addCallback(_check_publish)
         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
         def _check_retrieve(res):

-            self.failUnless("Mutable File Retrieve Status" in res, res)
+            self.failUnlessIn("Mutable File Retrieve Status", res)

         d.addCallback(_check_retrieve)
 
         return d
         d.addCallback(_check_retrieve)
 
         return d


@@ -710,6 +1011,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                              self.PUT, base + "/@@name=/blah.txt", "")
         return d
 
                              self.PUT, base + "/@@name=/blah.txt", "")
         return d
 

+

     def test_GET_DIRURL_named_bad(self):
         base = "/file/%s" % urllib.quote(self._foo_uri)
         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
     def test_GET_DIRURL_named_bad(self):
         base = "/file/%s" % urllib.quote(self._foo_uri)
         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",


@@ -752,6 +1054,24 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(self.failUnlessIsBarDotTxt)
         return d
 
         d.addCallback(self.failUnlessIsBarDotTxt)
         return d
 

+    def test_GET_FILE_URI_mdmf(self):
+        base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
+        d = self.GET(base)
+        d.addCallback(self.failUnlessIsQuuxDotTxt)
+        return d
+
+    def test_GET_FILE_URI_mdmf_extensions(self):
+        base = "/uri/%s" % urllib.quote("%s:RANDOMSTUFF" % self._quux_txt_uri)
+        d = self.GET(base)
+        d.addCallback(self.failUnlessIsQuuxDotTxt)
+        return d
+
+    def test_GET_FILE_URI_mdmf_readonly(self):
+        base = "/uri/%s" % urllib.quote(self._quux_txt_readonly_uri)
+        d = self.GET(base)
+        d.addCallback(self.failUnlessIsQuuxDotTxt)
+        return d
+

     def test_GET_FILE_URI_badchild(self):
         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
         errmsg = "Files have no children, certainly not named 'boguschild'"
     def test_GET_FILE_URI_badchild(self):
         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
         errmsg = "Files have no children, certainly not named 'boguschild'"


@@ -768,6 +1088,140 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                              self.PUT, base, "")
         return d
 
                              self.PUT, base, "")
         return d
 

+    def test_PUT_FILE_URI_mdmf(self):
+        base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
+        self._quux_new_contents = "new_contents"
+        d = self.GET(base)
+        d.addCallback(lambda res:
+            self.failUnlessIsQuuxDotTxt(res))
+        d.addCallback(lambda ignored:
+            self.PUT(base, self._quux_new_contents))
+        d.addCallback(lambda ignored:
+            self.GET(base))
+        d.addCallback(lambda res:
+            self.failUnlessReallyEqual(res, self._quux_new_contents))
+        return d
+
+    def test_PUT_FILE_URI_mdmf_extensions(self):
+        base = "/uri/%s" % urllib.quote("%s:EXTENSIONSTUFF" % self._quux_txt_uri)
+        self._quux_new_contents = "new_contents"
+        d = self.GET(base)
+        d.addCallback(lambda res: self.failUnlessIsQuuxDotTxt(res))
+        d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
+        d.addCallback(lambda ignored: self.GET(base))
+        d.addCallback(lambda res: self.failUnlessEqual(self._quux_new_contents,
+                                                       res))
+        return d
+
+    def test_PUT_FILE_URI_mdmf_readonly(self):
+        # We're not allowed to PUT things to a readonly cap.
+        base = "/uri/%s" % self._quux_txt_readonly_uri
+        d = self.GET(base)
+        d.addCallback(lambda res:
+            self.failUnlessIsQuuxDotTxt(res))
+        # What should we get here? We get a 500 error now; that's not right.
+        d.addCallback(lambda ignored:
+            self.shouldFail2(error.Error, "test_PUT_FILE_URI_mdmf_readonly",
+                             "400 Bad Request", "read-only cap",
+                             self.PUT, base, "new data"))
+        return d
+
+    def test_PUT_FILE_URI_sdmf_readonly(self):
+        # We're not allowed to put things to a readonly cap.
+        base = "/uri/%s" % self._baz_txt_readonly_uri
+        d = self.GET(base)
+        d.addCallback(lambda res:
+            self.failUnlessIsBazDotTxt(res))
+        d.addCallback(lambda ignored:
+            self.shouldFail2(error.Error, "test_PUT_FILE_URI_sdmf_readonly",
+                             "400 Bad Request", "read-only cap",
+                             self.PUT, base, "new_data"))
+        return d
+
+    def test_GET_etags(self):
+
+        def _check_etags(uri):
+            d1 = _get_etag(uri)
+            d2 = _get_etag(uri, 'json')
+            d = defer.DeferredList([d1, d2], consumeErrors=True)
+            def _check(results):
+                # All deferred must succeed
+                self.failUnless(all([r[0] for r in results]))
+                # the etag for the t=json form should be just like the etag
+                # fo the default t='' form, but with a 'json' suffix
+                self.failUnlessEqual(results[0][1] + 'json', results[1][1])
+            d.addCallback(_check)
+            return d
+
+        def _get_etag(uri, t=''):
+            targetbase = "/uri/%s?t=%s" % (urllib.quote(uri.strip()), t)
+            d = self.GET(targetbase, return_response=True, followRedirect=True)
+            def _just_the_etag(result):
+                data, response, headers = result
+                etag = headers['etag'][0]
+                if uri.startswith('URI:DIR'):
+                    self.failUnless(etag.startswith('DIR:'), etag)
+                return etag
+            return d.addCallback(_just_the_etag)
+
+        # Check that etags work with immutable directories
+        (newkids, caps) = self._create_immutable_children()
+        d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
+                      simplejson.dumps(newkids))
+        def _stash_immdir_uri(uri):
+            self._immdir_uri = uri
+            return uri
+        d.addCallback(_stash_immdir_uri)
+        d.addCallback(_check_etags)
+
+        # Check that etags work with immutable files
+        d.addCallback(lambda _: _check_etags(self._bar_txt_uri))
+
+        # use the ETag on GET
+        def _check_match(ign):
+            uri = "/uri/%s" % self._bar_txt_uri
+            d = self.GET(uri, return_response=True)
+            # extract the ETag
+            d.addCallback(lambda (data, code, headers):
+                          headers['etag'][0])
+            # do a GET that's supposed to match the ETag
+            d.addCallback(lambda etag:
+                          self.GET(uri, return_response=True,
+                                   headers={"If-None-Match": etag}))
+            # make sure it short-circuited (304 instead of 200)
+            d.addCallback(lambda (data, code, headers):
+                          self.failUnlessEqual(int(code), http.NOT_MODIFIED))
+            return d
+        d.addCallback(_check_match)
+
+        def _no_etag(uri, t):
+            target = "/uri/%s?t=%s" % (uri, t)
+            d = self.GET(target, return_response=True, followRedirect=True)
+            d.addCallback(lambda (data, code, headers):
+                          self.failIf("etag" in headers, target))
+            return d
+        def _yes_etag(uri, t):
+            target = "/uri/%s?t=%s" % (uri, t)
+            d = self.GET(target, return_response=True, followRedirect=True)
+            d.addCallback(lambda (data, code, headers):
+                          self.failUnless("etag" in headers, target))
+            return d
+
+        d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, ""))
+        d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "json"))
+        d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "uri"))
+        d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "readonly-uri"))
+        d.addCallback(lambda ign: _no_etag(self._bar_txt_uri, "info"))
+
+        d.addCallback(lambda ign: _yes_etag(self._immdir_uri, ""))
+        d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "json"))
+        d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "uri"))
+        d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "readonly-uri"))
+        d.addCallback(lambda ign: _no_etag(self._immdir_uri, "info"))
+        d.addCallback(lambda ign: _no_etag(self._immdir_uri, "rename-form"))
+
+        return d
+

     # TODO: version of this with a Unicode filename
     def test_GET_FILEURL_save(self):
         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
     # TODO: version of this with a Unicode filename
     def test_GET_FILEURL_save(self):
         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",


@@ -784,6 +1238,41 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addBoth(self.should404, "test_GET_FILEURL_missing")
         return d
 
         d.addBoth(self.should404, "test_GET_FILEURL_missing")
         return d
 

+    def test_GET_FILEURL_info_mdmf(self):
+        d = self.GET("/uri/%s?t=info" % self._quux_txt_uri)
+        def _got(res):
+            self.failUnlessIn("mutable file (mdmf)", res)
+            self.failUnlessIn(self._quux_txt_uri, res)
+            self.failUnlessIn(self._quux_txt_readonly_uri, res)
+        d.addCallback(_got)
+        return d
+
+    def test_GET_FILEURL_info_mdmf_readonly(self):
+        d = self.GET("/uri/%s?t=info" % self._quux_txt_readonly_uri)
+        def _got(res):
+            self.failUnlessIn("mutable file (mdmf)", res)
+            self.failIfIn(self._quux_txt_uri, res)
+            self.failUnlessIn(self._quux_txt_readonly_uri, res)
+        d.addCallback(_got)
+        return d
+
+    def test_GET_FILEURL_info_sdmf(self):
+        d = self.GET("/uri/%s?t=info" % self._baz_txt_uri)
+        def _got(res):
+            self.failUnlessIn("mutable file (sdmf)", res)
+            self.failUnlessIn(self._baz_txt_uri, res)
+        d.addCallback(_got)
+        return d
+
+    def test_GET_FILEURL_info_mdmf_extensions(self):
+        d = self.GET("/uri/%s:STUFF?t=info" % self._quux_txt_uri)
+        def _got(res):
+            self.failUnlessIn("mutable file (mdmf)", res)
+            self.failUnlessIn(self._quux_txt_uri, res)
+            self.failUnlessIn(self._quux_txt_readonly_uri, res)
+        d.addCallback(_got)
+        return d
+

     def test_PUT_overwrite_only_files(self):
         # create a directory, put a file in that directory.
         contents, n, filecap = self.makefile(8)
     def test_PUT_overwrite_only_files(self):
         # create a directory, put a file in that directory.
         contents, n, filecap = self.makefile(8)


@@ -825,6 +1314,35 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                                                       self.NEWFILE_CONTENTS))
         return d
 
                                                       self.NEWFILE_CONTENTS))
         return d
 

+    def test_PUT_NEWFILEURL_unlinked_mdmf(self):
+        # this should get us a few segments of an MDMF mutable file,
+        # which we can then test for.
+        contents = self.NEWFILE_CONTENTS * 300000
+        d = self.PUT("/uri?format=mdmf",
+                     contents)
+        def _got_filecap(filecap):
+            self.failUnless(filecap.startswith("URI:MDMF"))
+            return filecap
+        d.addCallback(_got_filecap)
+        d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
+        d.addCallback(lambda json: self.failUnlessIn("MDMF", json))
+        return d
+
+    def test_PUT_NEWFILEURL_unlinked_sdmf(self):
+        contents = self.NEWFILE_CONTENTS * 300000
+        d = self.PUT("/uri?format=sdmf",
+                     contents)
+        d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
+        d.addCallback(lambda json: self.failUnlessIn("SDMF", json))
+        return d
+
+    def test_PUT_NEWFILEURL_unlinked_bad_format(self):
+        contents = self.NEWFILE_CONTENTS * 300000
+        return self.shouldHTTPError("PUT_NEWFILEURL_unlinked_bad_format",
+                                    400, "Bad Request", "Unknown format: foo",
+                                    self.PUT, "/uri?format=foo",
+                                    contents)
+

     def test_PUT_NEWFILEURL_range_bad(self):
         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
         target = self.public_url + "/foo/new.txt"
     def test_PUT_NEWFILEURL_range_bad(self):
         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
         target = self.public_url + "/foo/new.txt"


@@ -857,12 +1375,10 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         return d
 
     def test_PUT_NEWFILEURL_mutable_toobig(self):
         return d
 
     def test_PUT_NEWFILEURL_mutable_toobig(self):

-        d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_mutable_toobig",
-                             "413 Request Entity Too Large",
-                             "SDMF is limited to one segment, and 10001 > 10000",
-                             self.PUT,
-                             self.public_url + "/foo/new.txt?mutable=true",
-                             "b" * (self.s.MUTABLE_SIZELIMIT+1))
+        # It is okay to upload large mutable files, so we should be able
+        # to do that.
+        d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
+                     "b" * (self.s.MUTABLE_SIZELIMIT + 1))

         return d
 
     def test_PUT_NEWFILEURL_replace(self):
         return d
 
     def test_PUT_NEWFILEURL_replace(self):


@@ -957,6 +1473,35 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(_check1)
         return d
 
         d.addCallback(_check1)
         return d
 

+    def test_GET_FILEURL_json_mutable_type(self):
+        # The JSON should include format, which says whether the
+        # file is SDMF or MDMF
+        d = self.PUT("/uri?format=mdmf",
+                     self.NEWFILE_CONTENTS * 300000)
+        d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
+        def _got_json(json, version):
+            data = simplejson.loads(json)
+            assert "filenode" == data[0]
+            data = data[1]
+            assert isinstance(data, dict)
+
+            self.failUnlessIn("format", data)
+            self.failUnlessEqual(data["format"], version)
+
+        d.addCallback(_got_json, "MDMF")
+        # Now make an SDMF file and check that it is reported correctly.
+        d.addCallback(lambda ignored:
+            self.PUT("/uri?format=sdmf",
+                      self.NEWFILE_CONTENTS * 300000))
+        d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
+        d.addCallback(_got_json, "SDMF")
+        return d
+
+    def test_GET_FILEURL_json_mdmf(self):
+        d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
+        d.addCallback(self.failUnlessIsQuuxJSON)
+        return d
+

     def test_GET_FILEURL_json_missing(self):
         d = self.GET(self.public_url + "/foo/missing?json")
         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
     def test_GET_FILEURL_json_missing(self):
         d = self.GET(self.public_url + "/foo/missing?json")
         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")


@@ -983,33 +1528,75 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         return d
 
     def test_CSS_FILE(self):
         return d
 
     def test_CSS_FILE(self):

-        d = self.GET("/tahoe_css", followRedirect=True)
+        d = self.GET("/tahoe.css", followRedirect=True)

         def _check(res):
             CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
             self.failUnless(CSS_STYLE.search(res), res)
         d.addCallback(_check)
         return d
         def _check(res):
             CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
             self.failUnless(CSS_STYLE.search(res), res)
         d.addCallback(_check)
         return d

-    
+

     def test_GET_FILEURL_uri_missing(self):
         d = self.GET(self.public_url + "/foo/missing?t=uri")
         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
         return d
 
     def test_GET_FILEURL_uri_missing(self):
         d = self.GET(self.public_url + "/foo/missing?t=uri")
         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
         return d
 

-    def test_GET_DIRECTORY_html_banner(self):
+    def _check_upload_and_mkdir_forms(self, html):
+        # We should have a form to create a file, with radio buttons that allow
+        # the user to toggle whether it is a CHK/LIT (default), SDMF, or MDMF file.
+        self.failUnless(re.search('<input (name="t" |value="upload" |type="hidden" ){3}/>', html), html)
+        self.failUnless(re.search('<input [^/]*id="upload-chk"', html), html)
+        self.failUnless(re.search('<input [^/]*id="upload-sdmf"', html), html)
+        self.failUnless(re.search('<input [^/]*id="upload-mdmf"', html), html)
+
+        # We should also have the ability to create a mutable directory, with
+        # radio buttons that allow the user to toggle whether it is an SDMF (default)
+        # or MDMF directory.
+        self.failUnless(re.search('<input (name="t" |value="mkdir" |type="hidden" ){3}/>', html), html)
+        self.failUnless(re.search('<input [^/]*id="mkdir-sdmf"', html), html)
+        self.failUnless(re.search('<input [^/]*id="mkdir-mdmf"', html), html)
+
+        self.failUnlessIn(FAVICON_MARKUP, html)
+
+    def test_GET_DIRECTORY_html(self):
+        d = self.GET(self.public_url + "/foo", followRedirect=True)
+        def _check(html):
+            self.failUnlessIn('<li class="toolbar-item"><a href="../../..">Return to Welcome page</a></li>', html)
+            self._check_upload_and_mkdir_forms(html)
+            self.failUnlessIn("quux", html)
+        d.addCallback(_check)
+        return d
+
+    def test_GET_DIRECTORY_html_filenode_encoding(self):

         d = self.GET(self.public_url + "/foo", followRedirect=True)
         d = self.GET(self.public_url + "/foo", followRedirect=True)

-        def _check(res):
-            self.failUnlessIn('<div class="toolbar-item"><a href="../../..">Return to Welcome page</a></div>',res)
+        def _check(html):
+            # Check if encoded entries are there
+            self.failUnlessIn('@@named=/' + self._htmlname_urlencoded + '">'
+                              + self._htmlname_escaped + '</a>', html)
+            self.failUnlessIn('value="' + self._htmlname_escaped_attr + '"', html)
+            self.failIfIn(self._htmlname_escaped_double, html)
+            # Make sure that Nevow escaping actually works by checking for unsafe characters
+            # and that '&' is escaped.
+            for entity in '<>':
+                self.failUnlessIn(entity, self._htmlname_raw)
+                self.failIfIn(entity, self._htmlname_escaped)
+            self.failUnlessIn('&', re.sub(r'&(amp|lt|gt|quot|apos);', '', self._htmlname_raw))
+            self.failIfIn('&', re.sub(r'&(amp|lt|gt|quot|apos);', '', self._htmlname_escaped))

         d.addCallback(_check)
         return d
 
         d.addCallback(_check)
         return d
 

+    def test_GET_root_html(self):
+        d = self.GET("/")
+        d.addCallback(self._check_upload_and_mkdir_forms)
+        return d
+

     def test_GET_DIRURL(self):
         # the addSlash means we get a redirect here
         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
         ROOT = "../../.."
         d = self.GET(self.public_url + "/foo", followRedirect=True)
         def _check(res):
     def test_GET_DIRURL(self):
         # the addSlash means we get a redirect here
         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
         ROOT = "../../.."
         d = self.GET(self.public_url + "/foo", followRedirect=True)
         def _check(res):

-            self.failUnless(('<a href="%s">Return to Welcome page' % ROOT)
-                            in res, res)
+            self.failUnlessIn('<a href="%s">Return to Welcome page' % ROOT, res)
+

             # the FILE reference points to a URI, but it should end in bar.txt
             bar_url = ("%s/file/%s/@@named=/bar.txt" %
                        (ROOT, urllib.quote(self._bar_txt_uri)))
             # the FILE reference points to a URI, but it should end in bar.txt
             bar_url = ("%s/file/%s/@@named=/bar.txt" %
                        (ROOT, urllib.quote(self._bar_txt_uri)))


@@ -1017,24 +1604,28 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                                r'\s+<td>',
                                r'<a href="%s">bar.txt</a>' % bar_url,
                                r'</td>',
                                r'\s+<td>',
                                r'<a href="%s">bar.txt</a>' % bar_url,
                                r'</td>',

-                               r'\s+<td>%d</td>' % len(self.BAR_CONTENTS),
+                               r'\s+<td align="right">%d</td>' % len(self.BAR_CONTENTS),

                                ])
             self.failUnless(re.search(get_bar, res), res)
                                ])
             self.failUnless(re.search(get_bar, res), res)

-            for line in res.split("\n"):
-                # find the line that contains the delete button for bar.txt
-                if ("form action" in line and
-                    'value="delete"' in line and
-                    'value="bar.txt"' in line):
-                    # the form target should use a relative URL
-                    foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
-                    self.failUnless(('action="%s"' % foo_url) in line, line)
-                    # and the when_done= should too
-                    #done_url = urllib.quote(???)
-                    #self.failUnless(('name="when_done" value="%s"' % done_url)
-                    #                in line, line)
-                    break
-            else:
-                self.fail("unable to find delete-bar.txt line", res)
+            for label in ['unlink', 'rename/relink']:
+                for line in res.split("\n"):
+                    # find the line that contains the relevant button for bar.txt
+                    if ("form action" in line and
+                        ('value="%s"' % (label,)) in line and
+                        'value="bar.txt"' in line):
+                        # the form target should use a relative URL
+                        foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
+                        self.failUnlessIn('action="%s"' % foo_url, line)
+                        # and the when_done= should too
+                        #done_url = urllib.quote(???)
+                        #self.failUnlessIn('name="when_done" value="%s"' % done_url, line)
+
+                        # 'unlink' needs to use POST because it directly has a side effect
+                        if label == 'unlink':
+                            self.failUnlessIn('method="post"', line)
+                        break
+                else:
+                    self.fail("unable to find '%s bar.txt' line" % (label,))

 
             # the DIR reference just points to a URI
             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
 
             # the DIR reference just points to a URI
             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))


@@ -1043,12 +1634,12 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
             self.failUnless(re.search(get_sub, res), res)
         d.addCallback(_check)
 
             self.failUnless(re.search(get_sub, res), res)
         d.addCallback(_check)
 

-        # look at a readonly directory 
+        # look at a readonly directory

         d.addCallback(lambda res:
                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
         def _check2(res):
         d.addCallback(lambda res:
                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
         def _check2(res):

-            self.failUnless("(read-only)" in res, res)
-            self.failIf("Upload a file" in res, res)
+            self.failUnlessIn("(read-only)", res)
+            self.failIfIn("Upload a file", res)

         d.addCallback(_check2)
 
         # and at a directory that contains a readonly directory
         d.addCallback(_check2)
 
         # and at a directory that contains a readonly directory


@@ -1062,8 +1653,8 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         # and an empty directory
         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
         def _check4(res):
         # and an empty directory
         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
         def _check4(res):

-            self.failUnless("directory is empty" in res, res)
-            MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
+            self.failUnlessIn("directory is empty", res)
+            MKDIR_BUTTON_RE=re.compile('<input (type="hidden" |name="t" |value="mkdir" ){3}/>.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input (type="submit" |class="btn" |value="Create" ){3}/>', re.I)

             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
         d.addCallback(_check4)
 
             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
         d.addCallback(_check4)
 


@@ -1072,7 +1663,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(lambda res:
                       self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
         def _check5(res):
         d.addCallback(lambda res:
                       self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
         def _check5(res):

-            self.failUnless('(immutable)' in res, res)
+            self.failUnlessIn('(immutable)', res)

             self.failUnless(re.search('<td>FILE</td>'
                                       r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
         d.addCallback(_check5)
             self.failUnless(re.search('<td>FILE</td>'
                                       r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
         d.addCallback(_check5)


@@ -1091,6 +1682,35 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(self.failUnlessIsFooJSON)
         return d
 
         d.addCallback(self.failUnlessIsFooJSON)
         return d
 

+    def test_GET_DIRURL_json_format(self):
+        d = self.PUT(self.public_url + \
+                     "/foo/sdmf.txt?format=sdmf",
+                     self.NEWFILE_CONTENTS * 300000)
+        d.addCallback(lambda ignored:
+            self.PUT(self.public_url + \
+                     "/foo/mdmf.txt?format=mdmf",
+                     self.NEWFILE_CONTENTS * 300000))
+        # Now we have an MDMF and SDMF file in the directory. If we GET
+        # its JSON, we should see their encodings.
+        d.addCallback(lambda ignored:
+            self.GET(self.public_url + "/foo?t=json"))
+        def _got_json(json):
+            data = simplejson.loads(json)
+            assert data[0] == "dirnode"
+
+            data = data[1]
+            kids = data['children']
+
+            mdmf_data = kids['mdmf.txt'][1]
+            self.failUnlessIn("format", mdmf_data)
+            self.failUnlessEqual(mdmf_data["format"], "MDMF")
+
+            sdmf_data = kids['sdmf.txt'][1]
+            self.failUnlessIn("format", sdmf_data)
+            self.failUnlessEqual(sdmf_data["format"], "SDMF")
+        d.addCallback(_got_json)
+        return d
+

 
     def test_POST_DIRURL_manifest_no_ophandle(self):
         d = self.shouldFail2(error.Error,
 
     def test_POST_DIRURL_manifest_no_ophandle(self):
         d = self.shouldFail2(error.Error,


@@ -1110,10 +1730,11 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
             return d
         d.addCallback(getman, None)
         def _got_html(manifest):
             return d
         d.addCallback(getman, None)
         def _got_html(manifest):

-            self.failUnless("Manifest of SI=" in manifest)
-            self.failUnless("<td>sub</td>" in manifest)
-            self.failUnless(self._sub_uri in manifest)
-            self.failUnless("<td>sub/baz.txt</td>" in manifest)
+            self.failUnlessIn("Manifest of SI=", manifest)
+            self.failUnlessIn("<td>sub</td>", manifest)
+            self.failUnlessIn(self._sub_uri, manifest)
+            self.failUnlessIn("<td>sub/baz.txt</td>", manifest)
+            self.failUnlessIn(FAVICON_MARKUP, manifest)

         d.addCallback(_got_html)
 
         # both t=status and unadorned GET should be identical
         d.addCallback(_got_html)
 
         # both t=status and unadorned GET should be identical


@@ -1124,8 +1745,8 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(_got_html)
         d.addCallback(getman, "text")
         def _got_text(manifest):
         d.addCallback(_got_html)
         d.addCallback(getman, "text")
         def _got_text(manifest):

-            self.failUnless("\nsub " + self._sub_uri + "\n" in manifest)
-            self.failUnless("\nsub/baz.txt URI:CHK:" in manifest)
+            self.failUnlessIn("\nsub " + self._sub_uri + "\n", manifest)
+            self.failUnlessIn("\nsub/baz.txt URI:CHK:", manifest)

         d.addCallback(_got_text)
         d.addCallback(getman, "JSON")
         def _got_json(res):
         d.addCallback(_got_text)
         d.addCallback(getman, "JSON")
         def _got_json(res):


@@ -1134,12 +1755,12 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
             for (path_list, cap) in data:
                 got[tuple(path_list)] = cap
             self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
             for (path_list, cap) in data:
                 got[tuple(path_list)] = cap
             self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)

-            self.failUnless((u"sub",u"baz.txt") in got)
-            self.failUnless("finished" in res)
-            self.failUnless("origin" in res)
-            self.failUnless("storage-index" in res)
-            self.failUnless("verifycaps" in res)
-            self.failUnless("stats" in res)
+            self.failUnlessIn((u"sub", u"baz.txt"), got)
+            self.failUnlessIn("finished", res)
+            self.failUnlessIn("origin", res)
+            self.failUnlessIn("storage-index", res)
+            self.failUnlessIn("verifycaps", res)
+            self.failUnlessIn("stats", res)

         d.addCallback(_got_json)
         return d
 
         d.addCallback(_got_json)
         return d
 


@@ -1185,16 +1806,16 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(self.wait_for_operation, "127")
         d.addCallback(self.get_operation_results, "127", "json")
         def _got_json(stats):
         d.addCallback(self.wait_for_operation, "127")
         d.addCallback(self.get_operation_results, "127", "json")
         def _got_json(stats):

-            expected = {"count-immutable-files": 3,
-                        "count-mutable-files": 0,
+            expected = {"count-immutable-files": 4,
+                        "count-mutable-files": 2,

                         "count-literal-files": 0,
                         "count-literal-files": 0,

-                        "count-files": 3,
+                        "count-files": 6,

                         "count-directories": 3,
                         "count-directories": 3,

-                        "size-immutable-files": 57,
+                        "size-immutable-files": 76,

                         "size-literal-files": 0,
                         #"size-directories": 1912, # varies
                         #"largest-directory": 1590,
                         "size-literal-files": 0,
                         #"size-directories": 1912, # varies
                         #"largest-directory": 1590,

-                        "largest-directory-children": 5,
+                        "largest-directory-children": 8,

                         "largest-immutable-file": 19,
                         }
             for k,v in expected.iteritems():
                         "largest-immutable-file": 19,
                         }
             for k,v in expected.iteritems():


@@ -1202,7 +1823,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                                            "stats[%s] was %s, not %s" %
                                            (k, stats[k], v))
             self.failUnlessReallyEqual(stats["size-files-histogram"],
                                            "stats[%s] was %s, not %s" %
                                            (k, stats[k], v))
             self.failUnlessReallyEqual(stats["size-files-histogram"],

-                                       [ [11, 31, 3] ])
+                                       [ [11, 31, 4] ])

         d.addCallback(_got_json)
         return d
 
         d.addCallback(_got_json)
         return d
 


@@ -1211,7 +1832,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         def _check(res):
             self.failUnless(res.endswith("\n"))
             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
         def _check(res):
             self.failUnless(res.endswith("\n"))
             units = [simplejson.loads(t) for t in res[:-1].split("\n")]

-            self.failUnlessReallyEqual(len(units), 7)
+            self.failUnlessReallyEqual(len(units), 10)

             self.failUnlessEqual(units[-1]["type"], "stats")
             first = units[0]
             self.failUnlessEqual(first["path"], [])
             self.failUnlessEqual(units[-1]["type"], "stats")
             first = units[0]
             self.failUnlessEqual(first["path"], [])


@@ -1222,6 +1843,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
             self.failIfEqual(baz["storage-index"], None)
             self.failIfEqual(baz["verifycap"], None)
             self.failIfEqual(baz["repaircap"], None)
             self.failIfEqual(baz["storage-index"], None)
             self.failIfEqual(baz["verifycap"], None)
             self.failIfEqual(baz["repaircap"], None)

+            # XXX: Add quux and baz to this test.

             return
         d.addCallback(_check)
         return d
             return
         d.addCallback(_check)
         return d


@@ -1248,6 +1870,31 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(self.failUnlessNodeKeysAre, [])
         return d
 
         d.addCallback(self.failUnlessNodeKeysAre, [])
         return d
 

+    def test_PUT_NEWDIRURL_mdmf(self):
+        d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
+        d.addCallback(lambda res:
+                      self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
+        d.addCallback(lambda res: self._foo_node.get(u"newdir"))
+        d.addCallback(lambda node:
+            self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
+        return d
+
+    def test_PUT_NEWDIRURL_sdmf(self):
+        d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=sdmf",
+                     "")
+        d.addCallback(lambda res:
+                      self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
+        d.addCallback(lambda res: self._foo_node.get(u"newdir"))
+        d.addCallback(lambda node:
+            self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
+        return d
+
+    def test_PUT_NEWDIRURL_bad_format(self):
+        return self.shouldHTTPError("PUT_NEWDIRURL_bad_format",
+                                    400, "Bad Request", "Unknown format: foo",
+                                    self.PUT, self.public_url +
+                                    "/foo/newdir=?t=mkdir&format=foo", "")
+

     def test_POST_NEWDIRURL(self):
         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
         d.addCallback(lambda res:
     def test_POST_NEWDIRURL(self):
         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
         d.addCallback(lambda res:


@@ -1256,21 +1903,53 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(self.failUnlessNodeKeysAre, [])
         return d
 
         d.addCallback(self.failUnlessNodeKeysAre, [])
         return d
 

+    def test_POST_NEWDIRURL_mdmf(self):
+        d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
+        d.addCallback(lambda res:
+                      self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
+        d.addCallback(lambda res: self._foo_node.get(u"newdir"))
+        d.addCallback(lambda node:
+            self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
+        return d
+
+    def test_POST_NEWDIRURL_sdmf(self):
+        d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=sdmf", "")
+        d.addCallback(lambda res:
+            self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
+        d.addCallback(lambda res: self._foo_node.get(u"newdir"))
+        d.addCallback(lambda node:
+            self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
+        return d
+
+    def test_POST_NEWDIRURL_bad_format(self):
+        return self.shouldHTTPError("POST_NEWDIRURL_bad_format",
+                                    400, "Bad Request", "Unknown format: foo",
+                                    self.POST2, self.public_url + \
+                                    "/foo/newdir?t=mkdir&format=foo", "")
+

     def test_POST_NEWDIRURL_emptyname(self):
         # an empty pathname component (i.e. a double-slash) is disallowed
     def test_POST_NEWDIRURL_emptyname(self):
         # an empty pathname component (i.e. a double-slash) is disallowed

-        d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_emptyname",
+        d = self.shouldFail2(error.Error, "POST_NEWDIRURL_emptyname",

                              "400 Bad Request",
                              "The webapi does not allow empty pathname components, i.e. a double slash",
                              self.POST, self.public_url + "//?t=mkdir")
         return d
 
                              "400 Bad Request",
                              "The webapi does not allow empty pathname components, i.e. a double slash",
                              self.POST, self.public_url + "//?t=mkdir")
         return d
 

-    def test_POST_NEWDIRURL_initial_children(self):
+    def _do_POST_NEWDIRURL_initial_children_test(self, version=None):

         (newkids, caps) = self._create_initial_children()
         (newkids, caps) = self._create_initial_children()

-        d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-with-children",
+        query = "/foo/newdir?t=mkdir-with-children"
+        if version == MDMF_VERSION:
+            query += "&format=mdmf"
+        elif version == SDMF_VERSION:
+            query += "&format=sdmf"
+        else:
+            version = SDMF_VERSION # for later
+        d = self.POST2(self.public_url + query,

                        simplejson.dumps(newkids))
         def _check(uri):
             n = self.s.create_node_from_uri(uri.strip())
             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
                        simplejson.dumps(newkids))
         def _check(uri):
             n = self.s.create_node_from_uri(uri.strip())
             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())

+            self.failUnlessEqual(n._node.get_version(), version)

             d2.addCallback(lambda ign:
                            self.failUnlessROChildURIIs(n, u"child-imm",
                                                        caps['filecap1']))
             d2.addCallback(lambda ign:
                            self.failUnlessROChildURIIs(n, u"child-imm",
                                                        caps['filecap1']))


@@ -1308,6 +1987,23 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
         return d
 
         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
         return d
 

+    def test_POST_NEWDIRURL_initial_children(self):
+        return self._do_POST_NEWDIRURL_initial_children_test()
+
+    def test_POST_NEWDIRURL_initial_children_mdmf(self):
+        return self._do_POST_NEWDIRURL_initial_children_test(MDMF_VERSION)
+
+    def test_POST_NEWDIRURL_initial_children_sdmf(self):
+        return self._do_POST_NEWDIRURL_initial_children_test(SDMF_VERSION)
+
+    def test_POST_NEWDIRURL_initial_children_bad_format(self):
+        (newkids, caps) = self._create_initial_children()
+        return self.shouldHTTPError("POST_NEWDIRURL_initial_children_bad_format",
+                                    400, "Bad Request", "Unknown format: foo",
+                                    self.POST2, self.public_url + \
+                                    "/foo/newdir?t=mkdir-with-children&format=foo",
+                                    simplejson.dumps(newkids))
+

     def test_POST_NEWDIRURL_immutable(self):
         (newkids, caps) = self._create_immutable_children()
         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
     def test_POST_NEWDIRURL_immutable(self):
         (newkids, caps) = self._create_immutable_children()
         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",


@@ -1378,25 +2074,6 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
         return d
 
         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
         return d
 

-    def test_PUT_NEWDIRURL_mkdir_p(self):
-        d = defer.succeed(None)
-        d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
-        d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
-        d.addCallback(lambda res: self._foo_node.get(u"mkp"))
-        def mkdir_p(mkpnode):
-            url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
-            d = self.POST(url)
-            def made_subsub(ssuri):
-                d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
-                d.addCallback(lambda ssnode: self.failUnlessReallyEqual(ssnode.get_uri(), ssuri))
-                d = self.POST(url)
-                d.addCallback(lambda uri2: self.failUnlessReallyEqual(uri2, ssuri))
-                return d
-            d.addCallback(made_subsub)
-            return d
-        d.addCallback(mkdir_p)
-        return d
-

     def test_PUT_NEWDIRURL_mkdirs(self):
         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
         d.addCallback(lambda res:
     def test_PUT_NEWDIRURL_mkdirs(self):
         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
         d.addCallback(lambda res:


@@ -1408,6 +2085,49 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(self.failUnlessNodeKeysAre, [])
         return d
 
         d.addCallback(self.failUnlessNodeKeysAre, [])
         return d
 

+    def test_PUT_NEWDIRURL_mkdirs_mdmf(self):
+        d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=mdmf", "")
+        d.addCallback(lambda ignored:
+            self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
+        d.addCallback(lambda ignored:
+            self.failIfNodeHasChild(self._foo_node, u"newdir"))
+        d.addCallback(lambda ignored:
+            self._foo_node.get_child_at_path(u"subdir"))
+        def _got_subdir(subdir):
+            # XXX: What we want?
+            #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
+            self.failUnlessNodeHasChild(subdir, u"newdir")
+            return subdir.get_child_at_path(u"newdir")
+        d.addCallback(_got_subdir)
+        d.addCallback(lambda newdir:
+            self.failUnlessEqual(newdir._node.get_version(), MDMF_VERSION))
+        return d
+
+    def test_PUT_NEWDIRURL_mkdirs_sdmf(self):
+        d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=sdmf", "")
+        d.addCallback(lambda ignored:
+            self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
+        d.addCallback(lambda ignored:
+            self.failIfNodeHasChild(self._foo_node, u"newdir"))
+        d.addCallback(lambda ignored:
+            self._foo_node.get_child_at_path(u"subdir"))
+        def _got_subdir(subdir):
+            # XXX: What we want?
+            #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
+            self.failUnlessNodeHasChild(subdir, u"newdir")
+            return subdir.get_child_at_path(u"newdir")
+        d.addCallback(_got_subdir)
+        d.addCallback(lambda newdir:
+            self.failUnlessEqual(newdir._node.get_version(), SDMF_VERSION))
+        return d
+
+    def test_PUT_NEWDIRURL_mkdirs_bad_format(self):
+        return self.shouldHTTPError("PUT_NEWDIRURL_mkdirs_bad_format",
+                                    400, "Bad Request", "Unknown format: foo",
+                                    self.PUT, self.public_url + \
+                                    "/foo/subdir/newdir?t=mkdir&format=foo",
+                                    "")
+

     def test_DELETE_DIRURL(self):
         d = self.DELETE(self.public_url + "/foo")
         d.addCallback(lambda res:
     def test_DELETE_DIRURL(self):
         d = self.DELETE(self.public_url + "/foo")
         d.addCallback(lambda res:


@@ -1446,14 +2166,14 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         assert isinstance(name, unicode)
         d = node.list()
         def _check(children):
         assert isinstance(name, unicode)
         d = node.list()
         def _check(children):

-            self.failUnless(name in children)
+            self.failUnlessIn(name, children)

         d.addCallback(_check)
         return d
     def failIfNodeHasChild(self, node, name):
         assert isinstance(name, unicode)
         d = node.list()
         def _check(children):
         d.addCallback(_check)
         return d
     def failIfNodeHasChild(self, node, name):
         assert isinstance(name, unicode)
         d = node.list()
         def _check(children):

-            self.failIf(name in children)
+            self.failIfIn(name, children)

         d.addCallback(_check)
         return d
 
         d.addCallback(_check)
         return d
 


@@ -1524,7 +2244,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         return d
 
     def failUnlessCHKURIHasContents(self, got_uri, contents):
         return d
 
     def failUnlessCHKURIHasContents(self, got_uri, contents):

-        self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
+        self.failUnless(self.get_all_contents()[got_uri] == contents)

 
     def test_POST_upload(self):
         d = self.POST(self.public_url + "/foo", t="upload",
 
     def test_POST_upload(self):
         d = self.POST(self.public_url + "/foo", t="upload",


@@ -1575,8 +2295,8 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         def _check_upload_results(page):
             # this should be a page which describes the results of the upload
             # that just finished.
         def _check_upload_results(page):
             # this should be a page which describes the results of the upload
             # that just finished.

-            self.failUnless("Upload Results:" in page)
-            self.failUnless("URI:" in page)
+            self.failUnlessIn("Upload Results:", page)
+            self.failUnlessIn("URI:", page)

             uri_re = re.compile("URI: <tt><span>(.*)</span>")
             mo = uri_re.search(page)
             self.failUnless(mo, page)
             uri_re = re.compile("URI: <tt><span>(.*)</span>")
             mo = uri_re.search(page)
             self.failUnless(mo, page)


@@ -1610,10 +2330,11 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
             self.failUnless(target.startswith(self.webish_url), target)
             return client.getPage(target, method="GET")
             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
             self.failUnless(target.startswith(self.webish_url), target)
             return client.getPage(target, method="GET")

+        # We encode "uri" as "%75ri" to exercise a case affected by ticket #1860.

         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
                                  check,
                                  self.POST, "/uri", t="upload",
         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
                                  check,
                                  self.POST, "/uri", t="upload",

-                                 when_done="/uri/%(uri)s",
+                                 when_done="/%75ri/%(uri)s",

                                  file=("new.txt", self.NEWFILE_CONTENTS))
         d.addCallback(lambda res:
                       self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
                                  file=("new.txt", self.NEWFILE_CONTENTS))
         d.addCallback(lambda res:
                       self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))


@@ -1627,7 +2348,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
             self.filecap = filecap
             u = uri.WriteableSSKFileURI.init_from_string(filecap)
             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
             self.filecap = filecap
             u = uri.WriteableSSKFileURI.init_from_string(filecap)

-            self.failUnless(u.get_storage_index() in FakeMutableFileNode.all_contents)
+            self.failUnlessIn(u.get_storage_index(), self.get_all_contents())

             n = self.s.create_node_from_uri(filecap)
             return n.download_best_version()
         d.addCallback(_check)
             n = self.s.create_node_from_uri(filecap)
             return n.download_best_version()
         d.addCallback(_check)


@@ -1645,16 +2366,89 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         return d
 
     def test_POST_upload_no_link_mutable_toobig(self):
         return d
 
     def test_POST_upload_no_link_mutable_toobig(self):

-        d = self.shouldFail2(error.Error,
-                             "test_POST_upload_no_link_mutable_toobig",
-                             "413 Request Entity Too Large",
-                             "SDMF is limited to one segment, and 10001 > 10000",
-                             self.POST,
-                             "/uri", t="upload", mutable="true",
-                             file=("new.txt",
-                                   "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
+        # The SDMF size limit is no longer in place, so we should be
+        # able to upload mutable files that are as large as we want them
+        # to be.
+        d = self.POST("/uri", t="upload", mutable="true",
+                      file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
+        return d
+
+
+    def test_POST_upload_format_unlinked(self):
+        def _check_upload_unlinked(ign, format, uri_prefix):
+            filename = format + ".txt"
+            d = self.POST("/uri?t=upload&format=" + format,
+                          file=(filename, self.NEWFILE_CONTENTS * 300000))
+            def _got_results(results):
+                if format.upper() in ("SDMF", "MDMF"):
+                    # webapi.rst says this returns a filecap
+                    filecap = results
+                else:
+                    # for immutable, it returns an "upload results page", and
+                    # the filecap is buried inside
+                    line = [l for l in results.split("\n") if "URI: " in l][0]
+                    mo = re.search(r'<span>([^<]+)</span>', line)
+                    filecap = mo.group(1)
+                self.failUnless(filecap.startswith(uri_prefix),
+                                (uri_prefix, filecap))
+                return self.GET("/uri/%s?t=json" % filecap)
+            d.addCallback(_got_results)
+            def _got_json(json):
+                data = simplejson.loads(json)
+                data = data[1]
+                self.failUnlessIn("format", data)
+                self.failUnlessEqual(data["format"], format.upper())
+            d.addCallback(_got_json)
+            return d
+        d = defer.succeed(None)
+        d.addCallback(_check_upload_unlinked, "chk", "URI:CHK")
+        d.addCallback(_check_upload_unlinked, "CHK", "URI:CHK")
+        d.addCallback(_check_upload_unlinked, "sdmf", "URI:SSK")
+        d.addCallback(_check_upload_unlinked, "mdmf", "URI:MDMF")
+        return d
+
+    def test_POST_upload_bad_format_unlinked(self):
+        return self.shouldHTTPError("POST_upload_bad_format_unlinked",
+                                    400, "Bad Request", "Unknown format: foo",
+                                    self.POST,
+                                    "/uri?t=upload&format=foo",
+                                    file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
+
+    def test_POST_upload_format(self):
+        def _check_upload(ign, format, uri_prefix, fn=None):
+            filename = format + ".txt"
+            d = self.POST(self.public_url +
+                          "/foo?t=upload&format=" + format,
+                          file=(filename, self.NEWFILE_CONTENTS * 300000))
+            def _got_filecap(filecap):
+                if fn is not None:
+                    filenameu = unicode(filename)
+                    self.failUnlessURIMatchesRWChild(filecap, fn, filenameu)
+                self.failUnless(filecap.startswith(uri_prefix))
+                return self.GET(self.public_url + "/foo/%s?t=json" % filename)
+            d.addCallback(_got_filecap)
+            def _got_json(json):
+                data = simplejson.loads(json)
+                data = data[1]
+                self.failUnlessIn("format", data)
+                self.failUnlessEqual(data["format"], format.upper())
+            d.addCallback(_got_json)
+            return d
+
+        d = defer.succeed(None)
+        d.addCallback(_check_upload, "chk", "URI:CHK")
+        d.addCallback(_check_upload, "sdmf", "URI:SSK", self._foo_node)
+        d.addCallback(_check_upload, "mdmf", "URI:MDMF")
+        d.addCallback(_check_upload, "MDMF", "URI:MDMF")

         return d
 
         return d
 

+    def test_POST_upload_bad_format(self):
+        return self.shouldHTTPError("POST_upload_bad_format",
+                                    400, "Bad Request", "Unknown format: foo",
+                                    self.POST, self.public_url + \
+                                    "/foo?t=upload&format=foo",
+                                    file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
+

     def test_POST_upload_mutable(self):
         # this creates a mutable file
         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
     def test_POST_upload_mutable(self):
         # this creates a mutable file
         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",


@@ -1708,7 +2502,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                                followRedirect=True))
         def _check_page(res):
             # TODO: assert more about the contents
                                followRedirect=True))
         def _check_page(res):
             # TODO: assert more about the contents

-            self.failUnless("SSK" in res)
+            self.failUnlessIn("SSK", res)

             return res
         d.addCallback(_check_page)
 
             return res
         d.addCallback(_check_page)
 


@@ -1730,7 +2524,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
             children = dict( [(unicode(name),value)
                               for (name,value)
                               in parsed[1]["children"].iteritems()] )
             children = dict( [(unicode(name),value)
                               for (name,value)
                               in parsed[1]["children"].iteritems()] )

-            self.failUnless(u"new.txt" in children)
+            self.failUnlessIn(u"new.txt", children)

             new_json = children[u"new.txt"]
             self.failUnlessEqual(new_json[0], "filenode")
             self.failUnless(new_json[1]["mutable"])
             new_json = children[u"new.txt"]
             self.failUnlessEqual(new_json[0], "filenode")
             self.failUnless(new_json[1]["mutable"])


@@ -1779,32 +2573,24 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
         d.addCallback(_got_headers)
 
             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
         d.addCallback(_got_headers)
 

-        # make sure that size errors are displayed correctly for overwrite
-        d.addCallback(lambda res:
-                      self.shouldFail2(error.Error,
-                                       "test_POST_upload_mutable-toobig",
-                                       "413 Request Entity Too Large",
-                                       "SDMF is limited to one segment, and 10001 > 10000",
-                                       self.POST,
-                                       self.public_url + "/foo", t="upload",
-                                       mutable="true",
-                                       file=("new.txt",
-                                             "b" * (self.s.MUTABLE_SIZELIMIT+1)),
-                                       ))
-
+        # make sure that outdated size limits aren't enforced anymore.
+        d.addCallback(lambda ignored:
+            self.POST(self.public_url + "/foo", t="upload",
+                      mutable="true",
+                      file=("new.txt",
+                            "b" * (self.s.MUTABLE_SIZELIMIT+1))))

         d.addErrback(self.dump_error)
         return d
 
     def test_POST_upload_mutable_toobig(self):
         d.addErrback(self.dump_error)
         return d
 
     def test_POST_upload_mutable_toobig(self):

-        d = self.shouldFail2(error.Error,
-                             "test_POST_upload_mutable_toobig",
-                             "413 Request Entity Too Large",
-                             "SDMF is limited to one segment, and 10001 > 10000",
-                             self.POST,
-                             self.public_url + "/foo",
-                             t="upload", mutable="true",
-                             file=("new.txt",
-                                   "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
+        # SDMF had a size limti that was removed a while ago. MDMF has
+        # never had a size limit. Test to make sure that we do not
+        # encounter errors when trying to upload large mutable files,
+        # since there should be no coded prohibitions regarding large
+        # mutable files.
+        d = self.POST(self.public_url + "/foo",
+                      t="upload", mutable="true",
+                      file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))

         return d
 
     def dump_error(self, f):
         return d
 
     def dump_error(self, f):


@@ -1892,8 +2678,9 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         # make sure that nothing was added
         d.addCallback(lambda res:
                       self.failUnlessNodeKeysAre(self._foo_node,
         # make sure that nothing was added
         d.addCallback(lambda res:
                       self.failUnlessNodeKeysAre(self._foo_node,

-                                                 [u"bar.txt", u"blockingfile",
-                                                  u"empty", u"n\u00fc.txt",
+                                                 [self._htmlname_unicode,
+                                                  u"bar.txt", u"baz.txt", u"blockingfile",
+                                                  u"empty", u"n\u00fc.txt", u"quux.txt",

                                                   u"sub"]))
         return d
 
                                                   u"sub"]))
         return d
 


@@ -1901,7 +2688,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         bar_url = self.public_url + "/foo/bar.txt"
         d = self.POST(bar_url, t="check")
         def _check(res):
         bar_url = self.public_url + "/foo/bar.txt"
         d = self.POST(bar_url, t="check")
         def _check(res):

-            self.failUnless("Healthy :" in res)
+            self.failUnlessIn("Healthy :", res)

         d.addCallback(_check)
         redir_url = "http://allmydata.org/TARGET"
         def _check2(statuscode, target):
         d.addCallback(_check)
         redir_url = "http://allmydata.org/TARGET"
         def _check2(statuscode, target):


@@ -1916,16 +2703,16 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(lambda res:
                       self.POST(bar_url, t="check", return_to=redir_url))
         def _check3(res):
         d.addCallback(lambda res:
                       self.POST(bar_url, t="check", return_to=redir_url))
         def _check3(res):

-            self.failUnless("Healthy :" in res)
-            self.failUnless("Return to file" in res)
-            self.failUnless(redir_url in res)
+            self.failUnlessIn("Healthy :", res)
+            self.failUnlessIn("Return to file", res)
+            self.failUnlessIn(redir_url, res)

         d.addCallback(_check3)
 
         d.addCallback(lambda res:
                       self.POST(bar_url, t="check", output="JSON"))
         def _check_json(res):
             data = simplejson.loads(res)
         d.addCallback(_check3)
 
         d.addCallback(lambda res:
                       self.POST(bar_url, t="check", output="JSON"))
         def _check_json(res):
             data = simplejson.loads(res)

-            self.failUnless("storage-index" in data)
+            self.failUnlessIn("storage-index", data)

             self.failUnless(data["results"]["healthy"])
         d.addCallback(_check_json)
 
             self.failUnless(data["results"]["healthy"])
         d.addCallback(_check_json)
 


@@ -1935,7 +2722,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         bar_url = self.public_url + "/foo/bar.txt"
         d = self.POST(bar_url, t="check", repair="true")
         def _check(res):
         bar_url = self.public_url + "/foo/bar.txt"
         d = self.POST(bar_url, t="check", repair="true")
         def _check(res):

-            self.failUnless("Healthy :" in res)
+            self.failUnlessIn("Healthy :", res)

         d.addCallback(_check)
         redir_url = "http://allmydata.org/TARGET"
         def _check2(statuscode, target):
         d.addCallback(_check)
         redir_url = "http://allmydata.org/TARGET"
         def _check2(statuscode, target):


@@ -1950,9 +2737,9 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(lambda res:
                       self.POST(bar_url, t="check", return_to=redir_url))
         def _check3(res):
         d.addCallback(lambda res:
                       self.POST(bar_url, t="check", return_to=redir_url))
         def _check3(res):

-            self.failUnless("Healthy :" in res)
-            self.failUnless("Return to file" in res)
-            self.failUnless(redir_url in res)
+            self.failUnlessIn("Healthy :", res)
+            self.failUnlessIn("Return to file", res)
+            self.failUnlessIn(redir_url, res)

         d.addCallback(_check3)
         return d
 
         d.addCallback(_check3)
         return d
 


@@ -1960,7 +2747,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         foo_url = self.public_url + "/foo/"
         d = self.POST(foo_url, t="check")
         def _check(res):
         foo_url = self.public_url + "/foo/"
         d = self.POST(foo_url, t="check")
         def _check(res):

-            self.failUnless("Healthy :" in res, res)
+            self.failUnlessIn("Healthy :", res)

         d.addCallback(_check)
         redir_url = "http://allmydata.org/TARGET"
         def _check2(statuscode, target):
         d.addCallback(_check)
         redir_url = "http://allmydata.org/TARGET"
         def _check2(statuscode, target):


@@ -1975,16 +2762,16 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(lambda res:
                       self.POST(foo_url, t="check", return_to=redir_url))
         def _check3(res):
         d.addCallback(lambda res:
                       self.POST(foo_url, t="check", return_to=redir_url))
         def _check3(res):

-            self.failUnless("Healthy :" in res, res)
-            self.failUnless("Return to file/directory" in res)
-            self.failUnless(redir_url in res)
+            self.failUnlessIn("Healthy :", res)
+            self.failUnlessIn("Return to file/directory", res)
+            self.failUnlessIn(redir_url, res)

         d.addCallback(_check3)
 
         d.addCallback(lambda res:
                       self.POST(foo_url, t="check", output="JSON"))
         def _check_json(res):
             data = simplejson.loads(res)
         d.addCallback(_check3)
 
         d.addCallback(lambda res:
                       self.POST(foo_url, t="check", output="JSON"))
         def _check_json(res):
             data = simplejson.loads(res)

-            self.failUnless("storage-index" in data)
+            self.failUnlessIn("storage-index", data)

             self.failUnless(data["results"]["healthy"])
         d.addCallback(_check_json)
 
             self.failUnless(data["results"]["healthy"])
         d.addCallback(_check_json)
 


@@ -1994,7 +2781,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         foo_url = self.public_url + "/foo/"
         d = self.POST(foo_url, t="check", repair="true")
         def _check(res):
         foo_url = self.public_url + "/foo/"
         d = self.POST(foo_url, t="check", repair="true")
         def _check(res):

-            self.failUnless("Healthy :" in res, res)
+            self.failUnlessIn("Healthy :", res)

         d.addCallback(_check)
         redir_url = "http://allmydata.org/TARGET"
         def _check2(statuscode, target):
         d.addCallback(_check)
         redir_url = "http://allmydata.org/TARGET"
         def _check2(statuscode, target):


@@ -2009,12 +2796,36 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(lambda res:
                       self.POST(foo_url, t="check", return_to=redir_url))
         def _check3(res):
         d.addCallback(lambda res:
                       self.POST(foo_url, t="check", return_to=redir_url))
         def _check3(res):

-            self.failUnless("Healthy :" in res)
-            self.failUnless("Return to file/directory" in res)
-            self.failUnless(redir_url in res)
+            self.failUnlessIn("Healthy :", res)
+            self.failUnlessIn("Return to file/directory", res)
+            self.failUnlessIn(redir_url, res)

         d.addCallback(_check3)
         return d
 
         d.addCallback(_check3)
         return d
 

+    def test_POST_FILEURL_mdmf_check(self):
+        quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
+        d = self.POST(quux_url, t="check")
+        def _check(res):
+            self.failUnlessIn("Healthy", res)
+        d.addCallback(_check)
+        quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
+        d.addCallback(lambda ignored:
+                      self.POST(quux_extension_url, t="check"))
+        d.addCallback(_check)
+        return d
+
+    def test_POST_FILEURL_mdmf_check_and_repair(self):
+        quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
+        d = self.POST(quux_url, t="check", repair="true")
+        def _check(res):
+            self.failUnlessIn("Healthy", res)
+        d.addCallback(_check)
+        quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
+        d.addCallback(lambda ignored:
+                      self.POST(quux_extension_url, t="check", repair="true"))
+        d.addCallback(_check)
+        return d
+

     def wait_for_operation(self, ignored, ophandle):
         url = "/operations/" + ophandle
         url += "?t=status&output=JSON"
     def wait_for_operation(self, ignored, ophandle):
         url = "/operations/" + ophandle
         url += "?t=status&output=JSON"


@@ -2060,13 +2871,14 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(self.wait_for_operation, "123")
         def _check_json(data):
             self.failUnlessReallyEqual(data["finished"], True)
         d.addCallback(self.wait_for_operation, "123")
         def _check_json(data):
             self.failUnlessReallyEqual(data["finished"], True)

-            self.failUnlessReallyEqual(data["count-objects-checked"], 8)
-            self.failUnlessReallyEqual(data["count-objects-healthy"], 8)
+            self.failUnlessReallyEqual(data["count-objects-checked"], 11)
+            self.failUnlessReallyEqual(data["count-objects-healthy"], 11)

         d.addCallback(_check_json)
         d.addCallback(self.get_operation_results, "123", "html")
         def _check_html(res):
         d.addCallback(_check_json)
         d.addCallback(self.get_operation_results, "123", "html")
         def _check_html(res):

-            self.failUnless("Objects Checked: <span>8</span>" in res)
-            self.failUnless("Objects Healthy: <span>8</span>" in res)
+            self.failUnlessIn("Objects Checked: <span>11</span>", res)
+            self.failUnlessIn("Objects Healthy: <span>11</span>", res)
+            self.failUnlessIn(FAVICON_MARKUP, res)

         d.addCallback(_check_html)
 
         d.addCallback(lambda res:
         d.addCallback(_check_html)
 
         d.addCallback(lambda res:


@@ -2095,32 +2907,34 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(self.wait_for_operation, "124")
         def _check_json(data):
             self.failUnlessReallyEqual(data["finished"], True)
         d.addCallback(self.wait_for_operation, "124")
         def _check_json(data):
             self.failUnlessReallyEqual(data["finished"], True)

-            self.failUnlessReallyEqual(data["count-objects-checked"], 8)
-            self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 8)
+            self.failUnlessReallyEqual(data["count-objects-checked"], 11)
+            self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 11)

             self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
             self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
             self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
             self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
             self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
             self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
             self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
             self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
             self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
             self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)

-            self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 8)
+            self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 11)

             self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
             self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
         d.addCallback(_check_json)
         d.addCallback(self.get_operation_results, "124", "html")
         def _check_html(res):
             self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
             self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
         d.addCallback(_check_json)
         d.addCallback(self.get_operation_results, "124", "html")
         def _check_html(res):

-            self.failUnless("Objects Checked: <span>8</span>" in res)
+            self.failUnlessIn("Objects Checked: <span>11</span>", res)

 
 

-            self.failUnless("Objects Healthy (before repair): <span>8</span>" in res)
-            self.failUnless("Objects Unhealthy (before repair): <span>0</span>" in res)
-            self.failUnless("Corrupt Shares (before repair): <span>0</span>" in res)
+            self.failUnlessIn("Objects Healthy (before repair): <span>11</span>", res)
+            self.failUnlessIn("Objects Unhealthy (before repair): <span>0</span>", res)
+            self.failUnlessIn("Corrupt Shares (before repair): <span>0</span>", res)

 
 

-            self.failUnless("Repairs Attempted: <span>0</span>" in res)
-            self.failUnless("Repairs Successful: <span>0</span>" in res)
-            self.failUnless("Repairs Unsuccessful: <span>0</span>" in res)
+            self.failUnlessIn("Repairs Attempted: <span>0</span>", res)
+            self.failUnlessIn("Repairs Successful: <span>0</span>", res)
+            self.failUnlessIn("Repairs Unsuccessful: <span>0</span>", res)

 
 

-            self.failUnless("Objects Healthy (after repair): <span>8</span>" in res)
-            self.failUnless("Objects Unhealthy (after repair): <span>0</span>" in res)
-            self.failUnless("Corrupt Shares (after repair): <span>0</span>" in res)
+            self.failUnlessIn("Objects Healthy (after repair): <span>11</span>", res)
+            self.failUnlessIn("Objects Unhealthy (after repair): <span>0</span>", res)
+            self.failUnlessIn("Corrupt Shares (after repair): <span>0</span>", res)
+
+            self.failUnlessIn(FAVICON_MARKUP, res)

         d.addCallback(_check_html)
         return d
 
         d.addCallback(_check_html)
         return d
 


@@ -2137,6 +2951,26 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(self.failUnlessNodeKeysAre, [])
         return d
 
         d.addCallback(self.failUnlessNodeKeysAre, [])
         return d
 

+    def test_POST_mkdir_mdmf(self):
+        d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=mdmf")
+        d.addCallback(lambda res: self._foo_node.get(u"newdir"))
+        d.addCallback(lambda node:
+            self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
+        return d
+
+    def test_POST_mkdir_sdmf(self):
+        d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=sdmf")
+        d.addCallback(lambda res: self._foo_node.get(u"newdir"))
+        d.addCallback(lambda node:
+            self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
+        return d
+
+    def test_POST_mkdir_bad_format(self):
+        return self.shouldHTTPError("POST_mkdir_bad_format",
+                                    400, "Bad Request", "Unknown format: foo",
+                                    self.POST, self.public_url +
+                                    "/foo?t=mkdir&name=newdir&format=foo")
+

     def test_POST_mkdir_initial_children(self):
         (newkids, caps) = self._create_initial_children()
         d = self.POST2(self.public_url +
     def test_POST_mkdir_initial_children(self):
         (newkids, caps) = self._create_initial_children()
         d = self.POST2(self.public_url +


@@ -2150,6 +2984,45 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
         return d
 
         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
         return d
 

+    def test_POST_mkdir_initial_children_mdmf(self):
+        (newkids, caps) = self._create_initial_children()
+        d = self.POST2(self.public_url +
+                       "/foo?t=mkdir-with-children&name=newdir&format=mdmf",
+                       simplejson.dumps(newkids))
+        d.addCallback(lambda res:
+                      self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
+        d.addCallback(lambda res: self._foo_node.get(u"newdir"))
+        d.addCallback(lambda node:
+            self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
+        d.addCallback(lambda res: self._foo_node.get(u"newdir"))
+        d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
+                       caps['filecap1'])
+        return d
+
+    # XXX: Duplication.
+    def test_POST_mkdir_initial_children_sdmf(self):
+        (newkids, caps) = self._create_initial_children()
+        d = self.POST2(self.public_url +
+                       "/foo?t=mkdir-with-children&name=newdir&format=sdmf",
+                       simplejson.dumps(newkids))
+        d.addCallback(lambda res:
+                      self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
+        d.addCallback(lambda res: self._foo_node.get(u"newdir"))
+        d.addCallback(lambda node:
+            self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
+        d.addCallback(lambda res: self._foo_node.get(u"newdir"))
+        d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
+                       caps['filecap1'])
+        return d
+
+    def test_POST_mkdir_initial_children_bad_format(self):
+        (newkids, caps) = self._create_initial_children()
+        return self.shouldHTTPError("POST_mkdir_initial_children_bad_format",
+                                    400, "Bad Request", "Unknown format: foo",
+                                    self.POST, self.public_url + \
+                                    "/foo?t=mkdir-with-children&name=newdir&format=foo",
+                                    simplejson.dumps(newkids))
+

     def test_POST_mkdir_immutable(self):
         (newkids, caps) = self._create_immutable_children()
         d = self.POST2(self.public_url +
     def test_POST_mkdir_immutable(self):
         (newkids, caps) = self._create_immutable_children()
         d = self.POST2(self.public_url +


@@ -2173,7 +3046,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
 
     def test_POST_mkdir_immutable_bad(self):
         (newkids, caps) = self._create_initial_children()
 
     def test_POST_mkdir_immutable_bad(self):
         (newkids, caps) = self._create_initial_children()

-        d = self.shouldFail2(error.Error, "test_POST_mkdir_immutable_bad",
+        d = self.shouldFail2(error.Error, "POST_mkdir_immutable_bad",

                              "400 Bad Request",
                              "needed to be immutable but was not",
                              self.POST2,
                              "400 Bad Request",
                              "needed to be immutable but was not",
                              self.POST2,


@@ -2206,6 +3079,29 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(_after_mkdir)
         return d
 
         d.addCallback(_after_mkdir)
         return d
 

+    def test_POST_mkdir_no_parentdir_noredirect_mdmf(self):
+        d = self.POST("/uri?t=mkdir&format=mdmf")
+        def _after_mkdir(res):
+            u = uri.from_string(res)
+            # Check that this is an MDMF writecap
+            self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
+        d.addCallback(_after_mkdir)
+        return d
+
+    def test_POST_mkdir_no_parentdir_noredirect_sdmf(self):
+        d = self.POST("/uri?t=mkdir&format=sdmf")
+        def _after_mkdir(res):
+            u = uri.from_string(res)
+            self.failUnlessIsInstance(u, uri.DirectoryURI)
+        d.addCallback(_after_mkdir)
+        return d
+
+    def test_POST_mkdir_no_parentdir_noredirect_bad_format(self):
+        return self.shouldHTTPError("POST_mkdir_no_parentdir_noredirect_bad_format",
+                                    400, "Bad Request", "Unknown format: foo",
+                                    self.POST, self.public_url +
+                                    "/uri?t=mkdir&format=foo")
+

     def test_POST_mkdir_no_parentdir_noredirect2(self):
         # make sure form-based arguments (as on the welcome page) still work
         d = self.POST("/uri", t="mkdir")
     def test_POST_mkdir_no_parentdir_noredirect2(self):
         # make sure form-based arguments (as on the welcome page) still work
         d = self.POST("/uri", t="mkdir")


@@ -2248,6 +3144,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         filecap3 = node3.get_readonly_uri()
         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
         dircap = DirectoryNode(node4, None, None).get_uri()
         filecap3 = node3.get_readonly_uri()
         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
         dircap = DirectoryNode(node4, None, None).get_uri()

+        mdmfcap = make_mutable_file_uri(mdmf=True)

         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
         emptydircap = "URI:DIR2-LIT:"
         newkids = {u"child-imm":        ["filenode", {"rw_uri": filecap1,
         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
         emptydircap = "URI:DIR2-LIT:"
         newkids = {u"child-imm":        ["filenode", {"rw_uri": filecap1,


@@ -2264,6 +3161,8 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                                                       "ro_uri": self._make_readonly(dircap)}],
                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
                                                       "ro_uri": self._make_readonly(dircap)}],
                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],

+                   u"child-mutable-mdmf": ["filenode", {"rw_uri": mdmfcap,
+                                                        "ro_uri": self._make_readonly(mdmfcap)}],

                    }
         return newkids, {'filecap1': filecap1,
                          'filecap2': filecap2,
                    }
         return newkids, {'filecap1': filecap1,
                          'filecap2': filecap2,


@@ -2273,12 +3172,14 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                          'unknown_immcap': unknown_immcap,
                          'dircap': dircap,
                          'litdircap': litdircap,
                          'unknown_immcap': unknown_immcap,
                          'dircap': dircap,
                          'litdircap': litdircap,

-                         'emptydircap': emptydircap}
+                         'emptydircap': emptydircap,
+                         'mdmfcap': mdmfcap}

 
     def _create_immutable_children(self):
         contents, n, filecap1 = self.makefile(12)
         md1 = {"metakey1": "metavalue1"}
 
     def _create_immutable_children(self):
         contents, n, filecap1 = self.makefile(12)
         md1 = {"metakey1": "metavalue1"}

-        tnode = create_chk_filenode("immutable directory contents\n"*10)
+        tnode = create_chk_filenode("immutable directory contents\n"*10,
+                                    self.get_all_contents())

         dnode = DirectoryNode(tnode, None, None)
         assert not dnode.is_mutable()
         immdircap = dnode.get_uri()
         dnode = DirectoryNode(tnode, None, None)
         assert not dnode.is_mutable()
         immdircap = dnode.get_uri()


@@ -2333,7 +3234,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         # the regular /uri?t=mkdir operation is specified to ignore its body.
         # Only t=mkdir-with-children pays attention to it.
         (newkids, caps) = self._create_initial_children()
         # the regular /uri?t=mkdir operation is specified to ignore its body.
         # Only t=mkdir-with-children pays attention to it.
         (newkids, caps) = self._create_initial_children()

-        d = self.shouldHTTPError("POST t=mkdir unexpected children",
+        d = self.shouldHTTPError("POST_mkdir_no_parentdir_unexpected_children",

                                  400, "Bad Request",
                                  "t=mkdir does not accept children=, "
                                  "try t=mkdir-with-children instead",
                                  400, "Bad Request",
                                  "t=mkdir does not accept children=, "
                                  "try t=mkdir-with-children instead",


@@ -2342,7 +3243,8 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         return d
 
     def test_POST_noparent_bad(self):
         return d
 
     def test_POST_noparent_bad(self):

-        d = self.shouldHTTPError("POST /uri?t=bogus", 400, "Bad Request",
+        d = self.shouldHTTPError("POST_noparent_bad",
+                                 400, "Bad Request",

                                  "/uri accepts only PUT, PUT?t=mkdir, "
                                  "POST?t=upload, and POST?t=mkdir",
                                  self.POST, "/uri?t=bogus")
                                  "/uri accepts only PUT, PUT?t=mkdir, "
                                  "POST?t=upload, and POST?t=mkdir",
                                  self.POST, "/uri?t=bogus")


@@ -2390,12 +3292,13 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d = self.GET("/")
         def _after_get_welcome_page(res):
             MKDIR_BUTTON_RE = re.compile(
         d = self.GET("/")
         def _after_get_welcome_page(res):
             MKDIR_BUTTON_RE = re.compile(

-                '<form action="([^"]*)" method="post".*?'
-                '<input type="hidden" name="t" value="([^"]*)" />'
-                '<input type="hidden" name="([^"]*)" value="([^"]*)" />'
-                '<input type="submit" value="Create a directory" />',
-                re.I)
-            mo = MKDIR_BUTTON_RE.search(res)
+                '<form(?: action="([^"]*)"| method="post"| enctype="multipart/form-data"){3}>.*'
+                '<input (?:type="hidden" |name="t" |value="([^"]*?)" ){3}/>[ ]*'
+                '<input (?:type="hidden" |name="([^"]*)" |value="([^"]*)" ){3}/>[ ]*'
+                '<input (type="submit" |class="btn" |value="Create a directory[^"]*" ){3}/>')
+            html = res.replace('\n', ' ')
+            mo = MKDIR_BUTTON_RE.search(html)
+            self.failUnless(mo, html)

             formaction = mo.group(1)
             formt = mo.group(2)
             formaname = mo.group(3)
             formaction = mo.group(1)
             formt = mo.group(2)
             formaname = mo.group(3)


@@ -2454,7 +3357,8 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         return d
 
     def test_POST_bad_t(self):
         return d
 
     def test_POST_bad_t(self):

-        d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
+        d = self.shouldFail2(error.Error, "POST_bad_t",
+                             "400 Bad Request",

                              "POST to a directory with bad t=BOGUS",
                              self.POST, self.public_url + "/foo", t="BOGUS")
         return d
                              "POST to a directory with bad t=BOGUS",
                              self.POST, self.public_url + "/foo", t="BOGUS")
         return d


@@ -2565,14 +3469,21 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(self.failUnlessIsBarDotTxt)
         return d
 
         d.addCallback(self.failUnlessIsBarDotTxt)
         return d
 

-    def test_POST_delete(self):
-        d = self.POST(self.public_url + "/foo", t="delete", name="bar.txt")
+    def test_POST_delete(self, command_name='delete'):
+        d = self._foo_node.list()
+        def _check_before(children):
+            self.failUnlessIn(u"bar.txt", children)
+        d.addCallback(_check_before)
+        d.addCallback(lambda res: self.POST(self.public_url + "/foo", t=command_name, name="bar.txt"))

         d.addCallback(lambda res: self._foo_node.list())
         d.addCallback(lambda res: self._foo_node.list())

-        def _check(children):
-            self.failIf(u"bar.txt" in children)
-        d.addCallback(_check)
+        def _check_after(children):
+            self.failIfIn(u"bar.txt", children)
+        d.addCallback(_check_after)

         return d
 
         return d
 

+    def test_POST_unlink(self):
+        return self.test_POST_delete(command_name='unlink')
+

     def test_POST_rename_file(self):
         d = self.POST(self.public_url + "/foo", t="rename",
                       from_name="bar.txt", to_name='wibble.txt')
     def test_POST_rename_file(self):
         d = self.POST(self.public_url + "/foo", t="rename",
                       from_name="bar.txt", to_name='wibble.txt')


@@ -2637,16 +3548,50 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(self.failUnlessIsEmptyJSON)
         return d
 
         d.addCallback(self.failUnlessIsEmptyJSON)
         return d
 

+    def test_POST_rename_file_no_replace_same_link(self):
+        d = self.POST(self.public_url + "/foo", t="rename",
+                      replace="false", from_name="bar.txt", to_name="bar.txt")
+        d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
+        d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
+        d.addCallback(self.failUnlessIsBarDotTxt)
+        d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
+        d.addCallback(self.failUnlessIsBarJSON)
+        return d
+
+    def test_POST_rename_file_replace_only_files(self):
+        d = self.POST(self.public_url + "/foo", t="rename",
+                      replace="only-files", from_name="bar.txt",
+                      to_name="baz.txt")
+        d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
+        d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
+        d.addCallback(self.failUnlessIsBarDotTxt)
+        d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt?t=json"))
+        d.addCallback(self.failUnlessIsBarJSON)
+        return d
+
+    def test_POST_rename_file_replace_only_files_conflict(self):
+        d = self.shouldFail2(error.Error, "POST_relink_file_replace_only_files_conflict",
+                             "409 Conflict",
+                             "There was already a child by that name, and you asked me to not replace it.",
+                             self.POST, self.public_url + "/foo", t="relink",
+                             replace="only-files", from_name="bar.txt",
+                             to_name="empty")
+        d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
+        d.addCallback(self.failUnlessIsBarDotTxt)
+        d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
+        d.addCallback(self.failUnlessIsBarJSON)
+        return d
+

     def failUnlessIsEmptyJSON(self, res):
         data = simplejson.loads(res)
         self.failUnlessEqual(data[0], "dirnode", data)
         self.failUnlessReallyEqual(len(data[1]["children"]), 0)
 
     def failUnlessIsEmptyJSON(self, res):
         data = simplejson.loads(res)
         self.failUnlessEqual(data[0], "dirnode", data)
         self.failUnlessReallyEqual(len(data[1]["children"]), 0)
 

-    def test_POST_rename_file_slash_fail(self):
+    def test_POST_rename_file_to_slash_fail(self):

         d = self.POST(self.public_url + "/foo", t="rename",
                       from_name="bar.txt", to_name='kirk/spock.txt')
         d.addBoth(self.shouldFail, error.Error,
         d = self.POST(self.public_url + "/foo", t="rename",
                       from_name="bar.txt", to_name='kirk/spock.txt')
         d.addBoth(self.shouldFail, error.Error,

-                  "test_POST_rename_file_slash_fail",
+                  "test_POST_rename_file_to_slash_fail",

                   "400 Bad Request",
                   "to_name= may not contain a slash",
                   )
                   "400 Bad Request",
                   "to_name= may not contain a slash",
                   )


@@ -2654,6 +3599,18 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
         return d
 
                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
         return d
 

+    def test_POST_rename_file_from_slash_fail(self):
+        d = self.POST(self.public_url + "/foo", t="rename",
+                      from_name="sub/bar.txt", to_name='spock.txt')
+        d.addBoth(self.shouldFail, error.Error,
+                  "test_POST_rename_from_file_slash_fail",
+                  "400 Bad Request",
+                  "from_name= may not contain a slash",
+                  )
+        d.addCallback(lambda res:
+                      self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
+        return d
+

     def test_POST_rename_dir(self):
         d = self.POST(self.public_url, t="rename",
                       from_name="foo", to_name='plunk')
     def test_POST_rename_dir(self):
         d = self.POST(self.public_url, t="rename",
                       from_name="foo", to_name='plunk')


@@ -2665,6 +3622,241 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(self.failUnlessIsFooJSON)
         return d
 
         d.addCallback(self.failUnlessIsFooJSON)
         return d
 

+    def test_POST_relink_file(self):
+        d = self.POST(self.public_url + "/foo", t="relink",
+                      from_name="bar.txt",
+                      to_dir=self.public_root.get_uri() + "/foo/sub")
+        d.addCallback(lambda res:
+                      self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
+        d.addCallback(lambda res:
+                      self.failUnlessNodeHasChild(self._sub_node, u"bar.txt"))
+        d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
+        d.addCallback(self.failUnlessIsBarDotTxt)
+        d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
+        d.addCallback(self.failUnlessIsBarJSON)
+        return d
+
+    def test_POST_relink_file_new_name(self):
+        d = self.POST(self.public_url + "/foo", t="relink",
+                      from_name="bar.txt",
+                      to_name="wibble.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
+        d.addCallback(lambda res:
+                      self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
+        d.addCallback(lambda res:
+                      self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
+        d.addCallback(lambda res:
+                      self.failUnlessNodeHasChild(self._sub_node, u"wibble.txt"))
+        d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt"))
+        d.addCallback(self.failUnlessIsBarDotTxt)
+        d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt?t=json"))
+        d.addCallback(self.failUnlessIsBarJSON)
+        return d
+
+    def test_POST_relink_file_replace(self):
+        d = self.POST(self.public_url + "/foo", t="relink",
+                      from_name="bar.txt",
+                      to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
+        d.addCallback(lambda res:
+                      self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
+        d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
+        d.addCallback(self.failUnlessIsBarDotTxt)
+        d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt?t=json"))
+        d.addCallback(self.failUnlessIsBarJSON)
+        return d
+
+    def test_POST_relink_file_no_replace(self):
+        d = self.shouldFail2(error.Error, "POST_relink_file_no_replace",
+                             "409 Conflict",
+                             "There was already a child by that name, and you asked me to not replace it",
+                             self.POST, self.public_url + "/foo", t="relink",
+                             replace="false", from_name="bar.txt",
+                             to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
+        d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
+        d.addCallback(self.failUnlessIsBarDotTxt)
+        d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
+        d.addCallback(self.failUnlessIsBarJSON)
+        d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
+        d.addCallback(self.failUnlessIsSubBazDotTxt)
+        return d
+
+    def test_POST_relink_file_no_replace_explicitly_same_link(self):
+        d = self.POST(self.public_url + "/foo", t="relink",
+                      replace="false", from_name="bar.txt",
+                      to_name="bar.txt", to_dir=self.public_root.get_uri() + "/foo")
+        d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
+        d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
+        d.addCallback(self.failUnlessIsBarDotTxt)
+        d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
+        d.addCallback(self.failUnlessIsBarJSON)
+        return d
+
+    def test_POST_relink_file_replace_only_files(self):
+        d = self.POST(self.public_url + "/foo", t="relink",
+                      replace="only-files", from_name="bar.txt",
+                      to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
+        d.addCallback(lambda res:
+                      self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
+        d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
+        d.addCallback(self.failUnlessIsBarDotTxt)
+        d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt?t=json"))
+        d.addCallback(self.failUnlessIsBarJSON)
+        return d
+
+    def test_POST_relink_file_replace_only_files_conflict(self):
+        d = self.shouldFail2(error.Error, "POST_relink_file_replace_only_files_conflict",
+                             "409 Conflict",
+                             "There was already a child by that name, and you asked me to not replace it.",
+                             self.POST, self.public_url + "/foo", t="relink",
+                             replace="only-files", from_name="bar.txt",
+                             to_name="sub", to_dir=self.public_root.get_uri() + "/foo")
+        d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
+        d.addCallback(self.failUnlessIsBarDotTxt)
+        d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
+        d.addCallback(self.failUnlessIsBarJSON)
+        return d
+
+    def test_POST_relink_file_to_slash_fail(self):
+        d = self.shouldFail2(error.Error, "test_POST_rename_file_slash_fail",
+                             "400 Bad Request",
+                             "to_name= may not contain a slash",
+                             self.POST, self.public_url + "/foo", t="relink",
+                             from_name="bar.txt",
+                             to_name="slash/fail.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
+        d.addCallback(lambda res:
+                      self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
+        d.addCallback(lambda res:
+                      self.failIfNodeHasChild(self._sub_node, u"slash/fail.txt"))
+        d.addCallback(lambda ign:
+                      self.shouldFail2(error.Error,
+                                       "test_POST_rename_file_slash_fail2",
+                                       "400 Bad Request",
+                                       "from_name= may not contain a slash",
+                                       self.POST, self.public_url + "/foo",
+                                       t="relink",
+                                       from_name="nope/bar.txt",
+                                       to_name="fail.txt",
+                                       to_dir=self.public_root.get_uri() + "/foo/sub"))
+        return d
+
+    def test_POST_relink_file_explicitly_same_link(self):
+        d = self.POST(self.public_url + "/foo", t="relink",
+                      from_name="bar.txt",
+                      to_name="bar.txt", to_dir=self.public_root.get_uri() + "/foo")
+        d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
+        d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
+        d.addCallback(self.failUnlessIsBarDotTxt)
+        d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
+        d.addCallback(self.failUnlessIsBarJSON)
+        return d
+
+    def test_POST_relink_file_implicitly_same_link(self):
+        d = self.POST(self.public_url + "/foo", t="relink",
+                      from_name="bar.txt")
+        d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
+        d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
+        d.addCallback(self.failUnlessIsBarDotTxt)
+        d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
+        d.addCallback(self.failUnlessIsBarJSON)
+        return d
+
+    def test_POST_relink_file_same_dir(self):
+        d = self.POST(self.public_url + "/foo", t="relink",
+                      from_name="bar.txt",
+                      to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo")
+        d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
+        d.addCallback(lambda res: self.failUnlessNodeHasChild(self._sub_node, u"baz.txt"))
+        d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
+        d.addCallback(self.failUnlessIsBarDotTxt)
+        d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt?t=json"))
+        d.addCallback(self.failUnlessIsBarJSON)
+        return d
+
+    def test_POST_relink_file_bad_replace(self):
+        d = self.shouldFail2(error.Error, "test_POST_relink_file_bad_replace",
+                             "400 Bad Request", "invalid replace= argument: 'boogabooga'",
+                             self.POST,
+                             self.public_url + "/foo", t="relink",
+                             replace="boogabooga", from_name="bar.txt",
+                             to_dir=self.public_root.get_uri() + "/foo/sub")
+        return d
+
+    def test_POST_relink_file_multi_level(self):
+        d = self.POST(self.public_url + "/foo/sub/level2?t=mkdir", "")
+        d.addCallback(lambda res: self.POST(self.public_url + "/foo", t="relink",
+                      from_name="bar.txt", to_dir=self.public_root.get_uri() + "/foo/sub/level2"))
+        d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
+        d.addCallback(lambda res: self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
+        d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt"))
+        d.addCallback(self.failUnlessIsBarDotTxt)
+        d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt?t=json"))
+        d.addCallback(self.failUnlessIsBarJSON)
+        return d
+
+    def test_POST_relink_file_to_uri(self):
+        d = self.POST(self.public_url + "/foo", t="relink", target_type="uri",
+                      from_name="bar.txt", to_dir=self._sub_uri)
+        d.addCallback(lambda res:
+                      self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
+        d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
+        d.addCallback(self.failUnlessIsBarDotTxt)
+        d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
+        d.addCallback(self.failUnlessIsBarJSON)
+        return d
+
+    def test_POST_relink_file_to_nonexistent_dir(self):
+        d = self.shouldFail2(error.Error, "POST_relink_file_to_nonexistent_dir",
+                            "404 Not Found", "No such child: nopechucktesta",
+                            self.POST, self.public_url + "/foo", t="relink",
+                            from_name="bar.txt",
+                            to_dir=self.public_root.get_uri() + "/nopechucktesta")
+        return d
+
+    def test_POST_relink_file_into_file(self):
+        d = self.shouldFail2(error.Error, "POST_relink_file_into_file",
+                             "400 Bad Request", "to_dir is not a directory",
+                             self.POST, self.public_url + "/foo", t="relink",
+                             from_name="bar.txt",
+                             to_dir=self.public_root.get_uri() + "/foo/baz.txt")
+        d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
+        d.addCallback(self.failUnlessIsBazDotTxt)
+        d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
+        d.addCallback(self.failUnlessIsBarDotTxt)
+        d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
+        d.addCallback(self.failUnlessIsBarJSON)
+        return d
+
+    def test_POST_relink_file_to_bad_uri(self):
+        d =  self.shouldFail2(error.Error, "POST_relink_file_to_bad_uri",
+                              "400 Bad Request", "to_dir is not a directory",
+                              self.POST, self.public_url + "/foo", t="relink",
+                              from_name="bar.txt",
+                              to_dir="URI:DIR2:mn5jlyjnrjeuydyswlzyui72i:rmneifcj6k6sycjljjhj3f6majsq2zqffydnnul5hfa4j577arma")
+        d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
+        d.addCallback(self.failUnlessIsBarDotTxt)
+        d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
+        d.addCallback(self.failUnlessIsBarJSON)
+        return d
+
+    def test_POST_relink_dir(self):
+        d = self.POST(self.public_url + "/foo", t="relink",
+                      from_name="bar.txt",
+                      to_dir=self.public_root.get_uri() + "/foo/empty")
+        d.addCallback(lambda res: self.POST(self.public_url + "/foo",
+                      t="relink", from_name="empty",
+                      to_dir=self.public_root.get_uri() + "/foo/sub"))
+        d.addCallback(lambda res:
+                      self.failIfNodeHasChild(self._foo_node, u"empty"))
+        d.addCallback(lambda res:
+                      self.failUnlessNodeHasChild(self._sub_node, u"empty"))
+        d.addCallback(lambda res:
+                      self._sub_node.get_child_at_path(u"empty"))
+        d.addCallback(lambda node:
+                      self.failUnlessNodeHasChild(node, u"bar.txt"))
+        d.addCallback(lambda res:
+                      self.GET(self.public_url + "/foo/sub/empty/bar.txt"))
+        d.addCallback(self.failUnlessIsBarDotTxt)
+        return d
+

     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
         """ If target is not None then the redirection has to go to target.  If
         statuscode is not None then the redirection has to be accomplished with
     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
         """ If target is not None then the redirection has to go to target.  If
         statuscode is not None then the redirection has to be accomplished with


@@ -2716,8 +3908,9 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
                      followRedirect=True)
         def _check(res):
         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
                      followRedirect=True)
         def _check(res):

-            self.failUnless('name="when_done" value="."' in res, res)
-            self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
+            self.failUnless(re.search('<input (name="when_done" |value="." |type="hidden" ){3}/>', res), res)
+            self.failUnless(re.search(r'<input (readonly="true" |type="text" |name="from_name" |value="bar\.txt" ){4}/>', res), res)
+            self.failUnlessIn(FAVICON_MARKUP, res)

         d.addCallback(_check)
         return d
 
         d.addCallback(_check)
         return d
 


@@ -2790,8 +3983,8 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
 
     def test_PUT_DIRURL_bad_t(self):
         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
 
     def test_PUT_DIRURL_bad_t(self):
         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",

-                                 "400 Bad Request", "PUT to a directory",
-                                 self.PUT, self.public_url + "/foo?t=BOGUS", "")
+                             "400 Bad Request", "PUT to a directory",
+                             self.PUT, self.public_url + "/foo?t=BOGUS", "")

         d.addCallback(lambda res:
                       self.failUnlessRWChildURIIs(self.public_root,
                                                   u"foo",
         d.addCallback(lambda res:
                       self.failUnlessRWChildURIIs(self.public_root,
                                                   u"foo",


@@ -2807,6 +4000,46 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                                                       contents))
         return d
 
                                                       contents))
         return d
 

+    def test_PUT_NEWFILEURL_mdmf(self):
+        new_contents = self.NEWFILE_CONTENTS * 300000
+        d = self.PUT(self.public_url + \
+                     "/foo/mdmf.txt?format=mdmf",
+                     new_contents)
+        d.addCallback(lambda ignored:
+            self.GET(self.public_url + "/foo/mdmf.txt?t=json"))
+        def _got_json(json):
+            data = simplejson.loads(json)
+            data = data[1]
+            self.failUnlessIn("format", data)
+            self.failUnlessEqual(data["format"], "MDMF")
+            self.failUnless(data['rw_uri'].startswith("URI:MDMF"))
+            self.failUnless(data['ro_uri'].startswith("URI:MDMF"))
+        d.addCallback(_got_json)
+        return d
+
+    def test_PUT_NEWFILEURL_sdmf(self):
+        new_contents = self.NEWFILE_CONTENTS * 300000
+        d = self.PUT(self.public_url + \
+                     "/foo/sdmf.txt?format=sdmf",
+                     new_contents)
+        d.addCallback(lambda ignored:
+            self.GET(self.public_url + "/foo/sdmf.txt?t=json"))
+        def _got_json(json):
+            data = simplejson.loads(json)
+            data = data[1]
+            self.failUnlessIn("format", data)
+            self.failUnlessEqual(data["format"], "SDMF")
+        d.addCallback(_got_json)
+        return d
+
+    def test_PUT_NEWFILEURL_bad_format(self):
+        new_contents = self.NEWFILE_CONTENTS * 300000
+        return self.shouldHTTPError("PUT_NEWFILEURL_bad_format",
+                                    400, "Bad Request", "Unknown format: foo",
+                                    self.PUT, self.public_url + \
+                                    "/foo/foo.txt?format=foo",
+                                    new_contents)
+

     def test_PUT_NEWFILEURL_uri_replace(self):
         contents, n, new_uri = self.makefile(8)
         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
     def test_PUT_NEWFILEURL_uri_replace(self):
         contents, n, new_uri = self.makefile(8)
         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)


@@ -2819,7 +4052,8 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
     def test_PUT_NEWFILEURL_uri_no_replace(self):
         contents, n, new_uri = self.makefile(8)
         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
     def test_PUT_NEWFILEURL_uri_no_replace(self):
         contents, n, new_uri = self.makefile(8)
         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)

-        d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_uri_no_replace",
+        d.addBoth(self.shouldFail, error.Error,
+                  "PUT_NEWFILEURL_uri_no_replace",

                   "409 Conflict",
                   "There was already a child by that name, and you asked me "
                   "to not replace it")
                   "409 Conflict",
                   "There was already a child by that name, and you asked me "
                   "to not replace it")


@@ -2850,8 +4084,8 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d = self.PUT("/uri", file_contents)
         def _check(uri):
             assert isinstance(uri, str), uri
         d = self.PUT("/uri", file_contents)
         def _check(uri):
             assert isinstance(uri, str), uri

-            self.failUnless(uri in FakeCHKFileNode.all_contents)
-            self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
+            self.failUnlessIn(uri, self.get_all_contents())
+            self.failUnlessReallyEqual(self.get_all_contents()[uri],

                                        file_contents)
             return self.GET("/uri/%s" % uri)
         d.addCallback(_check)
                                        file_contents)
             return self.GET("/uri/%s" % uri)
         d.addCallback(_check)


@@ -2865,8 +4099,8 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d = self.PUT("/uri?mutable=false", file_contents)
         def _check(uri):
             assert isinstance(uri, str), uri
         d = self.PUT("/uri?mutable=false", file_contents)
         def _check(uri):
             assert isinstance(uri, str), uri

-            self.failUnless(uri in FakeCHKFileNode.all_contents)
-            self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
+            self.failUnlessIn(uri, self.get_all_contents())
+            self.failUnlessReallyEqual(self.get_all_contents()[uri],

                                        file_contents)
             return self.GET("/uri/%s" % uri)
         d.addCallback(_check)
                                        file_contents)
             return self.GET("/uri/%s" % uri)
         d.addCallback(_check)


@@ -2891,7 +4125,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
             self.filecap = filecap
             u = uri.WriteableSSKFileURI.init_from_string(filecap)
             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
             self.filecap = filecap
             u = uri.WriteableSSKFileURI.init_from_string(filecap)

-            self.failUnless(u.get_storage_index() in FakeMutableFileNode.all_contents)
+            self.failUnlessIn(u.get_storage_index(), self.get_all_contents())

             n = self.s.create_node_from_uri(filecap)
             return n.download_best_version()
         d.addCallback(_check1)
             n = self.s.create_node_from_uri(filecap)
             return n.download_best_version()
         d.addCallback(_check1)


@@ -2916,6 +4150,29 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(self.failUnlessIsEmptyJSON)
         return d
 
         d.addCallback(self.failUnlessIsEmptyJSON)
         return d
 

+    def test_PUT_mkdir_mdmf(self):
+        d = self.PUT("/uri?t=mkdir&format=mdmf", "")
+        def _got(res):
+            u = uri.from_string(res)
+            # Check that this is an MDMF writecap
+            self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
+        d.addCallback(_got)
+        return d
+
+    def test_PUT_mkdir_sdmf(self):
+        d = self.PUT("/uri?t=mkdir&format=sdmf", "")
+        def _got(res):
+            u = uri.from_string(res)
+            self.failUnlessIsInstance(u, uri.DirectoryURI)
+        d.addCallback(_got)
+        return d
+
+    def test_PUT_mkdir_bad_format(self):
+        return self.shouldHTTPError("PUT_mkdir_bad_format",
+                                    400, "Bad Request", "Unknown format: foo",
+                                    self.PUT, "/uri?t=mkdir&format=foo",
+                                    "")
+

     def test_POST_check(self):
         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
         def _done(res):
     def test_POST_check(self):
         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
         def _done(res):


@@ -2928,9 +4185,78 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(_done)
         return d
 
         d.addCallback(_done)
         return d
 

+
+    def test_PUT_update_at_offset(self):
+        file_contents = "test file" * 100000 # about 900 KiB
+        d = self.PUT("/uri?mutable=true", file_contents)
+        def _then(filecap):
+            self.filecap = filecap
+            new_data = file_contents[:100]
+            new = "replaced and so on"
+            new_data += new
+            new_data += file_contents[len(new_data):]
+            assert len(new_data) == len(file_contents)
+            self.new_data = new_data
+        d.addCallback(_then)
+        d.addCallback(lambda ignored:
+            self.PUT("/uri/%s?replace=True&offset=100" % self.filecap,
+                     "replaced and so on"))
+        def _get_data(filecap):
+            n = self.s.create_node_from_uri(filecap)
+            return n.download_best_version()
+        d.addCallback(_get_data)
+        d.addCallback(lambda results:
+            self.failUnlessEqual(results, self.new_data))
+        # Now try appending things to the file
+        d.addCallback(lambda ignored:
+            self.PUT("/uri/%s?offset=%d" % (self.filecap, len(self.new_data)),
+                     "puppies" * 100))
+        d.addCallback(_get_data)
+        d.addCallback(lambda results:
+            self.failUnlessEqual(results, self.new_data + ("puppies" * 100)))
+        # and try replacing the beginning of the file
+        d.addCallback(lambda ignored:
+            self.PUT("/uri/%s?offset=0" % self.filecap, "begin"))
+        d.addCallback(_get_data)
+        d.addCallback(lambda results:
+            self.failUnlessEqual(results, "begin"+self.new_data[len("begin"):]+("puppies"*100)))
+        return d
+
+    def test_PUT_update_at_invalid_offset(self):
+        file_contents = "test file" * 100000 # about 900 KiB
+        d = self.PUT("/uri?mutable=true", file_contents)
+        def _then(filecap):
+            self.filecap = filecap
+        d.addCallback(_then)
+        # Negative offsets should cause an error.
+        d.addCallback(lambda ignored:
+            self.shouldHTTPError("PUT_update_at_invalid_offset",
+                                 400, "Bad Request",
+                                 "Invalid offset",
+                                 self.PUT,
+                                 "/uri/%s?offset=-1" % self.filecap,
+                                 "foo"))
+        return d
+
+    def test_PUT_update_at_offset_immutable(self):
+        file_contents = "Test file" * 100000
+        d = self.PUT("/uri", file_contents)
+        def _then(filecap):
+            self.filecap = filecap
+        d.addCallback(_then)
+        d.addCallback(lambda ignored:
+            self.shouldHTTPError("PUT_update_at_offset_immutable",
+                                 400, "Bad Request",
+                                 "immutable",
+                                 self.PUT,
+                                 "/uri/%s?offset=50" % self.filecap,
+                                 "foo"))
+        return d
+
+

     def test_bad_method(self):
         url = self.webish_url + self.public_url + "/foo/bar.txt"
     def test_bad_method(self):
         url = self.webish_url + self.public_url + "/foo/bar.txt"

-        d = self.shouldHTTPError("test_bad_method",
+        d = self.shouldHTTPError("bad_method",

                                  501, "Not Implemented",
                                  "I don't know how to treat a BOGUS request.",
                                  client.getPage, url, method="BOGUS")
                                  501, "Not Implemented",
                                  "I don't know how to treat a BOGUS request.",
                                  client.getPage, url, method="BOGUS")


@@ -2938,14 +4264,14 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
 
     def test_short_url(self):
         url = self.webish_url + "/uri"
 
     def test_short_url(self):
         url = self.webish_url + "/uri"

-        d = self.shouldHTTPError("test_short_url", 501, "Not Implemented",
+        d = self.shouldHTTPError("short_url", 501, "Not Implemented",

                                  "I don't know how to treat a DELETE request.",
                                  client.getPage, url, method="DELETE")
         return d
 
     def test_ophandle_bad(self):
         url = self.webish_url + "/operations/bogus?t=status"
                                  "I don't know how to treat a DELETE request.",
                                  client.getPage, url, method="DELETE")
         return d
 
     def test_ophandle_bad(self):
         url = self.webish_url + "/operations/bogus?t=status"

-        d = self.shouldHTTPError("test_ophandle_bad", 404, "404 Not Found",
+        d = self.shouldHTTPError("ophandle_bad", 404, "404 Not Found",

                                  "unknown/expired handle 'bogus'",
                                  client.getPage, url)
         return d
                                  "unknown/expired handle 'bogus'",
                                  client.getPage, url)
         return d


@@ -2969,7 +4295,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
             return d
         d.addCallback(_check1)
         d.addCallback(lambda ignored:
             return d
         d.addCallback(_check1)
         d.addCallback(lambda ignored:

-                      self.shouldHTTPError("test_ophandle_cancel",
+                      self.shouldHTTPError("ophandle_cancel",

                                            404, "404 Not Found",
                                            "unknown/expired handle '128'",
                                            self.GET,
                                            404, "404 Not Found",
                                            "unknown/expired handle '128'",
                                            self.GET,


@@ -2989,7 +4315,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(lambda ign:
             self.clock.advance(2.0))
         d.addCallback(lambda ignored:
         d.addCallback(lambda ign:
             self.clock.advance(2.0))
         d.addCallback(lambda ignored:

-                      self.shouldHTTPError("test_ophandle_retainfor",
+                      self.shouldHTTPError("ophandle_retainfor",

                                            404, "404 Not Found",
                                            "unknown/expired handle '129'",
                                            self.GET,
                                            404, "404 Not Found",
                                            "unknown/expired handle '129'",
                                            self.GET,


@@ -3004,7 +4330,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
         # the release-after-complete=true will cause the handle to be expired
         d.addCallback(lambda ignored:
                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
         # the release-after-complete=true will cause the handle to be expired
         d.addCallback(lambda ignored:

-                      self.shouldHTTPError("test_ophandle_release_after_complete",
+                      self.shouldHTTPError("ophandle_release_after_complete",

                                            404, "404 Not Found",
                                            "unknown/expired handle '130'",
                                            self.GET,
                                            404, "404 Not Found",
                                            "unknown/expired handle '130'",
                                            self.GET,


@@ -3046,7 +4372,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(lambda ign:
             self.clock.advance(96*60*60))
         d.addCallback(lambda ign:
         d.addCallback(lambda ign:
             self.clock.advance(96*60*60))
         d.addCallback(lambda ign:

-            self.shouldHTTPError("test_uncollected_ophandle_expired_after_100_hours",
+            self.shouldHTTPError("uncollected_ophandle_expired_after_100_hours",

                                  404, "404 Not Found",
                                  "unknown/expired handle '132'",
                                  self.GET,
                                  404, "404 Not Found",
                                  "unknown/expired handle '132'",
                                  self.GET,


@@ -3059,10 +4385,10 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
             d = self.POST(self.public_url +
                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
                           followRedirect=True)
             d = self.POST(self.public_url +
                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
                           followRedirect=True)

-            # By following the initial redirect, we collect the ophandle 
+            # By following the initial redirect, we collect the ophandle

             # we've just created.
             return d
             # we've just created.
             return d

-        # Create a collected ophandle, then collect it after 23 hours 
+        # Create a collected ophandle, then collect it after 23 hours

         # and 59 seconds to make sure that it is still there.
         d = _make_collected_ophandle(133)
         d.addCallback(lambda ign:
         # and 59 seconds to make sure that it is still there.
         d = _make_collected_ophandle(133)
         d.addCallback(lambda ign:


@@ -3080,7 +4406,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(lambda ign:
             self.clock.advance(24*60*60))
         d.addCallback(lambda ign:
         d.addCallback(lambda ign:
             self.clock.advance(24*60*60))
         d.addCallback(lambda ign:

-            self.shouldHTTPError("test_collected_ophandle_expired_after_1000_minutes",
+            self.shouldHTTPError("collected_ophandle_expired_after_1_day",

                                  404, "404 Not Found",
                                  "unknown/expired handle '134'",
                                  self.GET,
                                  404, "404 Not Found",
                                  "unknown/expired handle '134'",
                                  self.GET,


@@ -3090,7 +4416,8 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
     def test_incident(self):
         d = self.POST("/report_incident", details="eek")
         def _done(res):
     def test_incident(self):
         d = self.POST("/report_incident", details="eek")
         def _done(res):

-            self.failUnless("Thank you for your report!" in res, res)
+            self.failIfIn("<html>", res)
+            self.failUnlessIn("An incident report has been saved", res)

         d.addCallback(_done)
         return d
 
         d.addCallback(_done)
         return d
 


@@ -3108,6 +4435,58 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         return d
 
 
         return d
 
 

+class IntroducerWeb(unittest.TestCase):
+    def setUp(self):
+        self.node = None
+
+    def tearDown(self):
+        d = defer.succeed(None)
+        if self.node:
+            d.addCallback(lambda ign: self.node.stopService())
+        d.addCallback(flushEventualQueue)
+        return d
+
+    def test_welcome(self):
+        basedir = "web.IntroducerWeb.test_welcome"
+        os.mkdir(basedir)
+        cfg = "\n".join(["[node]",
+                         "tub.location = 127.0.0.1:1",
+                         "web.port = tcp:0",
+                         ]) + "\n"
+        fileutil.write(os.path.join(basedir, "tahoe.cfg"), cfg)
+        self.node = IntroducerNode(basedir)
+        self.ws = self.node.getServiceNamed("webish")
+
+        d = fireEventually(None)
+        d.addCallback(lambda ign: self.node.startService())
+        d.addCallback(lambda ign: self.node.when_tub_ready())
+
+        d.addCallback(lambda ign: self.GET("/"))
+        def _check(res):
+            self.failUnlessIn('Welcome to the Tahoe-LAFS Introducer', res)
+            self.failUnlessIn(FAVICON_MARKUP, res)
+            self.failUnlessIn('Page rendered at', res)
+            self.failUnlessIn('Tahoe-LAFS code imported from:', res)
+        d.addCallback(_check)
+        return d
+
+    def GET(self, urlpath, followRedirect=False, return_response=False,
+            **kwargs):
+        # if return_response=True, this fires with (data, statuscode,
+        # respheaders) instead of just data.
+        assert not isinstance(urlpath, unicode)
+        url = self.ws.getURL().rstrip('/') + urlpath
+        factory = HTTPClientGETFactory(url, method="GET",
+                                       followRedirect=followRedirect, **kwargs)
+        reactor.connectTCP("localhost", self.ws.getPortnum(), factory)
+        d = factory.deferred
+        def _got_data(data):
+            return (data, factory.status, factory.response_headers)
+        if return_response:
+            d.addCallback(_got_data)
+        return factory.deferred
+
+

 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
     def test_load_file(self):
         # This will raise an exception unless a well-formed XML file is found under that name.
 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
     def test_load_file(self):
         # This will raise an exception unless a well-formed XML file is found under that name.


@@ -3118,8 +4497,7 @@ class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
         self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
         self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
                                    "only-files")
         self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
         self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
                                    "only-files")

-        self.shouldFail(AssertionError, "test_parse_replace_arg", "",
-                        common.parse_replace_arg, "only_fles")
+        self.failUnlessRaises(common.WebError, common.parse_replace_arg, "only_fles")

 
     def test_abbreviate_time(self):
         self.failUnlessReallyEqual(common.abbreviate_time(None), "")
 
     def test_abbreviate_time(self):
         self.failUnlessReallyEqual(common.abbreviate_time(None), "")


@@ -3127,6 +4505,23 @@ class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
         self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
         self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
         self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
         self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
         self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
         self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")

+        self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
+
+    def test_compute_rate(self):
+        self.failUnlessReallyEqual(common.compute_rate(None, None), None)
+        self.failUnlessReallyEqual(common.compute_rate(None, 1), None)
+        self.failUnlessReallyEqual(common.compute_rate(250000, None), None)
+        self.failUnlessReallyEqual(common.compute_rate(250000, 0), None)
+        self.failUnlessReallyEqual(common.compute_rate(250000, 10), 25000.0)
+        self.failUnlessReallyEqual(common.compute_rate(0, 10), 0.0)
+        self.shouldFail(AssertionError, "test_compute_rate", "",
+                        common.compute_rate, -100, 10)
+        self.shouldFail(AssertionError, "test_compute_rate", "",
+                        common.compute_rate, 100, -10)
+
+        # Sanity check
+        rate = common.compute_rate(10*1000*1000, 1)
+        self.failUnlessReallyEqual(common.abbreviate_rate(rate), "10.00MBps")

 
     def test_abbreviate_rate(self):
         self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
 
     def test_abbreviate_rate(self):
         self.failUnlessReallyEqual(common.abbreviate_rate(None), "")


@@ -3169,7 +4564,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
         DATA = "data" * 100
         d = c0.upload(upload.Data(DATA, convergence=""))
         def _stash_uri(ur, which):
         DATA = "data" * 100
         d = c0.upload(upload.Data(DATA, convergence=""))
         def _stash_uri(ur, which):

-            self.uris[which] = ur.uri
+            self.uris[which] = ur.get_uri()

         d.addCallback(_stash_uri, "good")
         d.addCallback(lambda ign:
                       c0.upload(upload.Data(DATA+"1", convergence="")))
         d.addCallback(_stash_uri, "good")
         d.addCallback(lambda ign:
                       c0.upload(upload.Data(DATA+"1", convergence="")))


@@ -3180,7 +4575,8 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
         def _stash_mutable_uri(n, which):
             self.uris[which] = n.get_uri()
             assert isinstance(self.uris[which], str)
         def _stash_mutable_uri(n, which):
             self.uris[which] = n.get_uri()
             assert isinstance(self.uris[which], str)

-        d.addCallback(lambda ign: c0.create_mutable_file(DATA+"3"))
+        d.addCallback(lambda ign:
+            c0.create_mutable_file(publish.MutableData(DATA+"3")))

         d.addCallback(_stash_mutable_uri, "corrupt")
         d.addCallback(lambda ign:
                       c0.upload(upload.Data("literal", convergence="")))
         d.addCallback(_stash_mutable_uri, "corrupt")
         d.addCallback(lambda ign:
                       c0.upload(upload.Data("literal", convergence="")))


@@ -3211,36 +4607,35 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
 
         d.addCallback(self.CHECK, "good", "t=check")
         def _got_html_good(res):
 
         d.addCallback(self.CHECK, "good", "t=check")
         def _got_html_good(res):

-            self.failUnless("Healthy" in res, res)
-            self.failIf("Not Healthy" in res, res)
+            self.failUnlessIn("Healthy", res)
+            self.failIfIn("Not Healthy", res)
+            self.failUnlessIn(FAVICON_MARKUP, res)

         d.addCallback(_got_html_good)
         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
         def _got_html_good_return_to(res):
         d.addCallback(_got_html_good)
         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
         def _got_html_good_return_to(res):

-            self.failUnless("Healthy" in res, res)
-            self.failIf("Not Healthy" in res, res)
-            self.failUnless('<a href="somewhere">Return to file'
-                            in res, res)
+            self.failUnlessIn("Healthy", res)
+            self.failIfIn("Not Healthy", res)
+            self.failUnlessIn('<a href="somewhere">Return to file', res)

         d.addCallback(_got_html_good_return_to)
         d.addCallback(self.CHECK, "good", "t=check&output=json")
         def _got_json_good(res):
             r = simplejson.loads(res)
             self.failUnlessEqual(r["summary"], "Healthy")
             self.failUnless(r["results"]["healthy"])
         d.addCallback(_got_html_good_return_to)
         d.addCallback(self.CHECK, "good", "t=check&output=json")
         def _got_json_good(res):
             r = simplejson.loads(res)
             self.failUnlessEqual(r["summary"], "Healthy")
             self.failUnless(r["results"]["healthy"])

-            self.failIf(r["results"]["needs-rebalancing"])
+            self.failIfIn("needs-rebalancing", r["results"])

             self.failUnless(r["results"]["recoverable"])
         d.addCallback(_got_json_good)
 
         d.addCallback(self.CHECK, "small", "t=check")
         def _got_html_small(res):
             self.failUnless(r["results"]["recoverable"])
         d.addCallback(_got_json_good)
 
         d.addCallback(self.CHECK, "small", "t=check")
         def _got_html_small(res):

-            self.failUnless("Literal files are always healthy" in res, res)
-            self.failIf("Not Healthy" in res, res)
+            self.failUnlessIn("Literal files are always healthy", res)
+            self.failIfIn("Not Healthy", res)

         d.addCallback(_got_html_small)
         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
         def _got_html_small_return_to(res):
         d.addCallback(_got_html_small)
         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
         def _got_html_small_return_to(res):

-            self.failUnless("Literal files are always healthy" in res, res)
-            self.failIf("Not Healthy" in res, res)
-            self.failUnless('<a href="somewhere">Return to file'
-                            in res, res)
+            self.failUnlessIn("Literal files are always healthy", res)
+            self.failIfIn("Not Healthy", res)
+            self.failUnlessIn('<a href="somewhere">Return to file', res)

         d.addCallback(_got_html_small_return_to)
         d.addCallback(self.CHECK, "small", "t=check&output=json")
         def _got_json_small(res):
         d.addCallback(_got_html_small_return_to)
         d.addCallback(self.CHECK, "small", "t=check&output=json")
         def _got_json_small(res):


@@ -3251,8 +4646,8 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
 
         d.addCallback(self.CHECK, "smalldir", "t=check")
         def _got_html_smalldir(res):
 
         d.addCallback(self.CHECK, "smalldir", "t=check")
         def _got_html_smalldir(res):

-            self.failUnless("Literal files are always healthy" in res, res)
-            self.failIf("Not Healthy" in res, res)
+            self.failUnlessIn("Literal files are always healthy", res)
+            self.failIfIn("Not Healthy", res)

         d.addCallback(_got_html_smalldir)
         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
         def _got_json_smalldir(res):
         d.addCallback(_got_html_smalldir)
         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
         def _got_json_smalldir(res):


@@ -3263,7 +4658,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
 
         d.addCallback(self.CHECK, "sick", "t=check")
         def _got_html_sick(res):
 
         d.addCallback(self.CHECK, "sick", "t=check")
         def _got_html_sick(res):

-            self.failUnless("Not Healthy" in res, res)
+            self.failUnlessIn("Not Healthy", res)

         d.addCallback(_got_html_sick)
         d.addCallback(self.CHECK, "sick", "t=check&output=json")
         def _got_json_sick(res):
         d.addCallback(_got_html_sick)
         d.addCallback(self.CHECK, "sick", "t=check&output=json")
         def _got_json_sick(res):


@@ -3271,13 +4666,13 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
             self.failUnlessEqual(r["summary"],
                                  "Not Healthy: 9 shares (enc 3-of-10)")
             self.failIf(r["results"]["healthy"])
             self.failUnlessEqual(r["summary"],
                                  "Not Healthy: 9 shares (enc 3-of-10)")
             self.failIf(r["results"]["healthy"])

-            self.failIf(r["results"]["needs-rebalancing"])

             self.failUnless(r["results"]["recoverable"])
             self.failUnless(r["results"]["recoverable"])

+            self.failIfIn("needs-rebalancing", r["results"])

         d.addCallback(_got_json_sick)
 
         d.addCallback(self.CHECK, "dead", "t=check")
         def _got_html_dead(res):
         d.addCallback(_got_json_sick)
 
         d.addCallback(self.CHECK, "dead", "t=check")
         def _got_html_dead(res):

-            self.failUnless("Not Healthy" in res, res)
+            self.failUnlessIn("Not Healthy", res)

         d.addCallback(_got_html_dead)
         d.addCallback(self.CHECK, "dead", "t=check&output=json")
         def _got_json_dead(res):
         d.addCallback(_got_html_dead)
         d.addCallback(self.CHECK, "dead", "t=check&output=json")
         def _got_json_dead(res):


@@ -3285,21 +4680,22 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
             self.failUnlessEqual(r["summary"],
                                  "Not Healthy: 1 shares (enc 3-of-10)")
             self.failIf(r["results"]["healthy"])
             self.failUnlessEqual(r["summary"],
                                  "Not Healthy: 1 shares (enc 3-of-10)")
             self.failIf(r["results"]["healthy"])

-            self.failIf(r["results"]["needs-rebalancing"])

             self.failIf(r["results"]["recoverable"])
             self.failIf(r["results"]["recoverable"])

+            self.failIfIn("needs-rebalancing", r["results"])

         d.addCallback(_got_json_dead)
 
         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
         def _got_html_corrupt(res):
         d.addCallback(_got_json_dead)
 
         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
         def _got_html_corrupt(res):

-            self.failUnless("Not Healthy! : Unhealthy" in res, res)
+            self.failUnlessIn("Not Healthy! : Unhealthy", res)

         d.addCallback(_got_html_corrupt)
         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
         def _got_json_corrupt(res):
             r = simplejson.loads(res)
         d.addCallback(_got_html_corrupt)
         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
         def _got_json_corrupt(res):
             r = simplejson.loads(res)

-            self.failUnless("Unhealthy: 9 shares (enc 3-of-10)" in r["summary"],
-                            r["summary"])
+            self.failUnlessIn("Unhealthy: 9 shares (enc 3-of-10)", r["summary"])

             self.failIf(r["results"]["healthy"])
             self.failUnless(r["results"]["recoverable"])
             self.failIf(r["results"]["healthy"])
             self.failUnless(r["results"]["recoverable"])

+            self.failIfIn("needs-rebalancing", r["results"])
+            self.failUnlessReallyEqual(r["results"]["count-happiness"], 9)

             self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
             self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
         d.addCallback(_got_json_corrupt)
             self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
             self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
         d.addCallback(_got_json_corrupt)


@@ -3315,7 +4711,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
         DATA = "data" * 100
         d = c0.upload(upload.Data(DATA, convergence=""))
         def _stash_uri(ur, which):
         DATA = "data" * 100
         d = c0.upload(upload.Data(DATA, convergence=""))
         def _stash_uri(ur, which):

-            self.uris[which] = ur.uri
+            self.uris[which] = ur.get_uri()

         d.addCallback(_stash_uri, "good")
         d.addCallback(lambda ign:
                       c0.upload(upload.Data(DATA+"1", convergence="")))
         d.addCallback(_stash_uri, "good")
         d.addCallback(lambda ign:
                       c0.upload(upload.Data(DATA+"1", convergence="")))


@@ -3326,7 +4722,8 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
         def _stash_mutable_uri(n, which):
             self.uris[which] = n.get_uri()
             assert isinstance(self.uris[which], str)
         def _stash_mutable_uri(n, which):
             self.uris[which] = n.get_uri()
             assert isinstance(self.uris[which], str)

-        d.addCallback(lambda ign: c0.create_mutable_file(DATA+"3"))
+        d.addCallback(lambda ign:
+            c0.create_mutable_file(publish.MutableData(DATA+"3")))

         d.addCallback(_stash_mutable_uri, "corrupt")
 
         def _compute_fileurls(ignored):
         d.addCallback(_stash_mutable_uri, "corrupt")
 
         def _compute_fileurls(ignored):


@@ -3352,16 +4749,17 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
 
         d.addCallback(self.CHECK, "good", "t=check&repair=true")
         def _got_html_good(res):
 
         d.addCallback(self.CHECK, "good", "t=check&repair=true")
         def _got_html_good(res):

-            self.failUnless("Healthy" in res, res)
-            self.failIf("Not Healthy" in res, res)
-            self.failUnless("No repair necessary" in res, res)
+            self.failUnlessIn("Healthy", res)
+            self.failIfIn("Not Healthy", res)
+            self.failUnlessIn("No repair necessary", res)
+            self.failUnlessIn(FAVICON_MARKUP, res)

         d.addCallback(_got_html_good)
 
         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
         def _got_html_sick(res):
         d.addCallback(_got_html_good)
 
         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
         def _got_html_sick(res):

-            self.failUnless("Healthy : healthy" in res, res)
-            self.failIf("Not Healthy" in res, res)
-            self.failUnless("Repair successful" in res, res)
+            self.failUnlessIn("Healthy : healthy", res)
+            self.failIfIn("Not Healthy", res)
+            self.failUnlessIn("Repair successful", res)

         d.addCallback(_got_html_sick)
 
         # repair of a dead file will fail, of course, but it isn't yet
         d.addCallback(_got_html_sick)
 
         # repair of a dead file will fail, of course, but it isn't yet


@@ -3371,16 +4769,16 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
         #def _got_html_dead(res):
         #    print res
         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
         #def _got_html_dead(res):
         #    print res

-        #    self.failUnless("Healthy : healthy" in res, res)
-        #    self.failIf("Not Healthy" in res, res)
-        #    self.failUnless("No repair necessary" in res, res)
+        #    self.failUnlessIn("Healthy : healthy", res)
+        #    self.failIfIn("Not Healthy", res)
+        #    self.failUnlessIn("No repair necessary", res)

         #d.addCallback(_got_html_dead)
 
         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
         def _got_html_corrupt(res):
         #d.addCallback(_got_html_dead)
 
         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
         def _got_html_corrupt(res):

-            self.failUnless("Healthy : Healthy" in res, res)
-            self.failIf("Not Healthy" in res, res)
-            self.failUnless("Repair successful" in res, res)
+            self.failUnlessIn("Healthy : Healthy", res)
+            self.failIfIn("Not Healthy", res)
+            self.failUnlessIn("Repair successful", res)

         d.addCallback(_got_html_corrupt)
 
         d.addErrback(self.explain_web_error)
         d.addCallback(_got_html_corrupt)
 
         d.addErrback(self.explain_web_error)


@@ -3394,7 +4792,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
         DATA = "data" * 100
         d = c0.upload(upload.Data(DATA+"1", convergence=""))
         def _stash_uri(ur, which):
         DATA = "data" * 100
         d = c0.upload(upload.Data(DATA+"1", convergence=""))
         def _stash_uri(ur, which):

-            self.uris[which] = ur.uri
+            self.uris[which] = ur.get_uri()

         d.addCallback(_stash_uri, "sick")
 
         def _compute_fileurls(ignored):
         d.addCallback(_stash_uri, "sick")
 
         def _compute_fileurls(ignored):


@@ -3484,7 +4882,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
             else:
                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
             else:
                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)

-            self.failUnless("metadata" in f[1])
+            self.failUnlessIn("metadata", f[1])

         d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
 
         def _check_info(res, expect_rw_uri, expect_ro_uri):
         d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
 
         def _check_info(res, expect_rw_uri, expect_ro_uri):


@@ -3528,10 +4926,10 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
                 self.failUnlessReallyEqual(data[1]["mutable"], True)
             else:
                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
                 self.failUnlessReallyEqual(data[1]["mutable"], True)
             else:
                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)

-                self.failIf("mutable" in data[1], data[1])
+                self.failIfIn("mutable", data[1])

 
             # TODO: check metadata contents
 
             # TODO: check metadata contents

-            self.failUnless("metadata" in data[1])
+            self.failUnlessIn("metadata", data[1])

 
         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
         d.addCallback(_check_json, expect_rw_uri=not immutable)
 
         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
         d.addCallback(_check_json, expect_rw_uri=not immutable)


@@ -3550,7 +4948,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
 
         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
         d.addCallback(_check_json, expect_rw_uri=False)
 
         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
         d.addCallback(_check_json, expect_rw_uri=False)

-        
+

         # TODO: check that getting t=info from the Info link in the ro directory
         # works, and does not include the writecap URI.
         return d
         # TODO: check that getting t=info from the Info link in the ro directory
         # works, and does not include the writecap URI.
         return d


@@ -3570,12 +4968,12 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
         lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
         mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
         lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
         mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"

-        
+

         # This method tests mainly dirnode, but we'd have to duplicate code in order to
         # test the dirnode and web layers separately.
         # This method tests mainly dirnode, but we'd have to duplicate code in order to
         # test the dirnode and web layers separately.

-        
+

         # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
         # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,

-        # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field. 
+        # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.

         # When the directory is read, the mutants should be silently disposed of, leaving
         # their lonely sibling.
         # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
         # When the directory is read, the mutants should be silently disposed of, leaving
         # their lonely sibling.
         # We don't test the case of a retrieving a cap from the encrypted rw_uri field,


@@ -3602,7 +5000,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
                 u"write-in-ro": (mutant_write_in_ro_child, {}),
                 }
         d = c.create_immutable_dirnode(kids)
                 u"write-in-ro": (mutant_write_in_ro_child, {}),
                 }
         d = c.create_immutable_dirnode(kids)

-        
+

         def _created(dn):
             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
             self.failIf(dn.is_mutable())
         def _created(dn):
             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
             self.failIf(dn.is_mutable())


@@ -3610,7 +5008,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
             # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
             self.failIf(hasattr(dn._node, 'get_writekey'))
             rep = str(dn)
             # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
             self.failIf(hasattr(dn._node, 'get_writekey'))
             rep = str(dn)

-            self.failUnless("RO-IMM" in rep)
+            self.failUnlessIn("RO-IMM", rep)

             cap = dn.get_cap()
             self.failUnlessIn("CHK", cap.to_string())
             self.cap = cap
             cap = dn.get_cap()
             self.failUnlessIn("CHK", cap.to_string())
             self.cap = cap


@@ -3631,8 +5029,8 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
                 entry = entries[0]
                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
                 name = name_utf8.decode("utf-8")
                 entry = entries[0]
                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
                 name = name_utf8.decode("utf-8")

-                self.failUnless(rwcapdata == "")
-                self.failUnless(name in kids)
+                self.failUnlessEqual(rwcapdata, "")
+                self.failUnlessIn(name, kids)

                 (expected_child, ign) = kids[name]
                 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
                 numkids += 1
                 (expected_child, ign) = kids[name]
                 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
                 numkids += 1


@@ -3640,7 +5038,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
             self.failUnlessReallyEqual(numkids, 3)
             return self.rootnode.list()
         d.addCallback(_check_data)
             self.failUnlessReallyEqual(numkids, 3)
             return self.rootnode.list()
         d.addCallback(_check_data)

-        
+

         # Now when we use the real directory listing code, the mutants should be absent.
         def _check_kids(children):
             self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
         # Now when we use the real directory listing code, the mutants should be absent.
         def _check_kids(children):
             self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])


@@ -3662,7 +5060,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
                                   r'\s+<td>',
                                   r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
                                   r'</td>',
                                   r'\s+<td>',
                                   r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
                                   r'</td>',

-                                  r'\s+<td>%d</td>' % len("one"),
+                                  r'\s+<td align="right">%d</td>' % len("one"),

                                   ])
             self.failUnless(re.search(get_lonely, res), res)
 
                                   ])
             self.failUnless(re.search(get_lonely, res), res)
 


@@ -3681,7 +5079,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
             self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
             ll_type, ll_data = listed_children[u"lonely"]
             self.failUnlessEqual(ll_type, "filenode")
             self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
             ll_type, ll_data = listed_children[u"lonely"]
             self.failUnlessEqual(ll_type, "filenode")

-            self.failIf("rw_uri" in ll_data)
+            self.failIfIn("rw_uri", ll_data)

             self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
         d.addCallback(_check_json)
         return d
             self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
         d.addCallback(_check_json)
         return d


@@ -3746,12 +5144,14 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
             self.failUnlessEqual(u0["type"], "directory")
             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
             u0cr = u0["check-results"]
             self.failUnlessEqual(u0["type"], "directory")
             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
             u0cr = u0["check-results"]

+            self.failUnlessReallyEqual(u0cr["results"]["count-happiness"], 10)

             self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
 
             ugood = [u for u in units
                      if u["type"] == "file" and u["path"] == [u"good"]][0]
             self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
             ugoodcr = ugood["check-results"]
             self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
 
             ugood = [u for u in units
                      if u["type"] == "file" and u["path"] == [u"good"]][0]
             self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
             ugoodcr = ugood["check-results"]

+            self.failUnlessReallyEqual(ugoodcr["results"]["count-happiness"], 10)

             self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
 
             stats = units[-1]
             self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
 
             stats = units[-1]


@@ -3850,6 +5250,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
             self.failUnlessEqual(last_unit["path"], ["subdir"])
             r = last_unit["check-results"]["results"]
             self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
             self.failUnlessEqual(last_unit["path"], ["subdir"])
             r = last_unit["check-results"]["results"]
             self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)

+            self.failUnlessReallyEqual(r["count-happiness"], 1)

             self.failUnlessReallyEqual(r["count-shares-good"], 1)
             self.failUnlessReallyEqual(r["recoverable"], False)
         d.addCallback(_check_broken_deepcheck)
             self.failUnlessReallyEqual(r["count-shares-good"], 1)
             self.failUnlessReallyEqual(r["recoverable"], False)
         d.addCallback(_check_broken_deepcheck)


@@ -3927,6 +5328,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
             u0crr = u0["check-and-repair-results"]
             self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
             u0crr = u0["check-and-repair-results"]
             self.failUnlessReallyEqual(u0crr["repair-attempted"], False)

+            self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-happiness"], 10)

             self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
 
             ugood = [u for u in units
             self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
 
             ugood = [u for u in units


@@ -3934,6 +5336,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
             self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
             ugoodcrr = ugood["check-and-repair-results"]
             self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
             self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
             ugoodcrr = ugood["check-and-repair-results"]
             self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)

+            self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-happiness"], 10)

             self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
 
             usick = [u for u in units
             self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
 
             usick = [u for u in units


@@ -3942,7 +5345,9 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
             usickcrr = usick["check-and-repair-results"]
             self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
             self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
             usickcrr = usick["check-and-repair-results"]
             self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
             self.failUnlessReallyEqual(usickcrr["repair-successful"], True)

+            self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-happiness"], 9)

             self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
             self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)

+            self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-happiness"], 10)

             self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
 
             stats = units[-1]
             self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
 
             stats = units[-1]


@@ -3963,7 +5368,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
         for shnum, serverid, fn in shares:
             sf = get_share_file(fn)
             num_leases = len(list(sf.get_leases()))
         for shnum, serverid, fn in shares:
             sf = get_share_file(fn)
             num_leases = len(list(sf.get_leases()))

-        lease_counts.append( (fn, num_leases) )
+            lease_counts.append( (fn, num_leases) )

         return lease_counts
 
     def _assert_leasecount(self, lease_counts, expected):
         return lease_counts
 
     def _assert_leasecount(self, lease_counts, expected):


@@ -3980,7 +5385,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
         DATA = "data" * 100
         d = c0.upload(upload.Data(DATA, convergence=""))
         def _stash_uri(ur, which):
         DATA = "data" * 100
         d = c0.upload(upload.Data(DATA, convergence=""))
         def _stash_uri(ur, which):

-            self.uris[which] = ur.uri
+            self.uris[which] = ur.get_uri()

         d.addCallback(_stash_uri, "one")
         d.addCallback(lambda ign:
                       c0.upload(upload.Data(DATA+"1", convergence="")))
         d.addCallback(_stash_uri, "one")
         d.addCallback(lambda ign:
                       c0.upload(upload.Data(DATA+"1", convergence="")))


@@ -3988,7 +5393,8 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
         def _stash_mutable_uri(n, which):
             self.uris[which] = n.get_uri()
             assert isinstance(self.uris[which], str)
         def _stash_mutable_uri(n, which):
             self.uris[which] = n.get_uri()
             assert isinstance(self.uris[which], str)

-        d.addCallback(lambda ign: c0.create_mutable_file(DATA+"2"))
+        d.addCallback(lambda ign:
+            c0.create_mutable_file(publish.MutableData(DATA+"2")))

         d.addCallback(_stash_mutable_uri, "mutable")
 
         def _compute_fileurls(ignored):
         d.addCallback(_stash_mutable_uri, "mutable")
 
         def _compute_fileurls(ignored):


@@ -4006,8 +5412,8 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
 
         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
         def _got_html_good(res):
 
         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
         def _got_html_good(res):

-            self.failUnless("Healthy" in res, res)
-            self.failIf("Not Healthy" in res, res)
+            self.failUnlessIn("Healthy", res)
+            self.failIfIn("Not Healthy", res)

         d.addCallback(_got_html_good)
 
         d.addCallback(self._count_leases, "one")
         d.addCallback(_got_html_good)
 
         d.addCallback(self._count_leases, "one")


@@ -4087,7 +5493,8 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
                                                         convergence="")))
         d.addCallback(_stash_uri, "small")
 
                                                         convergence="")))
         d.addCallback(_stash_uri, "small")
 

-        d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
+        d.addCallback(lambda ign:
+            c0.create_mutable_file(publish.MutableData("mutable")))

         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
         d.addCallback(_stash_uri, "mutable")
 
         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
         d.addCallback(_stash_uri, "mutable")
 


@@ -4136,7 +5543,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
         self.basedir = "web/Grid/exceptions"
         self.set_up_grid(num_clients=1, num_servers=2)
         c0 = self.g.clients[0]
         self.basedir = "web/Grid/exceptions"
         self.set_up_grid(num_clients=1, num_servers=2)
         c0 = self.g.clients[0]

-        c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 2
+        c0.encoding_params['happy'] = 2

         self.fileurls = {}
         DATA = "data" * 100
         d = c0.create_dirnode()
         self.fileurls = {}
         DATA = "data" * 100
         d = c0.create_dirnode()


@@ -4147,10 +5554,10 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
         d.addCallback(_stash_root)
         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
         def _stash_bad(ur):
         d.addCallback(_stash_root)
         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
         def _stash_bad(ur):

-            self.fileurls["1share"] = "uri/" + urllib.quote(ur.uri)
-            self.delete_shares_numbered(ur.uri, range(1,10))
+            self.fileurls["1share"] = "uri/" + urllib.quote(ur.get_uri())
+            self.delete_shares_numbered(ur.get_uri(), range(1,10))

 
 

-            u = uri.from_string(ur.uri)
+            u = uri.from_string(ur.get_uri())

             u.key = testutil.flip_bit(u.key, 0)
             baduri = u.to_string()
             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
             u.key = testutil.flip_bit(u.key, 0)
             baduri = u.to_string()
             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)


@@ -4179,14 +5586,14 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
                                            410, "Gone", "NoSharesError",
                                            self.GET, self.fileurls["0shares"]))
         def _check_zero_shares(body):
                                            410, "Gone", "NoSharesError",
                                            self.GET, self.fileurls["0shares"]))
         def _check_zero_shares(body):

-            self.failIf("<html>" in body, body)
+            self.failIfIn("<html>", body)

             body = " ".join(body.strip().split())
             exp = ("NoSharesError: no shares could be found. "
                    "Zero shares usually indicates a corrupt URI, or that "
                    "no servers were connected, but it might also indicate "
                    "severe corruption. You should perform a filecheck on "
                    "this object to learn more. The full error message is: "
             body = " ".join(body.strip().split())
             exp = ("NoSharesError: no shares could be found. "
                    "Zero shares usually indicates a corrupt URI, or that "
                    "no servers were connected, but it might also indicate "
                    "severe corruption. You should perform a filecheck on "
                    "this object to learn more. The full error message is: "

-                   "Failed to get enough shareholders: have 0, need 3")
+                   "no shares (need 3). Last failure: None")

             self.failUnlessReallyEqual(exp, body)
         d.addCallback(_check_zero_shares)
 
             self.failUnlessReallyEqual(exp, body)
         d.addCallback(_check_zero_shares)
 


@@ -4196,15 +5603,23 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
                                            410, "Gone", "NotEnoughSharesError",
                                            self.GET, self.fileurls["1share"]))
         def _check_one_share(body):
                                            410, "Gone", "NotEnoughSharesError",
                                            self.GET, self.fileurls["1share"]))
         def _check_one_share(body):

-            self.failIf("<html>" in body, body)
+            self.failIfIn("<html>", body)

             body = " ".join(body.strip().split())
             body = " ".join(body.strip().split())

-            exp = ("NotEnoughSharesError: This indicates that some "
-                   "servers were unavailable, or that shares have been "
-                   "lost to server departure, hard drive failure, or disk "
-                   "corruption. You should perform a filecheck on "
-                   "this object to learn more. The full error message is:"
-                   " Failed to get enough shareholders: have 1, need 3")
-            self.failUnlessReallyEqual(exp, body)
+            msgbase = ("NotEnoughSharesError: This indicates that some "
+                       "servers were unavailable, or that shares have been "
+                       "lost to server departure, hard drive failure, or disk "
+                       "corruption. You should perform a filecheck on "
+                       "this object to learn more. The full error message is:"
+                       )
+            msg1 = msgbase + (" ran out of shares:"
+                              " complete=sh0"
+                              " pending="
+                              " overdue= unused= need 3. Last failure: None")
+            msg2 = msgbase + (" ran out of shares:"
+                              " complete="
+                              " pending=Share(sh0-on-xgru5)"
+                              " overdue= unused= need 3. Last failure: None")
+            self.failUnless(body == msg1 or body == msg2, body)

         d.addCallback(_check_one_share)
 
         d.addCallback(lambda ignored:
         d.addCallback(_check_one_share)
 
         d.addCallback(lambda ignored:


@@ -4212,12 +5627,12 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
                                            404, "Not Found", None,
                                            self.GET, self.fileurls["imaginary"]))
         def _missing_child(body):
                                            404, "Not Found", None,
                                            self.GET, self.fileurls["imaginary"]))
         def _missing_child(body):

-            self.failUnless("No such child: imaginary" in body, body)
+            self.failUnlessIn("No such child: imaginary", body)

         d.addCallback(_missing_child)
 
         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
         def _check_0shares_dir_html(body):
         d.addCallback(_missing_child)
 
         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
         def _check_0shares_dir_html(body):

-            self.failUnless("<html>" in body, body)
+            self.failUnlessIn(DIR_HTML_TAG, body)

             # we should see the regular page, but without the child table or
             # the dirops forms
             body = " ".join(body.strip().split())
             # we should see the regular page, but without the child table or
             # the dirops forms
             body = " ".join(body.strip().split())


@@ -4240,7 +5655,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
             # and some-shares like we did for immutable files (since there
             # are different sorts of advice to offer in each case). For now,
             # they present the same way.
             # and some-shares like we did for immutable files (since there
             # are different sorts of advice to offer in each case). For now,
             # they present the same way.

-            self.failUnless("<html>" in body, body)
+            self.failUnlessIn(DIR_HTML_TAG, body)

             body = " ".join(body.strip().split())
             self.failUnlessIn('href="?t=info">More info on this directory',
                               body)
             body = " ".join(body.strip().split())
             self.failUnlessIn('href="?t=info">More info on this directory',
                               body)


@@ -4261,7 +5676,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
                                            self.GET,
                                            self.fileurls["dir-0share-json"]))
         def _check_unrecoverable_file(body):
                                            self.GET,
                                            self.fileurls["dir-0share-json"]))
         def _check_unrecoverable_file(body):

-            self.failIf("<html>" in body, body)
+            self.failIfIn("<html>", body)

             body = " ".join(body.strip().split())
             exp = ("UnrecoverableFileError: the directory (or mutable file) "
                    "could not be retrieved, because there were insufficient "
             body = " ".join(body.strip().split())
             exp = ("UnrecoverableFileError: the directory (or mutable file) "
                    "could not be retrieved, because there were insufficient "


@@ -4299,18 +5714,18 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
                       self.shouldHTTPError("GET errorboom_html",
                                            500, "Internal Server Error", None,
                                            self.GET, "ERRORBOOM",
                       self.shouldHTTPError("GET errorboom_html",
                                            500, "Internal Server Error", None,
                                            self.GET, "ERRORBOOM",

-                                           headers={"accept": ["*/*"]}))
+                                           headers={"accept": "*/*"}))

         def _internal_error_html1(body):
         def _internal_error_html1(body):

-            self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
+            self.failUnlessIn("<html>", "expected HTML, not '%s'" % body)

         d.addCallback(_internal_error_html1)
 
         d.addCallback(lambda ignored:
                       self.shouldHTTPError("GET errorboom_text",
                                            500, "Internal Server Error", None,
                                            self.GET, "ERRORBOOM",
         d.addCallback(_internal_error_html1)
 
         d.addCallback(lambda ignored:
                       self.shouldHTTPError("GET errorboom_text",
                                            500, "Internal Server Error", None,
                                            self.GET, "ERRORBOOM",

-                                           headers={"accept": ["text/plain"]}))
+                                           headers={"accept": "text/plain"}))

         def _internal_error_text2(body):
         def _internal_error_text2(body):

-            self.failIf("<html>" in body, body)
+            self.failIfIn("<html>", body)

             self.failUnless(body.startswith("Traceback "), body)
         d.addCallback(_internal_error_text2)
 
             self.failUnless(body.startswith("Traceback "), body)
         d.addCallback(_internal_error_text2)
 


@@ -4319,9 +5734,9 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
                       self.shouldHTTPError("GET errorboom_text",
                                            500, "Internal Server Error", None,
                                            self.GET, "ERRORBOOM",
                       self.shouldHTTPError("GET errorboom_text",
                                            500, "Internal Server Error", None,
                                            self.GET, "ERRORBOOM",

-                                           headers={"accept": [CLI_accepts]}))
+                                           headers={"accept": CLI_accepts}))

         def _internal_error_text3(body):
         def _internal_error_text3(body):

-            self.failIf("<html>" in body, body)
+            self.failIfIn("<html>", body)

             self.failUnless(body.startswith("Traceback "), body)
         d.addCallback(_internal_error_text3)
 
             self.failUnless(body.startswith("Traceback "), body)
         d.addCallback(_internal_error_text3)
 


@@ -4330,7 +5745,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
                                            500, "Internal Server Error", None,
                                            self.GET, "ERRORBOOM"))
         def _internal_error_html4(body):
                                            500, "Internal Server Error", None,
                                            self.GET, "ERRORBOOM"))
         def _internal_error_html4(body):

-            self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
+            self.failUnlessIn("<html>", body)

         d.addCallback(_internal_error_html4)
 
         def _flush_errors(res):
         d.addCallback(_internal_error_html4)
 
         def _flush_errors(res):


@@ -4341,6 +5756,146 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
 
         return d
 
 
         return d
 

+    def test_blacklist(self):
+        # download from a blacklisted URI, get an error
+        self.basedir = "web/Grid/blacklist"
+        self.set_up_grid()
+        c0 = self.g.clients[0]
+        c0_basedir = c0.basedir
+        fn = os.path.join(c0_basedir, "access.blacklist")
+        self.uris = {}
+        DATA = "off-limits " * 50
+
+        d = c0.upload(upload.Data(DATA, convergence=""))
+        def _stash_uri_and_create_dir(ur):
+            self.uri = ur.get_uri()
+            self.url = "uri/"+self.uri
+            u = uri.from_string_filenode(self.uri)
+            self.si = u.get_storage_index()
+            childnode = c0.create_node_from_uri(self.uri, None)
+            return c0.create_dirnode({u"blacklisted.txt": (childnode,{}) })
+        d.addCallback(_stash_uri_and_create_dir)
+        def _stash_dir(node):
+            self.dir_node = node
+            self.dir_uri = node.get_uri()
+            self.dir_url = "uri/"+self.dir_uri
+        d.addCallback(_stash_dir)
+        d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
+        def _check_dir_html(body):
+            self.failUnlessIn(DIR_HTML_TAG, body)
+            self.failUnlessIn("blacklisted.txt</a>", body)
+        d.addCallback(_check_dir_html)
+        d.addCallback(lambda ign: self.GET(self.url))
+        d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
+
+        def _blacklist(ign):
+            f = open(fn, "w")
+            f.write(" # this is a comment\n")
+            f.write(" \n")
+            f.write("\n") # also exercise blank lines
+            f.write("%s %s\n" % (base32.b2a(self.si), "off-limits to you"))
+            f.close()
+            # clients should be checking the blacklist each time, so we don't
+            # need to restart the client
+        d.addCallback(_blacklist)
+        d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_uri",
+                                                       403, "Forbidden",
+                                                       "Access Prohibited: off-limits",
+                                                       self.GET, self.url))
+
+        # We should still be able to list the parent directory, in HTML...
+        d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
+        def _check_dir_html2(body):
+            self.failUnlessIn(DIR_HTML_TAG, body)
+            self.failUnlessIn("blacklisted.txt</strike>", body)
+        d.addCallback(_check_dir_html2)
+
+        # ... and in JSON (used by CLI).
+        d.addCallback(lambda ign: self.GET(self.dir_url+"?t=json", followRedirect=True))
+        def _check_dir_json(res):
+            data = simplejson.loads(res)
+            self.failUnless(isinstance(data, list), data)
+            self.failUnlessEqual(data[0], "dirnode")
+            self.failUnless(isinstance(data[1], dict), data)
+            self.failUnlessIn("children", data[1])
+            self.failUnlessIn("blacklisted.txt", data[1]["children"])
+            childdata = data[1]["children"]["blacklisted.txt"]
+            self.failUnless(isinstance(childdata, list), data)
+            self.failUnlessEqual(childdata[0], "filenode")
+            self.failUnless(isinstance(childdata[1], dict), data)
+        d.addCallback(_check_dir_json)
+
+        def _unblacklist(ign):
+            open(fn, "w").close()
+            # the Blacklist object watches mtime to tell when the file has
+            # changed, but on windows this test will run faster than the
+            # filesystem's mtime resolution. So we edit Blacklist.last_mtime
+            # to force a reload.
+            self.g.clients[0].blacklist.last_mtime -= 2.0
+        d.addCallback(_unblacklist)
+
+        # now a read should work
+        d.addCallback(lambda ign: self.GET(self.url))
+        d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
+
+        # read again to exercise the blacklist-is-unchanged logic
+        d.addCallback(lambda ign: self.GET(self.url))
+        d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
+
+        # now add a blacklisted directory, and make sure files under it are
+        # refused too
+        def _add_dir(ign):
+            childnode = c0.create_node_from_uri(self.uri, None)
+            return c0.create_dirnode({u"child": (childnode,{}) })
+        d.addCallback(_add_dir)
+        def _get_dircap(dn):
+            self.dir_si_b32 = base32.b2a(dn.get_storage_index())
+            self.dir_url_base = "uri/"+dn.get_write_uri()
+            self.dir_url_json1 = "uri/"+dn.get_write_uri()+"?t=json"
+            self.dir_url_json2 = "uri/"+dn.get_write_uri()+"/?t=json"
+            self.dir_url_json_ro = "uri/"+dn.get_readonly_uri()+"/?t=json"
+            self.child_url = "uri/"+dn.get_readonly_uri()+"/child"
+        d.addCallback(_get_dircap)
+        d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True))
+        d.addCallback(lambda body: self.failUnlessIn(DIR_HTML_TAG, body))
+        d.addCallback(lambda ign: self.GET(self.dir_url_json1))
+        d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
+        d.addCallback(lambda ign: self.GET(self.dir_url_json2))
+        d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
+        d.addCallback(lambda ign: self.GET(self.dir_url_json_ro))
+        d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
+        d.addCallback(lambda ign: self.GET(self.child_url))
+        d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
+
+        def _block_dir(ign):
+            f = open(fn, "w")
+            f.write("%s %s\n" % (self.dir_si_b32, "dir-off-limits to you"))
+            f.close()
+            self.g.clients[0].blacklist.last_mtime -= 2.0
+        d.addCallback(_block_dir)
+        d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir base",
+                                                       403, "Forbidden",
+                                                       "Access Prohibited: dir-off-limits",
+                                                       self.GET, self.dir_url_base))
+        d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json1",
+                                                       403, "Forbidden",
+                                                       "Access Prohibited: dir-off-limits",
+                                                       self.GET, self.dir_url_json1))
+        d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json2",
+                                                       403, "Forbidden",
+                                                       "Access Prohibited: dir-off-limits",
+                                                       self.GET, self.dir_url_json2))
+        d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json_ro",
+                                                       403, "Forbidden",
+                                                       "Access Prohibited: dir-off-limits",
+                                                       self.GET, self.dir_url_json_ro))
+        d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir child",
+                                                       403, "Forbidden",
+                                                       "Access Prohibited: dir-off-limits",
+                                                       self.GET, self.child_url))
+        return d
+
+

 class CompletelyUnhandledError(Exception):
     pass
 class ErrorBoom(rend.Page):
 class CompletelyUnhandledError(Exception):
     pass
 class ErrorBoom(rend.Page):