src/allmydata/test/test_web.py

   1 import os.path, re, urllib, time
   2 import simplejson
   3 from StringIO import StringIO
   4
   5 from twisted.application import service
   6 from twisted.trial import unittest
   7 from twisted.internet import defer, reactor
   8 from twisted.internet.task import Clock
   9 from twisted.web import client, error, http, html
  10 from twisted.python import failure, log
  11
  12 from foolscap.api import fireEventually, flushEventualQueue
  13
  14 from nevow.util import escapeToXML
  15 from nevow import rend
  16
  17 from allmydata import interfaces, uri, webish, dirnode
  18 from allmydata.storage.shares import get_share_file
  19 from allmydata.storage_client import StorageFarmBroker, StubServer
  20 from allmydata.immutable import upload
  21 from allmydata.immutable.downloader.status import DownloadStatus
  22 from allmydata.dirnode import DirectoryNode
  23 from allmydata.nodemaker import NodeMaker
  24 from allmydata.unknown import UnknownNode
  25 from allmydata.web import status, common
  26 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  27 from allmydata.util import fileutil, base32, hashutil
  28 from allmydata.util.consumer import download_to_data
  29 from allmydata.util.netstring import split_netstring
  30 from allmydata.util.encodingutil import to_str
  31 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  32      create_chk_filenode, WebErrorMixin, ShouldFailMixin, \
  33      make_mutable_file_uri, create_mutable_filenode
  34 from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
  35 from allmydata.mutable import servermap, publish, retrieve
  36 import allmydata.test.common_util as testutil
  37 from allmydata.test.no_network import GridTestMixin
  38 from allmydata.test.common_web import HTTPClientGETFactory, \
  39      HTTPClientHEADFactory
  40 from allmydata.client import Client, SecretHolder
  41 from allmydata.introducer import IntroducerNode
  42
  43 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  44 # create a webserver that works against them
  45
  46 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  47
  48 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
  49 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
  50 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
  51
  52 FAVICON_MARKUP = '<link href="/icon.png" rel="shortcut icon" />'
  53
  54
  55 class FakeStatsProvider:
  56     def get_stats(self):
  57         stats = {'stats': {}, 'counters': {}}
  58         return stats
  59
  60 class FakeNodeMaker(NodeMaker):
  61     encoding_params = {
  62         'k': 3,
  63         'n': 10,
  64         'happy': 7,
  65         'max_segment_size':128*1024 # 1024=KiB
  66     }
  67     def _create_lit(self, cap):
  68         return FakeCHKFileNode(cap, self.all_contents)
  69     def _create_immutable(self, cap):
  70         return FakeCHKFileNode(cap, self.all_contents)
  71     def _create_mutable(self, cap):
  72         return FakeMutableFileNode(None, None,
  73                                    self.encoding_params, None,
  74                                    self.all_contents).init_from_cap(cap)
  75     def create_mutable_file(self, contents="", keysize=None,
  76                             version=SDMF_VERSION):
  77         n = FakeMutableFileNode(None, None, self.encoding_params, None,
  78                                 self.all_contents)
  79         return n.create(contents, version=version)
  80
  81 class FakeUploader(service.Service):
  82     name = "uploader"
  83     def upload(self, uploadable):
  84         d = uploadable.get_size()
  85         d.addCallback(lambda size: uploadable.read(size))
  86         def _got_data(datav):
  87             data = "".join(datav)
  88             n = create_chk_filenode(data, self.all_contents)
  89             ur = upload.UploadResults(file_size=len(data),
  90                                       ciphertext_fetched=0,
  91                                       preexisting_shares=0,
  92                                       pushed_shares=10,
  93                                       sharemap={},
  94                                       servermap={},
  95                                       timings={},
  96                                       uri_extension_data={},
  97                                       uri_extension_hash="fake",
  98                                       verifycapstr="fakevcap")
  99             ur.set_uri(n.get_uri())
 100             return ur
 101         d.addCallback(_got_data)
 102         return d
 103     def get_helper_info(self):
 104         return (None, False)
 105
 106 def build_one_ds():
 107     ds = DownloadStatus("storage_index", 1234)
 108     now = time.time()
 109
 110     serverA = StubServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
 111     serverB = StubServer(hashutil.tagged_hash("foo", "serverid_b")[:20])
 112     storage_index = hashutil.storage_index_hash("SI")
 113     e0 = ds.add_segment_request(0, now)
 114     e0.activate(now+0.5)
 115     e0.deliver(now+1, 0, 100, 0.5) # when, start,len, decodetime
 116     e1 = ds.add_segment_request(1, now+2)
 117     e1.error(now+3)
 118     # two outstanding requests
 119     e2 = ds.add_segment_request(2, now+4)
 120     e3 = ds.add_segment_request(3, now+5)
 121     del e2,e3 # hush pyflakes
 122
 123     # simulate a segment which gets delivered faster than a system clock tick (ticket #1166)
 124     e = ds.add_segment_request(4, now)
 125     e.activate(now)
 126     e.deliver(now, 0, 140, 0.5)
 127
 128     e = ds.add_dyhb_request(serverA, now)
 129     e.finished([1,2], now+1)
 130     e = ds.add_dyhb_request(serverB, now+2) # left unfinished
 131
 132     e = ds.add_read_event(0, 120, now)
 133     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 134     e.finished(now+1)
 135     e = ds.add_read_event(120, 30, now+2) # left unfinished
 136
 137     e = ds.add_block_request(serverA, 1, 100, 20, now)
 138     e.finished(20, now+1)
 139     e = ds.add_block_request(serverB, 1, 120, 30, now+1) # left unfinished
 140
 141     # make sure that add_read_event() can come first too
 142     ds1 = DownloadStatus(storage_index, 1234)
 143     e = ds1.add_read_event(0, 120, now)
 144     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 145     e.finished(now+1)
 146
 147     return ds
 148
 149 class FakeHistory:
 150     _all_upload_status = [upload.UploadStatus()]
 151     _all_download_status = [build_one_ds()]
 152     _all_mapupdate_statuses = [servermap.UpdateStatus()]
 153     _all_publish_statuses = [publish.PublishStatus()]
 154     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
 155
 156     def list_all_upload_statuses(self):
 157         return self._all_upload_status
 158     def list_all_download_statuses(self):
 159         return self._all_download_status
 160     def list_all_mapupdate_statuses(self):
 161         return self._all_mapupdate_statuses
 162     def list_all_publish_statuses(self):
 163         return self._all_publish_statuses
 164     def list_all_retrieve_statuses(self):
 165         return self._all_retrieve_statuses
 166     def list_all_helper_statuses(self):
 167         return []
 168
 169 class FakeClient(Client):
 170     def __init__(self):
 171         # don't upcall to Client.__init__, since we only want to initialize a
 172         # minimal subset
 173         service.MultiService.__init__(self)
 174         self.all_contents = {}
 175         self.nodeid = "fake_nodeid"
 176         self.nickname = "fake_nickname"
 177         self.introducer_furl = "None"
 178         self.stats_provider = FakeStatsProvider()
 179         self._secret_holder = SecretHolder("lease secret", "convergence secret")
 180         self.helper = None
 181         self.convergence = "some random string"
 182         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 183         self.introducer_client = None
 184         self.history = FakeHistory()
 185         self.uploader = FakeUploader()
 186         self.uploader.all_contents = self.all_contents
 187         self.uploader.setServiceParent(self)
 188         self.blacklist = None
 189         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 190                                        self.uploader, None,
 191                                        None, None, None)
 192         self.nodemaker.all_contents = self.all_contents
 193         self.mutable_file_default = SDMF_VERSION
 194
 195     def startService(self):
 196         return service.MultiService.startService(self)
 197     def stopService(self):
 198         return service.MultiService.stopService(self)
 199
 200     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 201
 202 class WebMixin(object):
 203     def setUp(self):
 204         self.s = FakeClient()
 205         self.s.startService()
 206         self.staticdir = self.mktemp()
 207         self.clock = Clock()
 208         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
 209                                       clock=self.clock)
 210         self.ws.setServiceParent(self.s)
 211         self.webish_port = self.ws.getPortnum()
 212         self.webish_url = self.ws.getURL()
 213         assert self.webish_url.endswith("/")
 214         self.webish_url = self.webish_url[:-1] # these tests add their own /
 215
 216         l = [ self.s.create_dirnode() for x in range(6) ]
 217         d = defer.DeferredList(l)
 218         def _then(res):
 219             self.public_root = res[0][1]
 220             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 221             self.public_url = "/uri/" + self.public_root.get_uri()
 222             self.private_root = res[1][1]
 223
 224             foo = res[2][1]
 225             self._foo_node = foo
 226             self._foo_uri = foo.get_uri()
 227             self._foo_readonly_uri = foo.get_readonly_uri()
 228             self._foo_verifycap = foo.get_verify_cap().to_string()
 229             # NOTE: we ignore the deferred on all set_uri() calls, because we
 230             # know the fake nodes do these synchronously
 231             self.public_root.set_uri(u"foo", foo.get_uri(),
 232                                      foo.get_readonly_uri())
 233
 234             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 235             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 236             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 237
 238             # sdmf
 239             # XXX: Do we ever use this?
 240             self.BAZ_CONTENTS, n, self._baz_txt_uri, self._baz_txt_readonly_uri = self.makefile_mutable(0)
 241
 242             foo.set_uri(u"baz.txt", self._baz_txt_uri, self._baz_txt_readonly_uri)
 243
 244             # mdmf
 245             self.QUUX_CONTENTS, n, self._quux_txt_uri, self._quux_txt_readonly_uri = self.makefile_mutable(0, mdmf=True)
 246             assert self._quux_txt_uri.startswith("URI:MDMF")
 247             foo.set_uri(u"quux.txt", self._quux_txt_uri, self._quux_txt_readonly_uri)
 248
 249             foo.set_uri(u"empty", res[3][1].get_uri(),
 250                         res[3][1].get_readonly_uri())
 251             sub_uri = res[4][1].get_uri()
 252             self._sub_uri = sub_uri
 253             foo.set_uri(u"sub", sub_uri, sub_uri)
 254             sub = self.s.create_node_from_uri(sub_uri)
 255             self._sub_node = sub
 256
 257             _ign, n, blocking_uri = self.makefile(1)
 258             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 259
 260             # filenode to test for html encoding issues
 261             self._htmlname_unicode = u"<&weirdly'named\"file>>>_<iframe />.txt"
 262             self._htmlname_raw = self._htmlname_unicode.encode('utf-8')
 263             self._htmlname_urlencoded = urllib.quote(self._htmlname_raw, '')
 264             self._htmlname_escaped = escapeToXML(self._htmlname_raw)
 265             self._htmlname_escaped_attr = html.escape(self._htmlname_raw)
 266             self._htmlname_escaped_double = escapeToXML(html.escape(self._htmlname_raw))
 267             self.HTMLNAME_CONTENTS, n, self._htmlname_txt_uri = self.makefile(0)
 268             foo.set_uri(self._htmlname_unicode, self._htmlname_txt_uri, self._htmlname_txt_uri)
 269
 270             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 271             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 272             # still think of it as an umlaut
 273             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 274
 275             self.SUBBAZ_CONTENTS, n, baz_file = self.makefile(2)
 276             self._baz_file_uri = baz_file
 277             sub.set_uri(u"baz.txt", baz_file, baz_file)
 278
 279             _ign, n, self._bad_file_uri = self.makefile(3)
 280             # this uri should not be downloadable
 281             del self.s.all_contents[self._bad_file_uri]
 282
 283             rodir = res[5][1]
 284             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 285                                      rodir.get_readonly_uri())
 286             rodir.set_uri(u"nor", baz_file, baz_file)
 287
 288             # public/
 289             # public/foo/
 290             # public/foo/bar.txt
 291             # public/foo/baz.txt
 292             # public/foo/quux.txt
 293             # public/foo/blockingfile
 294             # public/foo/<&weirdly'named\"file>>>_<iframe />.txt
 295             # public/foo/empty/
 296             # public/foo/sub/
 297             # public/foo/sub/baz.txt
 298             # public/reedownlee/
 299             # public/reedownlee/nor
 300             self.NEWFILE_CONTENTS = "newfile contents\n"
 301
 302             return foo.get_metadata_for(u"bar.txt")
 303         d.addCallback(_then)
 304         def _got_metadata(metadata):
 305             self._bar_txt_metadata = metadata
 306         d.addCallback(_got_metadata)
 307         return d
 308
 309     def get_all_contents(self):
 310         return self.s.all_contents
 311
 312     def makefile(self, number):
 313         contents = "contents of file %s\n" % number
 314         n = create_chk_filenode(contents, self.get_all_contents())
 315         return contents, n, n.get_uri()
 316
 317     def makefile_mutable(self, number, mdmf=False):
 318         contents = "contents of mutable file %s\n" % number
 319         n = create_mutable_filenode(contents, mdmf, self.s.all_contents)
 320         return contents, n, n.get_uri(), n.get_readonly_uri()
 321
 322     def tearDown(self):
 323         return self.s.stopService()
 324
 325     def failUnlessIsBarDotTxt(self, res):
 326         self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
 327
 328     def failUnlessIsQuuxDotTxt(self, res):
 329         self.failUnlessReallyEqual(res, self.QUUX_CONTENTS, res)
 330
 331     def failUnlessIsBazDotTxt(self, res):
 332         self.failUnlessReallyEqual(res, self.BAZ_CONTENTS, res)
 333
 334     def failUnlessIsSubBazDotTxt(self, res):
 335         self.failUnlessReallyEqual(res, self.SUBBAZ_CONTENTS, res)
 336
 337     def failUnlessIsBarJSON(self, res):
 338         data = simplejson.loads(res)
 339         self.failUnless(isinstance(data, list))
 340         self.failUnlessEqual(data[0], "filenode")
 341         self.failUnless(isinstance(data[1], dict))
 342         self.failIf(data[1]["mutable"])
 343         self.failIfIn("rw_uri", data[1]) # immutable
 344         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
 345         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
 346         self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
 347
 348     def failUnlessIsQuuxJSON(self, res, readonly=False):
 349         data = simplejson.loads(res)
 350         self.failUnless(isinstance(data, list))
 351         self.failUnlessEqual(data[0], "filenode")
 352         self.failUnless(isinstance(data[1], dict))
 353         metadata = data[1]
 354         return self.failUnlessIsQuuxDotTxtMetadata(metadata, readonly)
 355
 356     def failUnlessIsQuuxDotTxtMetadata(self, metadata, readonly):
 357         self.failUnless(metadata['mutable'])
 358         if readonly:
 359             self.failIfIn("rw_uri", metadata)
 360         else:
 361             self.failUnlessIn("rw_uri", metadata)
 362             self.failUnlessEqual(metadata['rw_uri'], self._quux_txt_uri)
 363         self.failUnlessIn("ro_uri", metadata)
 364         self.failUnlessEqual(metadata['ro_uri'], self._quux_txt_readonly_uri)
 365         self.failUnlessReallyEqual(metadata['size'], len(self.QUUX_CONTENTS))
 366
 367     def failUnlessIsFooJSON(self, res):
 368         data = simplejson.loads(res)
 369         self.failUnless(isinstance(data, list))
 370         self.failUnlessEqual(data[0], "dirnode", res)
 371         self.failUnless(isinstance(data[1], dict))
 372         self.failUnless(data[1]["mutable"])
 373         self.failUnlessIn("rw_uri", data[1]) # mutable
 374         self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
 375         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
 376         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
 377
 378         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 379         self.failUnlessEqual(kidnames,
 380                              [self._htmlname_unicode, u"bar.txt", u"baz.txt",
 381                               u"blockingfile", u"empty", u"n\u00fc.txt", u"quux.txt", u"sub"])
 382         kids = dict( [(unicode(name),value)
 383                       for (name,value)
 384                       in data[1]["children"].iteritems()] )
 385         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 386         self.failUnlessIn("metadata", kids[u"sub"][1])
 387         self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
 388         tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
 389         self.failUnlessIn("linkcrtime", tahoe_md)
 390         self.failUnlessIn("linkmotime", tahoe_md)
 391         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 392         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 393         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri)
 394         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]),
 395                                    self._bar_txt_verifycap)
 396         self.failUnlessIn("metadata", kids[u"bar.txt"][1])
 397         self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
 398         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
 399                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
 400         self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
 401                                    self._bar_txt_uri)
 402         self.failUnlessIn("quux.txt", kids)
 403         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["rw_uri"]),
 404                                    self._quux_txt_uri)
 405         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["ro_uri"]),
 406                                    self._quux_txt_readonly_uri)
 407
 408     def GET(self, urlpath, followRedirect=False, return_response=False,
 409             **kwargs):
 410         # if return_response=True, this fires with (data, statuscode,
 411         # respheaders) instead of just data.
 412         assert not isinstance(urlpath, unicode)
 413         url = self.webish_url + urlpath
 414         factory = HTTPClientGETFactory(url, method="GET",
 415                                        followRedirect=followRedirect, **kwargs)
 416         reactor.connectTCP("localhost", self.webish_port, factory)
 417         d = factory.deferred
 418         def _got_data(data):
 419             return (data, factory.status, factory.response_headers)
 420         if return_response:
 421             d.addCallback(_got_data)
 422         return factory.deferred
 423
 424     def HEAD(self, urlpath, return_response=False, **kwargs):
 425         # this requires some surgery, because twisted.web.client doesn't want
 426         # to give us back the response headers.
 427         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 428         reactor.connectTCP("localhost", self.webish_port, factory)
 429         d = factory.deferred
 430         def _got_data(data):
 431             return (data, factory.status, factory.response_headers)
 432         if return_response:
 433             d.addCallback(_got_data)
 434         return factory.deferred
 435
 436     def PUT(self, urlpath, data, **kwargs):
 437         url = self.webish_url + urlpath
 438         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 439
 440     def DELETE(self, urlpath):
 441         url = self.webish_url + urlpath
 442         return client.getPage(url, method="DELETE")
 443
 444     def POST(self, urlpath, followRedirect=False, **fields):
 445         sepbase = "boogabooga"
 446         sep = "--" + sepbase
 447         form = []
 448         form.append(sep)
 449         form.append('Content-Disposition: form-data; name="_charset"')
 450         form.append('')
 451         form.append('UTF-8')
 452         form.append(sep)
 453         for name, value in fields.iteritems():
 454             if isinstance(value, tuple):
 455                 filename, value = value
 456                 form.append('Content-Disposition: form-data; name="%s"; '
 457                             'filename="%s"' % (name, filename.encode("utf-8")))
 458             else:
 459                 form.append('Content-Disposition: form-data; name="%s"' % name)
 460             form.append('')
 461             if isinstance(value, unicode):
 462                 value = value.encode("utf-8")
 463             else:
 464                 value = str(value)
 465             assert isinstance(value, str)
 466             form.append(value)
 467             form.append(sep)
 468         form[-1] += "--"
 469         body = ""
 470         headers = {}
 471         if fields:
 472             body = "\r\n".join(form) + "\r\n"
 473             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 474         return self.POST2(urlpath, body, headers, followRedirect)
 475
 476     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 477         url = self.webish_url + urlpath
 478         return client.getPage(url, method="POST", postdata=body,
 479                               headers=headers, followRedirect=followRedirect)
 480
 481     def shouldFail(self, res, expected_failure, which,
 482                    substring=None, response_substring=None):
 483         if isinstance(res, failure.Failure):
 484             res.trap(expected_failure)
 485             if substring:
 486                 self.failUnlessIn(substring, str(res), which)
 487             if response_substring:
 488                 self.failUnlessIn(response_substring, res.value.response, which)
 489         else:
 490             self.fail("%s was supposed to raise %s, not get '%s'" %
 491                       (which, expected_failure, res))
 492
 493     def shouldFail2(self, expected_failure, which, substring,
 494                     response_substring,
 495                     callable, *args, **kwargs):
 496         assert substring is None or isinstance(substring, str)
 497         assert response_substring is None or isinstance(response_substring, str)
 498         d = defer.maybeDeferred(callable, *args, **kwargs)
 499         def done(res):
 500             if isinstance(res, failure.Failure):
 501                 res.trap(expected_failure)
 502                 if substring:
 503                     self.failUnlessIn(substring, str(res),
 504                                       "'%s' not in '%s' for test '%s'" % \
 505                                       (substring, str(res), which))
 506                 if response_substring:
 507                     self.failUnlessIn(response_substring, res.value.response,
 508                                       "'%s' not in '%s' for test '%s'" % \
 509                                       (response_substring, res.value.response,
 510                                        which))
 511             else:
 512                 self.fail("%s was supposed to raise %s, not get '%s'" %
 513                           (which, expected_failure, res))
 514         d.addBoth(done)
 515         return d
 516
 517     def should404(self, res, which):
 518         if isinstance(res, failure.Failure):
 519             res.trap(error.Error)
 520             self.failUnlessReallyEqual(res.value.status, "404")
 521         else:
 522             self.fail("%s was supposed to Error(404), not get '%s'" %
 523                       (which, res))
 524
 525     def should302(self, res, which):
 526         if isinstance(res, failure.Failure):
 527             res.trap(error.Error)
 528             self.failUnlessReallyEqual(res.value.status, "302")
 529         else:
 530             self.fail("%s was supposed to Error(302), not get '%s'" %
 531                       (which, res))
 532
 533
 534 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
 535     def test_create(self):
 536         pass
 537
 538     def test_welcome(self):
 539         d = self.GET("/")
 540         def _check(res):
 541             self.failUnlessIn('Welcome to Tahoe-LAFS', res)
 542             self.failUnlessIn(FAVICON_MARKUP, res)
 543             self.failUnlessIn('href="https://tahoe-lafs.org/"', res)
 544
 545             self.s.basedir = 'web/test_welcome'
 546             fileutil.make_dirs("web/test_welcome")
 547             fileutil.make_dirs("web/test_welcome/private")
 548             return self.GET("/")
 549         d.addCallback(_check)
 550         return d
 551
 552     def test_status(self):
 553         h = self.s.get_history()
 554         dl_num = h.list_all_download_statuses()[0].get_counter()
 555         ul_num = h.list_all_upload_statuses()[0].get_counter()
 556         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 557         pub_num = h.list_all_publish_statuses()[0].get_counter()
 558         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 559         d = self.GET("/status", followRedirect=True)
 560         def _check(res):
 561             self.failUnlessIn('Upload and Download Status', res)
 562             self.failUnlessIn('"down-%d"' % dl_num, res)
 563             self.failUnlessIn('"up-%d"' % ul_num, res)
 564             self.failUnlessIn('"mapupdate-%d"' % mu_num, res)
 565             self.failUnlessIn('"publish-%d"' % pub_num, res)
 566             self.failUnlessIn('"retrieve-%d"' % ret_num, res)
 567         d.addCallback(_check)
 568         d.addCallback(lambda res: self.GET("/status/?t=json"))
 569         def _check_json(res):
 570             data = simplejson.loads(res)
 571             self.failUnless(isinstance(data, dict))
 572             #active = data["active"]
 573             # TODO: test more. We need a way to fake an active operation
 574             # here.
 575         d.addCallback(_check_json)
 576
 577         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 578         def _check_dl(res):
 579             self.failUnlessIn("File Download Status", res)
 580         d.addCallback(_check_dl)
 581         d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
 582         def _check_dl_json(res):
 583             data = simplejson.loads(res)
 584             self.failUnless(isinstance(data, dict))
 585             self.failUnlessIn("read", data)
 586             self.failUnlessEqual(data["read"][0]["length"], 120)
 587             self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
 588             self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
 589             self.failUnlessEqual(data["segment"][2]["finish_time"], None)
 590             phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
 591             cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
 592             # serverids[] keys are strings, since that's what JSON does, but
 593             # we'd really like them to be ints
 594             self.failUnlessEqual(data["serverids"]["0"], "phwrsjte")
 595             self.failUnless(data["serverids"].has_key("1"),
 596                             str(data["serverids"]))
 597             self.failUnlessEqual(data["serverids"]["1"], "cmpuvkjm",
 598                                  str(data["serverids"]))
 599             self.failUnlessEqual(data["server_info"][phwr_id]["short"],
 600                                  "phwrsjte")
 601             self.failUnlessEqual(data["server_info"][cmpu_id]["short"],
 602                                  "cmpuvkjm")
 603             self.failUnlessIn("dyhb", data)
 604             self.failUnlessIn("misc", data)
 605         d.addCallback(_check_dl_json)
 606         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 607         def _check_ul(res):
 608             self.failUnlessIn("File Upload Status", res)
 609         d.addCallback(_check_ul)
 610         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 611         def _check_mapupdate(res):
 612             self.failUnlessIn("Mutable File Servermap Update Status", res)
 613         d.addCallback(_check_mapupdate)
 614         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 615         def _check_publish(res):
 616             self.failUnlessIn("Mutable File Publish Status", res)
 617         d.addCallback(_check_publish)
 618         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 619         def _check_retrieve(res):
 620             self.failUnlessIn("Mutable File Retrieve Status", res)
 621         d.addCallback(_check_retrieve)
 622
 623         return d
 624
 625     def test_status_numbers(self):
 626         drrm = status.DownloadResultsRendererMixin()
 627         self.failUnlessReallyEqual(drrm.render_time(None, None), "")
 628         self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
 629         self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
 630         self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
 631         self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
 632         self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
 633         self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 634         self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
 635         self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
 636
 637         urrm = status.UploadResultsRendererMixin()
 638         self.failUnlessReallyEqual(urrm.render_time(None, None), "")
 639         self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
 640         self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
 641         self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
 642         self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
 643         self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
 644         self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 645         self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
 646         self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
 647
 648     def test_GET_FILEURL(self):
 649         d = self.GET(self.public_url + "/foo/bar.txt")
 650         d.addCallback(self.failUnlessIsBarDotTxt)
 651         return d
 652
 653     def test_GET_FILEURL_range(self):
 654         headers = {"range": "bytes=1-10"}
 655         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 656                      return_response=True)
 657         def _got((res, status, headers)):
 658             self.failUnlessReallyEqual(int(status), 206)
 659             self.failUnless(headers.has_key("content-range"))
 660             self.failUnlessReallyEqual(headers["content-range"][0],
 661                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 662             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
 663         d.addCallback(_got)
 664         return d
 665
 666     def test_GET_FILEURL_partial_range(self):
 667         headers = {"range": "bytes=5-"}
 668         length  = len(self.BAR_CONTENTS)
 669         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 670                      return_response=True)
 671         def _got((res, status, headers)):
 672             self.failUnlessReallyEqual(int(status), 206)
 673             self.failUnless(headers.has_key("content-range"))
 674             self.failUnlessReallyEqual(headers["content-range"][0],
 675                                        "bytes 5-%d/%d" % (length-1, length))
 676             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
 677         d.addCallback(_got)
 678         return d
 679
 680     def test_GET_FILEURL_partial_end_range(self):
 681         headers = {"range": "bytes=-5"}
 682         length  = len(self.BAR_CONTENTS)
 683         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 684                      return_response=True)
 685         def _got((res, status, headers)):
 686             self.failUnlessReallyEqual(int(status), 206)
 687             self.failUnless(headers.has_key("content-range"))
 688             self.failUnlessReallyEqual(headers["content-range"][0],
 689                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 690             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
 691         d.addCallback(_got)
 692         return d
 693
 694     def test_GET_FILEURL_partial_range_overrun(self):
 695         headers = {"range": "bytes=100-200"}
 696         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
 697                              "416 Requested Range not satisfiable",
 698                              "First beyond end of file",
 699                              self.GET, self.public_url + "/foo/bar.txt",
 700                              headers=headers)
 701         return d
 702
 703     def test_HEAD_FILEURL_range(self):
 704         headers = {"range": "bytes=1-10"}
 705         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 706                      return_response=True)
 707         def _got((res, status, headers)):
 708             self.failUnlessReallyEqual(res, "")
 709             self.failUnlessReallyEqual(int(status), 206)
 710             self.failUnless(headers.has_key("content-range"))
 711             self.failUnlessReallyEqual(headers["content-range"][0],
 712                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 713         d.addCallback(_got)
 714         return d
 715
 716     def test_HEAD_FILEURL_partial_range(self):
 717         headers = {"range": "bytes=5-"}
 718         length  = len(self.BAR_CONTENTS)
 719         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 720                      return_response=True)
 721         def _got((res, status, headers)):
 722             self.failUnlessReallyEqual(int(status), 206)
 723             self.failUnless(headers.has_key("content-range"))
 724             self.failUnlessReallyEqual(headers["content-range"][0],
 725                                        "bytes 5-%d/%d" % (length-1, length))
 726         d.addCallback(_got)
 727         return d
 728
 729     def test_HEAD_FILEURL_partial_end_range(self):
 730         headers = {"range": "bytes=-5"}
 731         length  = len(self.BAR_CONTENTS)
 732         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 733                      return_response=True)
 734         def _got((res, status, headers)):
 735             self.failUnlessReallyEqual(int(status), 206)
 736             self.failUnless(headers.has_key("content-range"))
 737             self.failUnlessReallyEqual(headers["content-range"][0],
 738                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 739         d.addCallback(_got)
 740         return d
 741
 742     def test_HEAD_FILEURL_partial_range_overrun(self):
 743         headers = {"range": "bytes=100-200"}
 744         d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
 745                              "416 Requested Range not satisfiable",
 746                              "",
 747                              self.HEAD, self.public_url + "/foo/bar.txt",
 748                              headers=headers)
 749         return d
 750
 751     def test_GET_FILEURL_range_bad(self):
 752         headers = {"range": "BOGUS=fizbop-quarnak"}
 753         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 754                      return_response=True)
 755         def _got((res, status, headers)):
 756             self.failUnlessReallyEqual(int(status), 200)
 757             self.failUnless(not headers.has_key("content-range"))
 758             self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
 759         d.addCallback(_got)
 760         return d
 761
 762     def test_HEAD_FILEURL(self):
 763         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 764         def _got((res, status, headers)):
 765             self.failUnlessReallyEqual(res, "")
 766             self.failUnlessReallyEqual(headers["content-length"][0],
 767                                        str(len(self.BAR_CONTENTS)))
 768             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
 769         d.addCallback(_got)
 770         return d
 771
 772     def test_GET_FILEURL_named(self):
 773         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 774         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 775         d = self.GET(base + "/@@name=/blah.txt")
 776         d.addCallback(self.failUnlessIsBarDotTxt)
 777         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 778         d.addCallback(self.failUnlessIsBarDotTxt)
 779         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 780         d.addCallback(self.failUnlessIsBarDotTxt)
 781         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 782         d.addCallback(self.failUnlessIsBarDotTxt)
 783         save_url = base + "?save=true&filename=blah.txt"
 784         d.addCallback(lambda res: self.GET(save_url))
 785         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 786         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 787         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 788         u_url = base + "?save=true&filename=" + u_fn_e
 789         d.addCallback(lambda res: self.GET(u_url))
 790         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 791         return d
 792
 793     def test_PUT_FILEURL_named_bad(self):
 794         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 795         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 796                              "400 Bad Request",
 797                              "/file can only be used with GET or HEAD",
 798                              self.PUT, base + "/@@name=/blah.txt", "")
 799         return d
 800
 801
 802     def test_GET_DIRURL_named_bad(self):
 803         base = "/file/%s" % urllib.quote(self._foo_uri)
 804         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 805                              "400 Bad Request",
 806                              "is not a file-cap",
 807                              self.GET, base + "/@@name=/blah.txt")
 808         return d
 809
 810     def test_GET_slash_file_bad(self):
 811         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 812                              "404 Not Found",
 813                              "/file must be followed by a file-cap and a name",
 814                              self.GET, "/file")
 815         return d
 816
 817     def test_GET_unhandled_URI_named(self):
 818         contents, n, newuri = self.makefile(12)
 819         verifier_cap = n.get_verify_cap().to_string()
 820         base = "/file/%s" % urllib.quote(verifier_cap)
 821         # client.create_node_from_uri() can't handle verify-caps
 822         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 823                              "400 Bad Request", "is not a file-cap",
 824                              self.GET, base)
 825         return d
 826
 827     def test_GET_unhandled_URI(self):
 828         contents, n, newuri = self.makefile(12)
 829         verifier_cap = n.get_verify_cap().to_string()
 830         base = "/uri/%s" % urllib.quote(verifier_cap)
 831         # client.create_node_from_uri() can't handle verify-caps
 832         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 833                              "400 Bad Request",
 834                              "GET unknown URI type: can only do t=info",
 835                              self.GET, base)
 836         return d
 837
 838     def test_GET_FILE_URI(self):
 839         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 840         d = self.GET(base)
 841         d.addCallback(self.failUnlessIsBarDotTxt)
 842         return d
 843
 844     def test_GET_FILE_URI_mdmf(self):
 845         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 846         d = self.GET(base)
 847         d.addCallback(self.failUnlessIsQuuxDotTxt)
 848         return d
 849
 850     def test_GET_FILE_URI_mdmf_extensions(self):
 851         base = "/uri/%s" % urllib.quote("%s:RANDOMSTUFF" % self._quux_txt_uri)
 852         d = self.GET(base)
 853         d.addCallback(self.failUnlessIsQuuxDotTxt)
 854         return d
 855
 856     def test_GET_FILE_URI_mdmf_readonly(self):
 857         base = "/uri/%s" % urllib.quote(self._quux_txt_readonly_uri)
 858         d = self.GET(base)
 859         d.addCallback(self.failUnlessIsQuuxDotTxt)
 860         return d
 861
 862     def test_GET_FILE_URI_badchild(self):
 863         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 864         errmsg = "Files have no children, certainly not named 'boguschild'"
 865         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 866                              "400 Bad Request", errmsg,
 867                              self.GET, base)
 868         return d
 869
 870     def test_PUT_FILE_URI_badchild(self):
 871         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 872         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 873         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 874                              "400 Bad Request", errmsg,
 875                              self.PUT, base, "")
 876         return d
 877
 878     def test_PUT_FILE_URI_mdmf(self):
 879         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 880         self._quux_new_contents = "new_contents"
 881         d = self.GET(base)
 882         d.addCallback(lambda res:
 883             self.failUnlessIsQuuxDotTxt(res))
 884         d.addCallback(lambda ignored:
 885             self.PUT(base, self._quux_new_contents))
 886         d.addCallback(lambda ignored:
 887             self.GET(base))
 888         d.addCallback(lambda res:
 889             self.failUnlessReallyEqual(res, self._quux_new_contents))
 890         return d
 891
 892     def test_PUT_FILE_URI_mdmf_extensions(self):
 893         base = "/uri/%s" % urllib.quote("%s:EXTENSIONSTUFF" % self._quux_txt_uri)
 894         self._quux_new_contents = "new_contents"
 895         d = self.GET(base)
 896         d.addCallback(lambda res: self.failUnlessIsQuuxDotTxt(res))
 897         d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
 898         d.addCallback(lambda ignored: self.GET(base))
 899         d.addCallback(lambda res: self.failUnlessEqual(self._quux_new_contents,
 900                                                        res))
 901         return d
 902
 903     def test_PUT_FILE_URI_mdmf_readonly(self):
 904         # We're not allowed to PUT things to a readonly cap.
 905         base = "/uri/%s" % self._quux_txt_readonly_uri
 906         d = self.GET(base)
 907         d.addCallback(lambda res:
 908             self.failUnlessIsQuuxDotTxt(res))
 909         # What should we get here? We get a 500 error now; that's not right.
 910         d.addCallback(lambda ignored:
 911             self.shouldFail2(error.Error, "test_PUT_FILE_URI_mdmf_readonly",
 912                              "400 Bad Request", "read-only cap",
 913                              self.PUT, base, "new data"))
 914         return d
 915
 916     def test_PUT_FILE_URI_sdmf_readonly(self):
 917         # We're not allowed to put things to a readonly cap.
 918         base = "/uri/%s" % self._baz_txt_readonly_uri
 919         d = self.GET(base)
 920         d.addCallback(lambda res:
 921             self.failUnlessIsBazDotTxt(res))
 922         d.addCallback(lambda ignored:
 923             self.shouldFail2(error.Error, "test_PUT_FILE_URI_sdmf_readonly",
 924                              "400 Bad Request", "read-only cap",
 925                              self.PUT, base, "new_data"))
 926         return d
 927
 928     def test_GET_etags(self):
 929
 930         def _check_etags(uri):
 931             d1 = _get_etag(uri)
 932             d2 = _get_etag(uri, 'json')
 933             d = defer.DeferredList([d1, d2], consumeErrors=True)
 934             def _check(results):
 935                 # All deferred must succeed
 936                 self.failUnless(all([r[0] for r in results]))
 937                 # the etag for the t=json form should be just like the etag
 938                 # fo the default t='' form, but with a 'json' suffix
 939                 self.failUnlessEqual(results[0][1] + 'json', results[1][1])
 940             d.addCallback(_check)
 941             return d
 942
 943         def _get_etag(uri, t=''):
 944             targetbase = "/uri/%s?t=%s" % (urllib.quote(uri.strip()), t)
 945             d = self.GET(targetbase, return_response=True, followRedirect=True)
 946             def _just_the_etag(result):
 947                 data, response, headers = result
 948                 etag = headers['etag'][0]
 949                 if uri.startswith('URI:DIR'):
 950                     self.failUnless(etag.startswith('DIR:'), etag)
 951                 return etag
 952             return d.addCallback(_just_the_etag)
 953
 954         # Check that etags work with immutable directories
 955         (newkids, caps) = self._create_immutable_children()
 956         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
 957                       simplejson.dumps(newkids))
 958         def _stash_immdir_uri(uri):
 959             self._immdir_uri = uri
 960             return uri
 961         d.addCallback(_stash_immdir_uri)
 962         d.addCallback(_check_etags)
 963
 964         # Check that etags work with immutable files
 965         d.addCallback(lambda _: _check_etags(self._bar_txt_uri))
 966
 967         # use the ETag on GET
 968         def _check_match(ign):
 969             uri = "/uri/%s" % self._bar_txt_uri
 970             d = self.GET(uri, return_response=True)
 971             # extract the ETag
 972             d.addCallback(lambda (data, code, headers):
 973                           headers['etag'][0])
 974             # do a GET that's supposed to match the ETag
 975             d.addCallback(lambda etag:
 976                           self.GET(uri, return_response=True,
 977                                    headers={"If-None-Match": etag}))
 978             # make sure it short-circuited (304 instead of 200)
 979             d.addCallback(lambda (data, code, headers):
 980                           self.failUnlessEqual(int(code), http.NOT_MODIFIED))
 981             return d
 982         d.addCallback(_check_match)
 983
 984         def _no_etag(uri, t):
 985             target = "/uri/%s?t=%s" % (uri, t)
 986             d = self.GET(target, return_response=True, followRedirect=True)
 987             d.addCallback(lambda (data, code, headers):
 988                           self.failIf("etag" in headers, target))
 989             return d
 990         def _yes_etag(uri, t):
 991             target = "/uri/%s?t=%s" % (uri, t)
 992             d = self.GET(target, return_response=True, followRedirect=True)
 993             d.addCallback(lambda (data, code, headers):
 994                           self.failUnless("etag" in headers, target))
 995             return d
 996
 997         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, ""))
 998         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "json"))
 999         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "uri"))
1000         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "readonly-uri"))
1001         d.addCallback(lambda ign: _no_etag(self._bar_txt_uri, "info"))
1002
1003         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, ""))
1004         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "json"))
1005         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "uri"))
1006         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "readonly-uri"))
1007         d.addCallback(lambda ign: _no_etag(self._immdir_uri, "info"))
1008         d.addCallback(lambda ign: _no_etag(self._immdir_uri, "rename-form"))
1009
1010         return d
1011
1012     # TODO: version of this with a Unicode filename
1013     def test_GET_FILEURL_save(self):
1014         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
1015                      return_response=True)
1016         def _got((res, statuscode, headers)):
1017             content_disposition = headers["content-disposition"][0]
1018             self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
1019             self.failUnlessIsBarDotTxt(res)
1020         d.addCallback(_got)
1021         return d
1022
1023     def test_GET_FILEURL_missing(self):
1024         d = self.GET(self.public_url + "/foo/missing")
1025         d.addBoth(self.should404, "test_GET_FILEURL_missing")
1026         return d
1027
1028     def test_GET_FILEURL_info_mdmf(self):
1029         d = self.GET("/uri/%s?t=info" % self._quux_txt_uri)
1030         def _got(res):
1031             self.failUnlessIn("mutable file (mdmf)", res)
1032             self.failUnlessIn(self._quux_txt_uri, res)
1033             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1034         d.addCallback(_got)
1035         return d
1036
1037     def test_GET_FILEURL_info_mdmf_readonly(self):
1038         d = self.GET("/uri/%s?t=info" % self._quux_txt_readonly_uri)
1039         def _got(res):
1040             self.failUnlessIn("mutable file (mdmf)", res)
1041             self.failIfIn(self._quux_txt_uri, res)
1042             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1043         d.addCallback(_got)
1044         return d
1045
1046     def test_GET_FILEURL_info_sdmf(self):
1047         d = self.GET("/uri/%s?t=info" % self._baz_txt_uri)
1048         def _got(res):
1049             self.failUnlessIn("mutable file (sdmf)", res)
1050             self.failUnlessIn(self._baz_txt_uri, res)
1051         d.addCallback(_got)
1052         return d
1053
1054     def test_GET_FILEURL_info_mdmf_extensions(self):
1055         d = self.GET("/uri/%s:STUFF?t=info" % self._quux_txt_uri)
1056         def _got(res):
1057             self.failUnlessIn("mutable file (mdmf)", res)
1058             self.failUnlessIn(self._quux_txt_uri, res)
1059             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1060         d.addCallback(_got)
1061         return d
1062
1063     def test_PUT_overwrite_only_files(self):
1064         # create a directory, put a file in that directory.
1065         contents, n, filecap = self.makefile(8)
1066         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
1067         d.addCallback(lambda res:
1068             self.PUT(self.public_url + "/foo/dir/file1.txt",
1069                      self.NEWFILE_CONTENTS))
1070         # try to overwrite the file with replace=only-files
1071         # (this should work)
1072         d.addCallback(lambda res:
1073             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
1074                      filecap))
1075         d.addCallback(lambda res:
1076             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
1077                  "There was already a child by that name, and you asked me "
1078                  "to not replace it",
1079                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
1080                  filecap))
1081         return d
1082
1083     def test_PUT_NEWFILEURL(self):
1084         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
1085         # TODO: we lose the response code, so we can't check this
1086         #self.failUnlessReallyEqual(responsecode, 201)
1087         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1088         d.addCallback(lambda res:
1089                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1090                                                       self.NEWFILE_CONTENTS))
1091         return d
1092
1093     def test_PUT_NEWFILEURL_not_mutable(self):
1094         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
1095                      self.NEWFILE_CONTENTS)
1096         # TODO: we lose the response code, so we can't check this
1097         #self.failUnlessReallyEqual(responsecode, 201)
1098         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1099         d.addCallback(lambda res:
1100                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1101                                                       self.NEWFILE_CONTENTS))
1102         return d
1103
1104     def test_PUT_NEWFILEURL_unlinked_mdmf(self):
1105         # this should get us a few segments of an MDMF mutable file,
1106         # which we can then test for.
1107         contents = self.NEWFILE_CONTENTS * 300000
1108         d = self.PUT("/uri?format=mdmf",
1109                      contents)
1110         def _got_filecap(filecap):
1111             self.failUnless(filecap.startswith("URI:MDMF"))
1112             return filecap
1113         d.addCallback(_got_filecap)
1114         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1115         d.addCallback(lambda json: self.failUnlessIn("MDMF", json))
1116         return d
1117
1118     def test_PUT_NEWFILEURL_unlinked_sdmf(self):
1119         contents = self.NEWFILE_CONTENTS * 300000
1120         d = self.PUT("/uri?format=sdmf",
1121                      contents)
1122         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1123         d.addCallback(lambda json: self.failUnlessIn("SDMF", json))
1124         return d
1125
1126     def test_PUT_NEWFILEURL_unlinked_bad_format(self):
1127         contents = self.NEWFILE_CONTENTS * 300000
1128         return self.shouldHTTPError("PUT_NEWFILEURL_unlinked_bad_format",
1129                                     400, "Bad Request", "Unknown format: foo",
1130                                     self.PUT, "/uri?format=foo",
1131                                     contents)
1132
1133     def test_PUT_NEWFILEURL_range_bad(self):
1134         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
1135         target = self.public_url + "/foo/new.txt"
1136         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
1137                              "501 Not Implemented",
1138                              "Content-Range in PUT not yet supported",
1139                              # (and certainly not for immutable files)
1140                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
1141                              headers=headers)
1142         d.addCallback(lambda res:
1143                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
1144         return d
1145
1146     def test_PUT_NEWFILEURL_mutable(self):
1147         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1148                      self.NEWFILE_CONTENTS)
1149         # TODO: we lose the response code, so we can't check this
1150         #self.failUnlessReallyEqual(responsecode, 201)
1151         def _check_uri(res):
1152             u = uri.from_string_mutable_filenode(res)
1153             self.failUnless(u.is_mutable())
1154             self.failIf(u.is_readonly())
1155             return res
1156         d.addCallback(_check_uri)
1157         d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
1158         d.addCallback(lambda res:
1159                       self.failUnlessMutableChildContentsAre(self._foo_node,
1160                                                              u"new.txt",
1161                                                              self.NEWFILE_CONTENTS))
1162         return d
1163
1164     def test_PUT_NEWFILEURL_mutable_toobig(self):
1165         # It is okay to upload large mutable files, so we should be able
1166         # to do that.
1167         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1168                      "b" * (self.s.MUTABLE_SIZELIMIT + 1))
1169         return d
1170
1171     def test_PUT_NEWFILEURL_replace(self):
1172         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
1173         # TODO: we lose the response code, so we can't check this
1174         #self.failUnlessReallyEqual(responsecode, 200)
1175         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
1176         d.addCallback(lambda res:
1177                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1178                                                       self.NEWFILE_CONTENTS))
1179         return d
1180
1181     def test_PUT_NEWFILEURL_bad_t(self):
1182         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
1183                              "PUT to a file: bad t=bogus",
1184                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
1185                              "contents")
1186         return d
1187
1188     def test_PUT_NEWFILEURL_no_replace(self):
1189         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
1190                      self.NEWFILE_CONTENTS)
1191         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
1192                   "409 Conflict",
1193                   "There was already a child by that name, and you asked me "
1194                   "to not replace it")
1195         return d
1196
1197     def test_PUT_NEWFILEURL_mkdirs(self):
1198         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
1199         fn = self._foo_node
1200         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
1201         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
1202         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
1203         d.addCallback(lambda res:
1204                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
1205                                                       self.NEWFILE_CONTENTS))
1206         return d
1207
1208     def test_PUT_NEWFILEURL_blocked(self):
1209         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
1210                      self.NEWFILE_CONTENTS)
1211         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
1212                   "409 Conflict",
1213                   "Unable to create directory 'blockingfile': a file was in the way")
1214         return d
1215
1216     def test_PUT_NEWFILEURL_emptyname(self):
1217         # an empty pathname component (i.e. a double-slash) is disallowed
1218         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
1219                              "400 Bad Request",
1220                              "The webapi does not allow empty pathname components",
1221                              self.PUT, self.public_url + "/foo//new.txt", "")
1222         return d
1223
1224     def test_DELETE_FILEURL(self):
1225         d = self.DELETE(self.public_url + "/foo/bar.txt")
1226         d.addCallback(lambda res:
1227                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1228         return d
1229
1230     def test_DELETE_FILEURL_missing(self):
1231         d = self.DELETE(self.public_url + "/foo/missing")
1232         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
1233         return d
1234
1235     def test_DELETE_FILEURL_missing2(self):
1236         d = self.DELETE(self.public_url + "/missing/missing")
1237         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
1238         return d
1239
1240     def failUnlessHasBarDotTxtMetadata(self, res):
1241         data = simplejson.loads(res)
1242         self.failUnless(isinstance(data, list))
1243         self.failUnlessIn("metadata", data[1])
1244         self.failUnlessIn("tahoe", data[1]["metadata"])
1245         self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
1246         self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
1247         self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
1248                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
1249
1250     def test_GET_FILEURL_json(self):
1251         # twisted.web.http.parse_qs ignores any query args without an '=', so
1252         # I can't do "GET /path?json", I have to do "GET /path/t=json"
1253         # instead. This may make it tricky to emulate the S3 interface
1254         # completely.
1255         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
1256         def _check1(data):
1257             self.failUnlessIsBarJSON(data)
1258             self.failUnlessHasBarDotTxtMetadata(data)
1259             return
1260         d.addCallback(_check1)
1261         return d
1262
1263     def test_GET_FILEURL_json_mutable_type(self):
1264         # The JSON should include format, which says whether the
1265         # file is SDMF or MDMF
1266         d = self.PUT("/uri?format=mdmf",
1267                      self.NEWFILE_CONTENTS * 300000)
1268         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1269         def _got_json(json, version):
1270             data = simplejson.loads(json)
1271             assert "filenode" == data[0]
1272             data = data[1]
1273             assert isinstance(data, dict)
1274
1275             self.failUnlessIn("format", data)
1276             self.failUnlessEqual(data["format"], version)
1277
1278         d.addCallback(_got_json, "MDMF")
1279         # Now make an SDMF file and check that it is reported correctly.
1280         d.addCallback(lambda ignored:
1281             self.PUT("/uri?format=sdmf",
1282                       self.NEWFILE_CONTENTS * 300000))
1283         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1284         d.addCallback(_got_json, "SDMF")
1285         return d
1286
1287     def test_GET_FILEURL_json_mdmf(self):
1288         d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
1289         d.addCallback(self.failUnlessIsQuuxJSON)
1290         return d
1291
1292     def test_GET_FILEURL_json_missing(self):
1293         d = self.GET(self.public_url + "/foo/missing?json")
1294         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
1295         return d
1296
1297     def test_GET_FILEURL_uri(self):
1298         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
1299         def _check(res):
1300             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1301         d.addCallback(_check)
1302         d.addCallback(lambda res:
1303                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
1304         def _check2(res):
1305             # for now, for files, uris and readonly-uris are the same
1306             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1307         d.addCallback(_check2)
1308         return d
1309
1310     def test_GET_FILEURL_badtype(self):
1311         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
1312                                  "bad t=bogus",
1313                                  self.GET,
1314                                  self.public_url + "/foo/bar.txt?t=bogus")
1315         return d
1316
1317     def test_CSS_FILE(self):
1318         d = self.GET("/tahoe.css", followRedirect=True)
1319         def _check(res):
1320             CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
1321             self.failUnless(CSS_STYLE.search(res), res)
1322         d.addCallback(_check)
1323         return d
1324
1325     def test_GET_FILEURL_uri_missing(self):
1326         d = self.GET(self.public_url + "/foo/missing?t=uri")
1327         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
1328         return d
1329
1330     def _check_upload_and_mkdir_forms(self, html):
1331         # We should have a form to create a file, with radio buttons that allow
1332         # the user to toggle whether it is a CHK/LIT (default), SDMF, or MDMF file.
1333         self.failUnlessIn('name="t" value="upload"', html)
1334         self.failUnlessIn('input checked="checked" type="radio" id="upload-chk" value="chk" name="format"', html)
1335         self.failUnlessIn('input type="radio" id="upload-sdmf" value="sdmf" name="format"', html)
1336         self.failUnlessIn('input type="radio" id="upload-mdmf" value="mdmf" name="format"', html)
1337
1338         # We should also have the ability to create a mutable directory, with
1339         # radio buttons that allow the user to toggle whether it is an SDMF (default)
1340         # or MDMF directory.
1341         self.failUnlessIn('name="t" value="mkdir"', html)
1342         self.failUnlessIn('input checked="checked" type="radio" id="mkdir-sdmf" value="sdmf" name="format"', html)
1343         self.failUnlessIn('input type="radio" id="mkdir-mdmf" value="mdmf" name="format"', html)
1344
1345         self.failUnlessIn(FAVICON_MARKUP, html)
1346
1347     def test_GET_DIRECTORY_html(self):
1348         d = self.GET(self.public_url + "/foo", followRedirect=True)
1349         def _check(html):
1350             self.failUnlessIn('<div class="toolbar-item"><a href="../../..">Return to Welcome page</a></div>', html)
1351             self._check_upload_and_mkdir_forms(html)
1352             self.failUnlessIn("quux", html)
1353         d.addCallback(_check)
1354         return d
1355
1356     def test_GET_DIRECTORY_html_filenode_encoding(self):
1357         d = self.GET(self.public_url + "/foo", followRedirect=True)
1358         def _check(html):
1359             # Check if encoded entries are there
1360             self.failUnlessIn('@@named=/' + self._htmlname_urlencoded + '">'
1361                               + self._htmlname_escaped + '</a>', html)
1362             self.failUnlessIn('value="' + self._htmlname_escaped_attr + '"', html)
1363             self.failIfIn(self._htmlname_escaped_double, html)
1364             # Make sure that Nevow escaping actually works by checking for unsafe characters
1365             # and that '&' is escaped.
1366             for entity in '<>':
1367                 self.failUnlessIn(entity, self._htmlname_raw)
1368                 self.failIfIn(entity, self._htmlname_escaped)
1369             self.failUnlessIn('&', re.sub(r'&(amp|lt|gt|quot|apos);', '', self._htmlname_raw))
1370             self.failIfIn('&', re.sub(r'&(amp|lt|gt|quot|apos);', '', self._htmlname_escaped))
1371         d.addCallback(_check)
1372         return d
1373
1374     def test_GET_root_html(self):
1375         d = self.GET("/")
1376         d.addCallback(self._check_upload_and_mkdir_forms)
1377         return d
1378
1379     def test_GET_DIRURL(self):
1380         # the addSlash means we get a redirect here
1381         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1382         ROOT = "../../.."
1383         d = self.GET(self.public_url + "/foo", followRedirect=True)
1384         def _check(res):
1385             self.failUnlessIn('<a href="%s">Return to Welcome page' % ROOT, res)
1386
1387             # the FILE reference points to a URI, but it should end in bar.txt
1388             bar_url = ("%s/file/%s/@@named=/bar.txt" %
1389                        (ROOT, urllib.quote(self._bar_txt_uri)))
1390             get_bar = "".join([r'<td>FILE</td>',
1391                                r'\s+<td>',
1392                                r'<a href="%s">bar.txt</a>' % bar_url,
1393                                r'</td>',
1394                                r'\s+<td align="right">%d</td>' % len(self.BAR_CONTENTS),
1395                                ])
1396             self.failUnless(re.search(get_bar, res), res)
1397             for label in ['unlink', 'rename/move']:
1398                 for line in res.split("\n"):
1399                     # find the line that contains the relevant button for bar.txt
1400                     if ("form action" in line and
1401                         ('value="%s"' % (label,)) in line and
1402                         'value="bar.txt"' in line):
1403                         # the form target should use a relative URL
1404                         foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1405                         self.failUnlessIn('action="%s"' % foo_url, line)
1406                         # and the when_done= should too
1407                         #done_url = urllib.quote(???)
1408                         #self.failUnlessIn('name="when_done" value="%s"' % done_url, line)
1409
1410                         # 'unlink' needs to use POST because it directly has a side effect
1411                         if label == 'unlink':
1412                             self.failUnlessIn('method="post"', line)
1413                         break
1414                 else:
1415                     self.fail("unable to find '%s bar.txt' line" % (label,))
1416
1417             # the DIR reference just points to a URI
1418             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1419             get_sub = ((r'<td>DIR</td>')
1420                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1421             self.failUnless(re.search(get_sub, res), res)
1422         d.addCallback(_check)
1423
1424         # look at a readonly directory
1425         d.addCallback(lambda res:
1426                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
1427         def _check2(res):
1428             self.failUnlessIn("(read-only)", res)
1429             self.failIfIn("Upload a file", res)
1430         d.addCallback(_check2)
1431
1432         # and at a directory that contains a readonly directory
1433         d.addCallback(lambda res:
1434                       self.GET(self.public_url, followRedirect=True))
1435         def _check3(res):
1436             self.failUnless(re.search('<td>DIR-RO</td>'
1437                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1438         d.addCallback(_check3)
1439
1440         # and an empty directory
1441         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1442         def _check4(res):
1443             self.failUnlessIn("directory is empty", res)
1444             MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
1445             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1446         d.addCallback(_check4)
1447
1448         # and at a literal directory
1449         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1450         d.addCallback(lambda res:
1451                       self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1452         def _check5(res):
1453             self.failUnlessIn('(immutable)', res)
1454             self.failUnless(re.search('<td>FILE</td>'
1455                                       r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1456         d.addCallback(_check5)
1457         return d
1458
1459     def test_GET_DIRURL_badtype(self):
1460         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1461                                  400, "Bad Request",
1462                                  "bad t=bogus",
1463                                  self.GET,
1464                                  self.public_url + "/foo?t=bogus")
1465         return d
1466
1467     def test_GET_DIRURL_json(self):
1468         d = self.GET(self.public_url + "/foo?t=json")
1469         d.addCallback(self.failUnlessIsFooJSON)
1470         return d
1471
1472     def test_GET_DIRURL_json_format(self):
1473         d = self.PUT(self.public_url + \
1474                      "/foo/sdmf.txt?format=sdmf",
1475                      self.NEWFILE_CONTENTS * 300000)
1476         d.addCallback(lambda ignored:
1477             self.PUT(self.public_url + \
1478                      "/foo/mdmf.txt?format=mdmf",
1479                      self.NEWFILE_CONTENTS * 300000))
1480         # Now we have an MDMF and SDMF file in the directory. If we GET
1481         # its JSON, we should see their encodings.
1482         d.addCallback(lambda ignored:
1483             self.GET(self.public_url + "/foo?t=json"))
1484         def _got_json(json):
1485             data = simplejson.loads(json)
1486             assert data[0] == "dirnode"
1487
1488             data = data[1]
1489             kids = data['children']
1490
1491             mdmf_data = kids['mdmf.txt'][1]
1492             self.failUnlessIn("format", mdmf_data)
1493             self.failUnlessEqual(mdmf_data["format"], "MDMF")
1494
1495             sdmf_data = kids['sdmf.txt'][1]
1496             self.failUnlessIn("format", sdmf_data)
1497             self.failUnlessEqual(sdmf_data["format"], "SDMF")
1498         d.addCallback(_got_json)
1499         return d
1500
1501
1502     def test_POST_DIRURL_manifest_no_ophandle(self):
1503         d = self.shouldFail2(error.Error,
1504                              "test_POST_DIRURL_manifest_no_ophandle",
1505                              "400 Bad Request",
1506                              "slow operation requires ophandle=",
1507                              self.POST, self.public_url, t="start-manifest")
1508         return d
1509
1510     def test_POST_DIRURL_manifest(self):
1511         d = defer.succeed(None)
1512         def getman(ignored, output):
1513             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1514                           followRedirect=True)
1515             d.addCallback(self.wait_for_operation, "125")
1516             d.addCallback(self.get_operation_results, "125", output)
1517             return d
1518         d.addCallback(getman, None)
1519         def _got_html(manifest):
1520             self.failUnlessIn("Manifest of SI=", manifest)
1521             self.failUnlessIn("<td>sub</td>", manifest)
1522             self.failUnlessIn(self._sub_uri, manifest)
1523             self.failUnlessIn("<td>sub/baz.txt</td>", manifest)
1524             self.failUnlessIn(FAVICON_MARKUP, manifest)
1525         d.addCallback(_got_html)
1526
1527         # both t=status and unadorned GET should be identical
1528         d.addCallback(lambda res: self.GET("/operations/125"))
1529         d.addCallback(_got_html)
1530
1531         d.addCallback(getman, "html")
1532         d.addCallback(_got_html)
1533         d.addCallback(getman, "text")
1534         def _got_text(manifest):
1535             self.failUnlessIn("\nsub " + self._sub_uri + "\n", manifest)
1536             self.failUnlessIn("\nsub/baz.txt URI:CHK:", manifest)
1537         d.addCallback(_got_text)
1538         d.addCallback(getman, "JSON")
1539         def _got_json(res):
1540             data = res["manifest"]
1541             got = {}
1542             for (path_list, cap) in data:
1543                 got[tuple(path_list)] = cap
1544             self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
1545             self.failUnlessIn((u"sub", u"baz.txt"), got)
1546             self.failUnlessIn("finished", res)
1547             self.failUnlessIn("origin", res)
1548             self.failUnlessIn("storage-index", res)
1549             self.failUnlessIn("verifycaps", res)
1550             self.failUnlessIn("stats", res)
1551         d.addCallback(_got_json)
1552         return d
1553
1554     def test_POST_DIRURL_deepsize_no_ophandle(self):
1555         d = self.shouldFail2(error.Error,
1556                              "test_POST_DIRURL_deepsize_no_ophandle",
1557                              "400 Bad Request",
1558                              "slow operation requires ophandle=",
1559                              self.POST, self.public_url, t="start-deep-size")
1560         return d
1561
1562     def test_POST_DIRURL_deepsize(self):
1563         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1564                       followRedirect=True)
1565         d.addCallback(self.wait_for_operation, "126")
1566         d.addCallback(self.get_operation_results, "126", "json")
1567         def _got_json(data):
1568             self.failUnlessReallyEqual(data["finished"], True)
1569             size = data["size"]
1570             self.failUnless(size > 1000)
1571         d.addCallback(_got_json)
1572         d.addCallback(self.get_operation_results, "126", "text")
1573         def _got_text(res):
1574             mo = re.search(r'^size: (\d+)$', res, re.M)
1575             self.failUnless(mo, res)
1576             size = int(mo.group(1))
1577             # with directories, the size varies.
1578             self.failUnless(size > 1000)
1579         d.addCallback(_got_text)
1580         return d
1581
1582     def test_POST_DIRURL_deepstats_no_ophandle(self):
1583         d = self.shouldFail2(error.Error,
1584                              "test_POST_DIRURL_deepstats_no_ophandle",
1585                              "400 Bad Request",
1586                              "slow operation requires ophandle=",
1587                              self.POST, self.public_url, t="start-deep-stats")
1588         return d
1589
1590     def test_POST_DIRURL_deepstats(self):
1591         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1592                       followRedirect=True)
1593         d.addCallback(self.wait_for_operation, "127")
1594         d.addCallback(self.get_operation_results, "127", "json")
1595         def _got_json(stats):
1596             expected = {"count-immutable-files": 4,
1597                         "count-mutable-files": 2,
1598                         "count-literal-files": 0,
1599                         "count-files": 6,
1600                         "count-directories": 3,
1601                         "size-immutable-files": 76,
1602                         "size-literal-files": 0,
1603                         #"size-directories": 1912, # varies
1604                         #"largest-directory": 1590,
1605                         "largest-directory-children": 8,
1606                         "largest-immutable-file": 19,
1607                         }
1608             for k,v in expected.iteritems():
1609                 self.failUnlessReallyEqual(stats[k], v,
1610                                            "stats[%s] was %s, not %s" %
1611                                            (k, stats[k], v))
1612             self.failUnlessReallyEqual(stats["size-files-histogram"],
1613                                        [ [11, 31, 4] ])
1614         d.addCallback(_got_json)
1615         return d
1616
1617     def test_POST_DIRURL_stream_manifest(self):
1618         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1619         def _check(res):
1620             self.failUnless(res.endswith("\n"))
1621             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1622             self.failUnlessReallyEqual(len(units), 10)
1623             self.failUnlessEqual(units[-1]["type"], "stats")
1624             first = units[0]
1625             self.failUnlessEqual(first["path"], [])
1626             self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri)
1627             self.failUnlessEqual(first["type"], "directory")
1628             baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0]
1629             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1630             self.failIfEqual(baz["storage-index"], None)
1631             self.failIfEqual(baz["verifycap"], None)
1632             self.failIfEqual(baz["repaircap"], None)
1633             # XXX: Add quux and baz to this test.
1634             return
1635         d.addCallback(_check)
1636         return d
1637
1638     def test_GET_DIRURL_uri(self):
1639         d = self.GET(self.public_url + "/foo?t=uri")
1640         def _check(res):
1641             self.failUnlessReallyEqual(to_str(res), self._foo_uri)
1642         d.addCallback(_check)
1643         return d
1644
1645     def test_GET_DIRURL_readonly_uri(self):
1646         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1647         def _check(res):
1648             self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri)
1649         d.addCallback(_check)
1650         return d
1651
1652     def test_PUT_NEWDIRURL(self):
1653         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1654         d.addCallback(lambda res:
1655                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1656         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1657         d.addCallback(self.failUnlessNodeKeysAre, [])
1658         return d
1659
1660     def test_PUT_NEWDIRURL_mdmf(self):
1661         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1662         d.addCallback(lambda res:
1663                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1664         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1665         d.addCallback(lambda node:
1666             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1667         return d
1668
1669     def test_PUT_NEWDIRURL_sdmf(self):
1670         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=sdmf",
1671                      "")
1672         d.addCallback(lambda res:
1673                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1674         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1675         d.addCallback(lambda node:
1676             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1677         return d
1678
1679     def test_PUT_NEWDIRURL_bad_format(self):
1680         return self.shouldHTTPError("PUT_NEWDIRURL_bad_format",
1681                                     400, "Bad Request", "Unknown format: foo",
1682                                     self.PUT, self.public_url +
1683                                     "/foo/newdir=?t=mkdir&format=foo", "")
1684
1685     def test_POST_NEWDIRURL(self):
1686         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1687         d.addCallback(lambda res:
1688                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1689         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1690         d.addCallback(self.failUnlessNodeKeysAre, [])
1691         return d
1692
1693     def test_POST_NEWDIRURL_mdmf(self):
1694         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1695         d.addCallback(lambda res:
1696                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1697         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1698         d.addCallback(lambda node:
1699             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1700         return d
1701
1702     def test_POST_NEWDIRURL_sdmf(self):
1703         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=sdmf", "")
1704         d.addCallback(lambda res:
1705             self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1706         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1707         d.addCallback(lambda node:
1708             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1709         return d
1710
1711     def test_POST_NEWDIRURL_bad_format(self):
1712         return self.shouldHTTPError("POST_NEWDIRURL_bad_format",
1713                                     400, "Bad Request", "Unknown format: foo",
1714                                     self.POST2, self.public_url + \
1715                                     "/foo/newdir?t=mkdir&format=foo", "")
1716
1717     def test_POST_NEWDIRURL_emptyname(self):
1718         # an empty pathname component (i.e. a double-slash) is disallowed
1719         d = self.shouldFail2(error.Error, "POST_NEWDIRURL_emptyname",
1720                              "400 Bad Request",
1721                              "The webapi does not allow empty pathname components, i.e. a double slash",
1722                              self.POST, self.public_url + "//?t=mkdir")
1723         return d
1724
1725     def _do_POST_NEWDIRURL_initial_children_test(self, version=None):
1726         (newkids, caps) = self._create_initial_children()
1727         query = "/foo/newdir?t=mkdir-with-children"
1728         if version == MDMF_VERSION:
1729             query += "&format=mdmf"
1730         elif version == SDMF_VERSION:
1731             query += "&format=sdmf"
1732         else:
1733             version = SDMF_VERSION # for later
1734         d = self.POST2(self.public_url + query,
1735                        simplejson.dumps(newkids))
1736         def _check(uri):
1737             n = self.s.create_node_from_uri(uri.strip())
1738             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1739             self.failUnlessEqual(n._node.get_version(), version)
1740             d2.addCallback(lambda ign:
1741                            self.failUnlessROChildURIIs(n, u"child-imm",
1742                                                        caps['filecap1']))
1743             d2.addCallback(lambda ign:
1744                            self.failUnlessRWChildURIIs(n, u"child-mutable",
1745                                                        caps['filecap2']))
1746             d2.addCallback(lambda ign:
1747                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1748                                                        caps['filecap3']))
1749             d2.addCallback(lambda ign:
1750                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1751                                                        caps['unknown_rocap']))
1752             d2.addCallback(lambda ign:
1753                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1754                                                        caps['unknown_rwcap']))
1755             d2.addCallback(lambda ign:
1756                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1757                                                        caps['unknown_immcap']))
1758             d2.addCallback(lambda ign:
1759                            self.failUnlessRWChildURIIs(n, u"dirchild",
1760                                                        caps['dircap']))
1761             d2.addCallback(lambda ign:
1762                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1763                                                        caps['litdircap']))
1764             d2.addCallback(lambda ign:
1765                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1766                                                        caps['emptydircap']))
1767             return d2
1768         d.addCallback(_check)
1769         d.addCallback(lambda res:
1770                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1771         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1772         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1773         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1774         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1775         return d
1776
1777     def test_POST_NEWDIRURL_initial_children(self):
1778         return self._do_POST_NEWDIRURL_initial_children_test()
1779
1780     def test_POST_NEWDIRURL_initial_children_mdmf(self):
1781         return self._do_POST_NEWDIRURL_initial_children_test(MDMF_VERSION)
1782
1783     def test_POST_NEWDIRURL_initial_children_sdmf(self):
1784         return self._do_POST_NEWDIRURL_initial_children_test(SDMF_VERSION)
1785
1786     def test_POST_NEWDIRURL_initial_children_bad_format(self):
1787         (newkids, caps) = self._create_initial_children()
1788         return self.shouldHTTPError("POST_NEWDIRURL_initial_children_bad_format",
1789                                     400, "Bad Request", "Unknown format: foo",
1790                                     self.POST2, self.public_url + \
1791                                     "/foo/newdir?t=mkdir-with-children&format=foo",
1792                                     simplejson.dumps(newkids))
1793
1794     def test_POST_NEWDIRURL_immutable(self):
1795         (newkids, caps) = self._create_immutable_children()
1796         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1797                        simplejson.dumps(newkids))
1798         def _check(uri):
1799             n = self.s.create_node_from_uri(uri.strip())
1800             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1801             d2.addCallback(lambda ign:
1802                            self.failUnlessROChildURIIs(n, u"child-imm",
1803                                                        caps['filecap1']))
1804             d2.addCallback(lambda ign:
1805                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1806                                                        caps['unknown_immcap']))
1807             d2.addCallback(lambda ign:
1808                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
1809                                                        caps['immdircap']))
1810             d2.addCallback(lambda ign:
1811                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1812                                                        caps['litdircap']))
1813             d2.addCallback(lambda ign:
1814                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1815                                                        caps['emptydircap']))
1816             return d2
1817         d.addCallback(_check)
1818         d.addCallback(lambda res:
1819                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1820         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1821         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1822         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1823         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1824         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1825         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1826         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1827         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
1828         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1829         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
1830         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1831         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
1832         d.addErrback(self.explain_web_error)
1833         return d
1834
1835     def test_POST_NEWDIRURL_immutable_bad(self):
1836         (newkids, caps) = self._create_initial_children()
1837         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
1838                              "400 Bad Request",
1839                              "needed to be immutable but was not",
1840                              self.POST2,
1841                              self.public_url + "/foo/newdir?t=mkdir-immutable",
1842                              simplejson.dumps(newkids))
1843         return d
1844
1845     def test_PUT_NEWDIRURL_exists(self):
1846         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1847         d.addCallback(lambda res:
1848                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1849         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1850         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1851         return d
1852
1853     def test_PUT_NEWDIRURL_blocked(self):
1854         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1855                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1856                              self.PUT,
1857                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1858         d.addCallback(lambda res:
1859                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1860         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1861         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1862         return d
1863
1864     def test_PUT_NEWDIRURL_mkdirs(self):
1865         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1866         d.addCallback(lambda res:
1867                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1868         d.addCallback(lambda res:
1869                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1870         d.addCallback(lambda res:
1871                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1872         d.addCallback(self.failUnlessNodeKeysAre, [])
1873         return d
1874
1875     def test_PUT_NEWDIRURL_mkdirs_mdmf(self):
1876         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=mdmf", "")
1877         d.addCallback(lambda ignored:
1878             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1879         d.addCallback(lambda ignored:
1880             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1881         d.addCallback(lambda ignored:
1882             self._foo_node.get_child_at_path(u"subdir"))
1883         def _got_subdir(subdir):
1884             # XXX: What we want?
1885             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1886             self.failUnlessNodeHasChild(subdir, u"newdir")
1887             return subdir.get_child_at_path(u"newdir")
1888         d.addCallback(_got_subdir)
1889         d.addCallback(lambda newdir:
1890             self.failUnlessEqual(newdir._node.get_version(), MDMF_VERSION))
1891         return d
1892
1893     def test_PUT_NEWDIRURL_mkdirs_sdmf(self):
1894         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=sdmf", "")
1895         d.addCallback(lambda ignored:
1896             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1897         d.addCallback(lambda ignored:
1898             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1899         d.addCallback(lambda ignored:
1900             self._foo_node.get_child_at_path(u"subdir"))
1901         def _got_subdir(subdir):
1902             # XXX: What we want?
1903             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1904             self.failUnlessNodeHasChild(subdir, u"newdir")
1905             return subdir.get_child_at_path(u"newdir")
1906         d.addCallback(_got_subdir)
1907         d.addCallback(lambda newdir:
1908             self.failUnlessEqual(newdir._node.get_version(), SDMF_VERSION))
1909         return d
1910
1911     def test_PUT_NEWDIRURL_mkdirs_bad_format(self):
1912         return self.shouldHTTPError("PUT_NEWDIRURL_mkdirs_bad_format",
1913                                     400, "Bad Request", "Unknown format: foo",
1914                                     self.PUT, self.public_url + \
1915                                     "/foo/subdir/newdir?t=mkdir&format=foo",
1916                                     "")
1917
1918     def test_DELETE_DIRURL(self):
1919         d = self.DELETE(self.public_url + "/foo")
1920         d.addCallback(lambda res:
1921                       self.failIfNodeHasChild(self.public_root, u"foo"))
1922         return d
1923
1924     def test_DELETE_DIRURL_missing(self):
1925         d = self.DELETE(self.public_url + "/foo/missing")
1926         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1927         d.addCallback(lambda res:
1928                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1929         return d
1930
1931     def test_DELETE_DIRURL_missing2(self):
1932         d = self.DELETE(self.public_url + "/missing")
1933         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1934         return d
1935
1936     def dump_root(self):
1937         print "NODEWALK"
1938         w = webish.DirnodeWalkerMixin()
1939         def visitor(childpath, childnode, metadata):
1940             print childpath
1941         d = w.walk(self.public_root, visitor)
1942         return d
1943
1944     def failUnlessNodeKeysAre(self, node, expected_keys):
1945         for k in expected_keys:
1946             assert isinstance(k, unicode)
1947         d = node.list()
1948         def _check(children):
1949             self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
1950         d.addCallback(_check)
1951         return d
1952     def failUnlessNodeHasChild(self, node, name):
1953         assert isinstance(name, unicode)
1954         d = node.list()
1955         def _check(children):
1956             self.failUnlessIn(name, children)
1957         d.addCallback(_check)
1958         return d
1959     def failIfNodeHasChild(self, node, name):
1960         assert isinstance(name, unicode)
1961         d = node.list()
1962         def _check(children):
1963             self.failIfIn(name, children)
1964         d.addCallback(_check)
1965         return d
1966
1967     def failUnlessChildContentsAre(self, node, name, expected_contents):
1968         assert isinstance(name, unicode)
1969         d = node.get_child_at_path(name)
1970         d.addCallback(lambda node: download_to_data(node))
1971         def _check(contents):
1972             self.failUnlessReallyEqual(contents, expected_contents)
1973         d.addCallback(_check)
1974         return d
1975
1976     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1977         assert isinstance(name, unicode)
1978         d = node.get_child_at_path(name)
1979         d.addCallback(lambda node: node.download_best_version())
1980         def _check(contents):
1981             self.failUnlessReallyEqual(contents, expected_contents)
1982         d.addCallback(_check)
1983         return d
1984
1985     def failUnlessRWChildURIIs(self, node, name, expected_uri):
1986         assert isinstance(name, unicode)
1987         d = node.get_child_at_path(name)
1988         def _check(child):
1989             self.failUnless(child.is_unknown() or not child.is_readonly())
1990             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1991             self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
1992             expected_ro_uri = self._make_readonly(expected_uri)
1993             if expected_ro_uri:
1994                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1995         d.addCallback(_check)
1996         return d
1997
1998     def failUnlessROChildURIIs(self, node, name, expected_uri):
1999         assert isinstance(name, unicode)
2000         d = node.get_child_at_path(name)
2001         def _check(child):
2002             self.failUnless(child.is_unknown() or child.is_readonly())
2003             self.failUnlessReallyEqual(child.get_write_uri(), None)
2004             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
2005             self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
2006         d.addCallback(_check)
2007         return d
2008
2009     def failUnlessURIMatchesRWChild(self, got_uri, node, name):
2010         assert isinstance(name, unicode)
2011         d = node.get_child_at_path(name)
2012         def _check(child):
2013             self.failUnless(child.is_unknown() or not child.is_readonly())
2014             self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
2015             self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
2016             expected_ro_uri = self._make_readonly(got_uri)
2017             if expected_ro_uri:
2018                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
2019         d.addCallback(_check)
2020         return d
2021
2022     def failUnlessURIMatchesROChild(self, got_uri, node, name):
2023         assert isinstance(name, unicode)
2024         d = node.get_child_at_path(name)
2025         def _check(child):
2026             self.failUnless(child.is_unknown() or child.is_readonly())
2027             self.failUnlessReallyEqual(child.get_write_uri(), None)
2028             self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
2029             self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
2030         d.addCallback(_check)
2031         return d
2032
2033     def failUnlessCHKURIHasContents(self, got_uri, contents):
2034         self.failUnless(self.get_all_contents()[got_uri] == contents)
2035
2036     def test_POST_upload(self):
2037         d = self.POST(self.public_url + "/foo", t="upload",
2038                       file=("new.txt", self.NEWFILE_CONTENTS))
2039         fn = self._foo_node
2040         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2041         d.addCallback(lambda res:
2042                       self.failUnlessChildContentsAre(fn, u"new.txt",
2043                                                       self.NEWFILE_CONTENTS))
2044         return d
2045
2046     def test_POST_upload_unicode(self):
2047         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2048         d = self.POST(self.public_url + "/foo", t="upload",
2049                       file=(filename, self.NEWFILE_CONTENTS))
2050         fn = self._foo_node
2051         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2052         d.addCallback(lambda res:
2053                       self.failUnlessChildContentsAre(fn, filename,
2054                                                       self.NEWFILE_CONTENTS))
2055         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2056         d.addCallback(lambda res: self.GET(target_url))
2057         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2058                                                                   self.NEWFILE_CONTENTS,
2059                                                                   contents))
2060         return d
2061
2062     def test_POST_upload_unicode_named(self):
2063         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2064         d = self.POST(self.public_url + "/foo", t="upload",
2065                       name=filename,
2066                       file=("overridden", self.NEWFILE_CONTENTS))
2067         fn = self._foo_node
2068         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2069         d.addCallback(lambda res:
2070                       self.failUnlessChildContentsAre(fn, filename,
2071                                                       self.NEWFILE_CONTENTS))
2072         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2073         d.addCallback(lambda res: self.GET(target_url))
2074         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2075                                                                   self.NEWFILE_CONTENTS,
2076                                                                   contents))
2077         return d
2078
2079     def test_POST_upload_no_link(self):
2080         d = self.POST("/uri", t="upload",
2081                       file=("new.txt", self.NEWFILE_CONTENTS))
2082         def _check_upload_results(page):
2083             # this should be a page which describes the results of the upload
2084             # that just finished.
2085             self.failUnlessIn("Upload Results:", page)
2086             self.failUnlessIn("URI:", page)
2087             uri_re = re.compile("URI: <tt><span>(.*)</span>")
2088             mo = uri_re.search(page)
2089             self.failUnless(mo, page)
2090             new_uri = mo.group(1)
2091             return new_uri
2092         d.addCallback(_check_upload_results)
2093         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
2094         return d
2095
2096     def test_POST_upload_no_link_whendone(self):
2097         d = self.POST("/uri", t="upload", when_done="/",
2098                       file=("new.txt", self.NEWFILE_CONTENTS))
2099         d.addBoth(self.shouldRedirect, "/")
2100         return d
2101
2102     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
2103         d = defer.maybeDeferred(callable, *args, **kwargs)
2104         def done(res):
2105             if isinstance(res, failure.Failure):
2106                 res.trap(error.PageRedirect)
2107                 statuscode = res.value.status
2108                 target = res.value.location
2109                 return checker(statuscode, target)
2110             self.fail("%s: callable was supposed to redirect, not return '%s'"
2111                       % (which, res))
2112         d.addBoth(done)
2113         return d
2114
2115     def test_POST_upload_no_link_whendone_results(self):
2116         def check(statuscode, target):
2117             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2118             self.failUnless(target.startswith(self.webish_url), target)
2119             return client.getPage(target, method="GET")
2120         # We encode "uri" as "%75ri" to exercise a case affected by ticket #1860.
2121         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
2122                                  check,
2123                                  self.POST, "/uri", t="upload",
2124                                  when_done="/%75ri/%(uri)s",
2125                                  file=("new.txt", self.NEWFILE_CONTENTS))
2126         d.addCallback(lambda res:
2127                       self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
2128         return d
2129
2130     def test_POST_upload_no_link_mutable(self):
2131         d = self.POST("/uri", t="upload", mutable="true",
2132                       file=("new.txt", self.NEWFILE_CONTENTS))
2133         def _check(filecap):
2134             filecap = filecap.strip()
2135             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2136             self.filecap = filecap
2137             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2138             self.failUnlessIn(u.get_storage_index(), self.get_all_contents())
2139             n = self.s.create_node_from_uri(filecap)
2140             return n.download_best_version()
2141         d.addCallback(_check)
2142         def _check2(data):
2143             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2144             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2145         d.addCallback(_check2)
2146         def _check3(data):
2147             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2148             return self.GET("/file/%s" % urllib.quote(self.filecap))
2149         d.addCallback(_check3)
2150         def _check4(data):
2151             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2152         d.addCallback(_check4)
2153         return d
2154
2155     def test_POST_upload_no_link_mutable_toobig(self):
2156         # The SDMF size limit is no longer in place, so we should be
2157         # able to upload mutable files that are as large as we want them
2158         # to be.
2159         d = self.POST("/uri", t="upload", mutable="true",
2160                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2161         return d
2162
2163
2164     def test_POST_upload_format_unlinked(self):
2165         def _check_upload_unlinked(ign, format, uri_prefix):
2166             filename = format + ".txt"
2167             d = self.POST("/uri?t=upload&format=" + format,
2168                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2169             def _got_results(results):
2170                 if format.upper() in ("SDMF", "MDMF"):
2171                     # webapi.rst says this returns a filecap
2172                     filecap = results
2173                 else:
2174                     # for immutable, it returns an "upload results page", and
2175                     # the filecap is buried inside
2176                     line = [l for l in results.split("\n") if "URI: " in l][0]
2177                     mo = re.search(r'<span>([^<]+)</span>', line)
2178                     filecap = mo.group(1)
2179                 self.failUnless(filecap.startswith(uri_prefix),
2180                                 (uri_prefix, filecap))
2181                 return self.GET("/uri/%s?t=json" % filecap)
2182             d.addCallback(_got_results)
2183             def _got_json(json):
2184                 data = simplejson.loads(json)
2185                 data = data[1]
2186                 self.failUnlessIn("format", data)
2187                 self.failUnlessEqual(data["format"], format.upper())
2188             d.addCallback(_got_json)
2189             return d
2190         d = defer.succeed(None)
2191         d.addCallback(_check_upload_unlinked, "chk", "URI:CHK")
2192         d.addCallback(_check_upload_unlinked, "CHK", "URI:CHK")
2193         d.addCallback(_check_upload_unlinked, "sdmf", "URI:SSK")
2194         d.addCallback(_check_upload_unlinked, "mdmf", "URI:MDMF")
2195         return d
2196
2197     def test_POST_upload_bad_format_unlinked(self):
2198         return self.shouldHTTPError("POST_upload_bad_format_unlinked",
2199                                     400, "Bad Request", "Unknown format: foo",
2200                                     self.POST,
2201                                     "/uri?t=upload&format=foo",
2202                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2203
2204     def test_POST_upload_format(self):
2205         def _check_upload(ign, format, uri_prefix, fn=None):
2206             filename = format + ".txt"
2207             d = self.POST(self.public_url +
2208                           "/foo?t=upload&format=" + format,
2209                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2210             def _got_filecap(filecap):
2211                 if fn is not None:
2212                     filenameu = unicode(filename)
2213                     self.failUnlessURIMatchesRWChild(filecap, fn, filenameu)
2214                 self.failUnless(filecap.startswith(uri_prefix))
2215                 return self.GET(self.public_url + "/foo/%s?t=json" % filename)
2216             d.addCallback(_got_filecap)
2217             def _got_json(json):
2218                 data = simplejson.loads(json)
2219                 data = data[1]
2220                 self.failUnlessIn("format", data)
2221                 self.failUnlessEqual(data["format"], format.upper())
2222             d.addCallback(_got_json)
2223             return d
2224
2225         d = defer.succeed(None)
2226         d.addCallback(_check_upload, "chk", "URI:CHK")
2227         d.addCallback(_check_upload, "sdmf", "URI:SSK", self._foo_node)
2228         d.addCallback(_check_upload, "mdmf", "URI:MDMF")
2229         d.addCallback(_check_upload, "MDMF", "URI:MDMF")
2230         return d
2231
2232     def test_POST_upload_bad_format(self):
2233         return self.shouldHTTPError("POST_upload_bad_format",
2234                                     400, "Bad Request", "Unknown format: foo",
2235                                     self.POST, self.public_url + \
2236                                     "/foo?t=upload&format=foo",
2237                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2238
2239     def test_POST_upload_mutable(self):
2240         # this creates a mutable file
2241         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
2242                       file=("new.txt", self.NEWFILE_CONTENTS))
2243         fn = self._foo_node
2244         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2245         d.addCallback(lambda res:
2246                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2247                                                              self.NEWFILE_CONTENTS))
2248         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2249         def _got(newnode):
2250             self.failUnless(IMutableFileNode.providedBy(newnode))
2251             self.failUnless(newnode.is_mutable())
2252             self.failIf(newnode.is_readonly())
2253             self._mutable_node = newnode
2254             self._mutable_uri = newnode.get_uri()
2255         d.addCallback(_got)
2256
2257         # now upload it again and make sure that the URI doesn't change
2258         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
2259         d.addCallback(lambda res:
2260                       self.POST(self.public_url + "/foo", t="upload",
2261                                 mutable="true",
2262                                 file=("new.txt", NEWER_CONTENTS)))
2263         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2264         d.addCallback(lambda res:
2265                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2266                                                              NEWER_CONTENTS))
2267         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2268         def _got2(newnode):
2269             self.failUnless(IMutableFileNode.providedBy(newnode))
2270             self.failUnless(newnode.is_mutable())
2271             self.failIf(newnode.is_readonly())
2272             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2273         d.addCallback(_got2)
2274
2275         # upload a second time, using PUT instead of POST
2276         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
2277         d.addCallback(lambda res:
2278                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
2279         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2280         d.addCallback(lambda res:
2281                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2282                                                              NEW2_CONTENTS))
2283
2284         # finally list the directory, since mutable files are displayed
2285         # slightly differently
2286
2287         d.addCallback(lambda res:
2288                       self.GET(self.public_url + "/foo/",
2289                                followRedirect=True))
2290         def _check_page(res):
2291             # TODO: assert more about the contents
2292             self.failUnlessIn("SSK", res)
2293             return res
2294         d.addCallback(_check_page)
2295
2296         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2297         def _got3(newnode):
2298             self.failUnless(IMutableFileNode.providedBy(newnode))
2299             self.failUnless(newnode.is_mutable())
2300             self.failIf(newnode.is_readonly())
2301             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2302         d.addCallback(_got3)
2303
2304         # look at the JSON form of the enclosing directory
2305         d.addCallback(lambda res:
2306                       self.GET(self.public_url + "/foo/?t=json",
2307                                followRedirect=True))
2308         def _check_page_json(res):
2309             parsed = simplejson.loads(res)
2310             self.failUnlessEqual(parsed[0], "dirnode")
2311             children = dict( [(unicode(name),value)
2312                               for (name,value)
2313                               in parsed[1]["children"].iteritems()] )
2314             self.failUnlessIn(u"new.txt", children)
2315             new_json = children[u"new.txt"]
2316             self.failUnlessEqual(new_json[0], "filenode")
2317             self.failUnless(new_json[1]["mutable"])
2318             self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri)
2319             ro_uri = self._mutable_node.get_readonly().to_string()
2320             self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri)
2321         d.addCallback(_check_page_json)
2322
2323         # and the JSON form of the file
2324         d.addCallback(lambda res:
2325                       self.GET(self.public_url + "/foo/new.txt?t=json"))
2326         def _check_file_json(res):
2327             parsed = simplejson.loads(res)
2328             self.failUnlessEqual(parsed[0], "filenode")
2329             self.failUnless(parsed[1]["mutable"])
2330             self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri)
2331             ro_uri = self._mutable_node.get_readonly().to_string()
2332             self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri)
2333         d.addCallback(_check_file_json)
2334
2335         # and look at t=uri and t=readonly-uri
2336         d.addCallback(lambda res:
2337                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
2338         d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
2339         d.addCallback(lambda res:
2340                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
2341         def _check_ro_uri(res):
2342             ro_uri = self._mutable_node.get_readonly().to_string()
2343             self.failUnlessReallyEqual(res, ro_uri)
2344         d.addCallback(_check_ro_uri)
2345
2346         # make sure we can get to it from /uri/URI
2347         d.addCallback(lambda res:
2348                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
2349         d.addCallback(lambda res:
2350                       self.failUnlessReallyEqual(res, NEW2_CONTENTS))
2351
2352         # and that HEAD computes the size correctly
2353         d.addCallback(lambda res:
2354                       self.HEAD(self.public_url + "/foo/new.txt",
2355                                 return_response=True))
2356         def _got_headers((res, status, headers)):
2357             self.failUnlessReallyEqual(res, "")
2358             self.failUnlessReallyEqual(headers["content-length"][0],
2359                                        str(len(NEW2_CONTENTS)))
2360             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
2361         d.addCallback(_got_headers)
2362
2363         # make sure that outdated size limits aren't enforced anymore.
2364         d.addCallback(lambda ignored:
2365             self.POST(self.public_url + "/foo", t="upload",
2366                       mutable="true",
2367                       file=("new.txt",
2368                             "b" * (self.s.MUTABLE_SIZELIMIT+1))))
2369         d.addErrback(self.dump_error)
2370         return d
2371
2372     def test_POST_upload_mutable_toobig(self):
2373         # SDMF had a size limti that was removed a while ago. MDMF has
2374         # never had a size limit. Test to make sure that we do not
2375         # encounter errors when trying to upload large mutable files,
2376         # since there should be no coded prohibitions regarding large
2377         # mutable files.
2378         d = self.POST(self.public_url + "/foo",
2379                       t="upload", mutable="true",
2380                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2381         return d
2382
2383     def dump_error(self, f):
2384         # if the web server returns an error code (like 400 Bad Request),
2385         # web.client.getPage puts the HTTP response body into the .response
2386         # attribute of the exception object that it gives back. It does not
2387         # appear in the Failure's repr(), so the ERROR that trial displays
2388         # will be rather terse and unhelpful. addErrback this method to the
2389         # end of your chain to get more information out of these errors.
2390         if f.check(error.Error):
2391             print "web.error.Error:"
2392             print f
2393             print f.value.response
2394         return f
2395
2396     def test_POST_upload_replace(self):
2397         d = self.POST(self.public_url + "/foo", t="upload",
2398                       file=("bar.txt", self.NEWFILE_CONTENTS))
2399         fn = self._foo_node
2400         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
2401         d.addCallback(lambda res:
2402                       self.failUnlessChildContentsAre(fn, u"bar.txt",
2403                                                       self.NEWFILE_CONTENTS))
2404         return d
2405
2406     def test_POST_upload_no_replace_ok(self):
2407         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2408                       file=("new.txt", self.NEWFILE_CONTENTS))
2409         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
2410         d.addCallback(lambda res: self.failUnlessReallyEqual(res,
2411                                                              self.NEWFILE_CONTENTS))
2412         return d
2413
2414     def test_POST_upload_no_replace_queryarg(self):
2415         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2416                       file=("bar.txt", self.NEWFILE_CONTENTS))
2417         d.addBoth(self.shouldFail, error.Error,
2418                   "POST_upload_no_replace_queryarg",
2419                   "409 Conflict",
2420                   "There was already a child by that name, and you asked me "
2421                   "to not replace it")
2422         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2423         d.addCallback(self.failUnlessIsBarDotTxt)
2424         return d
2425
2426     def test_POST_upload_no_replace_field(self):
2427         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
2428                       file=("bar.txt", self.NEWFILE_CONTENTS))
2429         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
2430                   "409 Conflict",
2431                   "There was already a child by that name, and you asked me "
2432                   "to not replace it")
2433         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2434         d.addCallback(self.failUnlessIsBarDotTxt)
2435         return d
2436
2437     def test_POST_upload_whendone(self):
2438         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
2439                       file=("new.txt", self.NEWFILE_CONTENTS))
2440         d.addBoth(self.shouldRedirect, "/THERE")
2441         fn = self._foo_node
2442         d.addCallback(lambda res:
2443                       self.failUnlessChildContentsAre(fn, u"new.txt",
2444                                                       self.NEWFILE_CONTENTS))
2445         return d
2446
2447     def test_POST_upload_named(self):
2448         fn = self._foo_node
2449         d = self.POST(self.public_url + "/foo", t="upload",
2450                       name="new.txt", file=self.NEWFILE_CONTENTS)
2451         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2452         d.addCallback(lambda res:
2453                       self.failUnlessChildContentsAre(fn, u"new.txt",
2454                                                       self.NEWFILE_CONTENTS))
2455         return d
2456
2457     def test_POST_upload_named_badfilename(self):
2458         d = self.POST(self.public_url + "/foo", t="upload",
2459                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
2460         d.addBoth(self.shouldFail, error.Error,
2461                   "test_POST_upload_named_badfilename",
2462                   "400 Bad Request",
2463                   "name= may not contain a slash",
2464                   )
2465         # make sure that nothing was added
2466         d.addCallback(lambda res:
2467                       self.failUnlessNodeKeysAre(self._foo_node,
2468                                                  [self._htmlname_unicode,
2469                                                   u"bar.txt", u"baz.txt", u"blockingfile",
2470                                                   u"empty", u"n\u00fc.txt", u"quux.txt",
2471                                                   u"sub"]))
2472         return d
2473
2474     def test_POST_FILEURL_check(self):
2475         bar_url = self.public_url + "/foo/bar.txt"
2476         d = self.POST(bar_url, t="check")
2477         def _check(res):
2478             self.failUnlessIn("Healthy :", res)
2479         d.addCallback(_check)
2480         redir_url = "http://allmydata.org/TARGET"
2481         def _check2(statuscode, target):
2482             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2483             self.failUnlessReallyEqual(target, redir_url)
2484         d.addCallback(lambda res:
2485                       self.shouldRedirect2("test_POST_FILEURL_check",
2486                                            _check2,
2487                                            self.POST, bar_url,
2488                                            t="check",
2489                                            when_done=redir_url))
2490         d.addCallback(lambda res:
2491                       self.POST(bar_url, t="check", return_to=redir_url))
2492         def _check3(res):
2493             self.failUnlessIn("Healthy :", res)
2494             self.failUnlessIn("Return to file", res)
2495             self.failUnlessIn(redir_url, res)
2496         d.addCallback(_check3)
2497
2498         d.addCallback(lambda res:
2499                       self.POST(bar_url, t="check", output="JSON"))
2500         def _check_json(res):
2501             data = simplejson.loads(res)
2502             self.failUnlessIn("storage-index", data)
2503             self.failUnless(data["results"]["healthy"])
2504         d.addCallback(_check_json)
2505
2506         return d
2507
2508     def test_POST_FILEURL_check_and_repair(self):
2509         bar_url = self.public_url + "/foo/bar.txt"
2510         d = self.POST(bar_url, t="check", repair="true")
2511         def _check(res):
2512             self.failUnlessIn("Healthy :", res)
2513         d.addCallback(_check)
2514         redir_url = "http://allmydata.org/TARGET"
2515         def _check2(statuscode, target):
2516             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2517             self.failUnlessReallyEqual(target, redir_url)
2518         d.addCallback(lambda res:
2519                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
2520                                            _check2,
2521                                            self.POST, bar_url,
2522                                            t="check", repair="true",
2523                                            when_done=redir_url))
2524         d.addCallback(lambda res:
2525                       self.POST(bar_url, t="check", return_to=redir_url))
2526         def _check3(res):
2527             self.failUnlessIn("Healthy :", res)
2528             self.failUnlessIn("Return to file", res)
2529             self.failUnlessIn(redir_url, res)
2530         d.addCallback(_check3)
2531         return d
2532
2533     def test_POST_DIRURL_check(self):
2534         foo_url = self.public_url + "/foo/"
2535         d = self.POST(foo_url, t="check")
2536         def _check(res):
2537             self.failUnlessIn("Healthy :", res)
2538         d.addCallback(_check)
2539         redir_url = "http://allmydata.org/TARGET"
2540         def _check2(statuscode, target):
2541             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2542             self.failUnlessReallyEqual(target, redir_url)
2543         d.addCallback(lambda res:
2544                       self.shouldRedirect2("test_POST_DIRURL_check",
2545                                            _check2,
2546                                            self.POST, foo_url,
2547                                            t="check",
2548                                            when_done=redir_url))
2549         d.addCallback(lambda res:
2550                       self.POST(foo_url, t="check", return_to=redir_url))
2551         def _check3(res):
2552             self.failUnlessIn("Healthy :", res)
2553             self.failUnlessIn("Return to file/directory", res)
2554             self.failUnlessIn(redir_url, res)
2555         d.addCallback(_check3)
2556
2557         d.addCallback(lambda res:
2558                       self.POST(foo_url, t="check", output="JSON"))
2559         def _check_json(res):
2560             data = simplejson.loads(res)
2561             self.failUnlessIn("storage-index", data)
2562             self.failUnless(data["results"]["healthy"])
2563         d.addCallback(_check_json)
2564
2565         return d
2566
2567     def test_POST_DIRURL_check_and_repair(self):
2568         foo_url = self.public_url + "/foo/"
2569         d = self.POST(foo_url, t="check", repair="true")
2570         def _check(res):
2571             self.failUnlessIn("Healthy :", res)
2572         d.addCallback(_check)
2573         redir_url = "http://allmydata.org/TARGET"
2574         def _check2(statuscode, target):
2575             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2576             self.failUnlessReallyEqual(target, redir_url)
2577         d.addCallback(lambda res:
2578                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2579                                            _check2,
2580                                            self.POST, foo_url,
2581                                            t="check", repair="true",
2582                                            when_done=redir_url))
2583         d.addCallback(lambda res:
2584                       self.POST(foo_url, t="check", return_to=redir_url))
2585         def _check3(res):
2586             self.failUnlessIn("Healthy :", res)
2587             self.failUnlessIn("Return to file/directory", res)
2588             self.failUnlessIn(redir_url, res)
2589         d.addCallback(_check3)
2590         return d
2591
2592     def test_POST_FILEURL_mdmf_check(self):
2593         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2594         d = self.POST(quux_url, t="check")
2595         def _check(res):
2596             self.failUnlessIn("Healthy", res)
2597         d.addCallback(_check)
2598         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2599         d.addCallback(lambda ignored:
2600                       self.POST(quux_extension_url, t="check"))
2601         d.addCallback(_check)
2602         return d
2603
2604     def test_POST_FILEURL_mdmf_check_and_repair(self):
2605         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2606         d = self.POST(quux_url, t="check", repair="true")
2607         def _check(res):
2608             self.failUnlessIn("Healthy", res)
2609         d.addCallback(_check)
2610         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2611         d.addCallback(lambda ignored:
2612                       self.POST(quux_extension_url, t="check", repair="true"))
2613         d.addCallback(_check)
2614         return d
2615
2616     def wait_for_operation(self, ignored, ophandle):
2617         url = "/operations/" + ophandle
2618         url += "?t=status&output=JSON"
2619         d = self.GET(url)
2620         def _got(res):
2621             data = simplejson.loads(res)
2622             if not data["finished"]:
2623                 d = self.stall(delay=1.0)
2624                 d.addCallback(self.wait_for_operation, ophandle)
2625                 return d
2626             return data
2627         d.addCallback(_got)
2628         return d
2629
2630     def get_operation_results(self, ignored, ophandle, output=None):
2631         url = "/operations/" + ophandle
2632         url += "?t=status"
2633         if output:
2634             url += "&output=" + output
2635         d = self.GET(url)
2636         def _got(res):
2637             if output and output.lower() == "json":
2638                 return simplejson.loads(res)
2639             return res
2640         d.addCallback(_got)
2641         return d
2642
2643     def test_POST_DIRURL_deepcheck_no_ophandle(self):
2644         d = self.shouldFail2(error.Error,
2645                              "test_POST_DIRURL_deepcheck_no_ophandle",
2646                              "400 Bad Request",
2647                              "slow operation requires ophandle=",
2648                              self.POST, self.public_url, t="start-deep-check")
2649         return d
2650
2651     def test_POST_DIRURL_deepcheck(self):
2652         def _check_redirect(statuscode, target):
2653             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2654             self.failUnless(target.endswith("/operations/123"))
2655         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2656                                  self.POST, self.public_url,
2657                                  t="start-deep-check", ophandle="123")
2658         d.addCallback(self.wait_for_operation, "123")
2659         def _check_json(data):
2660             self.failUnlessReallyEqual(data["finished"], True)
2661             self.failUnlessReallyEqual(data["count-objects-checked"], 11)
2662             self.failUnlessReallyEqual(data["count-objects-healthy"], 11)
2663         d.addCallback(_check_json)
2664         d.addCallback(self.get_operation_results, "123", "html")
2665         def _check_html(res):
2666             self.failUnlessIn("Objects Checked: <span>11</span>", res)
2667             self.failUnlessIn("Objects Healthy: <span>11</span>", res)
2668             self.failUnlessIn(FAVICON_MARKUP, res)
2669         d.addCallback(_check_html)
2670
2671         d.addCallback(lambda res:
2672                       self.GET("/operations/123/"))
2673         d.addCallback(_check_html) # should be the same as without the slash
2674
2675         d.addCallback(lambda res:
2676                       self.shouldFail2(error.Error, "one", "404 Not Found",
2677                                        "No detailed results for SI bogus",
2678                                        self.GET, "/operations/123/bogus"))
2679
2680         foo_si = self._foo_node.get_storage_index()
2681         foo_si_s = base32.b2a(foo_si)
2682         d.addCallback(lambda res:
2683                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2684         def _check_foo_json(res):
2685             data = simplejson.loads(res)
2686             self.failUnlessEqual(data["storage-index"], foo_si_s)
2687             self.failUnless(data["results"]["healthy"])
2688         d.addCallback(_check_foo_json)
2689         return d
2690
2691     def test_POST_DIRURL_deepcheck_and_repair(self):
2692         d = self.POST(self.public_url, t="start-deep-check", repair="true",
2693                       ophandle="124", output="json", followRedirect=True)
2694         d.addCallback(self.wait_for_operation, "124")
2695         def _check_json(data):
2696             self.failUnlessReallyEqual(data["finished"], True)
2697             self.failUnlessReallyEqual(data["count-objects-checked"], 11)
2698             self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 11)
2699             self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2700             self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2701             self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2702             self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2703             self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2704             self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 11)
2705             self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2706             self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2707         d.addCallback(_check_json)
2708         d.addCallback(self.get_operation_results, "124", "html")
2709         def _check_html(res):
2710             self.failUnlessIn("Objects Checked: <span>11</span>", res)
2711
2712             self.failUnlessIn("Objects Healthy (before repair): <span>11</span>", res)
2713             self.failUnlessIn("Objects Unhealthy (before repair): <span>0</span>", res)
2714             self.failUnlessIn("Corrupt Shares (before repair): <span>0</span>", res)
2715
2716             self.failUnlessIn("Repairs Attempted: <span>0</span>", res)
2717             self.failUnlessIn("Repairs Successful: <span>0</span>", res)
2718             self.failUnlessIn("Repairs Unsuccessful: <span>0</span>", res)
2719
2720             self.failUnlessIn("Objects Healthy (after repair): <span>11</span>", res)
2721             self.failUnlessIn("Objects Unhealthy (after repair): <span>0</span>", res)
2722             self.failUnlessIn("Corrupt Shares (after repair): <span>0</span>", res)
2723
2724             self.failUnlessIn(FAVICON_MARKUP, res)
2725         d.addCallback(_check_html)
2726         return d
2727
2728     def test_POST_FILEURL_bad_t(self):
2729         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2730                              "POST to file: bad t=bogus",
2731                              self.POST, self.public_url + "/foo/bar.txt",
2732                              t="bogus")
2733         return d
2734
2735     def test_POST_mkdir(self): # return value?
2736         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2737         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2738         d.addCallback(self.failUnlessNodeKeysAre, [])
2739         return d
2740
2741     def test_POST_mkdir_mdmf(self):
2742         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=mdmf")
2743         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2744         d.addCallback(lambda node:
2745             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2746         return d
2747
2748     def test_POST_mkdir_sdmf(self):
2749         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=sdmf")
2750         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2751         d.addCallback(lambda node:
2752             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2753         return d
2754
2755     def test_POST_mkdir_bad_format(self):
2756         return self.shouldHTTPError("POST_mkdir_bad_format",
2757                                     400, "Bad Request", "Unknown format: foo",
2758                                     self.POST, self.public_url +
2759                                     "/foo?t=mkdir&name=newdir&format=foo")
2760
2761     def test_POST_mkdir_initial_children(self):
2762         (newkids, caps) = self._create_initial_children()
2763         d = self.POST2(self.public_url +
2764                        "/foo?t=mkdir-with-children&name=newdir",
2765                        simplejson.dumps(newkids))
2766         d.addCallback(lambda res:
2767                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2768         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2769         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2770         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2771         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2772         return d
2773
2774     def test_POST_mkdir_initial_children_mdmf(self):
2775         (newkids, caps) = self._create_initial_children()
2776         d = self.POST2(self.public_url +
2777                        "/foo?t=mkdir-with-children&name=newdir&format=mdmf",
2778                        simplejson.dumps(newkids))
2779         d.addCallback(lambda res:
2780                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2781         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2782         d.addCallback(lambda node:
2783             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2784         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2785         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2786                        caps['filecap1'])
2787         return d
2788
2789     # XXX: Duplication.
2790     def test_POST_mkdir_initial_children_sdmf(self):
2791         (newkids, caps) = self._create_initial_children()
2792         d = self.POST2(self.public_url +
2793                        "/foo?t=mkdir-with-children&name=newdir&format=sdmf",
2794                        simplejson.dumps(newkids))
2795         d.addCallback(lambda res:
2796                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2797         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2798         d.addCallback(lambda node:
2799             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2800         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2801         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2802                        caps['filecap1'])
2803         return d
2804
2805     def test_POST_mkdir_initial_children_bad_format(self):
2806         (newkids, caps) = self._create_initial_children()
2807         return self.shouldHTTPError("POST_mkdir_initial_children_bad_format",
2808                                     400, "Bad Request", "Unknown format: foo",
2809                                     self.POST, self.public_url + \
2810                                     "/foo?t=mkdir-with-children&name=newdir&format=foo",
2811                                     simplejson.dumps(newkids))
2812
2813     def test_POST_mkdir_immutable(self):
2814         (newkids, caps) = self._create_immutable_children()
2815         d = self.POST2(self.public_url +
2816                        "/foo?t=mkdir-immutable&name=newdir",
2817                        simplejson.dumps(newkids))
2818         d.addCallback(lambda res:
2819                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2820         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2821         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2822         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2823         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2824         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2825         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2826         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2827         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2828         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2829         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2830         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2831         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2832         return d
2833
2834     def test_POST_mkdir_immutable_bad(self):
2835         (newkids, caps) = self._create_initial_children()
2836         d = self.shouldFail2(error.Error, "POST_mkdir_immutable_bad",
2837                              "400 Bad Request",
2838                              "needed to be immutable but was not",
2839                              self.POST2,
2840                              self.public_url +
2841                              "/foo?t=mkdir-immutable&name=newdir",
2842                              simplejson.dumps(newkids))
2843         return d
2844
2845     def test_POST_mkdir_2(self):
2846         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
2847         d.addCallback(lambda res:
2848                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2849         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2850         d.addCallback(self.failUnlessNodeKeysAre, [])
2851         return d
2852
2853     def test_POST_mkdirs_2(self):
2854         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
2855         d.addCallback(lambda res:
2856                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
2857         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
2858         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
2859         d.addCallback(self.failUnlessNodeKeysAre, [])
2860         return d
2861
2862     def test_POST_mkdir_no_parentdir_noredirect(self):
2863         d = self.POST("/uri?t=mkdir")
2864         def _after_mkdir(res):
2865             uri.DirectoryURI.init_from_string(res)
2866         d.addCallback(_after_mkdir)
2867         return d
2868
2869     def test_POST_mkdir_no_parentdir_noredirect_mdmf(self):
2870         d = self.POST("/uri?t=mkdir&format=mdmf")
2871         def _after_mkdir(res):
2872             u = uri.from_string(res)
2873             # Check that this is an MDMF writecap
2874             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
2875         d.addCallback(_after_mkdir)
2876         return d
2877
2878     def test_POST_mkdir_no_parentdir_noredirect_sdmf(self):
2879         d = self.POST("/uri?t=mkdir&format=sdmf")
2880         def _after_mkdir(res):
2881             u = uri.from_string(res)
2882             self.failUnlessIsInstance(u, uri.DirectoryURI)
2883         d.addCallback(_after_mkdir)
2884         return d
2885
2886     def test_POST_mkdir_no_parentdir_noredirect_bad_format(self):
2887         return self.shouldHTTPError("POST_mkdir_no_parentdir_noredirect_bad_format",
2888                                     400, "Bad Request", "Unknown format: foo",
2889                                     self.POST, self.public_url +
2890                                     "/uri?t=mkdir&format=foo")
2891
2892     def test_POST_mkdir_no_parentdir_noredirect2(self):
2893         # make sure form-based arguments (as on the welcome page) still work
2894         d = self.POST("/uri", t="mkdir")
2895         def _after_mkdir(res):
2896             uri.DirectoryURI.init_from_string(res)
2897         d.addCallback(_after_mkdir)
2898         d.addErrback(self.explain_web_error)
2899         return d
2900
2901     def test_POST_mkdir_no_parentdir_redirect(self):
2902         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
2903         d.addBoth(self.shouldRedirect, None, statuscode='303')
2904         def _check_target(target):
2905             target = urllib.unquote(target)
2906             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2907         d.addCallback(_check_target)
2908         return d
2909
2910     def test_POST_mkdir_no_parentdir_redirect2(self):
2911         d = self.POST("/uri", t="mkdir", redirect_to_result="true")
2912         d.addBoth(self.shouldRedirect, None, statuscode='303')
2913         def _check_target(target):
2914             target = urllib.unquote(target)
2915             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2916         d.addCallback(_check_target)
2917         d.addErrback(self.explain_web_error)
2918         return d
2919
2920     def _make_readonly(self, u):
2921         ro_uri = uri.from_string(u).get_readonly()
2922         if ro_uri is None:
2923             return None
2924         return ro_uri.to_string()
2925
2926     def _create_initial_children(self):
2927         contents, n, filecap1 = self.makefile(12)
2928         md1 = {"metakey1": "metavalue1"}
2929         filecap2 = make_mutable_file_uri()
2930         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
2931         filecap3 = node3.get_readonly_uri()
2932         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
2933         dircap = DirectoryNode(node4, None, None).get_uri()
2934         mdmfcap = make_mutable_file_uri(mdmf=True)
2935         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2936         emptydircap = "URI:DIR2-LIT:"
2937         newkids = {u"child-imm":        ["filenode", {"rw_uri": filecap1,
2938                                                       "ro_uri": self._make_readonly(filecap1),
2939                                                       "metadata": md1, }],
2940                    u"child-mutable":    ["filenode", {"rw_uri": filecap2,
2941                                                       "ro_uri": self._make_readonly(filecap2)}],
2942                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
2943                    u"unknownchild-rw":  ["unknown",  {"rw_uri": unknown_rwcap,
2944                                                       "ro_uri": unknown_rocap}],
2945                    u"unknownchild-ro":  ["unknown",  {"ro_uri": unknown_rocap}],
2946                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2947                    u"dirchild":         ["dirnode",  {"rw_uri": dircap,
2948                                                       "ro_uri": self._make_readonly(dircap)}],
2949                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2950                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2951                    u"child-mutable-mdmf": ["filenode", {"rw_uri": mdmfcap,
2952                                                         "ro_uri": self._make_readonly(mdmfcap)}],
2953                    }
2954         return newkids, {'filecap1': filecap1,
2955                          'filecap2': filecap2,
2956                          'filecap3': filecap3,
2957                          'unknown_rwcap': unknown_rwcap,
2958                          'unknown_rocap': unknown_rocap,
2959                          'unknown_immcap': unknown_immcap,
2960                          'dircap': dircap,
2961                          'litdircap': litdircap,
2962                          'emptydircap': emptydircap,
2963                          'mdmfcap': mdmfcap}
2964
2965     def _create_immutable_children(self):
2966         contents, n, filecap1 = self.makefile(12)
2967         md1 = {"metakey1": "metavalue1"}
2968         tnode = create_chk_filenode("immutable directory contents\n"*10,
2969                                     self.get_all_contents())
2970         dnode = DirectoryNode(tnode, None, None)
2971         assert not dnode.is_mutable()
2972         immdircap = dnode.get_uri()
2973         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2974         emptydircap = "URI:DIR2-LIT:"
2975         newkids = {u"child-imm":        ["filenode", {"ro_uri": filecap1,
2976                                                       "metadata": md1, }],
2977                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2978                    u"dirchild-imm":     ["dirnode",  {"ro_uri": immdircap}],
2979                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2980                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2981                    }
2982         return newkids, {'filecap1': filecap1,
2983                          'unknown_immcap': unknown_immcap,
2984                          'immdircap': immdircap,
2985                          'litdircap': litdircap,
2986                          'emptydircap': emptydircap}
2987
2988     def test_POST_mkdir_no_parentdir_initial_children(self):
2989         (newkids, caps) = self._create_initial_children()
2990         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
2991         def _after_mkdir(res):
2992             self.failUnless(res.startswith("URI:DIR"), res)
2993             n = self.s.create_node_from_uri(res)
2994             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2995             d2.addCallback(lambda ign:
2996                            self.failUnlessROChildURIIs(n, u"child-imm",
2997                                                        caps['filecap1']))
2998             d2.addCallback(lambda ign:
2999                            self.failUnlessRWChildURIIs(n, u"child-mutable",
3000                                                        caps['filecap2']))
3001             d2.addCallback(lambda ign:
3002                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
3003                                                        caps['filecap3']))
3004             d2.addCallback(lambda ign:
3005                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
3006                                                        caps['unknown_rwcap']))
3007             d2.addCallback(lambda ign:
3008                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
3009                                                        caps['unknown_rocap']))
3010             d2.addCallback(lambda ign:
3011                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3012                                                        caps['unknown_immcap']))
3013             d2.addCallback(lambda ign:
3014                            self.failUnlessRWChildURIIs(n, u"dirchild",
3015                                                        caps['dircap']))
3016             return d2
3017         d.addCallback(_after_mkdir)
3018         return d
3019
3020     def test_POST_mkdir_no_parentdir_unexpected_children(self):
3021         # the regular /uri?t=mkdir operation is specified to ignore its body.
3022         # Only t=mkdir-with-children pays attention to it.
3023         (newkids, caps) = self._create_initial_children()
3024         d = self.shouldHTTPError("POST_mkdir_no_parentdir_unexpected_children",
3025                                  400, "Bad Request",
3026                                  "t=mkdir does not accept children=, "
3027                                  "try t=mkdir-with-children instead",
3028                                  self.POST2, "/uri?t=mkdir", # without children
3029                                  simplejson.dumps(newkids))
3030         return d
3031
3032     def test_POST_noparent_bad(self):
3033         d = self.shouldHTTPError("POST_noparent_bad",
3034                                  400, "Bad Request",
3035                                  "/uri accepts only PUT, PUT?t=mkdir, "
3036                                  "POST?t=upload, and POST?t=mkdir",
3037                                  self.POST, "/uri?t=bogus")
3038         return d
3039
3040     def test_POST_mkdir_no_parentdir_immutable(self):
3041         (newkids, caps) = self._create_immutable_children()
3042         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
3043         def _after_mkdir(res):
3044             self.failUnless(res.startswith("URI:DIR"), res)
3045             n = self.s.create_node_from_uri(res)
3046             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3047             d2.addCallback(lambda ign:
3048                            self.failUnlessROChildURIIs(n, u"child-imm",
3049                                                           caps['filecap1']))
3050             d2.addCallback(lambda ign:
3051                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3052                                                           caps['unknown_immcap']))
3053             d2.addCallback(lambda ign:
3054                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
3055                                                           caps['immdircap']))
3056             d2.addCallback(lambda ign:
3057                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
3058                                                           caps['litdircap']))
3059             d2.addCallback(lambda ign:
3060                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
3061                                                           caps['emptydircap']))
3062             return d2
3063         d.addCallback(_after_mkdir)
3064         return d
3065
3066     def test_POST_mkdir_no_parentdir_immutable_bad(self):
3067         (newkids, caps) = self._create_initial_children()
3068         d = self.shouldFail2(error.Error,
3069                              "test_POST_mkdir_no_parentdir_immutable_bad",
3070                              "400 Bad Request",
3071                              "needed to be immutable but was not",
3072                              self.POST2,
3073                              "/uri?t=mkdir-immutable",
3074                              simplejson.dumps(newkids))
3075         return d
3076
3077     def test_welcome_page_mkdir_button(self):
3078         # Fetch the welcome page.
3079         d = self.GET("/")
3080         def _after_get_welcome_page(res):
3081             MKDIR_BUTTON_RE = re.compile(
3082                 '<form action="([^"]*)" method="post".*?'
3083                 '<input type="hidden" name="t" value="([^"]*)" />'
3084                 '<input type="hidden" name="([^"]*)" value="([^"]*)" />'
3085                 '<input type="submit" value="Create a directory" />',
3086                 re.I)
3087             mo = MKDIR_BUTTON_RE.search(res)
3088             formaction = mo.group(1)
3089             formt = mo.group(2)
3090             formaname = mo.group(3)
3091             formavalue = mo.group(4)
3092             return (formaction, formt, formaname, formavalue)
3093         d.addCallback(_after_get_welcome_page)
3094         def _after_parse_form(res):
3095             (formaction, formt, formaname, formavalue) = res
3096             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
3097         d.addCallback(_after_parse_form)
3098         d.addBoth(self.shouldRedirect, None, statuscode='303')
3099         return d
3100
3101     def test_POST_mkdir_replace(self): # return value?
3102         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
3103         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3104         d.addCallback(self.failUnlessNodeKeysAre, [])
3105         return d
3106
3107     def test_POST_mkdir_no_replace_queryarg(self): # return value?
3108         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
3109         d.addBoth(self.shouldFail, error.Error,
3110                   "POST_mkdir_no_replace_queryarg",
3111                   "409 Conflict",
3112                   "There was already a child by that name, and you asked me "
3113                   "to not replace it")
3114         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3115         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3116         return d
3117
3118     def test_POST_mkdir_no_replace_field(self): # return value?
3119         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
3120                       replace="false")
3121         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
3122                   "409 Conflict",
3123                   "There was already a child by that name, and you asked me "
3124                   "to not replace it")
3125         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3126         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3127         return d
3128
3129     def test_POST_mkdir_whendone_field(self):
3130         d = self.POST(self.public_url + "/foo",
3131                       t="mkdir", name="newdir", when_done="/THERE")
3132         d.addBoth(self.shouldRedirect, "/THERE")
3133         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3134         d.addCallback(self.failUnlessNodeKeysAre, [])
3135         return d
3136
3137     def test_POST_mkdir_whendone_queryarg(self):
3138         d = self.POST(self.public_url + "/foo?when_done=/THERE",
3139                       t="mkdir", name="newdir")
3140         d.addBoth(self.shouldRedirect, "/THERE")
3141         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3142         d.addCallback(self.failUnlessNodeKeysAre, [])
3143         return d
3144
3145     def test_POST_bad_t(self):
3146         d = self.shouldFail2(error.Error, "POST_bad_t",
3147                              "400 Bad Request",
3148                              "POST to a directory with bad t=BOGUS",
3149                              self.POST, self.public_url + "/foo", t="BOGUS")
3150         return d
3151
3152     def test_POST_set_children(self, command_name="set_children"):
3153         contents9, n9, newuri9 = self.makefile(9)
3154         contents10, n10, newuri10 = self.makefile(10)
3155         contents11, n11, newuri11 = self.makefile(11)
3156
3157         reqbody = """{
3158                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
3159                                                 "size": 0,
3160                                                 "metadata": {
3161                                                   "ctime": 1002777696.7564139,
3162                                                   "mtime": 1002777696.7564139
3163                                                  }
3164                                                } ],
3165                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
3166                                                 "size": 1,
3167                                                 "metadata": {
3168                                                   "ctime": 1002777696.7564139,
3169                                                   "mtime": 1002777696.7564139
3170                                                  }
3171                                                } ],
3172                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
3173                                                 "size": 2,
3174                                                 "metadata": {
3175                                                   "ctime": 1002777696.7564139,
3176                                                   "mtime": 1002777696.7564139
3177                                                  }
3178                                                } ]
3179                     }""" % (newuri9, newuri10, newuri11)
3180
3181         url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
3182
3183         d = client.getPage(url, method="POST", postdata=reqbody)
3184         def _then(res):
3185             self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
3186             self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
3187             self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
3188
3189         d.addCallback(_then)
3190         d.addErrback(self.dump_error)
3191         return d
3192
3193     def test_POST_set_children_with_hyphen(self):
3194         return self.test_POST_set_children(command_name="set-children")
3195
3196     def test_POST_link_uri(self):
3197         contents, n, newuri = self.makefile(8)
3198         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
3199         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
3200         d.addCallback(lambda res:
3201                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3202                                                       contents))
3203         return d
3204
3205     def test_POST_link_uri_replace(self):
3206         contents, n, newuri = self.makefile(8)
3207         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
3208         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
3209         d.addCallback(lambda res:
3210                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3211                                                       contents))
3212         return d
3213
3214     def test_POST_link_uri_unknown_bad(self):
3215         d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
3216         d.addBoth(self.shouldFail, error.Error,
3217                   "POST_link_uri_unknown_bad",
3218                   "400 Bad Request",
3219                   "unknown cap in a write slot")
3220         return d
3221
3222     def test_POST_link_uri_unknown_ro_good(self):
3223         d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
3224         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
3225         return d
3226
3227     def test_POST_link_uri_unknown_imm_good(self):
3228         d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
3229         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
3230         return d
3231
3232     def test_POST_link_uri_no_replace_queryarg(self):
3233         contents, n, newuri = self.makefile(8)
3234         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
3235                       name="bar.txt", uri=newuri)
3236         d.addBoth(self.shouldFail, error.Error,
3237                   "POST_link_uri_no_replace_queryarg",
3238                   "409 Conflict",
3239                   "There was already a child by that name, and you asked me "
3240                   "to not replace it")
3241         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3242         d.addCallback(self.failUnlessIsBarDotTxt)
3243         return d
3244
3245     def test_POST_link_uri_no_replace_field(self):
3246         contents, n, newuri = self.makefile(8)
3247         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
3248                       name="bar.txt", uri=newuri)
3249         d.addBoth(self.shouldFail, error.Error,
3250                   "POST_link_uri_no_replace_field",
3251                   "409 Conflict",
3252                   "There was already a child by that name, and you asked me "
3253                   "to not replace it")
3254         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3255         d.addCallback(self.failUnlessIsBarDotTxt)
3256         return d
3257
3258     def test_POST_delete(self, command_name='delete'):
3259         d = self._foo_node.list()
3260         def _check_before(children):
3261             self.failUnlessIn(u"bar.txt", children)
3262         d.addCallback(_check_before)
3263         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t=command_name, name="bar.txt"))
3264         d.addCallback(lambda res: self._foo_node.list())
3265         def _check_after(children):
3266             self.failIfIn(u"bar.txt", children)
3267         d.addCallback(_check_after)
3268         return d
3269
3270     def test_POST_unlink(self):
3271         return self.test_POST_delete(command_name='unlink')
3272
3273     def test_POST_rename_file(self):
3274         d = self.POST(self.public_url + "/foo", t="rename",
3275                       from_name="bar.txt", to_name='wibble.txt')
3276         d.addCallback(lambda res:
3277                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3278         d.addCallback(lambda res:
3279                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
3280         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
3281         d.addCallback(self.failUnlessIsBarDotTxt)
3282         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
3283         d.addCallback(self.failUnlessIsBarJSON)
3284         return d
3285
3286     def test_POST_rename_file_redundant(self):
3287         d = self.POST(self.public_url + "/foo", t="rename",
3288                       from_name="bar.txt", to_name='bar.txt')
3289         d.addCallback(lambda res:
3290                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3291         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3292         d.addCallback(self.failUnlessIsBarDotTxt)
3293         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3294         d.addCallback(self.failUnlessIsBarJSON)
3295         return d
3296
3297     def test_POST_rename_file_replace(self):
3298         # rename a file and replace a directory with it
3299         d = self.POST(self.public_url + "/foo", t="rename",
3300                       from_name="bar.txt", to_name='empty')
3301         d.addCallback(lambda res:
3302                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3303         d.addCallback(lambda res:
3304                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
3305         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
3306         d.addCallback(self.failUnlessIsBarDotTxt)
3307         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3308         d.addCallback(self.failUnlessIsBarJSON)
3309         return d
3310
3311     def test_POST_rename_file_no_replace_queryarg(self):
3312         # rename a file and replace a directory with it
3313         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
3314                       from_name="bar.txt", to_name='empty')
3315         d.addBoth(self.shouldFail, error.Error,
3316                   "POST_rename_file_no_replace_queryarg",
3317                   "409 Conflict",
3318                   "There was already a child by that name, and you asked me "
3319                   "to not replace it")
3320         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3321         d.addCallback(self.failUnlessIsEmptyJSON)
3322         return d
3323
3324     def test_POST_rename_file_no_replace_field(self):
3325         # rename a file and replace a directory with it
3326         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
3327                       from_name="bar.txt", to_name='empty')
3328         d.addBoth(self.shouldFail, error.Error,
3329                   "POST_rename_file_no_replace_field",
3330                   "409 Conflict",
3331                   "There was already a child by that name, and you asked me "
3332                   "to not replace it")
3333         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3334         d.addCallback(self.failUnlessIsEmptyJSON)
3335         return d
3336
3337     def failUnlessIsEmptyJSON(self, res):
3338         data = simplejson.loads(res)
3339         self.failUnlessEqual(data[0], "dirnode", data)
3340         self.failUnlessReallyEqual(len(data[1]["children"]), 0)
3341
3342     def test_POST_rename_file_slash_fail(self):
3343         d = self.POST(self.public_url + "/foo", t="rename",
3344                       from_name="bar.txt", to_name='kirk/spock.txt')
3345         d.addBoth(self.shouldFail, error.Error,
3346                   "test_POST_rename_file_slash_fail",
3347                   "400 Bad Request",
3348                   "to_name= may not contain a slash",
3349                   )
3350         d.addCallback(lambda res:
3351                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3352         return d
3353
3354     def test_POST_rename_dir(self):
3355         d = self.POST(self.public_url, t="rename",
3356                       from_name="foo", to_name='plunk')
3357         d.addCallback(lambda res:
3358                       self.failIfNodeHasChild(self.public_root, u"foo"))
3359         d.addCallback(lambda res:
3360                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
3361         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
3362         d.addCallback(self.failUnlessIsFooJSON)
3363         return d
3364
3365     def test_POST_move_file(self):
3366         d = self.POST(self.public_url + "/foo", t="move",
3367                       from_name="bar.txt", to_dir="sub")
3368         d.addCallback(lambda res:
3369                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3370         d.addCallback(lambda res:
3371                       self.failUnlessNodeHasChild(self._sub_node, u"bar.txt"))
3372         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3373         d.addCallback(self.failUnlessIsBarDotTxt)
3374         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3375         d.addCallback(self.failUnlessIsBarJSON)
3376         return d
3377
3378     def test_POST_move_file_new_name(self):
3379         d = self.POST(self.public_url + "/foo", t="move",
3380                       from_name="bar.txt", to_name="wibble.txt", to_dir="sub")
3381         d.addCallback(lambda res:
3382                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3383         d.addCallback(lambda res:
3384                       self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3385         d.addCallback(lambda res:
3386                       self.failUnlessNodeHasChild(self._sub_node, u"wibble.txt"))
3387         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt"))
3388         d.addCallback(self.failUnlessIsBarDotTxt)
3389         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt?t=json"))
3390         d.addCallback(self.failUnlessIsBarJSON)
3391         return d
3392
3393     def test_POST_move_file_replace(self):
3394         d = self.POST(self.public_url + "/foo", t="move",
3395                       from_name="bar.txt", to_name="baz.txt", to_dir="sub")
3396         d.addCallback(lambda res:
3397                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3398         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3399         d.addCallback(self.failUnlessIsBarDotTxt)
3400         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt?t=json"))
3401         d.addCallback(self.failUnlessIsBarJSON)
3402         return d
3403
3404     def test_POST_move_file_no_replace(self):
3405         d = self.shouldFail2(error.Error, "POST_move_file_no_replace",
3406                              "409 Conflict",
3407                              "There was already a child by that name, and you asked me to not replace it",
3408                              self.POST, self.public_url + "/foo", t="move",
3409                              replace="false", from_name="bar.txt",
3410                              to_name="baz.txt", to_dir="sub")
3411         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3412         d.addCallback(self.failUnlessIsBarDotTxt)
3413         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3414         d.addCallback(self.failUnlessIsBarJSON)
3415         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3416         d.addCallback(self.failUnlessIsSubBazDotTxt)
3417         return d
3418
3419     def test_POST_move_file_slash_fail(self):
3420         d = self.shouldFail2(error.Error, "test_POST_rename_file_slash_fail",
3421                              "400 Bad Request",
3422                              "to_name= may not contain a slash",
3423                              self.POST, self.public_url + "/foo", t="move",
3424                              from_name="bar.txt",
3425                              to_name="slash/fail.txt", to_dir="sub")
3426         d.addCallback(lambda res:
3427                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3428         d.addCallback(lambda res:
3429                       self.failIfNodeHasChild(self._sub_node, u"slash/fail.txt"))
3430         d.addCallback(lambda ign:
3431                       self.shouldFail2(error.Error,
3432                                        "test_POST_rename_file_slash_fail2",
3433                                        "400 Bad Request",
3434                                        "from_name= may not contain a slash",
3435                                        self.POST, self.public_url + "/foo",
3436                                        t="move",
3437                                        from_name="nope/bar.txt",
3438                                        to_name="fail.txt", to_dir="sub"))
3439         return d
3440
3441     def test_POST_move_file_no_target(self):
3442         d = self.shouldFail2(error.Error, "POST_move_file_no_target",
3443                              "400 Bad Request",
3444                              "move requires from_name and to_dir",
3445                              self.POST, self.public_url + "/foo", t="move",
3446                              from_name="bar.txt", to_name="baz.txt")
3447         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3448         d.addCallback(self.failUnlessIsBarDotTxt)
3449         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3450         d.addCallback(self.failUnlessIsBarJSON)
3451         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3452         d.addCallback(self.failUnlessIsBazDotTxt)
3453         return d
3454
3455     def test_POST_move_file_bad_target_type(self):
3456         d = self.shouldFail2(error.Error, "test_POST_move_file_bad_target_type",
3457                              "400 Bad Request", "invalid target_type parameter",
3458                              self.POST,
3459                              self.public_url + "/foo", t="move",
3460                              target_type="*D", from_name="bar.txt",
3461                              to_dir="sub")
3462         return d
3463
3464     def test_POST_move_file_multi_level(self):
3465         d = self.POST(self.public_url + "/foo/sub/level2?t=mkdir", "")
3466         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t="move",
3467                       from_name="bar.txt", to_dir="sub/level2"))
3468         d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3469         d.addCallback(lambda res: self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3470         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt"))
3471         d.addCallback(self.failUnlessIsBarDotTxt)
3472         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt?t=json"))
3473         d.addCallback(self.failUnlessIsBarJSON)
3474         return d
3475
3476     def test_POST_move_file_to_uri(self):
3477         d = self.POST(self.public_url + "/foo", t="move", target_type="uri",
3478                       from_name="bar.txt", to_dir=self._sub_uri)
3479         d.addCallback(lambda res:
3480                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3481         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3482         d.addCallback(self.failUnlessIsBarDotTxt)
3483         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3484         d.addCallback(self.failUnlessIsBarJSON)
3485         return d
3486
3487     def test_POST_move_file_to_nonexist_dir(self):
3488         d = self.shouldFail2(error.Error, "POST_move_file_to_nonexist_dir",
3489                             "404 Not Found", "No such child: nopechucktesta",
3490                             self.POST, self.public_url + "/foo", t="move",
3491                             from_name="bar.txt", to_dir="nopechucktesta")
3492         return d
3493
3494     def test_POST_move_file_into_file(self):
3495         d = self.shouldFail2(error.Error, "POST_move_file_into_file",
3496                              "400 Bad Request", "to_dir is not a directory",
3497                              self.POST, self.public_url + "/foo", t="move",
3498                              from_name="bar.txt", to_dir="baz.txt")
3499         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3500         d.addCallback(self.failUnlessIsBazDotTxt)
3501         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3502         d.addCallback(self.failUnlessIsBarDotTxt)
3503         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3504         d.addCallback(self.failUnlessIsBarJSON)
3505         return d
3506
3507     def test_POST_move_file_to_bad_uri(self):
3508         d =  self.shouldFail2(error.Error, "POST_move_file_to_bad_uri",
3509                               "400 Bad Request", "to_dir is not a directory",
3510                               self.POST, self.public_url + "/foo", t="move",
3511                               from_name="bar.txt", target_type="uri",
3512                               to_dir="URI:DIR2:mn5jlyjnrjeuydyswlzyui72i:rmneifcj6k6sycjljjhj3f6majsq2zqffydnnul5hfa4j577arma")
3513         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3514         d.addCallback(self.failUnlessIsBarDotTxt)
3515         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3516         d.addCallback(self.failUnlessIsBarJSON)
3517         return d
3518
3519     def test_POST_move_dir(self):
3520         d = self.POST(self.public_url + "/foo", t="move",
3521                       from_name="bar.txt", to_dir="empty")
3522         d.addCallback(lambda res: self.POST(self.public_url + "/foo",
3523                       t="move", from_name="empty", to_dir="sub"))
3524         d.addCallback(lambda res:
3525                       self.failIfNodeHasChild(self._foo_node, u"empty"))
3526         d.addCallback(lambda res:
3527                       self.failUnlessNodeHasChild(self._sub_node, u"empty"))
3528         d.addCallback(lambda res:
3529                       self._sub_node.get_child_at_path(u"empty"))
3530         d.addCallback(lambda node:
3531                       self.failUnlessNodeHasChild(node, u"bar.txt"))
3532         d.addCallback(lambda res:
3533                       self.GET(self.public_url + "/foo/sub/empty/bar.txt"))
3534         d.addCallback(self.failUnlessIsBarDotTxt)
3535         return d
3536
3537     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
3538         """ If target is not None then the redirection has to go to target.  If
3539         statuscode is not None then the redirection has to be accomplished with
3540         that HTTP status code."""
3541         if not isinstance(res, failure.Failure):
3542             to_where = (target is None) and "somewhere" or ("to " + target)
3543             self.fail("%s: we were expecting to get redirected %s, not get an"
3544                       " actual page: %s" % (which, to_where, res))
3545         res.trap(error.PageRedirect)
3546         if statuscode is not None:
3547             self.failUnlessReallyEqual(res.value.status, statuscode,
3548                                        "%s: not a redirect" % which)
3549         if target is not None:
3550             # the PageRedirect does not seem to capture the uri= query arg
3551             # properly, so we can't check for it.
3552             realtarget = self.webish_url + target
3553             self.failUnlessReallyEqual(res.value.location, realtarget,
3554                                        "%s: wrong target" % which)
3555         return res.value.location
3556
3557     def test_GET_URI_form(self):
3558         base = "/uri?uri=%s" % self._bar_txt_uri
3559         # this is supposed to give us a redirect to /uri/$URI, plus arguments
3560         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
3561         d = self.GET(base)
3562         d.addBoth(self.shouldRedirect, targetbase)
3563         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
3564         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
3565         d.addCallback(lambda res: self.GET(base+"&t=json"))
3566         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
3567         d.addCallback(self.log, "about to get file by uri")
3568         d.addCallback(lambda res: self.GET(base, followRedirect=True))
3569         d.addCallback(self.failUnlessIsBarDotTxt)
3570         d.addCallback(self.log, "got file by uri, about to get dir by uri")
3571         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
3572                                            followRedirect=True))
3573         d.addCallback(self.failUnlessIsFooJSON)
3574         d.addCallback(self.log, "got dir by uri")
3575
3576         return d
3577
3578     def test_GET_URI_form_bad(self):
3579         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
3580                              "400 Bad Request", "GET /uri requires uri=",
3581                              self.GET, "/uri")
3582         return d
3583
3584     def test_GET_rename_form(self):
3585         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
3586                      followRedirect=True)
3587         def _check(res):
3588             self.failUnlessIn('name="when_done" value="."', res)
3589             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
3590             self.failUnlessIn(FAVICON_MARKUP, res)
3591         d.addCallback(_check)
3592         return d
3593
3594     def log(self, res, msg):
3595         #print "MSG: %s  RES: %s" % (msg, res)
3596         log.msg(msg)
3597         return res
3598
3599     def test_GET_URI_URL(self):
3600         base = "/uri/%s" % self._bar_txt_uri
3601         d = self.GET(base)
3602         d.addCallback(self.failUnlessIsBarDotTxt)
3603         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
3604         d.addCallback(self.failUnlessIsBarDotTxt)
3605         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
3606         d.addCallback(self.failUnlessIsBarDotTxt)
3607         return d
3608
3609     def test_GET_URI_URL_dir(self):
3610         base = "/uri/%s?t=json" % self._foo_uri
3611         d = self.GET(base)
3612         d.addCallback(self.failUnlessIsFooJSON)
3613         return d
3614
3615     def test_GET_URI_URL_missing(self):
3616         base = "/uri/%s" % self._bad_file_uri
3617         d = self.shouldHTTPError("test_GET_URI_URL_missing",
3618                                  http.GONE, None, "NotEnoughSharesError",
3619                                  self.GET, base)
3620         # TODO: how can we exercise both sides of WebDownloadTarget.fail
3621         # here? we must arrange for a download to fail after target.open()
3622         # has been called, and then inspect the response to see that it is
3623         # shorter than we expected.
3624         return d
3625
3626     def test_PUT_DIRURL_uri(self):
3627         d = self.s.create_dirnode()
3628         def _made_dir(dn):
3629             new_uri = dn.get_uri()
3630             # replace /foo with a new (empty) directory
3631             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
3632             d.addCallback(lambda res:
3633                           self.failUnlessReallyEqual(res.strip(), new_uri))
3634             d.addCallback(lambda res:
3635                           self.failUnlessRWChildURIIs(self.public_root,
3636                                                       u"foo",
3637                                                       new_uri))
3638             return d
3639         d.addCallback(_made_dir)
3640         return d
3641
3642     def test_PUT_DIRURL_uri_noreplace(self):
3643         d = self.s.create_dirnode()
3644         def _made_dir(dn):
3645             new_uri = dn.get_uri()
3646             # replace /foo with a new (empty) directory, but ask that
3647             # replace=false, so it should fail
3648             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
3649                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
3650                                  self.PUT,
3651                                  self.public_url + "/foo?t=uri&replace=false",
3652                                  new_uri)
3653             d.addCallback(lambda res:
3654                           self.failUnlessRWChildURIIs(self.public_root,
3655                                                       u"foo",
3656                                                       self._foo_uri))
3657             return d
3658         d.addCallback(_made_dir)
3659         return d
3660
3661     def test_PUT_DIRURL_bad_t(self):
3662         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
3663                              "400 Bad Request", "PUT to a directory",
3664                              self.PUT, self.public_url + "/foo?t=BOGUS", "")
3665         d.addCallback(lambda res:
3666                       self.failUnlessRWChildURIIs(self.public_root,
3667                                                   u"foo",
3668                                                   self._foo_uri))
3669         return d
3670
3671     def test_PUT_NEWFILEURL_uri(self):
3672         contents, n, new_uri = self.makefile(8)
3673         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
3674         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3675         d.addCallback(lambda res:
3676                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3677                                                       contents))
3678         return d
3679
3680     def test_PUT_NEWFILEURL_mdmf(self):
3681         new_contents = self.NEWFILE_CONTENTS * 300000
3682         d = self.PUT(self.public_url + \
3683                      "/foo/mdmf.txt?format=mdmf",
3684                      new_contents)
3685         d.addCallback(lambda ignored:
3686             self.GET(self.public_url + "/foo/mdmf.txt?t=json"))
3687         def _got_json(json):
3688             data = simplejson.loads(json)
3689             data = data[1]
3690             self.failUnlessIn("format", data)
3691             self.failUnlessEqual(data["format"], "MDMF")
3692             self.failUnless(data['rw_uri'].startswith("URI:MDMF"))
3693             self.failUnless(data['ro_uri'].startswith("URI:MDMF"))
3694         d.addCallback(_got_json)
3695         return d
3696
3697     def test_PUT_NEWFILEURL_sdmf(self):
3698         new_contents = self.NEWFILE_CONTENTS * 300000
3699         d = self.PUT(self.public_url + \
3700                      "/foo/sdmf.txt?format=sdmf",
3701                      new_contents)
3702         d.addCallback(lambda ignored:
3703             self.GET(self.public_url + "/foo/sdmf.txt?t=json"))
3704         def _got_json(json):
3705             data = simplejson.loads(json)
3706             data = data[1]
3707             self.failUnlessIn("format", data)
3708             self.failUnlessEqual(data["format"], "SDMF")
3709         d.addCallback(_got_json)
3710         return d
3711
3712     def test_PUT_NEWFILEURL_bad_format(self):
3713         new_contents = self.NEWFILE_CONTENTS * 300000
3714         return self.shouldHTTPError("PUT_NEWFILEURL_bad_format",
3715                                     400, "Bad Request", "Unknown format: foo",
3716                                     self.PUT, self.public_url + \
3717                                     "/foo/foo.txt?format=foo",
3718                                     new_contents)
3719
3720     def test_PUT_NEWFILEURL_uri_replace(self):
3721         contents, n, new_uri = self.makefile(8)
3722         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
3723         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3724         d.addCallback(lambda res:
3725                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3726                                                       contents))
3727         return d
3728
3729     def test_PUT_NEWFILEURL_uri_no_replace(self):
3730         contents, n, new_uri = self.makefile(8)
3731         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
3732         d.addBoth(self.shouldFail, error.Error,
3733                   "PUT_NEWFILEURL_uri_no_replace",
3734                   "409 Conflict",
3735                   "There was already a child by that name, and you asked me "
3736                   "to not replace it")
3737         return d
3738
3739     def test_PUT_NEWFILEURL_uri_unknown_bad(self):
3740         d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
3741         d.addBoth(self.shouldFail, error.Error,
3742                   "POST_put_uri_unknown_bad",
3743                   "400 Bad Request",
3744                   "unknown cap in a write slot")
3745         return d
3746
3747     def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
3748         d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
3749         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3750                       u"put-future-ro.txt")
3751         return d
3752
3753     def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
3754         d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
3755         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3756                       u"put-future-imm.txt")
3757         return d
3758
3759     def test_PUT_NEWFILE_URI(self):
3760         file_contents = "New file contents here\n"
3761         d = self.PUT("/uri", file_contents)
3762         def _check(uri):
3763             assert isinstance(uri, str), uri
3764             self.failUnlessIn(uri, self.get_all_contents())
3765             self.failUnlessReallyEqual(self.get_all_contents()[uri],
3766                                        file_contents)
3767             return self.GET("/uri/%s" % uri)
3768         d.addCallback(_check)
3769         def _check2(res):
3770             self.failUnlessReallyEqual(res, file_contents)
3771         d.addCallback(_check2)
3772         return d
3773
3774     def test_PUT_NEWFILE_URI_not_mutable(self):
3775         file_contents = "New file contents here\n"
3776         d = self.PUT("/uri?mutable=false", file_contents)
3777         def _check(uri):
3778             assert isinstance(uri, str), uri
3779             self.failUnlessIn(uri, self.get_all_contents())
3780             self.failUnlessReallyEqual(self.get_all_contents()[uri],
3781                                        file_contents)
3782             return self.GET("/uri/%s" % uri)
3783         d.addCallback(_check)
3784         def _check2(res):
3785             self.failUnlessReallyEqual(res, file_contents)
3786         d.addCallback(_check2)
3787         return d
3788
3789     def test_PUT_NEWFILE_URI_only_PUT(self):
3790         d = self.PUT("/uri?t=bogus", "")
3791         d.addBoth(self.shouldFail, error.Error,
3792                   "PUT_NEWFILE_URI_only_PUT",
3793                   "400 Bad Request",
3794                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
3795         return d
3796
3797     def test_PUT_NEWFILE_URI_mutable(self):
3798         file_contents = "New file contents here\n"
3799         d = self.PUT("/uri?mutable=true", file_contents)
3800         def _check1(filecap):
3801             filecap = filecap.strip()
3802             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
3803             self.filecap = filecap
3804             u = uri.WriteableSSKFileURI.init_from_string(filecap)
3805             self.failUnlessIn(u.get_storage_index(), self.get_all_contents())
3806             n = self.s.create_node_from_uri(filecap)
3807             return n.download_best_version()
3808         d.addCallback(_check1)
3809         def _check2(data):
3810             self.failUnlessReallyEqual(data, file_contents)
3811             return self.GET("/uri/%s" % urllib.quote(self.filecap))
3812         d.addCallback(_check2)
3813         def _check3(res):
3814             self.failUnlessReallyEqual(res, file_contents)
3815         d.addCallback(_check3)
3816         return d
3817
3818     def test_PUT_mkdir(self):
3819         d = self.PUT("/uri?t=mkdir", "")
3820         def _check(uri):
3821             n = self.s.create_node_from_uri(uri.strip())
3822             d2 = self.failUnlessNodeKeysAre(n, [])
3823             d2.addCallback(lambda res:
3824                            self.GET("/uri/%s?t=json" % uri))
3825             return d2
3826         d.addCallback(_check)
3827         d.addCallback(self.failUnlessIsEmptyJSON)
3828         return d
3829
3830     def test_PUT_mkdir_mdmf(self):
3831         d = self.PUT("/uri?t=mkdir&format=mdmf", "")
3832         def _got(res):
3833             u = uri.from_string(res)
3834             # Check that this is an MDMF writecap
3835             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
3836         d.addCallback(_got)
3837         return d
3838
3839     def test_PUT_mkdir_sdmf(self):
3840         d = self.PUT("/uri?t=mkdir&format=sdmf", "")
3841         def _got(res):
3842             u = uri.from_string(res)
3843             self.failUnlessIsInstance(u, uri.DirectoryURI)
3844         d.addCallback(_got)
3845         return d
3846
3847     def test_PUT_mkdir_bad_format(self):
3848         return self.shouldHTTPError("PUT_mkdir_bad_format",
3849                                     400, "Bad Request", "Unknown format: foo",
3850                                     self.PUT, "/uri?t=mkdir&format=foo",
3851                                     "")
3852
3853     def test_POST_check(self):
3854         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
3855         def _done(res):
3856             # this returns a string form of the results, which are probably
3857             # None since we're using fake filenodes.
3858             # TODO: verify that the check actually happened, by changing
3859             # FakeCHKFileNode to count how many times .check() has been
3860             # called.
3861             pass
3862         d.addCallback(_done)
3863         return d
3864
3865
3866     def test_PUT_update_at_offset(self):
3867         file_contents = "test file" * 100000 # about 900 KiB
3868         d = self.PUT("/uri?mutable=true", file_contents)
3869         def _then(filecap):
3870             self.filecap = filecap
3871             new_data = file_contents[:100]
3872             new = "replaced and so on"
3873             new_data += new
3874             new_data += file_contents[len(new_data):]
3875             assert len(new_data) == len(file_contents)
3876             self.new_data = new_data
3877         d.addCallback(_then)
3878         d.addCallback(lambda ignored:
3879             self.PUT("/uri/%s?replace=True&offset=100" % self.filecap,
3880                      "replaced and so on"))
3881         def _get_data(filecap):
3882             n = self.s.create_node_from_uri(filecap)
3883             return n.download_best_version()
3884         d.addCallback(_get_data)
3885         d.addCallback(lambda results:
3886             self.failUnlessEqual(results, self.new_data))
3887         # Now try appending things to the file
3888         d.addCallback(lambda ignored:
3889             self.PUT("/uri/%s?offset=%d" % (self.filecap, len(self.new_data)),
3890                      "puppies" * 100))
3891         d.addCallback(_get_data)
3892         d.addCallback(lambda results:
3893             self.failUnlessEqual(results, self.new_data + ("puppies" * 100)))
3894         # and try replacing the beginning of the file
3895         d.addCallback(lambda ignored:
3896             self.PUT("/uri/%s?offset=0" % self.filecap, "begin"))
3897         d.addCallback(_get_data)
3898         d.addCallback(lambda results:
3899             self.failUnlessEqual(results, "begin"+self.new_data[len("begin"):]+("puppies"*100)))
3900         return d
3901
3902     def test_PUT_update_at_invalid_offset(self):
3903         file_contents = "test file" * 100000 # about 900 KiB
3904         d = self.PUT("/uri?mutable=true", file_contents)
3905         def _then(filecap):
3906             self.filecap = filecap
3907         d.addCallback(_then)
3908         # Negative offsets should cause an error.
3909         d.addCallback(lambda ignored:
3910             self.shouldHTTPError("PUT_update_at_invalid_offset",
3911                                  400, "Bad Request",
3912                                  "Invalid offset",
3913                                  self.PUT,
3914                                  "/uri/%s?offset=-1" % self.filecap,
3915                                  "foo"))
3916         return d
3917
3918     def test_PUT_update_at_offset_immutable(self):
3919         file_contents = "Test file" * 100000
3920         d = self.PUT("/uri", file_contents)
3921         def _then(filecap):
3922             self.filecap = filecap
3923         d.addCallback(_then)
3924         d.addCallback(lambda ignored:
3925             self.shouldHTTPError("PUT_update_at_offset_immutable",
3926                                  400, "Bad Request",
3927                                  "immutable",
3928                                  self.PUT,
3929                                  "/uri/%s?offset=50" % self.filecap,
3930                                  "foo"))
3931         return d
3932
3933
3934     def test_bad_method(self):
3935         url = self.webish_url + self.public_url + "/foo/bar.txt"
3936         d = self.shouldHTTPError("bad_method",
3937                                  501, "Not Implemented",
3938                                  "I don't know how to treat a BOGUS request.",
3939                                  client.getPage, url, method="BOGUS")
3940         return d
3941
3942     def test_short_url(self):
3943         url = self.webish_url + "/uri"
3944         d = self.shouldHTTPError("short_url", 501, "Not Implemented",
3945                                  "I don't know how to treat a DELETE request.",
3946                                  client.getPage, url, method="DELETE")
3947         return d
3948
3949     def test_ophandle_bad(self):
3950         url = self.webish_url + "/operations/bogus?t=status"
3951         d = self.shouldHTTPError("ophandle_bad", 404, "404 Not Found",
3952                                  "unknown/expired handle 'bogus'",
3953                                  client.getPage, url)
3954         return d
3955
3956     def test_ophandle_cancel(self):
3957         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
3958                       followRedirect=True)
3959         d.addCallback(lambda ignored:
3960                       self.GET("/operations/128?t=status&output=JSON"))
3961         def _check1(res):
3962             data = simplejson.loads(res)
3963             self.failUnless("finished" in data, res)
3964             monitor = self.ws.root.child_operations.handles["128"][0]
3965             d = self.POST("/operations/128?t=cancel&output=JSON")
3966             def _check2(res):
3967                 data = simplejson.loads(res)
3968                 self.failUnless("finished" in data, res)
3969                 # t=cancel causes the handle to be forgotten
3970                 self.failUnless(monitor.is_cancelled())
3971             d.addCallback(_check2)
3972             return d
3973         d.addCallback(_check1)
3974         d.addCallback(lambda ignored:
3975                       self.shouldHTTPError("ophandle_cancel",
3976                                            404, "404 Not Found",
3977                                            "unknown/expired handle '128'",
3978                                            self.GET,
3979                                            "/operations/128?t=status&output=JSON"))
3980         return d
3981
3982     def test_ophandle_retainfor(self):
3983         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
3984                       followRedirect=True)
3985         d.addCallback(lambda ignored:
3986                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
3987         def _check1(res):
3988             data = simplejson.loads(res)
3989             self.failUnless("finished" in data, res)
3990         d.addCallback(_check1)
3991         # the retain-for=0 will cause the handle to be expired very soon
3992         d.addCallback(lambda ign:
3993             self.clock.advance(2.0))
3994         d.addCallback(lambda ignored:
3995                       self.shouldHTTPError("ophandle_retainfor",
3996                                            404, "404 Not Found",
3997                                            "unknown/expired handle '129'",
3998                                            self.GET,
3999                                            "/operations/129?t=status&output=JSON"))
4000         return d
4001
4002     def test_ophandle_release_after_complete(self):
4003         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
4004                       followRedirect=True)
4005         d.addCallback(self.wait_for_operation, "130")
4006         d.addCallback(lambda ignored:
4007                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
4008         # the release-after-complete=true will cause the handle to be expired
4009         d.addCallback(lambda ignored:
4010                       self.shouldHTTPError("ophandle_release_after_complete",
4011                                            404, "404 Not Found",
4012                                            "unknown/expired handle '130'",
4013                                            self.GET,
4014                                            "/operations/130?t=status&output=JSON"))
4015         return d
4016
4017     def test_uncollected_ophandle_expiration(self):
4018         # uncollected ophandles should expire after 4 days
4019         def _make_uncollected_ophandle(ophandle):
4020             d = self.POST(self.public_url +
4021                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
4022                           followRedirect=False)
4023             # When we start the operation, the webapi server will want
4024             # to redirect us to the page for the ophandle, so we get
4025             # confirmation that the operation has started. If the
4026             # manifest operation has finished by the time we get there,
4027             # following that redirect (by setting followRedirect=True
4028             # above) has the side effect of collecting the ophandle that
4029             # we've just created, which means that we can't use the
4030             # ophandle to test the uncollected timeout anymore. So,
4031             # instead, catch the 302 here and don't follow it.
4032             d.addBoth(self.should302, "uncollected_ophandle_creation")
4033             return d
4034         # Create an ophandle, don't collect it, then advance the clock by
4035         # 4 days - 1 second and make sure that the ophandle is still there.
4036         d = _make_uncollected_ophandle(131)
4037         d.addCallback(lambda ign:
4038             self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
4039         d.addCallback(lambda ign:
4040             self.GET("/operations/131?t=status&output=JSON"))
4041         def _check1(res):
4042             data = simplejson.loads(res)
4043             self.failUnless("finished" in data, res)
4044         d.addCallback(_check1)
4045         # Create an ophandle, don't collect it, then try to collect it
4046         # after 4 days. It should be gone.
4047         d.addCallback(lambda ign:
4048             _make_uncollected_ophandle(132))
4049         d.addCallback(lambda ign:
4050             self.clock.advance(96*60*60))
4051         d.addCallback(lambda ign:
4052             self.shouldHTTPError("uncollected_ophandle_expired_after_100_hours",
4053                                  404, "404 Not Found",
4054                                  "unknown/expired handle '132'",
4055                                  self.GET,
4056                                  "/operations/132?t=status&output=JSON"))
4057         return d
4058
4059     def test_collected_ophandle_expiration(self):
4060         # collected ophandles should expire after 1 day
4061         def _make_collected_ophandle(ophandle):
4062             d = self.POST(self.public_url +
4063                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
4064                           followRedirect=True)
4065             # By following the initial redirect, we collect the ophandle
4066             # we've just created.
4067             return d
4068         # Create a collected ophandle, then collect it after 23 hours
4069         # and 59 seconds to make sure that it is still there.
4070         d = _make_collected_ophandle(133)
4071         d.addCallback(lambda ign:
4072             self.clock.advance((24*60*60) - 1))
4073         d.addCallback(lambda ign:
4074             self.GET("/operations/133?t=status&output=JSON"))
4075         def _check1(res):
4076             data = simplejson.loads(res)
4077             self.failUnless("finished" in data, res)
4078         d.addCallback(_check1)
4079         # Create another uncollected ophandle, then try to collect it
4080         # after 24 hours to make sure that it is gone.
4081         d.addCallback(lambda ign:
4082             _make_collected_ophandle(134))
4083         d.addCallback(lambda ign:
4084             self.clock.advance(24*60*60))
4085         d.addCallback(lambda ign:
4086             self.shouldHTTPError("collected_ophandle_expired_after_1_day",
4087                                  404, "404 Not Found",
4088                                  "unknown/expired handle '134'",
4089                                  self.GET,
4090                                  "/operations/134?t=status&output=JSON"))
4091         return d
4092
4093     def test_incident(self):
4094         d = self.POST("/report_incident", details="eek")
4095         def _done(res):
4096             self.failIfIn("<html>", res)
4097             self.failUnlessIn("Thank you for your report!", res)
4098         d.addCallback(_done)
4099         return d
4100
4101     def test_static(self):
4102         webdir = os.path.join(self.staticdir, "subdir")
4103         fileutil.make_dirs(webdir)
4104         f = open(os.path.join(webdir, "hello.txt"), "wb")
4105         f.write("hello")
4106         f.close()
4107
4108         d = self.GET("/static/subdir/hello.txt")
4109         def _check(res):
4110             self.failUnlessReallyEqual(res, "hello")
4111         d.addCallback(_check)
4112         return d
4113
4114
4115 class IntroducerWeb(unittest.TestCase):
4116     def setUp(self):
4117         self.node = None
4118
4119     def tearDown(self):
4120         d = defer.succeed(None)
4121         if self.node:
4122             d.addCallback(lambda ign: self.node.stopService())
4123         d.addCallback(flushEventualQueue)
4124         return d
4125
4126     def test_welcome(self):
4127         basedir = "web.IntroducerWeb.test_welcome"
4128         os.mkdir(basedir)
4129         fileutil.write(os.path.join(basedir, "tahoe.cfg"), "[node]\nweb.port = tcp:0\n")
4130         self.node = IntroducerNode(basedir)
4131         self.ws = self.node.getServiceNamed("webish")
4132
4133         d = fireEventually(None)
4134         d.addCallback(lambda ign: self.node.startService())
4135         d.addCallback(lambda ign: self.node.when_tub_ready())
4136
4137         d.addCallback(lambda ign: self.GET("/"))
4138         def _check(res):
4139             self.failUnlessIn('Welcome to the Tahoe-LAFS Introducer', res)
4140             self.failUnlessIn(FAVICON_MARKUP, res)
4141         d.addCallback(_check)
4142         return d
4143
4144     def GET(self, urlpath, followRedirect=False, return_response=False,
4145             **kwargs):
4146         # if return_response=True, this fires with (data, statuscode,
4147         # respheaders) instead of just data.
4148         assert not isinstance(urlpath, unicode)
4149         url = self.ws.getURL().rstrip('/') + urlpath
4150         factory = HTTPClientGETFactory(url, method="GET",
4151                                        followRedirect=followRedirect, **kwargs)
4152         reactor.connectTCP("localhost", self.ws.getPortnum(), factory)
4153         d = factory.deferred
4154         def _got_data(data):
4155             return (data, factory.status, factory.response_headers)
4156         if return_response:
4157             d.addCallback(_got_data)
4158         return factory.deferred
4159
4160
4161 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4162     def test_load_file(self):
4163         # This will raise an exception unless a well-formed XML file is found under that name.
4164         common.getxmlfile('directory.xhtml').load()
4165
4166     def test_parse_replace_arg(self):
4167         self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
4168         self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
4169         self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
4170                                    "only-files")
4171         self.shouldFail(AssertionError, "test_parse_replace_arg", "",
4172                         common.parse_replace_arg, "only_fles")
4173
4174     def test_abbreviate_time(self):
4175         self.failUnlessReallyEqual(common.abbreviate_time(None), "")
4176         self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
4177         self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
4178         self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
4179         self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
4180         self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
4181
4182     def test_compute_rate(self):
4183         self.failUnlessReallyEqual(common.compute_rate(None, None), None)
4184         self.failUnlessReallyEqual(common.compute_rate(None, 1), None)
4185         self.failUnlessReallyEqual(common.compute_rate(250000, None), None)
4186         self.failUnlessReallyEqual(common.compute_rate(250000, 0), None)
4187         self.failUnlessReallyEqual(common.compute_rate(250000, 10), 25000.0)
4188         self.failUnlessReallyEqual(common.compute_rate(0, 10), 0.0)
4189         self.shouldFail(AssertionError, "test_compute_rate", "",
4190                         common.compute_rate, -100, 10)
4191         self.shouldFail(AssertionError, "test_compute_rate", "",
4192                         common.compute_rate, 100, -10)
4193
4194         # Sanity check
4195         rate = common.compute_rate(10*1000*1000, 1)
4196         self.failUnlessReallyEqual(common.abbreviate_rate(rate), "10.00MBps")
4197
4198     def test_abbreviate_rate(self):
4199         self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
4200         self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
4201         self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
4202         self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
4203
4204     def test_abbreviate_size(self):
4205         self.failUnlessReallyEqual(common.abbreviate_size(None), "")
4206         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
4207         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
4208         self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
4209         self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
4210
4211     def test_plural(self):
4212         def convert(s):
4213             return "%d second%s" % (s, status.plural(s))
4214         self.failUnlessReallyEqual(convert(0), "0 seconds")
4215         self.failUnlessReallyEqual(convert(1), "1 second")
4216         self.failUnlessReallyEqual(convert(2), "2 seconds")
4217         def convert2(s):
4218             return "has share%s: %s" % (status.plural(s), ",".join(s))
4219         self.failUnlessReallyEqual(convert2([]), "has shares: ")
4220         self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
4221         self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
4222
4223
4224 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4225
4226     def CHECK(self, ign, which, args, clientnum=0):
4227         fileurl = self.fileurls[which]
4228         url = fileurl + "?" + args
4229         return self.GET(url, method="POST", clientnum=clientnum)
4230
4231     def test_filecheck(self):
4232         self.basedir = "web/Grid/filecheck"
4233         self.set_up_grid()
4234         c0 = self.g.clients[0]
4235         self.uris = {}
4236         DATA = "data" * 100
4237         d = c0.upload(upload.Data(DATA, convergence=""))
4238         def _stash_uri(ur, which):
4239             self.uris[which] = ur.get_uri()
4240         d.addCallback(_stash_uri, "good")
4241         d.addCallback(lambda ign:
4242                       c0.upload(upload.Data(DATA+"1", convergence="")))
4243         d.addCallback(_stash_uri, "sick")
4244         d.addCallback(lambda ign:
4245                       c0.upload(upload.Data(DATA+"2", convergence="")))
4246         d.addCallback(_stash_uri, "dead")
4247         def _stash_mutable_uri(n, which):
4248             self.uris[which] = n.get_uri()
4249             assert isinstance(self.uris[which], str)
4250         d.addCallback(lambda ign:
4251             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4252         d.addCallback(_stash_mutable_uri, "corrupt")
4253         d.addCallback(lambda ign:
4254                       c0.upload(upload.Data("literal", convergence="")))
4255         d.addCallback(_stash_uri, "small")
4256         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
4257         d.addCallback(_stash_mutable_uri, "smalldir")
4258
4259         def _compute_fileurls(ignored):
4260             self.fileurls = {}
4261             for which in self.uris:
4262                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4263         d.addCallback(_compute_fileurls)
4264
4265         def _clobber_shares(ignored):
4266             good_shares = self.find_uri_shares(self.uris["good"])
4267             self.failUnlessReallyEqual(len(good_shares), 10)
4268             sick_shares = self.find_uri_shares(self.uris["sick"])
4269             os.unlink(sick_shares[0][2])
4270             dead_shares = self.find_uri_shares(self.uris["dead"])
4271             for i in range(1, 10):
4272                 os.unlink(dead_shares[i][2])
4273             c_shares = self.find_uri_shares(self.uris["corrupt"])
4274             cso = CorruptShareOptions()
4275             cso.stdout = StringIO()
4276             cso.parseOptions([c_shares[0][2]])
4277             corrupt_share(cso)
4278         d.addCallback(_clobber_shares)
4279
4280         d.addCallback(self.CHECK, "good", "t=check")
4281         def _got_html_good(res):
4282             self.failUnlessIn("Healthy", res)
4283             self.failIfIn("Not Healthy", res)
4284             self.failUnlessIn(FAVICON_MARKUP, res)
4285         d.addCallback(_got_html_good)
4286         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
4287         def _got_html_good_return_to(res):
4288             self.failUnlessIn("Healthy", res)
4289             self.failIfIn("Not Healthy", res)
4290             self.failUnlessIn('<a href="somewhere">Return to file', res)
4291         d.addCallback(_got_html_good_return_to)
4292         d.addCallback(self.CHECK, "good", "t=check&output=json")
4293         def _got_json_good(res):
4294             r = simplejson.loads(res)
4295             self.failUnlessEqual(r["summary"], "Healthy")
4296             self.failUnless(r["results"]["healthy"])
4297             self.failIf(r["results"]["needs-rebalancing"])
4298             self.failUnless(r["results"]["recoverable"])
4299         d.addCallback(_got_json_good)
4300
4301         d.addCallback(self.CHECK, "small", "t=check")
4302         def _got_html_small(res):
4303             self.failUnlessIn("Literal files are always healthy", res)
4304             self.failIfIn("Not Healthy", res)
4305         d.addCallback(_got_html_small)
4306         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
4307         def _got_html_small_return_to(res):
4308             self.failUnlessIn("Literal files are always healthy", res)
4309             self.failIfIn("Not Healthy", res)
4310             self.failUnlessIn('<a href="somewhere">Return to file', res)
4311         d.addCallback(_got_html_small_return_to)
4312         d.addCallback(self.CHECK, "small", "t=check&output=json")
4313         def _got_json_small(res):
4314             r = simplejson.loads(res)
4315             self.failUnlessEqual(r["storage-index"], "")
4316             self.failUnless(r["results"]["healthy"])
4317         d.addCallback(_got_json_small)
4318
4319         d.addCallback(self.CHECK, "smalldir", "t=check")
4320         def _got_html_smalldir(res):
4321             self.failUnlessIn("Literal files are always healthy", res)
4322             self.failIfIn("Not Healthy", res)
4323         d.addCallback(_got_html_smalldir)
4324         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
4325         def _got_json_smalldir(res):
4326             r = simplejson.loads(res)
4327             self.failUnlessEqual(r["storage-index"], "")
4328             self.failUnless(r["results"]["healthy"])
4329         d.addCallback(_got_json_smalldir)
4330
4331         d.addCallback(self.CHECK, "sick", "t=check")
4332         def _got_html_sick(res):
4333             self.failUnlessIn("Not Healthy", res)
4334         d.addCallback(_got_html_sick)
4335         d.addCallback(self.CHECK, "sick", "t=check&output=json")
4336         def _got_json_sick(res):
4337             r = simplejson.loads(res)
4338             self.failUnlessEqual(r["summary"],
4339                                  "Not Healthy: 9 shares (enc 3-of-10)")
4340             self.failIf(r["results"]["healthy"])
4341             self.failIf(r["results"]["needs-rebalancing"])
4342             self.failUnless(r["results"]["recoverable"])
4343         d.addCallback(_got_json_sick)
4344
4345         d.addCallback(self.CHECK, "dead", "t=check")
4346         def _got_html_dead(res):
4347             self.failUnlessIn("Not Healthy", res)
4348         d.addCallback(_got_html_dead)
4349         d.addCallback(self.CHECK, "dead", "t=check&output=json")
4350         def _got_json_dead(res):
4351             r = simplejson.loads(res)
4352             self.failUnlessEqual(r["summary"],
4353                                  "Not Healthy: 1 shares (enc 3-of-10)")
4354             self.failIf(r["results"]["healthy"])
4355             self.failIf(r["results"]["needs-rebalancing"])
4356             self.failIf(r["results"]["recoverable"])
4357         d.addCallback(_got_json_dead)
4358
4359         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
4360         def _got_html_corrupt(res):
4361             self.failUnlessIn("Not Healthy! : Unhealthy", res)
4362         d.addCallback(_got_html_corrupt)
4363         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
4364         def _got_json_corrupt(res):
4365             r = simplejson.loads(res)
4366             self.failUnlessIn("Unhealthy: 9 shares (enc 3-of-10)", r["summary"])
4367             self.failIf(r["results"]["healthy"])
4368             self.failUnless(r["results"]["recoverable"])
4369             self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
4370             self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
4371         d.addCallback(_got_json_corrupt)
4372
4373         d.addErrback(self.explain_web_error)
4374         return d
4375
4376     def test_repair_html(self):
4377         self.basedir = "web/Grid/repair_html"
4378         self.set_up_grid()
4379         c0 = self.g.clients[0]
4380         self.uris = {}
4381         DATA = "data" * 100
4382         d = c0.upload(upload.Data(DATA, convergence=""))
4383         def _stash_uri(ur, which):
4384             self.uris[which] = ur.get_uri()
4385         d.addCallback(_stash_uri, "good")
4386         d.addCallback(lambda ign:
4387                       c0.upload(upload.Data(DATA+"1", convergence="")))
4388         d.addCallback(_stash_uri, "sick")
4389         d.addCallback(lambda ign:
4390                       c0.upload(upload.Data(DATA+"2", convergence="")))
4391         d.addCallback(_stash_uri, "dead")
4392         def _stash_mutable_uri(n, which):
4393             self.uris[which] = n.get_uri()
4394             assert isinstance(self.uris[which], str)
4395         d.addCallback(lambda ign:
4396             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4397         d.addCallback(_stash_mutable_uri, "corrupt")
4398
4399         def _compute_fileurls(ignored):
4400             self.fileurls = {}
4401             for which in self.uris:
4402                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4403         d.addCallback(_compute_fileurls)
4404
4405         def _clobber_shares(ignored):
4406             good_shares = self.find_uri_shares(self.uris["good"])
4407             self.failUnlessReallyEqual(len(good_shares), 10)
4408             sick_shares = self.find_uri_shares(self.uris["sick"])
4409             os.unlink(sick_shares[0][2])
4410             dead_shares = self.find_uri_shares(self.uris["dead"])
4411             for i in range(1, 10):
4412                 os.unlink(dead_shares[i][2])
4413             c_shares = self.find_uri_shares(self.uris["corrupt"])
4414             cso = CorruptShareOptions()
4415             cso.stdout = StringIO()
4416             cso.parseOptions([c_shares[0][2]])
4417             corrupt_share(cso)
4418         d.addCallback(_clobber_shares)
4419
4420         d.addCallback(self.CHECK, "good", "t=check&repair=true")
4421         def _got_html_good(res):
4422             self.failUnlessIn("Healthy", res)
4423             self.failIfIn("Not Healthy", res)
4424             self.failUnlessIn("No repair necessary", res)
4425             self.failUnlessIn(FAVICON_MARKUP, res)
4426         d.addCallback(_got_html_good)
4427
4428         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
4429         def _got_html_sick(res):
4430             self.failUnlessIn("Healthy : healthy", res)
4431             self.failIfIn("Not Healthy", res)
4432             self.failUnlessIn("Repair successful", res)
4433         d.addCallback(_got_html_sick)
4434
4435         # repair of a dead file will fail, of course, but it isn't yet
4436         # clear how this should be reported. Right now it shows up as
4437         # a "410 Gone".
4438         #
4439         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
4440         #def _got_html_dead(res):
4441         #    print res
4442         #    self.failUnlessIn("Healthy : healthy", res)
4443         #    self.failIfIn("Not Healthy", res)
4444         #    self.failUnlessIn("No repair necessary", res)
4445         #d.addCallback(_got_html_dead)
4446
4447         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
4448         def _got_html_corrupt(res):
4449             self.failUnlessIn("Healthy : Healthy", res)
4450             self.failIfIn("Not Healthy", res)
4451             self.failUnlessIn("Repair successful", res)
4452         d.addCallback(_got_html_corrupt)
4453
4454         d.addErrback(self.explain_web_error)
4455         return d
4456
4457     def test_repair_json(self):
4458         self.basedir = "web/Grid/repair_json"
4459         self.set_up_grid()
4460         c0 = self.g.clients[0]
4461         self.uris = {}
4462         DATA = "data" * 100
4463         d = c0.upload(upload.Data(DATA+"1", convergence=""))
4464         def _stash_uri(ur, which):
4465             self.uris[which] = ur.get_uri()
4466         d.addCallback(_stash_uri, "sick")
4467
4468         def _compute_fileurls(ignored):
4469             self.fileurls = {}
4470             for which in self.uris:
4471                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4472         d.addCallback(_compute_fileurls)
4473
4474         def _clobber_shares(ignored):
4475             sick_shares = self.find_uri_shares(self.uris["sick"])
4476             os.unlink(sick_shares[0][2])
4477         d.addCallback(_clobber_shares)
4478
4479         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
4480         def _got_json_sick(res):
4481             r = simplejson.loads(res)
4482             self.failUnlessReallyEqual(r["repair-attempted"], True)
4483             self.failUnlessReallyEqual(r["repair-successful"], True)
4484             self.failUnlessEqual(r["pre-repair-results"]["summary"],
4485                                  "Not Healthy: 9 shares (enc 3-of-10)")
4486             self.failIf(r["pre-repair-results"]["results"]["healthy"])
4487             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
4488             self.failUnless(r["post-repair-results"]["results"]["healthy"])
4489         d.addCallback(_got_json_sick)
4490
4491         d.addErrback(self.explain_web_error)
4492         return d
4493
4494     def test_unknown(self, immutable=False):
4495         self.basedir = "web/Grid/unknown"
4496         if immutable:
4497             self.basedir = "web/Grid/unknown-immutable"
4498
4499         self.set_up_grid()
4500         c0 = self.g.clients[0]
4501         self.uris = {}
4502         self.fileurls = {}
4503
4504         # the future cap format may contain slashes, which must be tolerated
4505         expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
4506                                                            safe="")
4507
4508         if immutable:
4509             name = u"future-imm"
4510             future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
4511             d = c0.create_immutable_dirnode({name: (future_node, {})})
4512         else:
4513             name = u"future"
4514             future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4515             d = c0.create_dirnode()
4516
4517         def _stash_root_and_create_file(n):
4518             self.rootnode = n
4519             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
4520             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
4521             if not immutable:
4522                 return self.rootnode.set_node(name, future_node)
4523         d.addCallback(_stash_root_and_create_file)
4524
4525         # make sure directory listing tolerates unknown nodes
4526         d.addCallback(lambda ign: self.GET(self.rooturl))
4527         def _check_directory_html(res, expected_type_suffix):
4528             pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
4529                                   '<td>%s</td>' % (expected_type_suffix, str(name)),
4530                                  re.DOTALL)
4531             self.failUnless(re.search(pattern, res), res)
4532             # find the More Info link for name, should be relative
4533             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4534             info_url = mo.group(1)
4535             self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
4536         if immutable:
4537             d.addCallback(_check_directory_html, "-IMM")
4538         else:
4539             d.addCallback(_check_directory_html, "")
4540
4541         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4542         def _check_directory_json(res, expect_rw_uri):
4543             data = simplejson.loads(res)
4544             self.failUnlessEqual(data[0], "dirnode")
4545             f = data[1]["children"][name]
4546             self.failUnlessEqual(f[0], "unknown")
4547             if expect_rw_uri:
4548                 self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data)
4549             else:
4550                 self.failIfIn("rw_uri", f[1])
4551             if immutable:
4552                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
4553             else:
4554                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
4555             self.failUnlessIn("metadata", f[1])
4556         d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
4557
4558         def _check_info(res, expect_rw_uri, expect_ro_uri):
4559             self.failUnlessIn("Object Type: <span>unknown</span>", res)
4560             if expect_rw_uri:
4561                 self.failUnlessIn(unknown_rwcap, res)
4562             if expect_ro_uri:
4563                 if immutable:
4564                     self.failUnlessIn(unknown_immcap, res)
4565                 else:
4566                     self.failUnlessIn(unknown_rocap, res)
4567             else:
4568                 self.failIfIn(unknown_rocap, res)
4569             self.failIfIn("Raw data as", res)
4570             self.failIfIn("Directory writecap", res)
4571             self.failIfIn("Checker Operations", res)
4572             self.failIfIn("Mutable File Operations", res)
4573             self.failIfIn("Directory Operations", res)
4574
4575         # FIXME: these should have expect_rw_uri=not immutable; I don't know
4576         # why they fail. Possibly related to ticket #922.
4577
4578         d.addCallback(lambda ign: self.GET(expected_info_url))
4579         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
4580         d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
4581         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
4582
4583         def _check_json(res, expect_rw_uri):
4584             data = simplejson.loads(res)
4585             self.failUnlessEqual(data[0], "unknown")
4586             if expect_rw_uri:
4587                 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data)
4588             else:
4589                 self.failIfIn("rw_uri", data[1])
4590
4591             if immutable:
4592                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data)
4593                 self.failUnlessReallyEqual(data[1]["mutable"], False)
4594             elif expect_rw_uri:
4595                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4596                 self.failUnlessReallyEqual(data[1]["mutable"], True)
4597             else:
4598                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4599                 self.failIfIn("mutable", data[1])
4600
4601             # TODO: check metadata contents
4602             self.failUnlessIn("metadata", data[1])
4603
4604         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
4605         d.addCallback(_check_json, expect_rw_uri=not immutable)
4606
4607         # and make sure that a read-only version of the directory can be
4608         # rendered too. This version will not have unknown_rwcap, whether
4609         # or not future_node was immutable.
4610         d.addCallback(lambda ign: self.GET(self.rourl))
4611         if immutable:
4612             d.addCallback(_check_directory_html, "-IMM")
4613         else:
4614             d.addCallback(_check_directory_html, "-RO")
4615
4616         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
4617         d.addCallback(_check_directory_json, expect_rw_uri=False)
4618
4619         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
4620         d.addCallback(_check_json, expect_rw_uri=False)
4621
4622         # TODO: check that getting t=info from the Info link in the ro directory
4623         # works, and does not include the writecap URI.
4624         return d
4625
4626     def test_immutable_unknown(self):
4627         return self.test_unknown(immutable=True)
4628
4629     def test_mutant_dirnodes_are_omitted(self):
4630         self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
4631
4632         self.set_up_grid()
4633         c = self.g.clients[0]
4634         nm = c.nodemaker
4635         self.uris = {}
4636         self.fileurls = {}
4637
4638         lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
4639         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
4640         mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
4641
4642         # This method tests mainly dirnode, but we'd have to duplicate code in order to
4643         # test the dirnode and web layers separately.
4644
4645         # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
4646         # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
4647         # When the directory is read, the mutants should be silently disposed of, leaving
4648         # their lonely sibling.
4649         # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
4650         # because immutable directories don't have a writecap and therefore that field
4651         # isn't (and can't be) decrypted.
4652         # TODO: The field still exists in the netstring. Technically we should check what
4653         # happens if something is put there (_unpack_contents should raise ValueError),
4654         # but that can wait.
4655
4656         lonely_child = nm.create_from_cap(lonely_uri)
4657         mutant_ro_child = nm.create_from_cap(mut_read_uri)
4658         mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
4659
4660         def _by_hook_or_by_crook():
4661             return True
4662         for n in [mutant_ro_child, mutant_write_in_ro_child]:
4663             n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
4664
4665         mutant_write_in_ro_child.get_write_uri    = lambda: None
4666         mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
4667
4668         kids = {u"lonely":      (lonely_child, {}),
4669                 u"ro":          (mutant_ro_child, {}),
4670                 u"write-in-ro": (mutant_write_in_ro_child, {}),
4671                 }
4672         d = c.create_immutable_dirnode(kids)
4673
4674         def _created(dn):
4675             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
4676             self.failIf(dn.is_mutable())
4677             self.failUnless(dn.is_readonly())
4678             # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
4679             self.failIf(hasattr(dn._node, 'get_writekey'))
4680             rep = str(dn)
4681             self.failUnlessIn("RO-IMM", rep)
4682             cap = dn.get_cap()
4683             self.failUnlessIn("CHK", cap.to_string())
4684             self.cap = cap
4685             self.rootnode = dn
4686             self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
4687             return download_to_data(dn._node)
4688         d.addCallback(_created)
4689
4690         def _check_data(data):
4691             # Decode the netstring representation of the directory to check that all children
4692             # are present. This is a bit of an abstraction violation, but there's not really
4693             # any other way to do it given that the real DirectoryNode._unpack_contents would
4694             # strip the mutant children out (which is what we're trying to test, later).
4695             position = 0
4696             numkids = 0
4697             while position < len(data):
4698                 entries, position = split_netstring(data, 1, position)
4699                 entry = entries[0]
4700                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
4701                 name = name_utf8.decode("utf-8")
4702                 self.failUnlessEqual(rwcapdata, "")
4703                 self.failUnlessIn(name, kids)
4704                 (expected_child, ign) = kids[name]
4705                 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
4706                 numkids += 1
4707
4708             self.failUnlessReallyEqual(numkids, 3)
4709             return self.rootnode.list()
4710         d.addCallback(_check_data)
4711
4712         # Now when we use the real directory listing code, the mutants should be absent.
4713         def _check_kids(children):
4714             self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
4715             lonely_node, lonely_metadata = children[u"lonely"]
4716
4717             self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
4718             self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
4719         d.addCallback(_check_kids)
4720
4721         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
4722         d.addCallback(lambda n: n.list())
4723         d.addCallback(_check_kids)  # again with dirnode recreated from cap
4724
4725         # Make sure the lonely child can be listed in HTML...
4726         d.addCallback(lambda ign: self.GET(self.rooturl))
4727         def _check_html(res):
4728             self.failIfIn("URI:SSK", res)
4729             get_lonely = "".join([r'<td>FILE</td>',
4730                                   r'\s+<td>',
4731                                   r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
4732                                   r'</td>',
4733                                   r'\s+<td align="right">%d</td>' % len("one"),
4734                                   ])
4735             self.failUnless(re.search(get_lonely, res), res)
4736
4737             # find the More Info link for name, should be relative
4738             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4739             info_url = mo.group(1)
4740             self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
4741         d.addCallback(_check_html)
4742
4743         # ... and in JSON.
4744         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4745         def _check_json(res):
4746             data = simplejson.loads(res)
4747             self.failUnlessEqual(data[0], "dirnode")
4748             listed_children = data[1]["children"]
4749             self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
4750             ll_type, ll_data = listed_children[u"lonely"]
4751             self.failUnlessEqual(ll_type, "filenode")
4752             self.failIfIn("rw_uri", ll_data)
4753             self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
4754         d.addCallback(_check_json)
4755         return d
4756
4757     def test_deep_check(self):
4758         self.basedir = "web/Grid/deep_check"
4759         self.set_up_grid()
4760         c0 = self.g.clients[0]
4761         self.uris = {}
4762         self.fileurls = {}
4763         DATA = "data" * 100
4764         d = c0.create_dirnode()
4765         def _stash_root_and_create_file(n):
4766             self.rootnode = n
4767             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4768             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4769         d.addCallback(_stash_root_and_create_file)
4770         def _stash_uri(fn, which):
4771             self.uris[which] = fn.get_uri()
4772             return fn
4773         d.addCallback(_stash_uri, "good")
4774         d.addCallback(lambda ign:
4775                       self.rootnode.add_file(u"small",
4776                                              upload.Data("literal",
4777                                                         convergence="")))
4778         d.addCallback(_stash_uri, "small")
4779         d.addCallback(lambda ign:
4780                       self.rootnode.add_file(u"sick",
4781                                              upload.Data(DATA+"1",
4782                                                         convergence="")))
4783         d.addCallback(_stash_uri, "sick")
4784
4785         # this tests that deep-check and stream-manifest will ignore
4786         # UnknownNode instances. Hopefully this will also cover deep-stats.
4787         future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4788         d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
4789
4790         def _clobber_shares(ignored):
4791             self.delete_shares_numbered(self.uris["sick"], [0,1])
4792         d.addCallback(_clobber_shares)
4793
4794         # root
4795         # root/good
4796         # root/small
4797         # root/sick
4798         # root/future
4799
4800         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4801         def _done(res):
4802             try:
4803                 units = [simplejson.loads(line)
4804                          for line in res.splitlines()
4805                          if line]
4806             except ValueError:
4807                 print "response is:", res
4808                 print "undecodeable line was '%s'" % line
4809                 raise
4810             self.failUnlessReallyEqual(len(units), 5+1)
4811             # should be parent-first
4812             u0 = units[0]
4813             self.failUnlessEqual(u0["path"], [])
4814             self.failUnlessEqual(u0["type"], "directory")
4815             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4816             u0cr = u0["check-results"]
4817             self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
4818
4819             ugood = [u for u in units
4820                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4821             self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
4822             ugoodcr = ugood["check-results"]
4823             self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
4824
4825             stats = units[-1]
4826             self.failUnlessEqual(stats["type"], "stats")
4827             s = stats["stats"]
4828             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4829             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4830             self.failUnlessReallyEqual(s["count-directories"], 1)
4831             self.failUnlessReallyEqual(s["count-unknown"], 1)
4832         d.addCallback(_done)
4833
4834         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4835         def _check_manifest(res):
4836             self.failUnless(res.endswith("\n"))
4837             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
4838             self.failUnlessReallyEqual(len(units), 5+1)
4839             self.failUnlessEqual(units[-1]["type"], "stats")
4840             first = units[0]
4841             self.failUnlessEqual(first["path"], [])
4842             self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri())
4843             self.failUnlessEqual(first["type"], "directory")
4844             stats = units[-1]["stats"]
4845             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4846             self.failUnlessReallyEqual(stats["count-literal-files"], 1)
4847             self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
4848             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4849             self.failUnlessReallyEqual(stats["count-unknown"], 1)
4850         d.addCallback(_check_manifest)
4851
4852         # now add root/subdir and root/subdir/grandchild, then make subdir
4853         # unrecoverable, then see what happens
4854
4855         d.addCallback(lambda ign:
4856                       self.rootnode.create_subdirectory(u"subdir"))
4857         d.addCallback(_stash_uri, "subdir")
4858         d.addCallback(lambda subdir_node:
4859                       subdir_node.add_file(u"grandchild",
4860                                            upload.Data(DATA+"2",
4861                                                        convergence="")))
4862         d.addCallback(_stash_uri, "grandchild")
4863
4864         d.addCallback(lambda ign:
4865                       self.delete_shares_numbered(self.uris["subdir"],
4866                                                   range(1, 10)))
4867
4868         # root
4869         # root/good
4870         # root/small
4871         # root/sick
4872         # root/future
4873         # root/subdir [unrecoverable]
4874         # root/subdir/grandchild
4875
4876         # how should a streaming-JSON API indicate fatal error?
4877         # answer: emit ERROR: instead of a JSON string
4878
4879         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4880         def _check_broken_manifest(res):
4881             lines = res.splitlines()
4882             error_lines = [i
4883                            for (i,line) in enumerate(lines)
4884                            if line.startswith("ERROR:")]
4885             if not error_lines:
4886                 self.fail("no ERROR: in output: %s" % (res,))
4887             first_error = error_lines[0]
4888             error_line = lines[first_error]
4889             error_msg = lines[first_error+1:]
4890             error_msg_s = "\n".join(error_msg) + "\n"
4891             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4892                               error_line)
4893             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4894             units = [simplejson.loads(line) for line in lines[:first_error]]
4895             self.failUnlessReallyEqual(len(units), 6) # includes subdir
4896             last_unit = units[-1]
4897             self.failUnlessEqual(last_unit["path"], ["subdir"])
4898         d.addCallback(_check_broken_manifest)
4899
4900         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4901         def _check_broken_deepcheck(res):
4902             lines = res.splitlines()
4903             error_lines = [i
4904                            for (i,line) in enumerate(lines)
4905                            if line.startswith("ERROR:")]
4906             if not error_lines:
4907                 self.fail("no ERROR: in output: %s" % (res,))
4908             first_error = error_lines[0]
4909             error_line = lines[first_error]
4910             error_msg = lines[first_error+1:]
4911             error_msg_s = "\n".join(error_msg) + "\n"
4912             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4913                               error_line)
4914             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4915             units = [simplejson.loads(line) for line in lines[:first_error]]
4916             self.failUnlessReallyEqual(len(units), 6) # includes subdir
4917             last_unit = units[-1]
4918             self.failUnlessEqual(last_unit["path"], ["subdir"])
4919             r = last_unit["check-results"]["results"]
4920             self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
4921             self.failUnlessReallyEqual(r["count-shares-good"], 1)
4922             self.failUnlessReallyEqual(r["recoverable"], False)
4923         d.addCallback(_check_broken_deepcheck)
4924
4925         d.addErrback(self.explain_web_error)
4926         return d
4927
4928     def test_deep_check_and_repair(self):
4929         self.basedir = "web/Grid/deep_check_and_repair"
4930         self.set_up_grid()
4931         c0 = self.g.clients[0]
4932         self.uris = {}
4933         self.fileurls = {}
4934         DATA = "data" * 100
4935         d = c0.create_dirnode()
4936         def _stash_root_and_create_file(n):
4937             self.rootnode = n
4938             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4939             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4940         d.addCallback(_stash_root_and_create_file)
4941         def _stash_uri(fn, which):
4942             self.uris[which] = fn.get_uri()
4943         d.addCallback(_stash_uri, "good")
4944         d.addCallback(lambda ign:
4945                       self.rootnode.add_file(u"small",
4946                                              upload.Data("literal",
4947                                                         convergence="")))
4948         d.addCallback(_stash_uri, "small")
4949         d.addCallback(lambda ign:
4950                       self.rootnode.add_file(u"sick",
4951                                              upload.Data(DATA+"1",
4952                                                         convergence="")))
4953         d.addCallback(_stash_uri, "sick")
4954         #d.addCallback(lambda ign:
4955         #              self.rootnode.add_file(u"dead",
4956         #                                     upload.Data(DATA+"2",
4957         #                                                convergence="")))
4958         #d.addCallback(_stash_uri, "dead")
4959
4960         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
4961         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
4962         #d.addCallback(_stash_uri, "corrupt")
4963
4964         def _clobber_shares(ignored):
4965             good_shares = self.find_uri_shares(self.uris["good"])
4966             self.failUnlessReallyEqual(len(good_shares), 10)
4967             sick_shares = self.find_uri_shares(self.uris["sick"])
4968             os.unlink(sick_shares[0][2])
4969             #dead_shares = self.find_uri_shares(self.uris["dead"])
4970             #for i in range(1, 10):
4971             #    os.unlink(dead_shares[i][2])
4972
4973             #c_shares = self.find_uri_shares(self.uris["corrupt"])
4974             #cso = CorruptShareOptions()
4975             #cso.stdout = StringIO()
4976             #cso.parseOptions([c_shares[0][2]])
4977             #corrupt_share(cso)
4978         d.addCallback(_clobber_shares)
4979
4980         # root
4981         # root/good   CHK, 10 shares
4982         # root/small  LIT
4983         # root/sick   CHK, 9 shares
4984
4985         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
4986         def _done(res):
4987             units = [simplejson.loads(line)
4988                      for line in res.splitlines()
4989                      if line]
4990             self.failUnlessReallyEqual(len(units), 4+1)
4991             # should be parent-first
4992             u0 = units[0]
4993             self.failUnlessEqual(u0["path"], [])
4994             self.failUnlessEqual(u0["type"], "directory")
4995             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4996             u0crr = u0["check-and-repair-results"]
4997             self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
4998             self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
4999
5000             ugood = [u for u in units
5001                      if u["type"] == "file" and u["path"] == [u"good"]][0]
5002             self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
5003             ugoodcrr = ugood["check-and-repair-results"]
5004             self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
5005             self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
5006
5007             usick = [u for u in units
5008                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
5009             self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"])
5010             usickcrr = usick["check-and-repair-results"]
5011             self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
5012             self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
5013             self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
5014             self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
5015
5016             stats = units[-1]
5017             self.failUnlessEqual(stats["type"], "stats")
5018             s = stats["stats"]
5019             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
5020             self.failUnlessReallyEqual(s["count-literal-files"], 1)
5021             self.failUnlessReallyEqual(s["count-directories"], 1)
5022         d.addCallback(_done)
5023
5024         d.addErrback(self.explain_web_error)
5025         return d
5026
5027     def _count_leases(self, ignored, which):
5028         u = self.uris[which]
5029         shares = self.find_uri_shares(u)
5030         lease_counts = []
5031         for shnum, serverid, fn in shares:
5032             sf = get_share_file(fn)
5033             num_leases = len(list(sf.get_leases()))
5034             lease_counts.append( (fn, num_leases) )
5035         return lease_counts
5036
5037     def _assert_leasecount(self, lease_counts, expected):
5038         for (fn, num_leases) in lease_counts:
5039             if num_leases != expected:
5040                 self.fail("expected %d leases, have %d, on %s" %
5041                           (expected, num_leases, fn))
5042
5043     def test_add_lease(self):
5044         self.basedir = "web/Grid/add_lease"
5045         self.set_up_grid(num_clients=2)
5046         c0 = self.g.clients[0]
5047         self.uris = {}
5048         DATA = "data" * 100
5049         d = c0.upload(upload.Data(DATA, convergence=""))
5050         def _stash_uri(ur, which):
5051             self.uris[which] = ur.get_uri()
5052         d.addCallback(_stash_uri, "one")
5053         d.addCallback(lambda ign:
5054                       c0.upload(upload.Data(DATA+"1", convergence="")))
5055         d.addCallback(_stash_uri, "two")
5056         def _stash_mutable_uri(n, which):
5057             self.uris[which] = n.get_uri()
5058             assert isinstance(self.uris[which], str)
5059         d.addCallback(lambda ign:
5060             c0.create_mutable_file(publish.MutableData(DATA+"2")))
5061         d.addCallback(_stash_mutable_uri, "mutable")
5062
5063         def _compute_fileurls(ignored):
5064             self.fileurls = {}
5065             for which in self.uris:
5066                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
5067         d.addCallback(_compute_fileurls)
5068
5069         d.addCallback(self._count_leases, "one")
5070         d.addCallback(self._assert_leasecount, 1)
5071         d.addCallback(self._count_leases, "two")
5072         d.addCallback(self._assert_leasecount, 1)
5073         d.addCallback(self._count_leases, "mutable")
5074         d.addCallback(self._assert_leasecount, 1)
5075
5076         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
5077         def _got_html_good(res):
5078             self.failUnlessIn("Healthy", res)
5079             self.failIfIn("Not Healthy", res)
5080         d.addCallback(_got_html_good)
5081
5082         d.addCallback(self._count_leases, "one")
5083         d.addCallback(self._assert_leasecount, 1)
5084         d.addCallback(self._count_leases, "two")
5085         d.addCallback(self._assert_leasecount, 1)
5086         d.addCallback(self._count_leases, "mutable")
5087         d.addCallback(self._assert_leasecount, 1)
5088
5089         # this CHECK uses the original client, which uses the same
5090         # lease-secrets, so it will just renew the original lease
5091         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
5092         d.addCallback(_got_html_good)
5093
5094         d.addCallback(self._count_leases, "one")
5095         d.addCallback(self._assert_leasecount, 1)
5096         d.addCallback(self._count_leases, "two")
5097         d.addCallback(self._assert_leasecount, 1)
5098         d.addCallback(self._count_leases, "mutable")
5099         d.addCallback(self._assert_leasecount, 1)
5100
5101         # this CHECK uses an alternate client, which adds a second lease
5102         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
5103         d.addCallback(_got_html_good)
5104
5105         d.addCallback(self._count_leases, "one")
5106         d.addCallback(self._assert_leasecount, 2)
5107         d.addCallback(self._count_leases, "two")
5108         d.addCallback(self._assert_leasecount, 1)
5109         d.addCallback(self._count_leases, "mutable")
5110         d.addCallback(self._assert_leasecount, 1)
5111
5112         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
5113         d.addCallback(_got_html_good)
5114
5115         d.addCallback(self._count_leases, "one")
5116         d.addCallback(self._assert_leasecount, 2)
5117         d.addCallback(self._count_leases, "two")
5118         d.addCallback(self._assert_leasecount, 1)
5119         d.addCallback(self._count_leases, "mutable")
5120         d.addCallback(self._assert_leasecount, 1)
5121
5122         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
5123                       clientnum=1)
5124         d.addCallback(_got_html_good)
5125
5126         d.addCallback(self._count_leases, "one")
5127         d.addCallback(self._assert_leasecount, 2)
5128         d.addCallback(self._count_leases, "two")
5129         d.addCallback(self._assert_leasecount, 1)
5130         d.addCallback(self._count_leases, "mutable")
5131         d.addCallback(self._assert_leasecount, 2)
5132
5133         d.addErrback(self.explain_web_error)
5134         return d
5135
5136     def test_deep_add_lease(self):
5137         self.basedir = "web/Grid/deep_add_lease"
5138         self.set_up_grid(num_clients=2)
5139         c0 = self.g.clients[0]
5140         self.uris = {}
5141         self.fileurls = {}
5142         DATA = "data" * 100
5143         d = c0.create_dirnode()
5144         def _stash_root_and_create_file(n):
5145             self.rootnode = n
5146             self.uris["root"] = n.get_uri()
5147             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5148             return n.add_file(u"one", upload.Data(DATA, convergence=""))
5149         d.addCallback(_stash_root_and_create_file)
5150         def _stash_uri(fn, which):
5151             self.uris[which] = fn.get_uri()
5152         d.addCallback(_stash_uri, "one")
5153         d.addCallback(lambda ign:
5154                       self.rootnode.add_file(u"small",
5155                                              upload.Data("literal",
5156                                                         convergence="")))
5157         d.addCallback(_stash_uri, "small")
5158
5159         d.addCallback(lambda ign:
5160             c0.create_mutable_file(publish.MutableData("mutable")))
5161         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
5162         d.addCallback(_stash_uri, "mutable")
5163
5164         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
5165         def _done(res):
5166             units = [simplejson.loads(line)
5167                      for line in res.splitlines()
5168                      if line]
5169             # root, one, small, mutable,   stats
5170             self.failUnlessReallyEqual(len(units), 4+1)
5171         d.addCallback(_done)
5172
5173         d.addCallback(self._count_leases, "root")
5174         d.addCallback(self._assert_leasecount, 1)
5175         d.addCallback(self._count_leases, "one")
5176         d.addCallback(self._assert_leasecount, 1)
5177         d.addCallback(self._count_leases, "mutable")
5178         d.addCallback(self._assert_leasecount, 1)
5179
5180         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
5181         d.addCallback(_done)
5182
5183         d.addCallback(self._count_leases, "root")
5184         d.addCallback(self._assert_leasecount, 1)
5185         d.addCallback(self._count_leases, "one")
5186         d.addCallback(self._assert_leasecount, 1)
5187         d.addCallback(self._count_leases, "mutable")
5188         d.addCallback(self._assert_leasecount, 1)
5189
5190         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
5191                       clientnum=1)
5192         d.addCallback(_done)
5193
5194         d.addCallback(self._count_leases, "root")
5195         d.addCallback(self._assert_leasecount, 2)
5196         d.addCallback(self._count_leases, "one")
5197         d.addCallback(self._assert_leasecount, 2)
5198         d.addCallback(self._count_leases, "mutable")
5199         d.addCallback(self._assert_leasecount, 2)
5200
5201         d.addErrback(self.explain_web_error)
5202         return d
5203
5204
5205     def test_exceptions(self):
5206         self.basedir = "web/Grid/exceptions"
5207         self.set_up_grid(num_clients=1, num_servers=2)
5208         c0 = self.g.clients[0]
5209         c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 2
5210         self.fileurls = {}
5211         DATA = "data" * 100
5212         d = c0.create_dirnode()
5213         def _stash_root(n):
5214             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5215             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
5216             return n
5217         d.addCallback(_stash_root)
5218         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
5219         def _stash_bad(ur):
5220             self.fileurls["1share"] = "uri/" + urllib.quote(ur.get_uri())
5221             self.delete_shares_numbered(ur.get_uri(), range(1,10))
5222
5223             u = uri.from_string(ur.get_uri())
5224             u.key = testutil.flip_bit(u.key, 0)
5225             baduri = u.to_string()
5226             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
5227         d.addCallback(_stash_bad)
5228         d.addCallback(lambda ign: c0.create_dirnode())
5229         def _mangle_dirnode_1share(n):
5230             u = n.get_uri()
5231             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
5232             self.fileurls["dir-1share-json"] = url + "?t=json"
5233             self.delete_shares_numbered(u, range(1,10))
5234         d.addCallback(_mangle_dirnode_1share)
5235         d.addCallback(lambda ign: c0.create_dirnode())
5236         def _mangle_dirnode_0share(n):
5237             u = n.get_uri()
5238             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
5239             self.fileurls["dir-0share-json"] = url + "?t=json"
5240             self.delete_shares_numbered(u, range(0,10))
5241         d.addCallback(_mangle_dirnode_0share)
5242
5243         # NotEnoughSharesError should be reported sensibly, with a
5244         # text/plain explanation of the problem, and perhaps some
5245         # information on which shares *could* be found.
5246
5247         d.addCallback(lambda ignored:
5248                       self.shouldHTTPError("GET unrecoverable",
5249                                            410, "Gone", "NoSharesError",
5250                                            self.GET, self.fileurls["0shares"]))
5251         def _check_zero_shares(body):
5252             self.failIfIn("<html>", body)
5253             body = " ".join(body.strip().split())
5254             exp = ("NoSharesError: no shares could be found. "
5255                    "Zero shares usually indicates a corrupt URI, or that "
5256                    "no servers were connected, but it might also indicate "
5257                    "severe corruption. You should perform a filecheck on "
5258                    "this object to learn more. The full error message is: "
5259                    "no shares (need 3). Last failure: None")
5260             self.failUnlessReallyEqual(exp, body)
5261         d.addCallback(_check_zero_shares)
5262
5263
5264         d.addCallback(lambda ignored:
5265                       self.shouldHTTPError("GET 1share",
5266                                            410, "Gone", "NotEnoughSharesError",
5267                                            self.GET, self.fileurls["1share"]))
5268         def _check_one_share(body):
5269             self.failIfIn("<html>", body)
5270             body = " ".join(body.strip().split())
5271             msgbase = ("NotEnoughSharesError: This indicates that some "
5272                        "servers were unavailable, or that shares have been "
5273                        "lost to server departure, hard drive failure, or disk "
5274                        "corruption. You should perform a filecheck on "
5275                        "this object to learn more. The full error message is:"
5276                        )
5277             msg1 = msgbase + (" ran out of shares:"
5278                               " complete=sh0"
5279                               " pending="
5280                               " overdue= unused= need 3. Last failure: None")
5281             msg2 = msgbase + (" ran out of shares:"
5282                               " complete="
5283                               " pending=Share(sh0-on-xgru5)"
5284                               " overdue= unused= need 3. Last failure: None")
5285             self.failUnless(body == msg1 or body == msg2, body)
5286         d.addCallback(_check_one_share)
5287
5288         d.addCallback(lambda ignored:
5289                       self.shouldHTTPError("GET imaginary",
5290                                            404, "Not Found", None,
5291                                            self.GET, self.fileurls["imaginary"]))
5292         def _missing_child(body):
5293             self.failUnlessIn("No such child: imaginary", body)
5294         d.addCallback(_missing_child)
5295
5296         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
5297         def _check_0shares_dir_html(body):
5298             self.failUnlessIn("<html>", body)
5299             # we should see the regular page, but without the child table or
5300             # the dirops forms
5301             body = " ".join(body.strip().split())
5302             self.failUnlessIn('href="?t=info">More info on this directory',
5303                               body)
5304             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5305                    "could not be retrieved, because there were insufficient "
5306                    "good shares. This might indicate that no servers were "
5307                    "connected, insufficient servers were connected, the URI "
5308                    "was corrupt, or that shares have been lost due to server "
5309                    "departure, hard drive failure, or disk corruption. You "
5310                    "should perform a filecheck on this object to learn more.")
5311             self.failUnlessIn(exp, body)
5312             self.failUnlessIn("No upload forms: directory is unreadable", body)
5313         d.addCallback(_check_0shares_dir_html)
5314
5315         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
5316         def _check_1shares_dir_html(body):
5317             # at some point, we'll split UnrecoverableFileError into 0-shares
5318             # and some-shares like we did for immutable files (since there
5319             # are different sorts of advice to offer in each case). For now,
5320             # they present the same way.
5321             self.failUnlessIn("<html>", body)
5322             body = " ".join(body.strip().split())
5323             self.failUnlessIn('href="?t=info">More info on this directory',
5324                               body)
5325             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5326                    "could not be retrieved, because there were insufficient "
5327                    "good shares. This might indicate that no servers were "
5328                    "connected, insufficient servers were connected, the URI "
5329                    "was corrupt, or that shares have been lost due to server "
5330                    "departure, hard drive failure, or disk corruption. You "
5331                    "should perform a filecheck on this object to learn more.")
5332             self.failUnlessIn(exp, body)
5333             self.failUnlessIn("No upload forms: directory is unreadable", body)
5334         d.addCallback(_check_1shares_dir_html)
5335
5336         d.addCallback(lambda ignored:
5337                       self.shouldHTTPError("GET dir-0share-json",
5338                                            410, "Gone", "UnrecoverableFileError",
5339                                            self.GET,
5340                                            self.fileurls["dir-0share-json"]))
5341         def _check_unrecoverable_file(body):
5342             self.failIfIn("<html>", body)
5343             body = " ".join(body.strip().split())
5344             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5345                    "could not be retrieved, because there were insufficient "
5346                    "good shares. This might indicate that no servers were "
5347                    "connected, insufficient servers were connected, the URI "
5348                    "was corrupt, or that shares have been lost due to server "
5349                    "departure, hard drive failure, or disk corruption. You "
5350                    "should perform a filecheck on this object to learn more.")
5351             self.failUnlessReallyEqual(exp, body)
5352         d.addCallback(_check_unrecoverable_file)
5353
5354         d.addCallback(lambda ignored:
5355                       self.shouldHTTPError("GET dir-1share-json",
5356                                            410, "Gone", "UnrecoverableFileError",
5357                                            self.GET,
5358                                            self.fileurls["dir-1share-json"]))
5359         d.addCallback(_check_unrecoverable_file)
5360
5361         d.addCallback(lambda ignored:
5362                       self.shouldHTTPError("GET imaginary",
5363                                            404, "Not Found", None,
5364                                            self.GET, self.fileurls["imaginary"]))
5365
5366         # attach a webapi child that throws a random error, to test how it
5367         # gets rendered.
5368         w = c0.getServiceNamed("webish")
5369         w.root.putChild("ERRORBOOM", ErrorBoom())
5370
5371         # "Accept: */*" :        should get a text/html stack trace
5372         # "Accept: text/plain" : should get a text/plain stack trace
5373         # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
5374         # no Accept header:      should get a text/html stack trace
5375
5376         d.addCallback(lambda ignored:
5377                       self.shouldHTTPError("GET errorboom_html",
5378                                            500, "Internal Server Error", None,
5379                                            self.GET, "ERRORBOOM",
5380                                            headers={"accept": "*/*"}))
5381         def _internal_error_html1(body):
5382             self.failUnlessIn("<html>", "expected HTML, not '%s'" % body)
5383         d.addCallback(_internal_error_html1)
5384
5385         d.addCallback(lambda ignored:
5386                       self.shouldHTTPError("GET errorboom_text",
5387                                            500, "Internal Server Error", None,
5388                                            self.GET, "ERRORBOOM",
5389                                            headers={"accept": "text/plain"}))
5390         def _internal_error_text2(body):
5391             self.failIfIn("<html>", body)
5392             self.failUnless(body.startswith("Traceback "), body)
5393         d.addCallback(_internal_error_text2)
5394
5395         CLI_accepts = "text/plain, application/octet-stream"
5396         d.addCallback(lambda ignored:
5397                       self.shouldHTTPError("GET errorboom_text",
5398                                            500, "Internal Server Error", None,
5399                                            self.GET, "ERRORBOOM",
5400                                            headers={"accept": CLI_accepts}))
5401         def _internal_error_text3(body):
5402             self.failIfIn("<html>", body)
5403             self.failUnless(body.startswith("Traceback "), body)
5404         d.addCallback(_internal_error_text3)
5405
5406         d.addCallback(lambda ignored:
5407                       self.shouldHTTPError("GET errorboom_text",
5408                                            500, "Internal Server Error", None,
5409                                            self.GET, "ERRORBOOM"))
5410         def _internal_error_html4(body):
5411             self.failUnlessIn("<html>", body)
5412         d.addCallback(_internal_error_html4)
5413
5414         def _flush_errors(res):
5415             # Trial: please ignore the CompletelyUnhandledError in the logs
5416             self.flushLoggedErrors(CompletelyUnhandledError)
5417             return res
5418         d.addBoth(_flush_errors)
5419
5420         return d
5421
5422     def test_blacklist(self):
5423         # download from a blacklisted URI, get an error
5424         self.basedir = "web/Grid/blacklist"
5425         self.set_up_grid()
5426         c0 = self.g.clients[0]
5427         c0_basedir = c0.basedir
5428         fn = os.path.join(c0_basedir, "access.blacklist")
5429         self.uris = {}
5430         DATA = "off-limits " * 50
5431
5432         d = c0.upload(upload.Data(DATA, convergence=""))
5433         def _stash_uri_and_create_dir(ur):
5434             self.uri = ur.get_uri()
5435             self.url = "uri/"+self.uri
5436             u = uri.from_string_filenode(self.uri)
5437             self.si = u.get_storage_index()
5438             childnode = c0.create_node_from_uri(self.uri, None)
5439             return c0.create_dirnode({u"blacklisted.txt": (childnode,{}) })
5440         d.addCallback(_stash_uri_and_create_dir)
5441         def _stash_dir(node):
5442             self.dir_node = node
5443             self.dir_uri = node.get_uri()
5444             self.dir_url = "uri/"+self.dir_uri
5445         d.addCallback(_stash_dir)
5446         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5447         def _check_dir_html(body):
5448             self.failUnlessIn("<html>", body)
5449             self.failUnlessIn("blacklisted.txt</a>", body)
5450         d.addCallback(_check_dir_html)
5451         d.addCallback(lambda ign: self.GET(self.url))
5452         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5453
5454         def _blacklist(ign):
5455             f = open(fn, "w")
5456             f.write(" # this is a comment\n")
5457             f.write(" \n")
5458             f.write("\n") # also exercise blank lines
5459             f.write("%s %s\n" % (base32.b2a(self.si), "off-limits to you"))
5460             f.close()
5461             # clients should be checking the blacklist each time, so we don't
5462             # need to restart the client
5463         d.addCallback(_blacklist)
5464         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_uri",
5465                                                        403, "Forbidden",
5466                                                        "Access Prohibited: off-limits",
5467                                                        self.GET, self.url))
5468
5469         # We should still be able to list the parent directory, in HTML...
5470         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5471         def _check_dir_html2(body):
5472             self.failUnlessIn("<html>", body)
5473             self.failUnlessIn("blacklisted.txt</strike>", body)
5474         d.addCallback(_check_dir_html2)
5475
5476         # ... and in JSON (used by CLI).
5477         d.addCallback(lambda ign: self.GET(self.dir_url+"?t=json", followRedirect=True))
5478         def _check_dir_json(res):
5479             data = simplejson.loads(res)
5480             self.failUnless(isinstance(data, list), data)
5481             self.failUnlessEqual(data[0], "dirnode")
5482             self.failUnless(isinstance(data[1], dict), data)
5483             self.failUnlessIn("children", data[1])
5484             self.failUnlessIn("blacklisted.txt", data[1]["children"])
5485             childdata = data[1]["children"]["blacklisted.txt"]
5486             self.failUnless(isinstance(childdata, list), data)
5487             self.failUnlessEqual(childdata[0], "filenode")
5488             self.failUnless(isinstance(childdata[1], dict), data)
5489         d.addCallback(_check_dir_json)
5490
5491         def _unblacklist(ign):
5492             open(fn, "w").close()
5493             # the Blacklist object watches mtime to tell when the file has
5494             # changed, but on windows this test will run faster than the
5495             # filesystem's mtime resolution. So we edit Blacklist.last_mtime
5496             # to force a reload.
5497             self.g.clients[0].blacklist.last_mtime -= 2.0
5498         d.addCallback(_unblacklist)
5499
5500         # now a read should work
5501         d.addCallback(lambda ign: self.GET(self.url))
5502         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5503
5504         # read again to exercise the blacklist-is-unchanged logic
5505         d.addCallback(lambda ign: self.GET(self.url))
5506         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5507
5508         # now add a blacklisted directory, and make sure files under it are
5509         # refused too
5510         def _add_dir(ign):
5511             childnode = c0.create_node_from_uri(self.uri, None)
5512             return c0.create_dirnode({u"child": (childnode,{}) })
5513         d.addCallback(_add_dir)
5514         def _get_dircap(dn):
5515             self.dir_si_b32 = base32.b2a(dn.get_storage_index())
5516             self.dir_url_base = "uri/"+dn.get_write_uri()
5517             self.dir_url_json1 = "uri/"+dn.get_write_uri()+"?t=json"
5518             self.dir_url_json2 = "uri/"+dn.get_write_uri()+"/?t=json"
5519             self.dir_url_json_ro = "uri/"+dn.get_readonly_uri()+"/?t=json"
5520             self.child_url = "uri/"+dn.get_readonly_uri()+"/child"
5521         d.addCallback(_get_dircap)
5522         d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True))
5523         d.addCallback(lambda body: self.failUnlessIn("<html>", body))
5524         d.addCallback(lambda ign: self.GET(self.dir_url_json1))
5525         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5526         d.addCallback(lambda ign: self.GET(self.dir_url_json2))
5527         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5528         d.addCallback(lambda ign: self.GET(self.dir_url_json_ro))
5529         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5530         d.addCallback(lambda ign: self.GET(self.child_url))
5531         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5532
5533         def _block_dir(ign):
5534             f = open(fn, "w")
5535             f.write("%s %s\n" % (self.dir_si_b32, "dir-off-limits to you"))
5536             f.close()
5537             self.g.clients[0].blacklist.last_mtime -= 2.0
5538         d.addCallback(_block_dir)
5539         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir base",
5540                                                        403, "Forbidden",
5541                                                        "Access Prohibited: dir-off-limits",
5542                                                        self.GET, self.dir_url_base))
5543         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json1",
5544                                                        403, "Forbidden",
5545                                                        "Access Prohibited: dir-off-limits",
5546                                                        self.GET, self.dir_url_json1))
5547         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json2",
5548                                                        403, "Forbidden",
5549                                                        "Access Prohibited: dir-off-limits",
5550                                                        self.GET, self.dir_url_json2))
5551         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json_ro",
5552                                                        403, "Forbidden",
5553                                                        "Access Prohibited: dir-off-limits",
5554                                                        self.GET, self.dir_url_json_ro))
5555         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir child",
5556                                                        403, "Forbidden",
5557                                                        "Access Prohibited: dir-off-limits",
5558                                                        self.GET, self.child_url))
5559         return d
5560
5561
5562 class CompletelyUnhandledError(Exception):
5563     pass
5564 class ErrorBoom(rend.Page):
5565     def beforeRender(self, ctx):
5566         raise CompletelyUnhandledError("whoops")