src/allmydata/test/test_web.py

   1 import os.path, re, urllib, time
   2 import simplejson
   3 from StringIO import StringIO
   4
   5 from twisted.application import service
   6 from twisted.trial import unittest
   7 from twisted.internet import defer, reactor
   8 from twisted.internet.task import Clock
   9 from twisted.web import client, error, http
  10 from twisted.python import failure, log
  11
  12 from foolscap.api import fireEventually, flushEventualQueue
  13
  14 from nevow import rend
  15
  16 from allmydata import interfaces, uri, webish, dirnode
  17 from allmydata.storage.shares import get_share_file
  18 from allmydata.storage_client import StorageFarmBroker, StubServer
  19 from allmydata.immutable import upload
  20 from allmydata.immutable.downloader.status import DownloadStatus
  21 from allmydata.dirnode import DirectoryNode
  22 from allmydata.nodemaker import NodeMaker
  23 from allmydata.unknown import UnknownNode
  24 from allmydata.web import status, common
  25 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  26 from allmydata.util import fileutil, base32, hashutil
  27 from allmydata.util.consumer import download_to_data
  28 from allmydata.util.netstring import split_netstring
  29 from allmydata.util.encodingutil import to_str
  30 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  31      create_chk_filenode, WebErrorMixin, ShouldFailMixin, \
  32      make_mutable_file_uri, create_mutable_filenode
  33 from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
  34 from allmydata.mutable import servermap, publish, retrieve
  35 import allmydata.test.common_util as testutil
  36 from allmydata.test.no_network import GridTestMixin
  37 from allmydata.test.common_web import HTTPClientGETFactory, \
  38      HTTPClientHEADFactory
  39 from allmydata.client import Client, SecretHolder
  40 from allmydata.introducer import IntroducerNode
  41
  42 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  43 # create a webserver that works against them
  44
  45 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  46
  47 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
  48 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
  49 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
  50
  51 FAVICON_MARKUP = '<link href="/icon.png" rel="shortcut icon" />'
  52
  53
  54 class FakeStatsProvider:
  55     def get_stats(self):
  56         stats = {'stats': {}, 'counters': {}}
  57         return stats
  58
  59 class FakeNodeMaker(NodeMaker):
  60     encoding_params = {
  61         'k': 3,
  62         'n': 10,
  63         'happy': 7,
  64         'max_segment_size':128*1024 # 1024=KiB
  65     }
  66     def _create_lit(self, cap):
  67         return FakeCHKFileNode(cap, self.all_contents)
  68     def _create_immutable(self, cap):
  69         return FakeCHKFileNode(cap, self.all_contents)
  70     def _create_mutable(self, cap):
  71         return FakeMutableFileNode(None, None,
  72                                    self.encoding_params, None,
  73                                    self.all_contents).init_from_cap(cap)
  74     def create_mutable_file(self, contents="", keysize=None,
  75                             version=SDMF_VERSION):
  76         n = FakeMutableFileNode(None, None, self.encoding_params, None,
  77                                 self.all_contents)
  78         return n.create(contents, version=version)
  79
  80 class FakeUploader(service.Service):
  81     name = "uploader"
  82     def upload(self, uploadable):
  83         d = uploadable.get_size()
  84         d.addCallback(lambda size: uploadable.read(size))
  85         def _got_data(datav):
  86             data = "".join(datav)
  87             n = create_chk_filenode(data, self.all_contents)
  88             ur = upload.UploadResults(file_size=len(data),
  89                                       ciphertext_fetched=0,
  90                                       preexisting_shares=0,
  91                                       pushed_shares=10,
  92                                       sharemap={},
  93                                       servermap={},
  94                                       timings={},
  95                                       uri_extension_data={},
  96                                       uri_extension_hash="fake",
  97                                       verifycapstr="fakevcap")
  98             ur.set_uri(n.get_uri())
  99             return ur
 100         d.addCallback(_got_data)
 101         return d
 102     def get_helper_info(self):
 103         return (None, False)
 104
 105 def build_one_ds():
 106     ds = DownloadStatus("storage_index", 1234)
 107     now = time.time()
 108
 109     serverA = StubServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
 110     serverB = StubServer(hashutil.tagged_hash("foo", "serverid_b")[:20])
 111     storage_index = hashutil.storage_index_hash("SI")
 112     e0 = ds.add_segment_request(0, now)
 113     e0.activate(now+0.5)
 114     e0.deliver(now+1, 0, 100, 0.5) # when, start,len, decodetime
 115     e1 = ds.add_segment_request(1, now+2)
 116     e1.error(now+3)
 117     # two outstanding requests
 118     e2 = ds.add_segment_request(2, now+4)
 119     e3 = ds.add_segment_request(3, now+5)
 120     del e2,e3 # hush pyflakes
 121
 122     # simulate a segment which gets delivered faster than a system clock tick (ticket #1166)
 123     e = ds.add_segment_request(4, now)
 124     e.activate(now)
 125     e.deliver(now, 0, 140, 0.5)
 126
 127     e = ds.add_dyhb_request(serverA, now)
 128     e.finished([1,2], now+1)
 129     e = ds.add_dyhb_request(serverB, now+2) # left unfinished
 130
 131     e = ds.add_read_event(0, 120, now)
 132     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 133     e.finished(now+1)
 134     e = ds.add_read_event(120, 30, now+2) # left unfinished
 135
 136     e = ds.add_block_request(serverA, 1, 100, 20, now)
 137     e.finished(20, now+1)
 138     e = ds.add_block_request(serverB, 1, 120, 30, now+1) # left unfinished
 139
 140     # make sure that add_read_event() can come first too
 141     ds1 = DownloadStatus(storage_index, 1234)
 142     e = ds1.add_read_event(0, 120, now)
 143     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 144     e.finished(now+1)
 145
 146     return ds
 147
 148 class FakeHistory:
 149     _all_upload_status = [upload.UploadStatus()]
 150     _all_download_status = [build_one_ds()]
 151     _all_mapupdate_statuses = [servermap.UpdateStatus()]
 152     _all_publish_statuses = [publish.PublishStatus()]
 153     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
 154
 155     def list_all_upload_statuses(self):
 156         return self._all_upload_status
 157     def list_all_download_statuses(self):
 158         return self._all_download_status
 159     def list_all_mapupdate_statuses(self):
 160         return self._all_mapupdate_statuses
 161     def list_all_publish_statuses(self):
 162         return self._all_publish_statuses
 163     def list_all_retrieve_statuses(self):
 164         return self._all_retrieve_statuses
 165     def list_all_helper_statuses(self):
 166         return []
 167
 168 class FakeClient(Client):
 169     def __init__(self):
 170         # don't upcall to Client.__init__, since we only want to initialize a
 171         # minimal subset
 172         service.MultiService.__init__(self)
 173         self.all_contents = {}
 174         self.nodeid = "fake_nodeid"
 175         self.nickname = "fake_nickname"
 176         self.introducer_furl = "None"
 177         self.stats_provider = FakeStatsProvider()
 178         self._secret_holder = SecretHolder("lease secret", "convergence secret")
 179         self.helper = None
 180         self.convergence = "some random string"
 181         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 182         self.introducer_client = None
 183         self.history = FakeHistory()
 184         self.uploader = FakeUploader()
 185         self.uploader.all_contents = self.all_contents
 186         self.uploader.setServiceParent(self)
 187         self.blacklist = None
 188         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 189                                        self.uploader, None,
 190                                        None, None, None)
 191         self.nodemaker.all_contents = self.all_contents
 192         self.mutable_file_default = SDMF_VERSION
 193
 194     def startService(self):
 195         return service.MultiService.startService(self)
 196     def stopService(self):
 197         return service.MultiService.stopService(self)
 198
 199     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 200
 201 class WebMixin(object):
 202     def setUp(self):
 203         self.s = FakeClient()
 204         self.s.startService()
 205         self.staticdir = self.mktemp()
 206         self.clock = Clock()
 207         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
 208                                       clock=self.clock)
 209         self.ws.setServiceParent(self.s)
 210         self.webish_port = self.ws.getPortnum()
 211         self.webish_url = self.ws.getURL()
 212         assert self.webish_url.endswith("/")
 213         self.webish_url = self.webish_url[:-1] # these tests add their own /
 214
 215         l = [ self.s.create_dirnode() for x in range(6) ]
 216         d = defer.DeferredList(l)
 217         def _then(res):
 218             self.public_root = res[0][1]
 219             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 220             self.public_url = "/uri/" + self.public_root.get_uri()
 221             self.private_root = res[1][1]
 222
 223             foo = res[2][1]
 224             self._foo_node = foo
 225             self._foo_uri = foo.get_uri()
 226             self._foo_readonly_uri = foo.get_readonly_uri()
 227             self._foo_verifycap = foo.get_verify_cap().to_string()
 228             # NOTE: we ignore the deferred on all set_uri() calls, because we
 229             # know the fake nodes do these synchronously
 230             self.public_root.set_uri(u"foo", foo.get_uri(),
 231                                      foo.get_readonly_uri())
 232
 233             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 234             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 235             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 236
 237             # sdmf
 238             # XXX: Do we ever use this?
 239             self.BAZ_CONTENTS, n, self._baz_txt_uri, self._baz_txt_readonly_uri = self.makefile_mutable(0)
 240
 241             foo.set_uri(u"baz.txt", self._baz_txt_uri, self._baz_txt_readonly_uri)
 242
 243             # mdmf
 244             self.QUUX_CONTENTS, n, self._quux_txt_uri, self._quux_txt_readonly_uri = self.makefile_mutable(0, mdmf=True)
 245             assert self._quux_txt_uri.startswith("URI:MDMF")
 246             foo.set_uri(u"quux.txt", self._quux_txt_uri, self._quux_txt_readonly_uri)
 247
 248             foo.set_uri(u"empty", res[3][1].get_uri(),
 249                         res[3][1].get_readonly_uri())
 250             sub_uri = res[4][1].get_uri()
 251             self._sub_uri = sub_uri
 252             foo.set_uri(u"sub", sub_uri, sub_uri)
 253             sub = self.s.create_node_from_uri(sub_uri)
 254             self._sub_node = sub
 255
 256             _ign, n, blocking_uri = self.makefile(1)
 257             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 258
 259             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 260             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 261             # still think of it as an umlaut
 262             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 263
 264             self.SUBBAZ_CONTENTS, n, baz_file = self.makefile(2)
 265             self._baz_file_uri = baz_file
 266             sub.set_uri(u"baz.txt", baz_file, baz_file)
 267
 268             _ign, n, self._bad_file_uri = self.makefile(3)
 269             # this uri should not be downloadable
 270             del self.s.all_contents[self._bad_file_uri]
 271
 272             rodir = res[5][1]
 273             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 274                                      rodir.get_readonly_uri())
 275             rodir.set_uri(u"nor", baz_file, baz_file)
 276
 277             # public/
 278             # public/foo/
 279             # public/foo/bar.txt
 280             # public/foo/baz.txt
 281             # public/foo/quux.txt
 282             # public/foo/blockingfile
 283             # public/foo/empty/
 284             # public/foo/sub/
 285             # public/foo/sub/baz.txt
 286             # public/reedownlee/
 287             # public/reedownlee/nor
 288             self.NEWFILE_CONTENTS = "newfile contents\n"
 289
 290             return foo.get_metadata_for(u"bar.txt")
 291         d.addCallback(_then)
 292         def _got_metadata(metadata):
 293             self._bar_txt_metadata = metadata
 294         d.addCallback(_got_metadata)
 295         return d
 296
 297     def get_all_contents(self):
 298         return self.s.all_contents
 299
 300     def makefile(self, number):
 301         contents = "contents of file %s\n" % number
 302         n = create_chk_filenode(contents, self.get_all_contents())
 303         return contents, n, n.get_uri()
 304
 305     def makefile_mutable(self, number, mdmf=False):
 306         contents = "contents of mutable file %s\n" % number
 307         n = create_mutable_filenode(contents, mdmf, self.s.all_contents)
 308         return contents, n, n.get_uri(), n.get_readonly_uri()
 309
 310     def tearDown(self):
 311         return self.s.stopService()
 312
 313     def failUnlessIsBarDotTxt(self, res):
 314         self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
 315
 316     def failUnlessIsQuuxDotTxt(self, res):
 317         self.failUnlessReallyEqual(res, self.QUUX_CONTENTS, res)
 318
 319     def failUnlessIsBazDotTxt(self, res):
 320         self.failUnlessReallyEqual(res, self.BAZ_CONTENTS, res)
 321
 322     def failUnlessIsSubBazDotTxt(self, res):
 323         self.failUnlessReallyEqual(res, self.SUBBAZ_CONTENTS, res)
 324
 325     def failUnlessIsBarJSON(self, res):
 326         data = simplejson.loads(res)
 327         self.failUnless(isinstance(data, list))
 328         self.failUnlessEqual(data[0], "filenode")
 329         self.failUnless(isinstance(data[1], dict))
 330         self.failIf(data[1]["mutable"])
 331         self.failIfIn("rw_uri", data[1]) # immutable
 332         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
 333         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
 334         self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
 335
 336     def failUnlessIsQuuxJSON(self, res, readonly=False):
 337         data = simplejson.loads(res)
 338         self.failUnless(isinstance(data, list))
 339         self.failUnlessEqual(data[0], "filenode")
 340         self.failUnless(isinstance(data[1], dict))
 341         metadata = data[1]
 342         return self.failUnlessIsQuuxDotTxtMetadata(metadata, readonly)
 343
 344     def failUnlessIsQuuxDotTxtMetadata(self, metadata, readonly):
 345         self.failUnless(metadata['mutable'])
 346         if readonly:
 347             self.failIfIn("rw_uri", metadata)
 348         else:
 349             self.failUnlessIn("rw_uri", metadata)
 350             self.failUnlessEqual(metadata['rw_uri'], self._quux_txt_uri)
 351         self.failUnlessIn("ro_uri", metadata)
 352         self.failUnlessEqual(metadata['ro_uri'], self._quux_txt_readonly_uri)
 353         self.failUnlessReallyEqual(metadata['size'], len(self.QUUX_CONTENTS))
 354
 355     def failUnlessIsFooJSON(self, res):
 356         data = simplejson.loads(res)
 357         self.failUnless(isinstance(data, list))
 358         self.failUnlessEqual(data[0], "dirnode", res)
 359         self.failUnless(isinstance(data[1], dict))
 360         self.failUnless(data[1]["mutable"])
 361         self.failUnlessIn("rw_uri", data[1]) # mutable
 362         self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
 363         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
 364         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
 365
 366         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 367         self.failUnlessEqual(kidnames,
 368                              [u"bar.txt", u"baz.txt", u"blockingfile",
 369                               u"empty", u"n\u00fc.txt", u"quux.txt", u"sub"])
 370         kids = dict( [(unicode(name),value)
 371                       for (name,value)
 372                       in data[1]["children"].iteritems()] )
 373         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 374         self.failUnlessIn("metadata", kids[u"sub"][1])
 375         self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
 376         tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
 377         self.failUnlessIn("linkcrtime", tahoe_md)
 378         self.failUnlessIn("linkmotime", tahoe_md)
 379         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 380         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 381         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri)
 382         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]),
 383                                    self._bar_txt_verifycap)
 384         self.failUnlessIn("metadata", kids[u"bar.txt"][1])
 385         self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
 386         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
 387                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
 388         self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
 389                                    self._bar_txt_uri)
 390         self.failUnlessIn("quux.txt", kids)
 391         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["rw_uri"]),
 392                                    self._quux_txt_uri)
 393         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["ro_uri"]),
 394                                    self._quux_txt_readonly_uri)
 395
 396     def GET(self, urlpath, followRedirect=False, return_response=False,
 397             **kwargs):
 398         # if return_response=True, this fires with (data, statuscode,
 399         # respheaders) instead of just data.
 400         assert not isinstance(urlpath, unicode)
 401         url = self.webish_url + urlpath
 402         factory = HTTPClientGETFactory(url, method="GET",
 403                                        followRedirect=followRedirect, **kwargs)
 404         reactor.connectTCP("localhost", self.webish_port, factory)
 405         d = factory.deferred
 406         def _got_data(data):
 407             return (data, factory.status, factory.response_headers)
 408         if return_response:
 409             d.addCallback(_got_data)
 410         return factory.deferred
 411
 412     def HEAD(self, urlpath, return_response=False, **kwargs):
 413         # this requires some surgery, because twisted.web.client doesn't want
 414         # to give us back the response headers.
 415         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 416         reactor.connectTCP("localhost", self.webish_port, factory)
 417         d = factory.deferred
 418         def _got_data(data):
 419             return (data, factory.status, factory.response_headers)
 420         if return_response:
 421             d.addCallback(_got_data)
 422         return factory.deferred
 423
 424     def PUT(self, urlpath, data, **kwargs):
 425         url = self.webish_url + urlpath
 426         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 427
 428     def DELETE(self, urlpath):
 429         url = self.webish_url + urlpath
 430         return client.getPage(url, method="DELETE")
 431
 432     def POST(self, urlpath, followRedirect=False, **fields):
 433         sepbase = "boogabooga"
 434         sep = "--" + sepbase
 435         form = []
 436         form.append(sep)
 437         form.append('Content-Disposition: form-data; name="_charset"')
 438         form.append('')
 439         form.append('UTF-8')
 440         form.append(sep)
 441         for name, value in fields.iteritems():
 442             if isinstance(value, tuple):
 443                 filename, value = value
 444                 form.append('Content-Disposition: form-data; name="%s"; '
 445                             'filename="%s"' % (name, filename.encode("utf-8")))
 446             else:
 447                 form.append('Content-Disposition: form-data; name="%s"' % name)
 448             form.append('')
 449             if isinstance(value, unicode):
 450                 value = value.encode("utf-8")
 451             else:
 452                 value = str(value)
 453             assert isinstance(value, str)
 454             form.append(value)
 455             form.append(sep)
 456         form[-1] += "--"
 457         body = ""
 458         headers = {}
 459         if fields:
 460             body = "\r\n".join(form) + "\r\n"
 461             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 462         return self.POST2(urlpath, body, headers, followRedirect)
 463
 464     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 465         url = self.webish_url + urlpath
 466         return client.getPage(url, method="POST", postdata=body,
 467                               headers=headers, followRedirect=followRedirect)
 468
 469     def shouldFail(self, res, expected_failure, which,
 470                    substring=None, response_substring=None):
 471         if isinstance(res, failure.Failure):
 472             res.trap(expected_failure)
 473             if substring:
 474                 self.failUnlessIn(substring, str(res), which)
 475             if response_substring:
 476                 self.failUnlessIn(response_substring, res.value.response, which)
 477         else:
 478             self.fail("%s was supposed to raise %s, not get '%s'" %
 479                       (which, expected_failure, res))
 480
 481     def shouldFail2(self, expected_failure, which, substring,
 482                     response_substring,
 483                     callable, *args, **kwargs):
 484         assert substring is None or isinstance(substring, str)
 485         assert response_substring is None or isinstance(response_substring, str)
 486         d = defer.maybeDeferred(callable, *args, **kwargs)
 487         def done(res):
 488             if isinstance(res, failure.Failure):
 489                 res.trap(expected_failure)
 490                 if substring:
 491                     self.failUnlessIn(substring, str(res),
 492                                       "'%s' not in '%s' for test '%s'" % \
 493                                       (substring, str(res), which))
 494                 if response_substring:
 495                     self.failUnlessIn(response_substring, res.value.response,
 496                                       "'%s' not in '%s' for test '%s'" % \
 497                                       (response_substring, res.value.response,
 498                                        which))
 499             else:
 500                 self.fail("%s was supposed to raise %s, not get '%s'" %
 501                           (which, expected_failure, res))
 502         d.addBoth(done)
 503         return d
 504
 505     def should404(self, res, which):
 506         if isinstance(res, failure.Failure):
 507             res.trap(error.Error)
 508             self.failUnlessReallyEqual(res.value.status, "404")
 509         else:
 510             self.fail("%s was supposed to Error(404), not get '%s'" %
 511                       (which, res))
 512
 513     def should302(self, res, which):
 514         if isinstance(res, failure.Failure):
 515             res.trap(error.Error)
 516             self.failUnlessReallyEqual(res.value.status, "302")
 517         else:
 518             self.fail("%s was supposed to Error(302), not get '%s'" %
 519                       (which, res))
 520
 521
 522 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
 523     def test_create(self):
 524         pass
 525
 526     def test_welcome(self):
 527         d = self.GET("/")
 528         def _check(res):
 529             self.failUnlessIn('Welcome to Tahoe-LAFS', res)
 530             self.failUnlessIn(FAVICON_MARKUP, res)
 531             self.failUnlessIn('href="https://tahoe-lafs.org/"', res)
 532
 533             self.s.basedir = 'web/test_welcome'
 534             fileutil.make_dirs("web/test_welcome")
 535             fileutil.make_dirs("web/test_welcome/private")
 536             return self.GET("/")
 537         d.addCallback(_check)
 538         return d
 539
 540     def test_status(self):
 541         h = self.s.get_history()
 542         dl_num = h.list_all_download_statuses()[0].get_counter()
 543         ul_num = h.list_all_upload_statuses()[0].get_counter()
 544         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 545         pub_num = h.list_all_publish_statuses()[0].get_counter()
 546         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 547         d = self.GET("/status", followRedirect=True)
 548         def _check(res):
 549             self.failUnlessIn('Upload and Download Status', res)
 550             self.failUnlessIn('"down-%d"' % dl_num, res)
 551             self.failUnlessIn('"up-%d"' % ul_num, res)
 552             self.failUnlessIn('"mapupdate-%d"' % mu_num, res)
 553             self.failUnlessIn('"publish-%d"' % pub_num, res)
 554             self.failUnlessIn('"retrieve-%d"' % ret_num, res)
 555         d.addCallback(_check)
 556         d.addCallback(lambda res: self.GET("/status/?t=json"))
 557         def _check_json(res):
 558             data = simplejson.loads(res)
 559             self.failUnless(isinstance(data, dict))
 560             #active = data["active"]
 561             # TODO: test more. We need a way to fake an active operation
 562             # here.
 563         d.addCallback(_check_json)
 564
 565         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 566         def _check_dl(res):
 567             self.failUnlessIn("File Download Status", res)
 568         d.addCallback(_check_dl)
 569         d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
 570         def _check_dl_json(res):
 571             data = simplejson.loads(res)
 572             self.failUnless(isinstance(data, dict))
 573             self.failUnlessIn("read", data)
 574             self.failUnlessEqual(data["read"][0]["length"], 120)
 575             self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
 576             self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
 577             self.failUnlessEqual(data["segment"][2]["finish_time"], None)
 578             phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
 579             cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
 580             # serverids[] keys are strings, since that's what JSON does, but
 581             # we'd really like them to be ints
 582             self.failUnlessEqual(data["serverids"]["0"], "phwrsjte")
 583             self.failUnless(data["serverids"].has_key("1"),
 584                             str(data["serverids"]))
 585             self.failUnlessEqual(data["serverids"]["1"], "cmpuvkjm",
 586                                  str(data["serverids"]))
 587             self.failUnlessEqual(data["server_info"][phwr_id]["short"],
 588                                  "phwrsjte")
 589             self.failUnlessEqual(data["server_info"][cmpu_id]["short"],
 590                                  "cmpuvkjm")
 591             self.failUnlessIn("dyhb", data)
 592             self.failUnlessIn("misc", data)
 593         d.addCallback(_check_dl_json)
 594         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 595         def _check_ul(res):
 596             self.failUnlessIn("File Upload Status", res)
 597         d.addCallback(_check_ul)
 598         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 599         def _check_mapupdate(res):
 600             self.failUnlessIn("Mutable File Servermap Update Status", res)
 601         d.addCallback(_check_mapupdate)
 602         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 603         def _check_publish(res):
 604             self.failUnlessIn("Mutable File Publish Status", res)
 605         d.addCallback(_check_publish)
 606         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 607         def _check_retrieve(res):
 608             self.failUnlessIn("Mutable File Retrieve Status", res)
 609         d.addCallback(_check_retrieve)
 610
 611         return d
 612
 613     def test_status_numbers(self):
 614         drrm = status.DownloadResultsRendererMixin()
 615         self.failUnlessReallyEqual(drrm.render_time(None, None), "")
 616         self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
 617         self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
 618         self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
 619         self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
 620         self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
 621         self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 622         self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
 623         self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
 624
 625         urrm = status.UploadResultsRendererMixin()
 626         self.failUnlessReallyEqual(urrm.render_time(None, None), "")
 627         self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
 628         self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
 629         self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
 630         self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
 631         self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
 632         self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 633         self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
 634         self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
 635
 636     def test_GET_FILEURL(self):
 637         d = self.GET(self.public_url + "/foo/bar.txt")
 638         d.addCallback(self.failUnlessIsBarDotTxt)
 639         return d
 640
 641     def test_GET_FILEURL_range(self):
 642         headers = {"range": "bytes=1-10"}
 643         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 644                      return_response=True)
 645         def _got((res, status, headers)):
 646             self.failUnlessReallyEqual(int(status), 206)
 647             self.failUnless(headers.has_key("content-range"))
 648             self.failUnlessReallyEqual(headers["content-range"][0],
 649                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 650             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
 651         d.addCallback(_got)
 652         return d
 653
 654     def test_GET_FILEURL_partial_range(self):
 655         headers = {"range": "bytes=5-"}
 656         length  = len(self.BAR_CONTENTS)
 657         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 658                      return_response=True)
 659         def _got((res, status, headers)):
 660             self.failUnlessReallyEqual(int(status), 206)
 661             self.failUnless(headers.has_key("content-range"))
 662             self.failUnlessReallyEqual(headers["content-range"][0],
 663                                        "bytes 5-%d/%d" % (length-1, length))
 664             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
 665         d.addCallback(_got)
 666         return d
 667
 668     def test_GET_FILEURL_partial_end_range(self):
 669         headers = {"range": "bytes=-5"}
 670         length  = len(self.BAR_CONTENTS)
 671         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 672                      return_response=True)
 673         def _got((res, status, headers)):
 674             self.failUnlessReallyEqual(int(status), 206)
 675             self.failUnless(headers.has_key("content-range"))
 676             self.failUnlessReallyEqual(headers["content-range"][0],
 677                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 678             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
 679         d.addCallback(_got)
 680         return d
 681
 682     def test_GET_FILEURL_partial_range_overrun(self):
 683         headers = {"range": "bytes=100-200"}
 684         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
 685                              "416 Requested Range not satisfiable",
 686                              "First beyond end of file",
 687                              self.GET, self.public_url + "/foo/bar.txt",
 688                              headers=headers)
 689         return d
 690
 691     def test_HEAD_FILEURL_range(self):
 692         headers = {"range": "bytes=1-10"}
 693         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 694                      return_response=True)
 695         def _got((res, status, headers)):
 696             self.failUnlessReallyEqual(res, "")
 697             self.failUnlessReallyEqual(int(status), 206)
 698             self.failUnless(headers.has_key("content-range"))
 699             self.failUnlessReallyEqual(headers["content-range"][0],
 700                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 701         d.addCallback(_got)
 702         return d
 703
 704     def test_HEAD_FILEURL_partial_range(self):
 705         headers = {"range": "bytes=5-"}
 706         length  = len(self.BAR_CONTENTS)
 707         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 708                      return_response=True)
 709         def _got((res, status, headers)):
 710             self.failUnlessReallyEqual(int(status), 206)
 711             self.failUnless(headers.has_key("content-range"))
 712             self.failUnlessReallyEqual(headers["content-range"][0],
 713                                        "bytes 5-%d/%d" % (length-1, length))
 714         d.addCallback(_got)
 715         return d
 716
 717     def test_HEAD_FILEURL_partial_end_range(self):
 718         headers = {"range": "bytes=-5"}
 719         length  = len(self.BAR_CONTENTS)
 720         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 721                      return_response=True)
 722         def _got((res, status, headers)):
 723             self.failUnlessReallyEqual(int(status), 206)
 724             self.failUnless(headers.has_key("content-range"))
 725             self.failUnlessReallyEqual(headers["content-range"][0],
 726                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 727         d.addCallback(_got)
 728         return d
 729
 730     def test_HEAD_FILEURL_partial_range_overrun(self):
 731         headers = {"range": "bytes=100-200"}
 732         d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
 733                              "416 Requested Range not satisfiable",
 734                              "",
 735                              self.HEAD, self.public_url + "/foo/bar.txt",
 736                              headers=headers)
 737         return d
 738
 739     def test_GET_FILEURL_range_bad(self):
 740         headers = {"range": "BOGUS=fizbop-quarnak"}
 741         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 742                      return_response=True)
 743         def _got((res, status, headers)):
 744             self.failUnlessReallyEqual(int(status), 200)
 745             self.failUnless(not headers.has_key("content-range"))
 746             self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
 747         d.addCallback(_got)
 748         return d
 749
 750     def test_HEAD_FILEURL(self):
 751         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 752         def _got((res, status, headers)):
 753             self.failUnlessReallyEqual(res, "")
 754             self.failUnlessReallyEqual(headers["content-length"][0],
 755                                        str(len(self.BAR_CONTENTS)))
 756             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
 757         d.addCallback(_got)
 758         return d
 759
 760     def test_GET_FILEURL_named(self):
 761         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 762         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 763         d = self.GET(base + "/@@name=/blah.txt")
 764         d.addCallback(self.failUnlessIsBarDotTxt)
 765         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 766         d.addCallback(self.failUnlessIsBarDotTxt)
 767         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 768         d.addCallback(self.failUnlessIsBarDotTxt)
 769         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 770         d.addCallback(self.failUnlessIsBarDotTxt)
 771         save_url = base + "?save=true&filename=blah.txt"
 772         d.addCallback(lambda res: self.GET(save_url))
 773         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 774         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 775         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 776         u_url = base + "?save=true&filename=" + u_fn_e
 777         d.addCallback(lambda res: self.GET(u_url))
 778         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 779         return d
 780
 781     def test_PUT_FILEURL_named_bad(self):
 782         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 783         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 784                              "400 Bad Request",
 785                              "/file can only be used with GET or HEAD",
 786                              self.PUT, base + "/@@name=/blah.txt", "")
 787         return d
 788
 789
 790     def test_GET_DIRURL_named_bad(self):
 791         base = "/file/%s" % urllib.quote(self._foo_uri)
 792         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 793                              "400 Bad Request",
 794                              "is not a file-cap",
 795                              self.GET, base + "/@@name=/blah.txt")
 796         return d
 797
 798     def test_GET_slash_file_bad(self):
 799         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 800                              "404 Not Found",
 801                              "/file must be followed by a file-cap and a name",
 802                              self.GET, "/file")
 803         return d
 804
 805     def test_GET_unhandled_URI_named(self):
 806         contents, n, newuri = self.makefile(12)
 807         verifier_cap = n.get_verify_cap().to_string()
 808         base = "/file/%s" % urllib.quote(verifier_cap)
 809         # client.create_node_from_uri() can't handle verify-caps
 810         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 811                              "400 Bad Request", "is not a file-cap",
 812                              self.GET, base)
 813         return d
 814
 815     def test_GET_unhandled_URI(self):
 816         contents, n, newuri = self.makefile(12)
 817         verifier_cap = n.get_verify_cap().to_string()
 818         base = "/uri/%s" % urllib.quote(verifier_cap)
 819         # client.create_node_from_uri() can't handle verify-caps
 820         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 821                              "400 Bad Request",
 822                              "GET unknown URI type: can only do t=info",
 823                              self.GET, base)
 824         return d
 825
 826     def test_GET_FILE_URI(self):
 827         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 828         d = self.GET(base)
 829         d.addCallback(self.failUnlessIsBarDotTxt)
 830         return d
 831
 832     def test_GET_FILE_URI_mdmf(self):
 833         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 834         d = self.GET(base)
 835         d.addCallback(self.failUnlessIsQuuxDotTxt)
 836         return d
 837
 838     def test_GET_FILE_URI_mdmf_extensions(self):
 839         base = "/uri/%s" % urllib.quote("%s:RANDOMSTUFF" % self._quux_txt_uri)
 840         d = self.GET(base)
 841         d.addCallback(self.failUnlessIsQuuxDotTxt)
 842         return d
 843
 844     def test_GET_FILE_URI_mdmf_readonly(self):
 845         base = "/uri/%s" % urllib.quote(self._quux_txt_readonly_uri)
 846         d = self.GET(base)
 847         d.addCallback(self.failUnlessIsQuuxDotTxt)
 848         return d
 849
 850     def test_GET_FILE_URI_badchild(self):
 851         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 852         errmsg = "Files have no children, certainly not named 'boguschild'"
 853         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 854                              "400 Bad Request", errmsg,
 855                              self.GET, base)
 856         return d
 857
 858     def test_PUT_FILE_URI_badchild(self):
 859         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 860         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 861         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 862                              "400 Bad Request", errmsg,
 863                              self.PUT, base, "")
 864         return d
 865
 866     def test_PUT_FILE_URI_mdmf(self):
 867         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 868         self._quux_new_contents = "new_contents"
 869         d = self.GET(base)
 870         d.addCallback(lambda res:
 871             self.failUnlessIsQuuxDotTxt(res))
 872         d.addCallback(lambda ignored:
 873             self.PUT(base, self._quux_new_contents))
 874         d.addCallback(lambda ignored:
 875             self.GET(base))
 876         d.addCallback(lambda res:
 877             self.failUnlessReallyEqual(res, self._quux_new_contents))
 878         return d
 879
 880     def test_PUT_FILE_URI_mdmf_extensions(self):
 881         base = "/uri/%s" % urllib.quote("%s:EXTENSIONSTUFF" % self._quux_txt_uri)
 882         self._quux_new_contents = "new_contents"
 883         d = self.GET(base)
 884         d.addCallback(lambda res: self.failUnlessIsQuuxDotTxt(res))
 885         d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
 886         d.addCallback(lambda ignored: self.GET(base))
 887         d.addCallback(lambda res: self.failUnlessEqual(self._quux_new_contents,
 888                                                        res))
 889         return d
 890
 891     def test_PUT_FILE_URI_mdmf_readonly(self):
 892         # We're not allowed to PUT things to a readonly cap.
 893         base = "/uri/%s" % self._quux_txt_readonly_uri
 894         d = self.GET(base)
 895         d.addCallback(lambda res:
 896             self.failUnlessIsQuuxDotTxt(res))
 897         # What should we get here? We get a 500 error now; that's not right.
 898         d.addCallback(lambda ignored:
 899             self.shouldFail2(error.Error, "test_PUT_FILE_URI_mdmf_readonly",
 900                              "400 Bad Request", "read-only cap",
 901                              self.PUT, base, "new data"))
 902         return d
 903
 904     def test_PUT_FILE_URI_sdmf_readonly(self):
 905         # We're not allowed to put things to a readonly cap.
 906         base = "/uri/%s" % self._baz_txt_readonly_uri
 907         d = self.GET(base)
 908         d.addCallback(lambda res:
 909             self.failUnlessIsBazDotTxt(res))
 910         d.addCallback(lambda ignored:
 911             self.shouldFail2(error.Error, "test_PUT_FILE_URI_sdmf_readonly",
 912                              "400 Bad Request", "read-only cap",
 913                              self.PUT, base, "new_data"))
 914         return d
 915
 916     def test_GET_etags(self):
 917
 918         def _check_etags(uri):
 919             d1 = _get_etag(uri)
 920             d2 = _get_etag(uri, 'json')
 921             d = defer.DeferredList([d1, d2], consumeErrors=True)
 922             def _check(results):
 923                 # All deferred must succeed
 924                 self.failUnless(all([r[0] for r in results]))
 925                 # the etag for the t=json form should be just like the etag
 926                 # fo the default t='' form, but with a 'json' suffix
 927                 self.failUnlessEqual(results[0][1] + 'json', results[1][1])
 928             d.addCallback(_check)
 929             return d
 930
 931         def _get_etag(uri, t=''):
 932             targetbase = "/uri/%s?t=%s" % (urllib.quote(uri.strip()), t)
 933             d = self.GET(targetbase, return_response=True, followRedirect=True)
 934             def _just_the_etag(result):
 935                 data, response, headers = result
 936                 etag = headers['etag'][0]
 937                 if uri.startswith('URI:DIR'):
 938                     self.failUnless(etag.startswith('DIR:'), etag)
 939                 return etag
 940             return d.addCallback(_just_the_etag)
 941
 942         # Check that etags work with immutable directories
 943         (newkids, caps) = self._create_immutable_children()
 944         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
 945                       simplejson.dumps(newkids))
 946         def _stash_immdir_uri(uri):
 947             self._immdir_uri = uri
 948             return uri
 949         d.addCallback(_stash_immdir_uri)
 950         d.addCallback(_check_etags)
 951
 952         # Check that etags work with immutable files
 953         d.addCallback(lambda _: _check_etags(self._bar_txt_uri))
 954
 955         # use the ETag on GET
 956         def _check_match(ign):
 957             uri = "/uri/%s" % self._bar_txt_uri
 958             d = self.GET(uri, return_response=True)
 959             # extract the ETag
 960             d.addCallback(lambda (data, code, headers):
 961                           headers['etag'][0])
 962             # do a GET that's supposed to match the ETag
 963             d.addCallback(lambda etag:
 964                           self.GET(uri, return_response=True,
 965                                    headers={"If-None-Match": etag}))
 966             # make sure it short-circuited (304 instead of 200)
 967             d.addCallback(lambda (data, code, headers):
 968                           self.failUnlessEqual(int(code), http.NOT_MODIFIED))
 969             return d
 970         d.addCallback(_check_match)
 971
 972         def _no_etag(uri, t):
 973             target = "/uri/%s?t=%s" % (uri, t)
 974             d = self.GET(target, return_response=True, followRedirect=True)
 975             d.addCallback(lambda (data, code, headers):
 976                           self.failIf("etag" in headers, target))
 977             return d
 978         def _yes_etag(uri, t):
 979             target = "/uri/%s?t=%s" % (uri, t)
 980             d = self.GET(target, return_response=True, followRedirect=True)
 981             d.addCallback(lambda (data, code, headers):
 982                           self.failUnless("etag" in headers, target))
 983             return d
 984
 985         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, ""))
 986         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "json"))
 987         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "uri"))
 988         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "readonly-uri"))
 989         d.addCallback(lambda ign: _no_etag(self._bar_txt_uri, "info"))
 990
 991         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, ""))
 992         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "json"))
 993         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "uri"))
 994         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "readonly-uri"))
 995         d.addCallback(lambda ign: _no_etag(self._immdir_uri, "info"))
 996         d.addCallback(lambda ign: _no_etag(self._immdir_uri, "rename-form"))
 997
 998         return d
 999
1000     # TODO: version of this with a Unicode filename
1001     def test_GET_FILEURL_save(self):
1002         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
1003                      return_response=True)
1004         def _got((res, statuscode, headers)):
1005             content_disposition = headers["content-disposition"][0]
1006             self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
1007             self.failUnlessIsBarDotTxt(res)
1008         d.addCallback(_got)
1009         return d
1010
1011     def test_GET_FILEURL_missing(self):
1012         d = self.GET(self.public_url + "/foo/missing")
1013         d.addBoth(self.should404, "test_GET_FILEURL_missing")
1014         return d
1015
1016     def test_GET_FILEURL_info_mdmf(self):
1017         d = self.GET("/uri/%s?t=info" % self._quux_txt_uri)
1018         def _got(res):
1019             self.failUnlessIn("mutable file (mdmf)", res)
1020             self.failUnlessIn(self._quux_txt_uri, res)
1021             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1022         d.addCallback(_got)
1023         return d
1024
1025     def test_GET_FILEURL_info_mdmf_readonly(self):
1026         d = self.GET("/uri/%s?t=info" % self._quux_txt_readonly_uri)
1027         def _got(res):
1028             self.failUnlessIn("mutable file (mdmf)", res)
1029             self.failIfIn(self._quux_txt_uri, res)
1030             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1031         d.addCallback(_got)
1032         return d
1033
1034     def test_GET_FILEURL_info_sdmf(self):
1035         d = self.GET("/uri/%s?t=info" % self._baz_txt_uri)
1036         def _got(res):
1037             self.failUnlessIn("mutable file (sdmf)", res)
1038             self.failUnlessIn(self._baz_txt_uri, res)
1039         d.addCallback(_got)
1040         return d
1041
1042     def test_GET_FILEURL_info_mdmf_extensions(self):
1043         d = self.GET("/uri/%s:STUFF?t=info" % self._quux_txt_uri)
1044         def _got(res):
1045             self.failUnlessIn("mutable file (mdmf)", res)
1046             self.failUnlessIn(self._quux_txt_uri, res)
1047             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1048         d.addCallback(_got)
1049         return d
1050
1051     def test_PUT_overwrite_only_files(self):
1052         # create a directory, put a file in that directory.
1053         contents, n, filecap = self.makefile(8)
1054         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
1055         d.addCallback(lambda res:
1056             self.PUT(self.public_url + "/foo/dir/file1.txt",
1057                      self.NEWFILE_CONTENTS))
1058         # try to overwrite the file with replace=only-files
1059         # (this should work)
1060         d.addCallback(lambda res:
1061             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
1062                      filecap))
1063         d.addCallback(lambda res:
1064             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
1065                  "There was already a child by that name, and you asked me "
1066                  "to not replace it",
1067                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
1068                  filecap))
1069         return d
1070
1071     def test_PUT_NEWFILEURL(self):
1072         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
1073         # TODO: we lose the response code, so we can't check this
1074         #self.failUnlessReallyEqual(responsecode, 201)
1075         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1076         d.addCallback(lambda res:
1077                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1078                                                       self.NEWFILE_CONTENTS))
1079         return d
1080
1081     def test_PUT_NEWFILEURL_not_mutable(self):
1082         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
1083                      self.NEWFILE_CONTENTS)
1084         # TODO: we lose the response code, so we can't check this
1085         #self.failUnlessReallyEqual(responsecode, 201)
1086         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1087         d.addCallback(lambda res:
1088                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1089                                                       self.NEWFILE_CONTENTS))
1090         return d
1091
1092     def test_PUT_NEWFILEURL_unlinked_mdmf(self):
1093         # this should get us a few segments of an MDMF mutable file,
1094         # which we can then test for.
1095         contents = self.NEWFILE_CONTENTS * 300000
1096         d = self.PUT("/uri?format=mdmf",
1097                      contents)
1098         def _got_filecap(filecap):
1099             self.failUnless(filecap.startswith("URI:MDMF"))
1100             return filecap
1101         d.addCallback(_got_filecap)
1102         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1103         d.addCallback(lambda json: self.failUnlessIn("MDMF", json))
1104         return d
1105
1106     def test_PUT_NEWFILEURL_unlinked_sdmf(self):
1107         contents = self.NEWFILE_CONTENTS * 300000
1108         d = self.PUT("/uri?format=sdmf",
1109                      contents)
1110         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1111         d.addCallback(lambda json: self.failUnlessIn("SDMF", json))
1112         return d
1113
1114     def test_PUT_NEWFILEURL_unlinked_bad_format(self):
1115         contents = self.NEWFILE_CONTENTS * 300000
1116         return self.shouldHTTPError("PUT_NEWFILEURL_unlinked_bad_format",
1117                                     400, "Bad Request", "Unknown format: foo",
1118                                     self.PUT, "/uri?format=foo",
1119                                     contents)
1120
1121     def test_PUT_NEWFILEURL_range_bad(self):
1122         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
1123         target = self.public_url + "/foo/new.txt"
1124         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
1125                              "501 Not Implemented",
1126                              "Content-Range in PUT not yet supported",
1127                              # (and certainly not for immutable files)
1128                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
1129                              headers=headers)
1130         d.addCallback(lambda res:
1131                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
1132         return d
1133
1134     def test_PUT_NEWFILEURL_mutable(self):
1135         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1136                      self.NEWFILE_CONTENTS)
1137         # TODO: we lose the response code, so we can't check this
1138         #self.failUnlessReallyEqual(responsecode, 201)
1139         def _check_uri(res):
1140             u = uri.from_string_mutable_filenode(res)
1141             self.failUnless(u.is_mutable())
1142             self.failIf(u.is_readonly())
1143             return res
1144         d.addCallback(_check_uri)
1145         d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
1146         d.addCallback(lambda res:
1147                       self.failUnlessMutableChildContentsAre(self._foo_node,
1148                                                              u"new.txt",
1149                                                              self.NEWFILE_CONTENTS))
1150         return d
1151
1152     def test_PUT_NEWFILEURL_mutable_toobig(self):
1153         # It is okay to upload large mutable files, so we should be able
1154         # to do that.
1155         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1156                      "b" * (self.s.MUTABLE_SIZELIMIT + 1))
1157         return d
1158
1159     def test_PUT_NEWFILEURL_replace(self):
1160         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
1161         # TODO: we lose the response code, so we can't check this
1162         #self.failUnlessReallyEqual(responsecode, 200)
1163         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
1164         d.addCallback(lambda res:
1165                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1166                                                       self.NEWFILE_CONTENTS))
1167         return d
1168
1169     def test_PUT_NEWFILEURL_bad_t(self):
1170         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
1171                              "PUT to a file: bad t=bogus",
1172                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
1173                              "contents")
1174         return d
1175
1176     def test_PUT_NEWFILEURL_no_replace(self):
1177         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
1178                      self.NEWFILE_CONTENTS)
1179         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
1180                   "409 Conflict",
1181                   "There was already a child by that name, and you asked me "
1182                   "to not replace it")
1183         return d
1184
1185     def test_PUT_NEWFILEURL_mkdirs(self):
1186         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
1187         fn = self._foo_node
1188         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
1189         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
1190         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
1191         d.addCallback(lambda res:
1192                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
1193                                                       self.NEWFILE_CONTENTS))
1194         return d
1195
1196     def test_PUT_NEWFILEURL_blocked(self):
1197         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
1198                      self.NEWFILE_CONTENTS)
1199         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
1200                   "409 Conflict",
1201                   "Unable to create directory 'blockingfile': a file was in the way")
1202         return d
1203
1204     def test_PUT_NEWFILEURL_emptyname(self):
1205         # an empty pathname component (i.e. a double-slash) is disallowed
1206         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
1207                              "400 Bad Request",
1208                              "The webapi does not allow empty pathname components",
1209                              self.PUT, self.public_url + "/foo//new.txt", "")
1210         return d
1211
1212     def test_DELETE_FILEURL(self):
1213         d = self.DELETE(self.public_url + "/foo/bar.txt")
1214         d.addCallback(lambda res:
1215                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1216         return d
1217
1218     def test_DELETE_FILEURL_missing(self):
1219         d = self.DELETE(self.public_url + "/foo/missing")
1220         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
1221         return d
1222
1223     def test_DELETE_FILEURL_missing2(self):
1224         d = self.DELETE(self.public_url + "/missing/missing")
1225         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
1226         return d
1227
1228     def failUnlessHasBarDotTxtMetadata(self, res):
1229         data = simplejson.loads(res)
1230         self.failUnless(isinstance(data, list))
1231         self.failUnlessIn("metadata", data[1])
1232         self.failUnlessIn("tahoe", data[1]["metadata"])
1233         self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
1234         self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
1235         self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
1236                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
1237
1238     def test_GET_FILEURL_json(self):
1239         # twisted.web.http.parse_qs ignores any query args without an '=', so
1240         # I can't do "GET /path?json", I have to do "GET /path/t=json"
1241         # instead. This may make it tricky to emulate the S3 interface
1242         # completely.
1243         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
1244         def _check1(data):
1245             self.failUnlessIsBarJSON(data)
1246             self.failUnlessHasBarDotTxtMetadata(data)
1247             return
1248         d.addCallback(_check1)
1249         return d
1250
1251     def test_GET_FILEURL_json_mutable_type(self):
1252         # The JSON should include format, which says whether the
1253         # file is SDMF or MDMF
1254         d = self.PUT("/uri?format=mdmf",
1255                      self.NEWFILE_CONTENTS * 300000)
1256         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1257         def _got_json(json, version):
1258             data = simplejson.loads(json)
1259             assert "filenode" == data[0]
1260             data = data[1]
1261             assert isinstance(data, dict)
1262
1263             self.failUnlessIn("format", data)
1264             self.failUnlessEqual(data["format"], version)
1265
1266         d.addCallback(_got_json, "MDMF")
1267         # Now make an SDMF file and check that it is reported correctly.
1268         d.addCallback(lambda ignored:
1269             self.PUT("/uri?format=sdmf",
1270                       self.NEWFILE_CONTENTS * 300000))
1271         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1272         d.addCallback(_got_json, "SDMF")
1273         return d
1274
1275     def test_GET_FILEURL_json_mdmf(self):
1276         d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
1277         d.addCallback(self.failUnlessIsQuuxJSON)
1278         return d
1279
1280     def test_GET_FILEURL_json_missing(self):
1281         d = self.GET(self.public_url + "/foo/missing?json")
1282         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
1283         return d
1284
1285     def test_GET_FILEURL_uri(self):
1286         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
1287         def _check(res):
1288             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1289         d.addCallback(_check)
1290         d.addCallback(lambda res:
1291                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
1292         def _check2(res):
1293             # for now, for files, uris and readonly-uris are the same
1294             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1295         d.addCallback(_check2)
1296         return d
1297
1298     def test_GET_FILEURL_badtype(self):
1299         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
1300                                  "bad t=bogus",
1301                                  self.GET,
1302                                  self.public_url + "/foo/bar.txt?t=bogus")
1303         return d
1304
1305     def test_CSS_FILE(self):
1306         d = self.GET("/tahoe.css", followRedirect=True)
1307         def _check(res):
1308             CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
1309             self.failUnless(CSS_STYLE.search(res), res)
1310         d.addCallback(_check)
1311         return d
1312
1313     def test_GET_FILEURL_uri_missing(self):
1314         d = self.GET(self.public_url + "/foo/missing?t=uri")
1315         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
1316         return d
1317
1318     def _check_upload_and_mkdir_forms(self, html):
1319         # We should have a form to create a file, with radio buttons that allow
1320         # the user to toggle whether it is a CHK/LIT (default), SDMF, or MDMF file.
1321         self.failUnlessIn('name="t" value="upload"', html)
1322         self.failUnlessIn('input checked="checked" type="radio" id="upload-chk" value="chk" name="format"', html)
1323         self.failUnlessIn('input type="radio" id="upload-sdmf" value="sdmf" name="format"', html)
1324         self.failUnlessIn('input type="radio" id="upload-mdmf" value="mdmf" name="format"', html)
1325
1326         # We should also have the ability to create a mutable directory, with
1327         # radio buttons that allow the user to toggle whether it is an SDMF (default)
1328         # or MDMF directory.
1329         self.failUnlessIn('name="t" value="mkdir"', html)
1330         self.failUnlessIn('input checked="checked" type="radio" id="mkdir-sdmf" value="sdmf" name="format"', html)
1331         self.failUnlessIn('input type="radio" id="mkdir-mdmf" value="mdmf" name="format"', html)
1332
1333         self.failUnlessIn(FAVICON_MARKUP, html)
1334
1335     def test_GET_DIRECTORY_html(self):
1336         d = self.GET(self.public_url + "/foo", followRedirect=True)
1337         def _check(html):
1338             self.failUnlessIn('<div class="toolbar-item"><a href="../../..">Return to Welcome page</a></div>', html)
1339             self._check_upload_and_mkdir_forms(html)
1340             self.failUnlessIn("quux", html)
1341         d.addCallback(_check)
1342         return d
1343
1344     def test_GET_root_html(self):
1345         d = self.GET("/")
1346         d.addCallback(self._check_upload_and_mkdir_forms)
1347         return d
1348
1349     def test_GET_DIRURL(self):
1350         # the addSlash means we get a redirect here
1351         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1352         ROOT = "../../.."
1353         d = self.GET(self.public_url + "/foo", followRedirect=True)
1354         def _check(res):
1355             self.failUnlessIn('<a href="%s">Return to Welcome page' % ROOT, res)
1356
1357             # the FILE reference points to a URI, but it should end in bar.txt
1358             bar_url = ("%s/file/%s/@@named=/bar.txt" %
1359                        (ROOT, urllib.quote(self._bar_txt_uri)))
1360             get_bar = "".join([r'<td>FILE</td>',
1361                                r'\s+<td>',
1362                                r'<a href="%s">bar.txt</a>' % bar_url,
1363                                r'</td>',
1364                                r'\s+<td align="right">%d</td>' % len(self.BAR_CONTENTS),
1365                                ])
1366             self.failUnless(re.search(get_bar, res), res)
1367             for label in ['unlink', 'rename/move']:
1368                 for line in res.split("\n"):
1369                     # find the line that contains the relevant button for bar.txt
1370                     if ("form action" in line and
1371                         ('value="%s"' % (label,)) in line and
1372                         'value="bar.txt"' in line):
1373                         # the form target should use a relative URL
1374                         foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1375                         self.failUnlessIn('action="%s"' % foo_url, line)
1376                         # and the when_done= should too
1377                         #done_url = urllib.quote(???)
1378                         #self.failUnlessIn('name="when_done" value="%s"' % done_url, line)
1379
1380                         # 'unlink' needs to use POST because it directly has a side effect
1381                         if label == 'unlink':
1382                             self.failUnlessIn('method="post"', line)
1383                         break
1384                 else:
1385                     self.fail("unable to find '%s bar.txt' line" % (label,))
1386
1387             # the DIR reference just points to a URI
1388             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1389             get_sub = ((r'<td>DIR</td>')
1390                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1391             self.failUnless(re.search(get_sub, res), res)
1392         d.addCallback(_check)
1393
1394         # look at a readonly directory
1395         d.addCallback(lambda res:
1396                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
1397         def _check2(res):
1398             self.failUnlessIn("(read-only)", res)
1399             self.failIfIn("Upload a file", res)
1400         d.addCallback(_check2)
1401
1402         # and at a directory that contains a readonly directory
1403         d.addCallback(lambda res:
1404                       self.GET(self.public_url, followRedirect=True))
1405         def _check3(res):
1406             self.failUnless(re.search('<td>DIR-RO</td>'
1407                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1408         d.addCallback(_check3)
1409
1410         # and an empty directory
1411         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1412         def _check4(res):
1413             self.failUnlessIn("directory is empty", res)
1414             MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
1415             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1416         d.addCallback(_check4)
1417
1418         # and at a literal directory
1419         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1420         d.addCallback(lambda res:
1421                       self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1422         def _check5(res):
1423             self.failUnlessIn('(immutable)', res)
1424             self.failUnless(re.search('<td>FILE</td>'
1425                                       r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1426         d.addCallback(_check5)
1427         return d
1428
1429     def test_GET_DIRURL_badtype(self):
1430         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1431                                  400, "Bad Request",
1432                                  "bad t=bogus",
1433                                  self.GET,
1434                                  self.public_url + "/foo?t=bogus")
1435         return d
1436
1437     def test_GET_DIRURL_json(self):
1438         d = self.GET(self.public_url + "/foo?t=json")
1439         d.addCallback(self.failUnlessIsFooJSON)
1440         return d
1441
1442     def test_GET_DIRURL_json_format(self):
1443         d = self.PUT(self.public_url + \
1444                      "/foo/sdmf.txt?format=sdmf",
1445                      self.NEWFILE_CONTENTS * 300000)
1446         d.addCallback(lambda ignored:
1447             self.PUT(self.public_url + \
1448                      "/foo/mdmf.txt?format=mdmf",
1449                      self.NEWFILE_CONTENTS * 300000))
1450         # Now we have an MDMF and SDMF file in the directory. If we GET
1451         # its JSON, we should see their encodings.
1452         d.addCallback(lambda ignored:
1453             self.GET(self.public_url + "/foo?t=json"))
1454         def _got_json(json):
1455             data = simplejson.loads(json)
1456             assert data[0] == "dirnode"
1457
1458             data = data[1]
1459             kids = data['children']
1460
1461             mdmf_data = kids['mdmf.txt'][1]
1462             self.failUnlessIn("format", mdmf_data)
1463             self.failUnlessEqual(mdmf_data["format"], "MDMF")
1464
1465             sdmf_data = kids['sdmf.txt'][1]
1466             self.failUnlessIn("format", sdmf_data)
1467             self.failUnlessEqual(sdmf_data["format"], "SDMF")
1468         d.addCallback(_got_json)
1469         return d
1470
1471
1472     def test_POST_DIRURL_manifest_no_ophandle(self):
1473         d = self.shouldFail2(error.Error,
1474                              "test_POST_DIRURL_manifest_no_ophandle",
1475                              "400 Bad Request",
1476                              "slow operation requires ophandle=",
1477                              self.POST, self.public_url, t="start-manifest")
1478         return d
1479
1480     def test_POST_DIRURL_manifest(self):
1481         d = defer.succeed(None)
1482         def getman(ignored, output):
1483             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1484                           followRedirect=True)
1485             d.addCallback(self.wait_for_operation, "125")
1486             d.addCallback(self.get_operation_results, "125", output)
1487             return d
1488         d.addCallback(getman, None)
1489         def _got_html(manifest):
1490             self.failUnlessIn("Manifest of SI=", manifest)
1491             self.failUnlessIn("<td>sub</td>", manifest)
1492             self.failUnlessIn(self._sub_uri, manifest)
1493             self.failUnlessIn("<td>sub/baz.txt</td>", manifest)
1494             self.failUnlessIn(FAVICON_MARKUP, manifest)
1495         d.addCallback(_got_html)
1496
1497         # both t=status and unadorned GET should be identical
1498         d.addCallback(lambda res: self.GET("/operations/125"))
1499         d.addCallback(_got_html)
1500
1501         d.addCallback(getman, "html")
1502         d.addCallback(_got_html)
1503         d.addCallback(getman, "text")
1504         def _got_text(manifest):
1505             self.failUnlessIn("\nsub " + self._sub_uri + "\n", manifest)
1506             self.failUnlessIn("\nsub/baz.txt URI:CHK:", manifest)
1507         d.addCallback(_got_text)
1508         d.addCallback(getman, "JSON")
1509         def _got_json(res):
1510             data = res["manifest"]
1511             got = {}
1512             for (path_list, cap) in data:
1513                 got[tuple(path_list)] = cap
1514             self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
1515             self.failUnlessIn((u"sub", u"baz.txt"), got)
1516             self.failUnlessIn("finished", res)
1517             self.failUnlessIn("origin", res)
1518             self.failUnlessIn("storage-index", res)
1519             self.failUnlessIn("verifycaps", res)
1520             self.failUnlessIn("stats", res)
1521         d.addCallback(_got_json)
1522         return d
1523
1524     def test_POST_DIRURL_deepsize_no_ophandle(self):
1525         d = self.shouldFail2(error.Error,
1526                              "test_POST_DIRURL_deepsize_no_ophandle",
1527                              "400 Bad Request",
1528                              "slow operation requires ophandle=",
1529                              self.POST, self.public_url, t="start-deep-size")
1530         return d
1531
1532     def test_POST_DIRURL_deepsize(self):
1533         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1534                       followRedirect=True)
1535         d.addCallback(self.wait_for_operation, "126")
1536         d.addCallback(self.get_operation_results, "126", "json")
1537         def _got_json(data):
1538             self.failUnlessReallyEqual(data["finished"], True)
1539             size = data["size"]
1540             self.failUnless(size > 1000)
1541         d.addCallback(_got_json)
1542         d.addCallback(self.get_operation_results, "126", "text")
1543         def _got_text(res):
1544             mo = re.search(r'^size: (\d+)$', res, re.M)
1545             self.failUnless(mo, res)
1546             size = int(mo.group(1))
1547             # with directories, the size varies.
1548             self.failUnless(size > 1000)
1549         d.addCallback(_got_text)
1550         return d
1551
1552     def test_POST_DIRURL_deepstats_no_ophandle(self):
1553         d = self.shouldFail2(error.Error,
1554                              "test_POST_DIRURL_deepstats_no_ophandle",
1555                              "400 Bad Request",
1556                              "slow operation requires ophandle=",
1557                              self.POST, self.public_url, t="start-deep-stats")
1558         return d
1559
1560     def test_POST_DIRURL_deepstats(self):
1561         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1562                       followRedirect=True)
1563         d.addCallback(self.wait_for_operation, "127")
1564         d.addCallback(self.get_operation_results, "127", "json")
1565         def _got_json(stats):
1566             expected = {"count-immutable-files": 3,
1567                         "count-mutable-files": 2,
1568                         "count-literal-files": 0,
1569                         "count-files": 5,
1570                         "count-directories": 3,
1571                         "size-immutable-files": 57,
1572                         "size-literal-files": 0,
1573                         #"size-directories": 1912, # varies
1574                         #"largest-directory": 1590,
1575                         "largest-directory-children": 7,
1576                         "largest-immutable-file": 19,
1577                         }
1578             for k,v in expected.iteritems():
1579                 self.failUnlessReallyEqual(stats[k], v,
1580                                            "stats[%s] was %s, not %s" %
1581                                            (k, stats[k], v))
1582             self.failUnlessReallyEqual(stats["size-files-histogram"],
1583                                        [ [11, 31, 3] ])
1584         d.addCallback(_got_json)
1585         return d
1586
1587     def test_POST_DIRURL_stream_manifest(self):
1588         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1589         def _check(res):
1590             self.failUnless(res.endswith("\n"))
1591             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1592             self.failUnlessReallyEqual(len(units), 9)
1593             self.failUnlessEqual(units[-1]["type"], "stats")
1594             first = units[0]
1595             self.failUnlessEqual(first["path"], [])
1596             self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri)
1597             self.failUnlessEqual(first["type"], "directory")
1598             baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0]
1599             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1600             self.failIfEqual(baz["storage-index"], None)
1601             self.failIfEqual(baz["verifycap"], None)
1602             self.failIfEqual(baz["repaircap"], None)
1603             # XXX: Add quux and baz to this test.
1604             return
1605         d.addCallback(_check)
1606         return d
1607
1608     def test_GET_DIRURL_uri(self):
1609         d = self.GET(self.public_url + "/foo?t=uri")
1610         def _check(res):
1611             self.failUnlessReallyEqual(to_str(res), self._foo_uri)
1612         d.addCallback(_check)
1613         return d
1614
1615     def test_GET_DIRURL_readonly_uri(self):
1616         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1617         def _check(res):
1618             self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri)
1619         d.addCallback(_check)
1620         return d
1621
1622     def test_PUT_NEWDIRURL(self):
1623         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1624         d.addCallback(lambda res:
1625                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1626         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1627         d.addCallback(self.failUnlessNodeKeysAre, [])
1628         return d
1629
1630     def test_PUT_NEWDIRURL_mdmf(self):
1631         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1632         d.addCallback(lambda res:
1633                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1634         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1635         d.addCallback(lambda node:
1636             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1637         return d
1638
1639     def test_PUT_NEWDIRURL_sdmf(self):
1640         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=sdmf",
1641                      "")
1642         d.addCallback(lambda res:
1643                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1644         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1645         d.addCallback(lambda node:
1646             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1647         return d
1648
1649     def test_PUT_NEWDIRURL_bad_format(self):
1650         return self.shouldHTTPError("PUT_NEWDIRURL_bad_format",
1651                                     400, "Bad Request", "Unknown format: foo",
1652                                     self.PUT, self.public_url +
1653                                     "/foo/newdir=?t=mkdir&format=foo", "")
1654
1655     def test_POST_NEWDIRURL(self):
1656         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1657         d.addCallback(lambda res:
1658                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1659         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1660         d.addCallback(self.failUnlessNodeKeysAre, [])
1661         return d
1662
1663     def test_POST_NEWDIRURL_mdmf(self):
1664         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1665         d.addCallback(lambda res:
1666                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1667         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1668         d.addCallback(lambda node:
1669             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1670         return d
1671
1672     def test_POST_NEWDIRURL_sdmf(self):
1673         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=sdmf", "")
1674         d.addCallback(lambda res:
1675             self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1676         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1677         d.addCallback(lambda node:
1678             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1679         return d
1680
1681     def test_POST_NEWDIRURL_bad_format(self):
1682         return self.shouldHTTPError("POST_NEWDIRURL_bad_format",
1683                                     400, "Bad Request", "Unknown format: foo",
1684                                     self.POST2, self.public_url + \
1685                                     "/foo/newdir?t=mkdir&format=foo", "")
1686
1687     def test_POST_NEWDIRURL_emptyname(self):
1688         # an empty pathname component (i.e. a double-slash) is disallowed
1689         d = self.shouldFail2(error.Error, "POST_NEWDIRURL_emptyname",
1690                              "400 Bad Request",
1691                              "The webapi does not allow empty pathname components, i.e. a double slash",
1692                              self.POST, self.public_url + "//?t=mkdir")
1693         return d
1694
1695     def _do_POST_NEWDIRURL_initial_children_test(self, version=None):
1696         (newkids, caps) = self._create_initial_children()
1697         query = "/foo/newdir?t=mkdir-with-children"
1698         if version == MDMF_VERSION:
1699             query += "&format=mdmf"
1700         elif version == SDMF_VERSION:
1701             query += "&format=sdmf"
1702         else:
1703             version = SDMF_VERSION # for later
1704         d = self.POST2(self.public_url + query,
1705                        simplejson.dumps(newkids))
1706         def _check(uri):
1707             n = self.s.create_node_from_uri(uri.strip())
1708             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1709             self.failUnlessEqual(n._node.get_version(), version)
1710             d2.addCallback(lambda ign:
1711                            self.failUnlessROChildURIIs(n, u"child-imm",
1712                                                        caps['filecap1']))
1713             d2.addCallback(lambda ign:
1714                            self.failUnlessRWChildURIIs(n, u"child-mutable",
1715                                                        caps['filecap2']))
1716             d2.addCallback(lambda ign:
1717                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1718                                                        caps['filecap3']))
1719             d2.addCallback(lambda ign:
1720                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1721                                                        caps['unknown_rocap']))
1722             d2.addCallback(lambda ign:
1723                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1724                                                        caps['unknown_rwcap']))
1725             d2.addCallback(lambda ign:
1726                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1727                                                        caps['unknown_immcap']))
1728             d2.addCallback(lambda ign:
1729                            self.failUnlessRWChildURIIs(n, u"dirchild",
1730                                                        caps['dircap']))
1731             d2.addCallback(lambda ign:
1732                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1733                                                        caps['litdircap']))
1734             d2.addCallback(lambda ign:
1735                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1736                                                        caps['emptydircap']))
1737             return d2
1738         d.addCallback(_check)
1739         d.addCallback(lambda res:
1740                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1741         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1742         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1743         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1744         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1745         return d
1746
1747     def test_POST_NEWDIRURL_initial_children(self):
1748         return self._do_POST_NEWDIRURL_initial_children_test()
1749
1750     def test_POST_NEWDIRURL_initial_children_mdmf(self):
1751         return self._do_POST_NEWDIRURL_initial_children_test(MDMF_VERSION)
1752
1753     def test_POST_NEWDIRURL_initial_children_sdmf(self):
1754         return self._do_POST_NEWDIRURL_initial_children_test(SDMF_VERSION)
1755
1756     def test_POST_NEWDIRURL_initial_children_bad_format(self):
1757         (newkids, caps) = self._create_initial_children()
1758         return self.shouldHTTPError("POST_NEWDIRURL_initial_children_bad_format",
1759                                     400, "Bad Request", "Unknown format: foo",
1760                                     self.POST2, self.public_url + \
1761                                     "/foo/newdir?t=mkdir-with-children&format=foo",
1762                                     simplejson.dumps(newkids))
1763
1764     def test_POST_NEWDIRURL_immutable(self):
1765         (newkids, caps) = self._create_immutable_children()
1766         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1767                        simplejson.dumps(newkids))
1768         def _check(uri):
1769             n = self.s.create_node_from_uri(uri.strip())
1770             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1771             d2.addCallback(lambda ign:
1772                            self.failUnlessROChildURIIs(n, u"child-imm",
1773                                                        caps['filecap1']))
1774             d2.addCallback(lambda ign:
1775                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1776                                                        caps['unknown_immcap']))
1777             d2.addCallback(lambda ign:
1778                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
1779                                                        caps['immdircap']))
1780             d2.addCallback(lambda ign:
1781                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1782                                                        caps['litdircap']))
1783             d2.addCallback(lambda ign:
1784                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1785                                                        caps['emptydircap']))
1786             return d2
1787         d.addCallback(_check)
1788         d.addCallback(lambda res:
1789                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1790         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1791         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1792         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1793         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1794         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1795         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1796         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1797         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
1798         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1799         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
1800         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1801         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
1802         d.addErrback(self.explain_web_error)
1803         return d
1804
1805     def test_POST_NEWDIRURL_immutable_bad(self):
1806         (newkids, caps) = self._create_initial_children()
1807         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
1808                              "400 Bad Request",
1809                              "needed to be immutable but was not",
1810                              self.POST2,
1811                              self.public_url + "/foo/newdir?t=mkdir-immutable",
1812                              simplejson.dumps(newkids))
1813         return d
1814
1815     def test_PUT_NEWDIRURL_exists(self):
1816         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1817         d.addCallback(lambda res:
1818                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1819         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1820         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1821         return d
1822
1823     def test_PUT_NEWDIRURL_blocked(self):
1824         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1825                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1826                              self.PUT,
1827                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1828         d.addCallback(lambda res:
1829                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1830         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1831         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1832         return d
1833
1834     def test_PUT_NEWDIRURL_mkdirs(self):
1835         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1836         d.addCallback(lambda res:
1837                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1838         d.addCallback(lambda res:
1839                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1840         d.addCallback(lambda res:
1841                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1842         d.addCallback(self.failUnlessNodeKeysAre, [])
1843         return d
1844
1845     def test_PUT_NEWDIRURL_mkdirs_mdmf(self):
1846         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=mdmf", "")
1847         d.addCallback(lambda ignored:
1848             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1849         d.addCallback(lambda ignored:
1850             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1851         d.addCallback(lambda ignored:
1852             self._foo_node.get_child_at_path(u"subdir"))
1853         def _got_subdir(subdir):
1854             # XXX: What we want?
1855             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1856             self.failUnlessNodeHasChild(subdir, u"newdir")
1857             return subdir.get_child_at_path(u"newdir")
1858         d.addCallback(_got_subdir)
1859         d.addCallback(lambda newdir:
1860             self.failUnlessEqual(newdir._node.get_version(), MDMF_VERSION))
1861         return d
1862
1863     def test_PUT_NEWDIRURL_mkdirs_sdmf(self):
1864         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=sdmf", "")
1865         d.addCallback(lambda ignored:
1866             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1867         d.addCallback(lambda ignored:
1868             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1869         d.addCallback(lambda ignored:
1870             self._foo_node.get_child_at_path(u"subdir"))
1871         def _got_subdir(subdir):
1872             # XXX: What we want?
1873             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1874             self.failUnlessNodeHasChild(subdir, u"newdir")
1875             return subdir.get_child_at_path(u"newdir")
1876         d.addCallback(_got_subdir)
1877         d.addCallback(lambda newdir:
1878             self.failUnlessEqual(newdir._node.get_version(), SDMF_VERSION))
1879         return d
1880
1881     def test_PUT_NEWDIRURL_mkdirs_bad_format(self):
1882         return self.shouldHTTPError("PUT_NEWDIRURL_mkdirs_bad_format",
1883                                     400, "Bad Request", "Unknown format: foo",
1884                                     self.PUT, self.public_url + \
1885                                     "/foo/subdir/newdir?t=mkdir&format=foo",
1886                                     "")
1887
1888     def test_DELETE_DIRURL(self):
1889         d = self.DELETE(self.public_url + "/foo")
1890         d.addCallback(lambda res:
1891                       self.failIfNodeHasChild(self.public_root, u"foo"))
1892         return d
1893
1894     def test_DELETE_DIRURL_missing(self):
1895         d = self.DELETE(self.public_url + "/foo/missing")
1896         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1897         d.addCallback(lambda res:
1898                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1899         return d
1900
1901     def test_DELETE_DIRURL_missing2(self):
1902         d = self.DELETE(self.public_url + "/missing")
1903         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1904         return d
1905
1906     def dump_root(self):
1907         print "NODEWALK"
1908         w = webish.DirnodeWalkerMixin()
1909         def visitor(childpath, childnode, metadata):
1910             print childpath
1911         d = w.walk(self.public_root, visitor)
1912         return d
1913
1914     def failUnlessNodeKeysAre(self, node, expected_keys):
1915         for k in expected_keys:
1916             assert isinstance(k, unicode)
1917         d = node.list()
1918         def _check(children):
1919             self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
1920         d.addCallback(_check)
1921         return d
1922     def failUnlessNodeHasChild(self, node, name):
1923         assert isinstance(name, unicode)
1924         d = node.list()
1925         def _check(children):
1926             self.failUnlessIn(name, children)
1927         d.addCallback(_check)
1928         return d
1929     def failIfNodeHasChild(self, node, name):
1930         assert isinstance(name, unicode)
1931         d = node.list()
1932         def _check(children):
1933             self.failIfIn(name, children)
1934         d.addCallback(_check)
1935         return d
1936
1937     def failUnlessChildContentsAre(self, node, name, expected_contents):
1938         assert isinstance(name, unicode)
1939         d = node.get_child_at_path(name)
1940         d.addCallback(lambda node: download_to_data(node))
1941         def _check(contents):
1942             self.failUnlessReallyEqual(contents, expected_contents)
1943         d.addCallback(_check)
1944         return d
1945
1946     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1947         assert isinstance(name, unicode)
1948         d = node.get_child_at_path(name)
1949         d.addCallback(lambda node: node.download_best_version())
1950         def _check(contents):
1951             self.failUnlessReallyEqual(contents, expected_contents)
1952         d.addCallback(_check)
1953         return d
1954
1955     def failUnlessRWChildURIIs(self, node, name, expected_uri):
1956         assert isinstance(name, unicode)
1957         d = node.get_child_at_path(name)
1958         def _check(child):
1959             self.failUnless(child.is_unknown() or not child.is_readonly())
1960             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1961             self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
1962             expected_ro_uri = self._make_readonly(expected_uri)
1963             if expected_ro_uri:
1964                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1965         d.addCallback(_check)
1966         return d
1967
1968     def failUnlessROChildURIIs(self, node, name, expected_uri):
1969         assert isinstance(name, unicode)
1970         d = node.get_child_at_path(name)
1971         def _check(child):
1972             self.failUnless(child.is_unknown() or child.is_readonly())
1973             self.failUnlessReallyEqual(child.get_write_uri(), None)
1974             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1975             self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
1976         d.addCallback(_check)
1977         return d
1978
1979     def failUnlessURIMatchesRWChild(self, got_uri, node, name):
1980         assert isinstance(name, unicode)
1981         d = node.get_child_at_path(name)
1982         def _check(child):
1983             self.failUnless(child.is_unknown() or not child.is_readonly())
1984             self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
1985             self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
1986             expected_ro_uri = self._make_readonly(got_uri)
1987             if expected_ro_uri:
1988                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1989         d.addCallback(_check)
1990         return d
1991
1992     def failUnlessURIMatchesROChild(self, got_uri, node, name):
1993         assert isinstance(name, unicode)
1994         d = node.get_child_at_path(name)
1995         def _check(child):
1996             self.failUnless(child.is_unknown() or child.is_readonly())
1997             self.failUnlessReallyEqual(child.get_write_uri(), None)
1998             self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
1999             self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
2000         d.addCallback(_check)
2001         return d
2002
2003     def failUnlessCHKURIHasContents(self, got_uri, contents):
2004         self.failUnless(self.get_all_contents()[got_uri] == contents)
2005
2006     def test_POST_upload(self):
2007         d = self.POST(self.public_url + "/foo", t="upload",
2008                       file=("new.txt", self.NEWFILE_CONTENTS))
2009         fn = self._foo_node
2010         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2011         d.addCallback(lambda res:
2012                       self.failUnlessChildContentsAre(fn, u"new.txt",
2013                                                       self.NEWFILE_CONTENTS))
2014         return d
2015
2016     def test_POST_upload_unicode(self):
2017         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2018         d = self.POST(self.public_url + "/foo", t="upload",
2019                       file=(filename, self.NEWFILE_CONTENTS))
2020         fn = self._foo_node
2021         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2022         d.addCallback(lambda res:
2023                       self.failUnlessChildContentsAre(fn, filename,
2024                                                       self.NEWFILE_CONTENTS))
2025         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2026         d.addCallback(lambda res: self.GET(target_url))
2027         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2028                                                                   self.NEWFILE_CONTENTS,
2029                                                                   contents))
2030         return d
2031
2032     def test_POST_upload_unicode_named(self):
2033         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2034         d = self.POST(self.public_url + "/foo", t="upload",
2035                       name=filename,
2036                       file=("overridden", self.NEWFILE_CONTENTS))
2037         fn = self._foo_node
2038         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2039         d.addCallback(lambda res:
2040                       self.failUnlessChildContentsAre(fn, filename,
2041                                                       self.NEWFILE_CONTENTS))
2042         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2043         d.addCallback(lambda res: self.GET(target_url))
2044         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2045                                                                   self.NEWFILE_CONTENTS,
2046                                                                   contents))
2047         return d
2048
2049     def test_POST_upload_no_link(self):
2050         d = self.POST("/uri", t="upload",
2051                       file=("new.txt", self.NEWFILE_CONTENTS))
2052         def _check_upload_results(page):
2053             # this should be a page which describes the results of the upload
2054             # that just finished.
2055             self.failUnlessIn("Upload Results:", page)
2056             self.failUnlessIn("URI:", page)
2057             uri_re = re.compile("URI: <tt><span>(.*)</span>")
2058             mo = uri_re.search(page)
2059             self.failUnless(mo, page)
2060             new_uri = mo.group(1)
2061             return new_uri
2062         d.addCallback(_check_upload_results)
2063         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
2064         return d
2065
2066     def test_POST_upload_no_link_whendone(self):
2067         d = self.POST("/uri", t="upload", when_done="/",
2068                       file=("new.txt", self.NEWFILE_CONTENTS))
2069         d.addBoth(self.shouldRedirect, "/")
2070         return d
2071
2072     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
2073         d = defer.maybeDeferred(callable, *args, **kwargs)
2074         def done(res):
2075             if isinstance(res, failure.Failure):
2076                 res.trap(error.PageRedirect)
2077                 statuscode = res.value.status
2078                 target = res.value.location
2079                 return checker(statuscode, target)
2080             self.fail("%s: callable was supposed to redirect, not return '%s'"
2081                       % (which, res))
2082         d.addBoth(done)
2083         return d
2084
2085     def test_POST_upload_no_link_whendone_results(self):
2086         def check(statuscode, target):
2087             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2088             self.failUnless(target.startswith(self.webish_url), target)
2089             return client.getPage(target, method="GET")
2090         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
2091                                  check,
2092                                  self.POST, "/uri", t="upload",
2093                                  when_done="/uri/%(uri)s",
2094                                  file=("new.txt", self.NEWFILE_CONTENTS))
2095         d.addCallback(lambda res:
2096                       self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
2097         return d
2098
2099     def test_POST_upload_no_link_mutable(self):
2100         d = self.POST("/uri", t="upload", mutable="true",
2101                       file=("new.txt", self.NEWFILE_CONTENTS))
2102         def _check(filecap):
2103             filecap = filecap.strip()
2104             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2105             self.filecap = filecap
2106             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2107             self.failUnlessIn(u.get_storage_index(), self.get_all_contents())
2108             n = self.s.create_node_from_uri(filecap)
2109             return n.download_best_version()
2110         d.addCallback(_check)
2111         def _check2(data):
2112             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2113             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2114         d.addCallback(_check2)
2115         def _check3(data):
2116             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2117             return self.GET("/file/%s" % urllib.quote(self.filecap))
2118         d.addCallback(_check3)
2119         def _check4(data):
2120             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2121         d.addCallback(_check4)
2122         return d
2123
2124     def test_POST_upload_no_link_mutable_toobig(self):
2125         # The SDMF size limit is no longer in place, so we should be
2126         # able to upload mutable files that are as large as we want them
2127         # to be.
2128         d = self.POST("/uri", t="upload", mutable="true",
2129                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2130         return d
2131
2132
2133     def test_POST_upload_format_unlinked(self):
2134         def _check_upload_unlinked(ign, format, uri_prefix):
2135             filename = format + ".txt"
2136             d = self.POST("/uri?t=upload&format=" + format,
2137                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2138             def _got_results(results):
2139                 if format.upper() in ("SDMF", "MDMF"):
2140                     # webapi.rst says this returns a filecap
2141                     filecap = results
2142                 else:
2143                     # for immutable, it returns an "upload results page", and
2144                     # the filecap is buried inside
2145                     line = [l for l in results.split("\n") if "URI: " in l][0]
2146                     mo = re.search(r'<span>([^<]+)</span>', line)
2147                     filecap = mo.group(1)
2148                 self.failUnless(filecap.startswith(uri_prefix),
2149                                 (uri_prefix, filecap))
2150                 return self.GET("/uri/%s?t=json" % filecap)
2151             d.addCallback(_got_results)
2152             def _got_json(json):
2153                 data = simplejson.loads(json)
2154                 data = data[1]
2155                 self.failUnlessIn("format", data)
2156                 self.failUnlessEqual(data["format"], format.upper())
2157             d.addCallback(_got_json)
2158             return d
2159         d = defer.succeed(None)
2160         d.addCallback(_check_upload_unlinked, "chk", "URI:CHK")
2161         d.addCallback(_check_upload_unlinked, "CHK", "URI:CHK")
2162         d.addCallback(_check_upload_unlinked, "sdmf", "URI:SSK")
2163         d.addCallback(_check_upload_unlinked, "mdmf", "URI:MDMF")
2164         return d
2165
2166     def test_POST_upload_bad_format_unlinked(self):
2167         return self.shouldHTTPError("POST_upload_bad_format_unlinked",
2168                                     400, "Bad Request", "Unknown format: foo",
2169                                     self.POST,
2170                                     "/uri?t=upload&format=foo",
2171                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2172
2173     def test_POST_upload_format(self):
2174         def _check_upload(ign, format, uri_prefix, fn=None):
2175             filename = format + ".txt"
2176             d = self.POST(self.public_url +
2177                           "/foo?t=upload&format=" + format,
2178                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2179             def _got_filecap(filecap):
2180                 if fn is not None:
2181                     filenameu = unicode(filename)
2182                     self.failUnlessURIMatchesRWChild(filecap, fn, filenameu)
2183                 self.failUnless(filecap.startswith(uri_prefix))
2184                 return self.GET(self.public_url + "/foo/%s?t=json" % filename)
2185             d.addCallback(_got_filecap)
2186             def _got_json(json):
2187                 data = simplejson.loads(json)
2188                 data = data[1]
2189                 self.failUnlessIn("format", data)
2190                 self.failUnlessEqual(data["format"], format.upper())
2191             d.addCallback(_got_json)
2192             return d
2193
2194         d = defer.succeed(None)
2195         d.addCallback(_check_upload, "chk", "URI:CHK")
2196         d.addCallback(_check_upload, "sdmf", "URI:SSK", self._foo_node)
2197         d.addCallback(_check_upload, "mdmf", "URI:MDMF")
2198         d.addCallback(_check_upload, "MDMF", "URI:MDMF")
2199         return d
2200
2201     def test_POST_upload_bad_format(self):
2202         return self.shouldHTTPError("POST_upload_bad_format",
2203                                     400, "Bad Request", "Unknown format: foo",
2204                                     self.POST, self.public_url + \
2205                                     "/foo?t=upload&format=foo",
2206                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2207
2208     def test_POST_upload_mutable(self):
2209         # this creates a mutable file
2210         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
2211                       file=("new.txt", self.NEWFILE_CONTENTS))
2212         fn = self._foo_node
2213         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2214         d.addCallback(lambda res:
2215                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2216                                                              self.NEWFILE_CONTENTS))
2217         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2218         def _got(newnode):
2219             self.failUnless(IMutableFileNode.providedBy(newnode))
2220             self.failUnless(newnode.is_mutable())
2221             self.failIf(newnode.is_readonly())
2222             self._mutable_node = newnode
2223             self._mutable_uri = newnode.get_uri()
2224         d.addCallback(_got)
2225
2226         # now upload it again and make sure that the URI doesn't change
2227         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
2228         d.addCallback(lambda res:
2229                       self.POST(self.public_url + "/foo", t="upload",
2230                                 mutable="true",
2231                                 file=("new.txt", NEWER_CONTENTS)))
2232         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2233         d.addCallback(lambda res:
2234                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2235                                                              NEWER_CONTENTS))
2236         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2237         def _got2(newnode):
2238             self.failUnless(IMutableFileNode.providedBy(newnode))
2239             self.failUnless(newnode.is_mutable())
2240             self.failIf(newnode.is_readonly())
2241             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2242         d.addCallback(_got2)
2243
2244         # upload a second time, using PUT instead of POST
2245         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
2246         d.addCallback(lambda res:
2247                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
2248         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2249         d.addCallback(lambda res:
2250                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2251                                                              NEW2_CONTENTS))
2252
2253         # finally list the directory, since mutable files are displayed
2254         # slightly differently
2255
2256         d.addCallback(lambda res:
2257                       self.GET(self.public_url + "/foo/",
2258                                followRedirect=True))
2259         def _check_page(res):
2260             # TODO: assert more about the contents
2261             self.failUnlessIn("SSK", res)
2262             return res
2263         d.addCallback(_check_page)
2264
2265         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2266         def _got3(newnode):
2267             self.failUnless(IMutableFileNode.providedBy(newnode))
2268             self.failUnless(newnode.is_mutable())
2269             self.failIf(newnode.is_readonly())
2270             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2271         d.addCallback(_got3)
2272
2273         # look at the JSON form of the enclosing directory
2274         d.addCallback(lambda res:
2275                       self.GET(self.public_url + "/foo/?t=json",
2276                                followRedirect=True))
2277         def _check_page_json(res):
2278             parsed = simplejson.loads(res)
2279             self.failUnlessEqual(parsed[0], "dirnode")
2280             children = dict( [(unicode(name),value)
2281                               for (name,value)
2282                               in parsed[1]["children"].iteritems()] )
2283             self.failUnlessIn(u"new.txt", children)
2284             new_json = children[u"new.txt"]
2285             self.failUnlessEqual(new_json[0], "filenode")
2286             self.failUnless(new_json[1]["mutable"])
2287             self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri)
2288             ro_uri = self._mutable_node.get_readonly().to_string()
2289             self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri)
2290         d.addCallback(_check_page_json)
2291
2292         # and the JSON form of the file
2293         d.addCallback(lambda res:
2294                       self.GET(self.public_url + "/foo/new.txt?t=json"))
2295         def _check_file_json(res):
2296             parsed = simplejson.loads(res)
2297             self.failUnlessEqual(parsed[0], "filenode")
2298             self.failUnless(parsed[1]["mutable"])
2299             self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri)
2300             ro_uri = self._mutable_node.get_readonly().to_string()
2301             self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri)
2302         d.addCallback(_check_file_json)
2303
2304         # and look at t=uri and t=readonly-uri
2305         d.addCallback(lambda res:
2306                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
2307         d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
2308         d.addCallback(lambda res:
2309                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
2310         def _check_ro_uri(res):
2311             ro_uri = self._mutable_node.get_readonly().to_string()
2312             self.failUnlessReallyEqual(res, ro_uri)
2313         d.addCallback(_check_ro_uri)
2314
2315         # make sure we can get to it from /uri/URI
2316         d.addCallback(lambda res:
2317                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
2318         d.addCallback(lambda res:
2319                       self.failUnlessReallyEqual(res, NEW2_CONTENTS))
2320
2321         # and that HEAD computes the size correctly
2322         d.addCallback(lambda res:
2323                       self.HEAD(self.public_url + "/foo/new.txt",
2324                                 return_response=True))
2325         def _got_headers((res, status, headers)):
2326             self.failUnlessReallyEqual(res, "")
2327             self.failUnlessReallyEqual(headers["content-length"][0],
2328                                        str(len(NEW2_CONTENTS)))
2329             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
2330         d.addCallback(_got_headers)
2331
2332         # make sure that outdated size limits aren't enforced anymore.
2333         d.addCallback(lambda ignored:
2334             self.POST(self.public_url + "/foo", t="upload",
2335                       mutable="true",
2336                       file=("new.txt",
2337                             "b" * (self.s.MUTABLE_SIZELIMIT+1))))
2338         d.addErrback(self.dump_error)
2339         return d
2340
2341     def test_POST_upload_mutable_toobig(self):
2342         # SDMF had a size limti that was removed a while ago. MDMF has
2343         # never had a size limit. Test to make sure that we do not
2344         # encounter errors when trying to upload large mutable files,
2345         # since there should be no coded prohibitions regarding large
2346         # mutable files.
2347         d = self.POST(self.public_url + "/foo",
2348                       t="upload", mutable="true",
2349                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2350         return d
2351
2352     def dump_error(self, f):
2353         # if the web server returns an error code (like 400 Bad Request),
2354         # web.client.getPage puts the HTTP response body into the .response
2355         # attribute of the exception object that it gives back. It does not
2356         # appear in the Failure's repr(), so the ERROR that trial displays
2357         # will be rather terse and unhelpful. addErrback this method to the
2358         # end of your chain to get more information out of these errors.
2359         if f.check(error.Error):
2360             print "web.error.Error:"
2361             print f
2362             print f.value.response
2363         return f
2364
2365     def test_POST_upload_replace(self):
2366         d = self.POST(self.public_url + "/foo", t="upload",
2367                       file=("bar.txt", self.NEWFILE_CONTENTS))
2368         fn = self._foo_node
2369         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
2370         d.addCallback(lambda res:
2371                       self.failUnlessChildContentsAre(fn, u"bar.txt",
2372                                                       self.NEWFILE_CONTENTS))
2373         return d
2374
2375     def test_POST_upload_no_replace_ok(self):
2376         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2377                       file=("new.txt", self.NEWFILE_CONTENTS))
2378         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
2379         d.addCallback(lambda res: self.failUnlessReallyEqual(res,
2380                                                              self.NEWFILE_CONTENTS))
2381         return d
2382
2383     def test_POST_upload_no_replace_queryarg(self):
2384         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2385                       file=("bar.txt", self.NEWFILE_CONTENTS))
2386         d.addBoth(self.shouldFail, error.Error,
2387                   "POST_upload_no_replace_queryarg",
2388                   "409 Conflict",
2389                   "There was already a child by that name, and you asked me "
2390                   "to not replace it")
2391         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2392         d.addCallback(self.failUnlessIsBarDotTxt)
2393         return d
2394
2395     def test_POST_upload_no_replace_field(self):
2396         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
2397                       file=("bar.txt", self.NEWFILE_CONTENTS))
2398         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
2399                   "409 Conflict",
2400                   "There was already a child by that name, and you asked me "
2401                   "to not replace it")
2402         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2403         d.addCallback(self.failUnlessIsBarDotTxt)
2404         return d
2405
2406     def test_POST_upload_whendone(self):
2407         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
2408                       file=("new.txt", self.NEWFILE_CONTENTS))
2409         d.addBoth(self.shouldRedirect, "/THERE")
2410         fn = self._foo_node
2411         d.addCallback(lambda res:
2412                       self.failUnlessChildContentsAre(fn, u"new.txt",
2413                                                       self.NEWFILE_CONTENTS))
2414         return d
2415
2416     def test_POST_upload_named(self):
2417         fn = self._foo_node
2418         d = self.POST(self.public_url + "/foo", t="upload",
2419                       name="new.txt", file=self.NEWFILE_CONTENTS)
2420         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2421         d.addCallback(lambda res:
2422                       self.failUnlessChildContentsAre(fn, u"new.txt",
2423                                                       self.NEWFILE_CONTENTS))
2424         return d
2425
2426     def test_POST_upload_named_badfilename(self):
2427         d = self.POST(self.public_url + "/foo", t="upload",
2428                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
2429         d.addBoth(self.shouldFail, error.Error,
2430                   "test_POST_upload_named_badfilename",
2431                   "400 Bad Request",
2432                   "name= may not contain a slash",
2433                   )
2434         # make sure that nothing was added
2435         d.addCallback(lambda res:
2436                       self.failUnlessNodeKeysAre(self._foo_node,
2437                                                  [u"bar.txt", u"baz.txt", u"blockingfile",
2438                                                   u"empty", u"n\u00fc.txt", u"quux.txt",
2439                                                   u"sub"]))
2440         return d
2441
2442     def test_POST_FILEURL_check(self):
2443         bar_url = self.public_url + "/foo/bar.txt"
2444         d = self.POST(bar_url, t="check")
2445         def _check(res):
2446             self.failUnlessIn("Healthy :", res)
2447         d.addCallback(_check)
2448         redir_url = "http://allmydata.org/TARGET"
2449         def _check2(statuscode, target):
2450             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2451             self.failUnlessReallyEqual(target, redir_url)
2452         d.addCallback(lambda res:
2453                       self.shouldRedirect2("test_POST_FILEURL_check",
2454                                            _check2,
2455                                            self.POST, bar_url,
2456                                            t="check",
2457                                            when_done=redir_url))
2458         d.addCallback(lambda res:
2459                       self.POST(bar_url, t="check", return_to=redir_url))
2460         def _check3(res):
2461             self.failUnlessIn("Healthy :", res)
2462             self.failUnlessIn("Return to file", res)
2463             self.failUnlessIn(redir_url, res)
2464         d.addCallback(_check3)
2465
2466         d.addCallback(lambda res:
2467                       self.POST(bar_url, t="check", output="JSON"))
2468         def _check_json(res):
2469             data = simplejson.loads(res)
2470             self.failUnlessIn("storage-index", data)
2471             self.failUnless(data["results"]["healthy"])
2472         d.addCallback(_check_json)
2473
2474         return d
2475
2476     def test_POST_FILEURL_check_and_repair(self):
2477         bar_url = self.public_url + "/foo/bar.txt"
2478         d = self.POST(bar_url, t="check", repair="true")
2479         def _check(res):
2480             self.failUnlessIn("Healthy :", res)
2481         d.addCallback(_check)
2482         redir_url = "http://allmydata.org/TARGET"
2483         def _check2(statuscode, target):
2484             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2485             self.failUnlessReallyEqual(target, redir_url)
2486         d.addCallback(lambda res:
2487                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
2488                                            _check2,
2489                                            self.POST, bar_url,
2490                                            t="check", repair="true",
2491                                            when_done=redir_url))
2492         d.addCallback(lambda res:
2493                       self.POST(bar_url, t="check", return_to=redir_url))
2494         def _check3(res):
2495             self.failUnlessIn("Healthy :", res)
2496             self.failUnlessIn("Return to file", res)
2497             self.failUnlessIn(redir_url, res)
2498         d.addCallback(_check3)
2499         return d
2500
2501     def test_POST_DIRURL_check(self):
2502         foo_url = self.public_url + "/foo/"
2503         d = self.POST(foo_url, t="check")
2504         def _check(res):
2505             self.failUnlessIn("Healthy :", res)
2506         d.addCallback(_check)
2507         redir_url = "http://allmydata.org/TARGET"
2508         def _check2(statuscode, target):
2509             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2510             self.failUnlessReallyEqual(target, redir_url)
2511         d.addCallback(lambda res:
2512                       self.shouldRedirect2("test_POST_DIRURL_check",
2513                                            _check2,
2514                                            self.POST, foo_url,
2515                                            t="check",
2516                                            when_done=redir_url))
2517         d.addCallback(lambda res:
2518                       self.POST(foo_url, t="check", return_to=redir_url))
2519         def _check3(res):
2520             self.failUnlessIn("Healthy :", res)
2521             self.failUnlessIn("Return to file/directory", res)
2522             self.failUnlessIn(redir_url, res)
2523         d.addCallback(_check3)
2524
2525         d.addCallback(lambda res:
2526                       self.POST(foo_url, t="check", output="JSON"))
2527         def _check_json(res):
2528             data = simplejson.loads(res)
2529             self.failUnlessIn("storage-index", data)
2530             self.failUnless(data["results"]["healthy"])
2531         d.addCallback(_check_json)
2532
2533         return d
2534
2535     def test_POST_DIRURL_check_and_repair(self):
2536         foo_url = self.public_url + "/foo/"
2537         d = self.POST(foo_url, t="check", repair="true")
2538         def _check(res):
2539             self.failUnlessIn("Healthy :", res)
2540         d.addCallback(_check)
2541         redir_url = "http://allmydata.org/TARGET"
2542         def _check2(statuscode, target):
2543             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2544             self.failUnlessReallyEqual(target, redir_url)
2545         d.addCallback(lambda res:
2546                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2547                                            _check2,
2548                                            self.POST, foo_url,
2549                                            t="check", repair="true",
2550                                            when_done=redir_url))
2551         d.addCallback(lambda res:
2552                       self.POST(foo_url, t="check", return_to=redir_url))
2553         def _check3(res):
2554             self.failUnlessIn("Healthy :", res)
2555             self.failUnlessIn("Return to file/directory", res)
2556             self.failUnlessIn(redir_url, res)
2557         d.addCallback(_check3)
2558         return d
2559
2560     def test_POST_FILEURL_mdmf_check(self):
2561         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2562         d = self.POST(quux_url, t="check")
2563         def _check(res):
2564             self.failUnlessIn("Healthy", res)
2565         d.addCallback(_check)
2566         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2567         d.addCallback(lambda ignored:
2568                       self.POST(quux_extension_url, t="check"))
2569         d.addCallback(_check)
2570         return d
2571
2572     def test_POST_FILEURL_mdmf_check_and_repair(self):
2573         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2574         d = self.POST(quux_url, t="check", repair="true")
2575         def _check(res):
2576             self.failUnlessIn("Healthy", res)
2577         d.addCallback(_check)
2578         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2579         d.addCallback(lambda ignored:
2580                       self.POST(quux_extension_url, t="check", repair="true"))
2581         d.addCallback(_check)
2582         return d
2583
2584     def wait_for_operation(self, ignored, ophandle):
2585         url = "/operations/" + ophandle
2586         url += "?t=status&output=JSON"
2587         d = self.GET(url)
2588         def _got(res):
2589             data = simplejson.loads(res)
2590             if not data["finished"]:
2591                 d = self.stall(delay=1.0)
2592                 d.addCallback(self.wait_for_operation, ophandle)
2593                 return d
2594             return data
2595         d.addCallback(_got)
2596         return d
2597
2598     def get_operation_results(self, ignored, ophandle, output=None):
2599         url = "/operations/" + ophandle
2600         url += "?t=status"
2601         if output:
2602             url += "&output=" + output
2603         d = self.GET(url)
2604         def _got(res):
2605             if output and output.lower() == "json":
2606                 return simplejson.loads(res)
2607             return res
2608         d.addCallback(_got)
2609         return d
2610
2611     def test_POST_DIRURL_deepcheck_no_ophandle(self):
2612         d = self.shouldFail2(error.Error,
2613                              "test_POST_DIRURL_deepcheck_no_ophandle",
2614                              "400 Bad Request",
2615                              "slow operation requires ophandle=",
2616                              self.POST, self.public_url, t="start-deep-check")
2617         return d
2618
2619     def test_POST_DIRURL_deepcheck(self):
2620         def _check_redirect(statuscode, target):
2621             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2622             self.failUnless(target.endswith("/operations/123"))
2623         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2624                                  self.POST, self.public_url,
2625                                  t="start-deep-check", ophandle="123")
2626         d.addCallback(self.wait_for_operation, "123")
2627         def _check_json(data):
2628             self.failUnlessReallyEqual(data["finished"], True)
2629             self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2630             self.failUnlessReallyEqual(data["count-objects-healthy"], 10)
2631         d.addCallback(_check_json)
2632         d.addCallback(self.get_operation_results, "123", "html")
2633         def _check_html(res):
2634             self.failUnlessIn("Objects Checked: <span>10</span>", res)
2635             self.failUnlessIn("Objects Healthy: <span>10</span>", res)
2636             self.failUnlessIn(FAVICON_MARKUP, res)
2637         d.addCallback(_check_html)
2638
2639         d.addCallback(lambda res:
2640                       self.GET("/operations/123/"))
2641         d.addCallback(_check_html) # should be the same as without the slash
2642
2643         d.addCallback(lambda res:
2644                       self.shouldFail2(error.Error, "one", "404 Not Found",
2645                                        "No detailed results for SI bogus",
2646                                        self.GET, "/operations/123/bogus"))
2647
2648         foo_si = self._foo_node.get_storage_index()
2649         foo_si_s = base32.b2a(foo_si)
2650         d.addCallback(lambda res:
2651                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2652         def _check_foo_json(res):
2653             data = simplejson.loads(res)
2654             self.failUnlessEqual(data["storage-index"], foo_si_s)
2655             self.failUnless(data["results"]["healthy"])
2656         d.addCallback(_check_foo_json)
2657         return d
2658
2659     def test_POST_DIRURL_deepcheck_and_repair(self):
2660         d = self.POST(self.public_url, t="start-deep-check", repair="true",
2661                       ophandle="124", output="json", followRedirect=True)
2662         d.addCallback(self.wait_for_operation, "124")
2663         def _check_json(data):
2664             self.failUnlessReallyEqual(data["finished"], True)
2665             self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2666             self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 10)
2667             self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2668             self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2669             self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2670             self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2671             self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2672             self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 10)
2673             self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2674             self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2675         d.addCallback(_check_json)
2676         d.addCallback(self.get_operation_results, "124", "html")
2677         def _check_html(res):
2678             self.failUnlessIn("Objects Checked: <span>10</span>", res)
2679
2680             self.failUnlessIn("Objects Healthy (before repair): <span>10</span>", res)
2681             self.failUnlessIn("Objects Unhealthy (before repair): <span>0</span>", res)
2682             self.failUnlessIn("Corrupt Shares (before repair): <span>0</span>", res)
2683
2684             self.failUnlessIn("Repairs Attempted: <span>0</span>", res)
2685             self.failUnlessIn("Repairs Successful: <span>0</span>", res)
2686             self.failUnlessIn("Repairs Unsuccessful: <span>0</span>", res)
2687
2688             self.failUnlessIn("Objects Healthy (after repair): <span>10</span>", res)
2689             self.failUnlessIn("Objects Unhealthy (after repair): <span>0</span>", res)
2690             self.failUnlessIn("Corrupt Shares (after repair): <span>0</span>", res)
2691
2692             self.failUnlessIn(FAVICON_MARKUP, res)
2693         d.addCallback(_check_html)
2694         return d
2695
2696     def test_POST_FILEURL_bad_t(self):
2697         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2698                              "POST to file: bad t=bogus",
2699                              self.POST, self.public_url + "/foo/bar.txt",
2700                              t="bogus")
2701         return d
2702
2703     def test_POST_mkdir(self): # return value?
2704         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2705         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2706         d.addCallback(self.failUnlessNodeKeysAre, [])
2707         return d
2708
2709     def test_POST_mkdir_mdmf(self):
2710         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=mdmf")
2711         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2712         d.addCallback(lambda node:
2713             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2714         return d
2715
2716     def test_POST_mkdir_sdmf(self):
2717         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=sdmf")
2718         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2719         d.addCallback(lambda node:
2720             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2721         return d
2722
2723     def test_POST_mkdir_bad_format(self):
2724         return self.shouldHTTPError("POST_mkdir_bad_format",
2725                                     400, "Bad Request", "Unknown format: foo",
2726                                     self.POST, self.public_url +
2727                                     "/foo?t=mkdir&name=newdir&format=foo")
2728
2729     def test_POST_mkdir_initial_children(self):
2730         (newkids, caps) = self._create_initial_children()
2731         d = self.POST2(self.public_url +
2732                        "/foo?t=mkdir-with-children&name=newdir",
2733                        simplejson.dumps(newkids))
2734         d.addCallback(lambda res:
2735                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2736         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2737         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2738         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2739         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2740         return d
2741
2742     def test_POST_mkdir_initial_children_mdmf(self):
2743         (newkids, caps) = self._create_initial_children()
2744         d = self.POST2(self.public_url +
2745                        "/foo?t=mkdir-with-children&name=newdir&format=mdmf",
2746                        simplejson.dumps(newkids))
2747         d.addCallback(lambda res:
2748                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2749         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2750         d.addCallback(lambda node:
2751             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2752         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2753         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2754                        caps['filecap1'])
2755         return d
2756
2757     # XXX: Duplication.
2758     def test_POST_mkdir_initial_children_sdmf(self):
2759         (newkids, caps) = self._create_initial_children()
2760         d = self.POST2(self.public_url +
2761                        "/foo?t=mkdir-with-children&name=newdir&format=sdmf",
2762                        simplejson.dumps(newkids))
2763         d.addCallback(lambda res:
2764                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2765         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2766         d.addCallback(lambda node:
2767             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2768         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2769         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2770                        caps['filecap1'])
2771         return d
2772
2773     def test_POST_mkdir_initial_children_bad_format(self):
2774         (newkids, caps) = self._create_initial_children()
2775         return self.shouldHTTPError("POST_mkdir_initial_children_bad_format",
2776                                     400, "Bad Request", "Unknown format: foo",
2777                                     self.POST, self.public_url + \
2778                                     "/foo?t=mkdir-with-children&name=newdir&format=foo",
2779                                     simplejson.dumps(newkids))
2780
2781     def test_POST_mkdir_immutable(self):
2782         (newkids, caps) = self._create_immutable_children()
2783         d = self.POST2(self.public_url +
2784                        "/foo?t=mkdir-immutable&name=newdir",
2785                        simplejson.dumps(newkids))
2786         d.addCallback(lambda res:
2787                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2788         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2789         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2790         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2791         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2792         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2793         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2794         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2795         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2796         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2797         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2798         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2799         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2800         return d
2801
2802     def test_POST_mkdir_immutable_bad(self):
2803         (newkids, caps) = self._create_initial_children()
2804         d = self.shouldFail2(error.Error, "POST_mkdir_immutable_bad",
2805                              "400 Bad Request",
2806                              "needed to be immutable but was not",
2807                              self.POST2,
2808                              self.public_url +
2809                              "/foo?t=mkdir-immutable&name=newdir",
2810                              simplejson.dumps(newkids))
2811         return d
2812
2813     def test_POST_mkdir_2(self):
2814         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
2815         d.addCallback(lambda res:
2816                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2817         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2818         d.addCallback(self.failUnlessNodeKeysAre, [])
2819         return d
2820
2821     def test_POST_mkdirs_2(self):
2822         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
2823         d.addCallback(lambda res:
2824                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
2825         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
2826         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
2827         d.addCallback(self.failUnlessNodeKeysAre, [])
2828         return d
2829
2830     def test_POST_mkdir_no_parentdir_noredirect(self):
2831         d = self.POST("/uri?t=mkdir")
2832         def _after_mkdir(res):
2833             uri.DirectoryURI.init_from_string(res)
2834         d.addCallback(_after_mkdir)
2835         return d
2836
2837     def test_POST_mkdir_no_parentdir_noredirect_mdmf(self):
2838         d = self.POST("/uri?t=mkdir&format=mdmf")
2839         def _after_mkdir(res):
2840             u = uri.from_string(res)
2841             # Check that this is an MDMF writecap
2842             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
2843         d.addCallback(_after_mkdir)
2844         return d
2845
2846     def test_POST_mkdir_no_parentdir_noredirect_sdmf(self):
2847         d = self.POST("/uri?t=mkdir&format=sdmf")
2848         def _after_mkdir(res):
2849             u = uri.from_string(res)
2850             self.failUnlessIsInstance(u, uri.DirectoryURI)
2851         d.addCallback(_after_mkdir)
2852         return d
2853
2854     def test_POST_mkdir_no_parentdir_noredirect_bad_format(self):
2855         return self.shouldHTTPError("POST_mkdir_no_parentdir_noredirect_bad_format",
2856                                     400, "Bad Request", "Unknown format: foo",
2857                                     self.POST, self.public_url +
2858                                     "/uri?t=mkdir&format=foo")
2859
2860     def test_POST_mkdir_no_parentdir_noredirect2(self):
2861         # make sure form-based arguments (as on the welcome page) still work
2862         d = self.POST("/uri", t="mkdir")
2863         def _after_mkdir(res):
2864             uri.DirectoryURI.init_from_string(res)
2865         d.addCallback(_after_mkdir)
2866         d.addErrback(self.explain_web_error)
2867         return d
2868
2869     def test_POST_mkdir_no_parentdir_redirect(self):
2870         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
2871         d.addBoth(self.shouldRedirect, None, statuscode='303')
2872         def _check_target(target):
2873             target = urllib.unquote(target)
2874             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2875         d.addCallback(_check_target)
2876         return d
2877
2878     def test_POST_mkdir_no_parentdir_redirect2(self):
2879         d = self.POST("/uri", t="mkdir", redirect_to_result="true")
2880         d.addBoth(self.shouldRedirect, None, statuscode='303')
2881         def _check_target(target):
2882             target = urllib.unquote(target)
2883             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2884         d.addCallback(_check_target)
2885         d.addErrback(self.explain_web_error)
2886         return d
2887
2888     def _make_readonly(self, u):
2889         ro_uri = uri.from_string(u).get_readonly()
2890         if ro_uri is None:
2891             return None
2892         return ro_uri.to_string()
2893
2894     def _create_initial_children(self):
2895         contents, n, filecap1 = self.makefile(12)
2896         md1 = {"metakey1": "metavalue1"}
2897         filecap2 = make_mutable_file_uri()
2898         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
2899         filecap3 = node3.get_readonly_uri()
2900         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
2901         dircap = DirectoryNode(node4, None, None).get_uri()
2902         mdmfcap = make_mutable_file_uri(mdmf=True)
2903         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2904         emptydircap = "URI:DIR2-LIT:"
2905         newkids = {u"child-imm":        ["filenode", {"rw_uri": filecap1,
2906                                                       "ro_uri": self._make_readonly(filecap1),
2907                                                       "metadata": md1, }],
2908                    u"child-mutable":    ["filenode", {"rw_uri": filecap2,
2909                                                       "ro_uri": self._make_readonly(filecap2)}],
2910                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
2911                    u"unknownchild-rw":  ["unknown",  {"rw_uri": unknown_rwcap,
2912                                                       "ro_uri": unknown_rocap}],
2913                    u"unknownchild-ro":  ["unknown",  {"ro_uri": unknown_rocap}],
2914                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2915                    u"dirchild":         ["dirnode",  {"rw_uri": dircap,
2916                                                       "ro_uri": self._make_readonly(dircap)}],
2917                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2918                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2919                    u"child-mutable-mdmf": ["filenode", {"rw_uri": mdmfcap,
2920                                                         "ro_uri": self._make_readonly(mdmfcap)}],
2921                    }
2922         return newkids, {'filecap1': filecap1,
2923                          'filecap2': filecap2,
2924                          'filecap3': filecap3,
2925                          'unknown_rwcap': unknown_rwcap,
2926                          'unknown_rocap': unknown_rocap,
2927                          'unknown_immcap': unknown_immcap,
2928                          'dircap': dircap,
2929                          'litdircap': litdircap,
2930                          'emptydircap': emptydircap,
2931                          'mdmfcap': mdmfcap}
2932
2933     def _create_immutable_children(self):
2934         contents, n, filecap1 = self.makefile(12)
2935         md1 = {"metakey1": "metavalue1"}
2936         tnode = create_chk_filenode("immutable directory contents\n"*10,
2937                                     self.get_all_contents())
2938         dnode = DirectoryNode(tnode, None, None)
2939         assert not dnode.is_mutable()
2940         immdircap = dnode.get_uri()
2941         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2942         emptydircap = "URI:DIR2-LIT:"
2943         newkids = {u"child-imm":        ["filenode", {"ro_uri": filecap1,
2944                                                       "metadata": md1, }],
2945                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2946                    u"dirchild-imm":     ["dirnode",  {"ro_uri": immdircap}],
2947                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2948                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2949                    }
2950         return newkids, {'filecap1': filecap1,
2951                          'unknown_immcap': unknown_immcap,
2952                          'immdircap': immdircap,
2953                          'litdircap': litdircap,
2954                          'emptydircap': emptydircap}
2955
2956     def test_POST_mkdir_no_parentdir_initial_children(self):
2957         (newkids, caps) = self._create_initial_children()
2958         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
2959         def _after_mkdir(res):
2960             self.failUnless(res.startswith("URI:DIR"), res)
2961             n = self.s.create_node_from_uri(res)
2962             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2963             d2.addCallback(lambda ign:
2964                            self.failUnlessROChildURIIs(n, u"child-imm",
2965                                                        caps['filecap1']))
2966             d2.addCallback(lambda ign:
2967                            self.failUnlessRWChildURIIs(n, u"child-mutable",
2968                                                        caps['filecap2']))
2969             d2.addCallback(lambda ign:
2970                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
2971                                                        caps['filecap3']))
2972             d2.addCallback(lambda ign:
2973                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
2974                                                        caps['unknown_rwcap']))
2975             d2.addCallback(lambda ign:
2976                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
2977                                                        caps['unknown_rocap']))
2978             d2.addCallback(lambda ign:
2979                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2980                                                        caps['unknown_immcap']))
2981             d2.addCallback(lambda ign:
2982                            self.failUnlessRWChildURIIs(n, u"dirchild",
2983                                                        caps['dircap']))
2984             return d2
2985         d.addCallback(_after_mkdir)
2986         return d
2987
2988     def test_POST_mkdir_no_parentdir_unexpected_children(self):
2989         # the regular /uri?t=mkdir operation is specified to ignore its body.
2990         # Only t=mkdir-with-children pays attention to it.
2991         (newkids, caps) = self._create_initial_children()
2992         d = self.shouldHTTPError("POST_mkdir_no_parentdir_unexpected_children",
2993                                  400, "Bad Request",
2994                                  "t=mkdir does not accept children=, "
2995                                  "try t=mkdir-with-children instead",
2996                                  self.POST2, "/uri?t=mkdir", # without children
2997                                  simplejson.dumps(newkids))
2998         return d
2999
3000     def test_POST_noparent_bad(self):
3001         d = self.shouldHTTPError("POST_noparent_bad",
3002                                  400, "Bad Request",
3003                                  "/uri accepts only PUT, PUT?t=mkdir, "
3004                                  "POST?t=upload, and POST?t=mkdir",
3005                                  self.POST, "/uri?t=bogus")
3006         return d
3007
3008     def test_POST_mkdir_no_parentdir_immutable(self):
3009         (newkids, caps) = self._create_immutable_children()
3010         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
3011         def _after_mkdir(res):
3012             self.failUnless(res.startswith("URI:DIR"), res)
3013             n = self.s.create_node_from_uri(res)
3014             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3015             d2.addCallback(lambda ign:
3016                            self.failUnlessROChildURIIs(n, u"child-imm",
3017                                                           caps['filecap1']))
3018             d2.addCallback(lambda ign:
3019                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3020                                                           caps['unknown_immcap']))
3021             d2.addCallback(lambda ign:
3022                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
3023                                                           caps['immdircap']))
3024             d2.addCallback(lambda ign:
3025                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
3026                                                           caps['litdircap']))
3027             d2.addCallback(lambda ign:
3028                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
3029                                                           caps['emptydircap']))
3030             return d2
3031         d.addCallback(_after_mkdir)
3032         return d
3033
3034     def test_POST_mkdir_no_parentdir_immutable_bad(self):
3035         (newkids, caps) = self._create_initial_children()
3036         d = self.shouldFail2(error.Error,
3037                              "test_POST_mkdir_no_parentdir_immutable_bad",
3038                              "400 Bad Request",
3039                              "needed to be immutable but was not",
3040                              self.POST2,
3041                              "/uri?t=mkdir-immutable",
3042                              simplejson.dumps(newkids))
3043         return d
3044
3045     def test_welcome_page_mkdir_button(self):
3046         # Fetch the welcome page.
3047         d = self.GET("/")
3048         def _after_get_welcome_page(res):
3049             MKDIR_BUTTON_RE = re.compile(
3050                 '<form action="([^"]*)" method="post".*?'
3051                 '<input type="hidden" name="t" value="([^"]*)" />'
3052                 '<input type="hidden" name="([^"]*)" value="([^"]*)" />'
3053                 '<input type="submit" value="Create a directory" />',
3054                 re.I)
3055             mo = MKDIR_BUTTON_RE.search(res)
3056             formaction = mo.group(1)
3057             formt = mo.group(2)
3058             formaname = mo.group(3)
3059             formavalue = mo.group(4)
3060             return (formaction, formt, formaname, formavalue)
3061         d.addCallback(_after_get_welcome_page)
3062         def _after_parse_form(res):
3063             (formaction, formt, formaname, formavalue) = res
3064             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
3065         d.addCallback(_after_parse_form)
3066         d.addBoth(self.shouldRedirect, None, statuscode='303')
3067         return d
3068
3069     def test_POST_mkdir_replace(self): # return value?
3070         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
3071         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3072         d.addCallback(self.failUnlessNodeKeysAre, [])
3073         return d
3074
3075     def test_POST_mkdir_no_replace_queryarg(self): # return value?
3076         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
3077         d.addBoth(self.shouldFail, error.Error,
3078                   "POST_mkdir_no_replace_queryarg",
3079                   "409 Conflict",
3080                   "There was already a child by that name, and you asked me "
3081                   "to not replace it")
3082         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3083         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3084         return d
3085
3086     def test_POST_mkdir_no_replace_field(self): # return value?
3087         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
3088                       replace="false")
3089         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
3090                   "409 Conflict",
3091                   "There was already a child by that name, and you asked me "
3092                   "to not replace it")
3093         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3094         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3095         return d
3096
3097     def test_POST_mkdir_whendone_field(self):
3098         d = self.POST(self.public_url + "/foo",
3099                       t="mkdir", name="newdir", when_done="/THERE")
3100         d.addBoth(self.shouldRedirect, "/THERE")
3101         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3102         d.addCallback(self.failUnlessNodeKeysAre, [])
3103         return d
3104
3105     def test_POST_mkdir_whendone_queryarg(self):
3106         d = self.POST(self.public_url + "/foo?when_done=/THERE",
3107                       t="mkdir", name="newdir")
3108         d.addBoth(self.shouldRedirect, "/THERE")
3109         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3110         d.addCallback(self.failUnlessNodeKeysAre, [])
3111         return d
3112
3113     def test_POST_bad_t(self):
3114         d = self.shouldFail2(error.Error, "POST_bad_t",
3115                              "400 Bad Request",
3116                              "POST to a directory with bad t=BOGUS",
3117                              self.POST, self.public_url + "/foo", t="BOGUS")
3118         return d
3119
3120     def test_POST_set_children(self, command_name="set_children"):
3121         contents9, n9, newuri9 = self.makefile(9)
3122         contents10, n10, newuri10 = self.makefile(10)
3123         contents11, n11, newuri11 = self.makefile(11)
3124
3125         reqbody = """{
3126                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
3127                                                 "size": 0,
3128                                                 "metadata": {
3129                                                   "ctime": 1002777696.7564139,
3130                                                   "mtime": 1002777696.7564139
3131                                                  }
3132                                                } ],
3133                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
3134                                                 "size": 1,
3135                                                 "metadata": {
3136                                                   "ctime": 1002777696.7564139,
3137                                                   "mtime": 1002777696.7564139
3138                                                  }
3139                                                } ],
3140                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
3141                                                 "size": 2,
3142                                                 "metadata": {
3143                                                   "ctime": 1002777696.7564139,
3144                                                   "mtime": 1002777696.7564139
3145                                                  }
3146                                                } ]
3147                     }""" % (newuri9, newuri10, newuri11)
3148
3149         url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
3150
3151         d = client.getPage(url, method="POST", postdata=reqbody)
3152         def _then(res):
3153             self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
3154             self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
3155             self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
3156
3157         d.addCallback(_then)
3158         d.addErrback(self.dump_error)
3159         return d
3160
3161     def test_POST_set_children_with_hyphen(self):
3162         return self.test_POST_set_children(command_name="set-children")
3163
3164     def test_POST_link_uri(self):
3165         contents, n, newuri = self.makefile(8)
3166         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
3167         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
3168         d.addCallback(lambda res:
3169                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3170                                                       contents))
3171         return d
3172
3173     def test_POST_link_uri_replace(self):
3174         contents, n, newuri = self.makefile(8)
3175         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
3176         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
3177         d.addCallback(lambda res:
3178                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3179                                                       contents))
3180         return d
3181
3182     def test_POST_link_uri_unknown_bad(self):
3183         d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
3184         d.addBoth(self.shouldFail, error.Error,
3185                   "POST_link_uri_unknown_bad",
3186                   "400 Bad Request",
3187                   "unknown cap in a write slot")
3188         return d
3189
3190     def test_POST_link_uri_unknown_ro_good(self):
3191         d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
3192         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
3193         return d
3194
3195     def test_POST_link_uri_unknown_imm_good(self):
3196         d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
3197         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
3198         return d
3199
3200     def test_POST_link_uri_no_replace_queryarg(self):
3201         contents, n, newuri = self.makefile(8)
3202         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
3203                       name="bar.txt", uri=newuri)
3204         d.addBoth(self.shouldFail, error.Error,
3205                   "POST_link_uri_no_replace_queryarg",
3206                   "409 Conflict",
3207                   "There was already a child by that name, and you asked me "
3208                   "to not replace it")
3209         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3210         d.addCallback(self.failUnlessIsBarDotTxt)
3211         return d
3212
3213     def test_POST_link_uri_no_replace_field(self):
3214         contents, n, newuri = self.makefile(8)
3215         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
3216                       name="bar.txt", uri=newuri)
3217         d.addBoth(self.shouldFail, error.Error,
3218                   "POST_link_uri_no_replace_field",
3219                   "409 Conflict",
3220                   "There was already a child by that name, and you asked me "
3221                   "to not replace it")
3222         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3223         d.addCallback(self.failUnlessIsBarDotTxt)
3224         return d
3225
3226     def test_POST_delete(self, command_name='delete'):
3227         d = self._foo_node.list()
3228         def _check_before(children):
3229             self.failUnlessIn(u"bar.txt", children)
3230         d.addCallback(_check_before)
3231         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t=command_name, name="bar.txt"))
3232         d.addCallback(lambda res: self._foo_node.list())
3233         def _check_after(children):
3234             self.failIfIn(u"bar.txt", children)
3235         d.addCallback(_check_after)
3236         return d
3237
3238     def test_POST_unlink(self):
3239         return self.test_POST_delete(command_name='unlink')
3240
3241     def test_POST_rename_file(self):
3242         d = self.POST(self.public_url + "/foo", t="rename",
3243                       from_name="bar.txt", to_name='wibble.txt')
3244         d.addCallback(lambda res:
3245                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3246         d.addCallback(lambda res:
3247                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
3248         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
3249         d.addCallback(self.failUnlessIsBarDotTxt)
3250         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
3251         d.addCallback(self.failUnlessIsBarJSON)
3252         return d
3253
3254     def test_POST_rename_file_redundant(self):
3255         d = self.POST(self.public_url + "/foo", t="rename",
3256                       from_name="bar.txt", to_name='bar.txt')
3257         d.addCallback(lambda res:
3258                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3259         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3260         d.addCallback(self.failUnlessIsBarDotTxt)
3261         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3262         d.addCallback(self.failUnlessIsBarJSON)
3263         return d
3264
3265     def test_POST_rename_file_replace(self):
3266         # rename a file and replace a directory with it
3267         d = self.POST(self.public_url + "/foo", t="rename",
3268                       from_name="bar.txt", to_name='empty')
3269         d.addCallback(lambda res:
3270                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3271         d.addCallback(lambda res:
3272                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
3273         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
3274         d.addCallback(self.failUnlessIsBarDotTxt)
3275         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3276         d.addCallback(self.failUnlessIsBarJSON)
3277         return d
3278
3279     def test_POST_rename_file_no_replace_queryarg(self):
3280         # rename a file and replace a directory with it
3281         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
3282                       from_name="bar.txt", to_name='empty')
3283         d.addBoth(self.shouldFail, error.Error,
3284                   "POST_rename_file_no_replace_queryarg",
3285                   "409 Conflict",
3286                   "There was already a child by that name, and you asked me "
3287                   "to not replace it")
3288         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3289         d.addCallback(self.failUnlessIsEmptyJSON)
3290         return d
3291
3292     def test_POST_rename_file_no_replace_field(self):
3293         # rename a file and replace a directory with it
3294         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
3295                       from_name="bar.txt", to_name='empty')
3296         d.addBoth(self.shouldFail, error.Error,
3297                   "POST_rename_file_no_replace_field",
3298                   "409 Conflict",
3299                   "There was already a child by that name, and you asked me "
3300                   "to not replace it")
3301         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3302         d.addCallback(self.failUnlessIsEmptyJSON)
3303         return d
3304
3305     def failUnlessIsEmptyJSON(self, res):
3306         data = simplejson.loads(res)
3307         self.failUnlessEqual(data[0], "dirnode", data)
3308         self.failUnlessReallyEqual(len(data[1]["children"]), 0)
3309
3310     def test_POST_rename_file_slash_fail(self):
3311         d = self.POST(self.public_url + "/foo", t="rename",
3312                       from_name="bar.txt", to_name='kirk/spock.txt')
3313         d.addBoth(self.shouldFail, error.Error,
3314                   "test_POST_rename_file_slash_fail",
3315                   "400 Bad Request",
3316                   "to_name= may not contain a slash",
3317                   )
3318         d.addCallback(lambda res:
3319                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3320         return d
3321
3322     def test_POST_rename_dir(self):
3323         d = self.POST(self.public_url, t="rename",
3324                       from_name="foo", to_name='plunk')
3325         d.addCallback(lambda res:
3326                       self.failIfNodeHasChild(self.public_root, u"foo"))
3327         d.addCallback(lambda res:
3328                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
3329         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
3330         d.addCallback(self.failUnlessIsFooJSON)
3331         return d
3332
3333     def test_POST_move_file(self):
3334         d = self.POST(self.public_url + "/foo", t="move",
3335                       from_name="bar.txt", to_dir="sub")
3336         d.addCallback(lambda res:
3337                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3338         d.addCallback(lambda res:
3339                       self.failUnlessNodeHasChild(self._sub_node, u"bar.txt"))
3340         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3341         d.addCallback(self.failUnlessIsBarDotTxt)
3342         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3343         d.addCallback(self.failUnlessIsBarJSON)
3344         return d
3345
3346     def test_POST_move_file_new_name(self):
3347         d = self.POST(self.public_url + "/foo", t="move",
3348                       from_name="bar.txt", to_name="wibble.txt", to_dir="sub")
3349         d.addCallback(lambda res:
3350                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3351         d.addCallback(lambda res:
3352                       self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3353         d.addCallback(lambda res:
3354                       self.failUnlessNodeHasChild(self._sub_node, u"wibble.txt"))
3355         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt"))
3356         d.addCallback(self.failUnlessIsBarDotTxt)
3357         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt?t=json"))
3358         d.addCallback(self.failUnlessIsBarJSON)
3359         return d
3360
3361     def test_POST_move_file_replace(self):
3362         d = self.POST(self.public_url + "/foo", t="move",
3363                       from_name="bar.txt", to_name="baz.txt", to_dir="sub")
3364         d.addCallback(lambda res:
3365                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3366         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3367         d.addCallback(self.failUnlessIsBarDotTxt)
3368         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt?t=json"))
3369         d.addCallback(self.failUnlessIsBarJSON)
3370         return d
3371
3372     def test_POST_move_file_no_replace(self):
3373         d = self.shouldFail2(error.Error, "POST_move_file_no_replace",
3374                              "409 Conflict",
3375                              "There was already a child by that name, and you asked me to not replace it",
3376                              self.POST, self.public_url + "/foo", t="move",
3377                              replace="false", from_name="bar.txt",
3378                              to_name="baz.txt", to_dir="sub")
3379         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3380         d.addCallback(self.failUnlessIsBarDotTxt)
3381         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3382         d.addCallback(self.failUnlessIsBarJSON)
3383         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3384         d.addCallback(self.failUnlessIsSubBazDotTxt)
3385         return d
3386
3387     def test_POST_move_file_slash_fail(self):
3388         d = self.shouldFail2(error.Error, "test_POST_rename_file_slash_fail",
3389                              "400 Bad Request",
3390                              "to_name= may not contain a slash",
3391                              self.POST, self.public_url + "/foo", t="move",
3392                              from_name="bar.txt",
3393                              to_name="slash/fail.txt", to_dir="sub")
3394         d.addCallback(lambda res:
3395                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3396         d.addCallback(lambda res:
3397                       self.failIfNodeHasChild(self._sub_node, u"slash/fail.txt"))
3398         d.addCallback(lambda ign:
3399                       self.shouldFail2(error.Error,
3400                                        "test_POST_rename_file_slash_fail2",
3401                                        "400 Bad Request",
3402                                        "from_name= may not contain a slash",
3403                                        self.POST, self.public_url + "/foo",
3404                                        t="move",
3405                                        from_name="nope/bar.txt",
3406                                        to_name="fail.txt", to_dir="sub"))
3407         return d
3408
3409     def test_POST_move_file_no_target(self):
3410         d = self.shouldFail2(error.Error, "POST_move_file_no_target",
3411                              "400 Bad Request",
3412                              "move requires from_name and to_dir",
3413                              self.POST, self.public_url + "/foo", t="move",
3414                              from_name="bar.txt", to_name="baz.txt")
3415         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3416         d.addCallback(self.failUnlessIsBarDotTxt)
3417         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3418         d.addCallback(self.failUnlessIsBarJSON)
3419         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3420         d.addCallback(self.failUnlessIsBazDotTxt)
3421         return d
3422
3423     def test_POST_move_file_bad_target_type(self):
3424         d = self.shouldFail2(error.Error, "test_POST_move_file_bad_target_type",
3425                              "400 Bad Request", "invalid target_type parameter",
3426                              self.POST,
3427                              self.public_url + "/foo", t="move",
3428                              target_type="*D", from_name="bar.txt",
3429                              to_dir="sub")
3430         return d
3431
3432     def test_POST_move_file_multi_level(self):
3433         d = self.POST(self.public_url + "/foo/sub/level2?t=mkdir", "")
3434         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t="move",
3435                       from_name="bar.txt", to_dir="sub/level2"))
3436         d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3437         d.addCallback(lambda res: self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3438         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt"))
3439         d.addCallback(self.failUnlessIsBarDotTxt)
3440         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt?t=json"))
3441         d.addCallback(self.failUnlessIsBarJSON)
3442         return d
3443
3444     def test_POST_move_file_to_uri(self):
3445         d = self.POST(self.public_url + "/foo", t="move", target_type="uri",
3446                       from_name="bar.txt", to_dir=self._sub_uri)
3447         d.addCallback(lambda res:
3448                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3449         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3450         d.addCallback(self.failUnlessIsBarDotTxt)
3451         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3452         d.addCallback(self.failUnlessIsBarJSON)
3453         return d
3454
3455     def test_POST_move_file_to_nonexist_dir(self):
3456         d = self.shouldFail2(error.Error, "POST_move_file_to_nonexist_dir",
3457                             "404 Not Found", "No such child: nopechucktesta",
3458                             self.POST, self.public_url + "/foo", t="move",
3459                             from_name="bar.txt", to_dir="nopechucktesta")
3460         return d
3461
3462     def test_POST_move_file_into_file(self):
3463         d = self.shouldFail2(error.Error, "POST_move_file_into_file",
3464                              "400 Bad Request", "to_dir is not a directory",
3465                              self.POST, self.public_url + "/foo", t="move",
3466                              from_name="bar.txt", to_dir="baz.txt")
3467         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3468         d.addCallback(self.failUnlessIsBazDotTxt)
3469         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3470         d.addCallback(self.failUnlessIsBarDotTxt)
3471         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3472         d.addCallback(self.failUnlessIsBarJSON)
3473         return d
3474
3475     def test_POST_move_file_to_bad_uri(self):
3476         d =  self.shouldFail2(error.Error, "POST_move_file_to_bad_uri",
3477                               "400 Bad Request", "to_dir is not a directory",
3478                               self.POST, self.public_url + "/foo", t="move",
3479                               from_name="bar.txt", target_type="uri",
3480                               to_dir="URI:DIR2:mn5jlyjnrjeuydyswlzyui72i:rmneifcj6k6sycjljjhj3f6majsq2zqffydnnul5hfa4j577arma")
3481         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3482         d.addCallback(self.failUnlessIsBarDotTxt)
3483         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3484         d.addCallback(self.failUnlessIsBarJSON)
3485         return d
3486
3487     def test_POST_move_dir(self):
3488         d = self.POST(self.public_url + "/foo", t="move",
3489                       from_name="bar.txt", to_dir="empty")
3490         d.addCallback(lambda res: self.POST(self.public_url + "/foo",
3491                       t="move", from_name="empty", to_dir="sub"))
3492         d.addCallback(lambda res:
3493                       self.failIfNodeHasChild(self._foo_node, u"empty"))
3494         d.addCallback(lambda res:
3495                       self.failUnlessNodeHasChild(self._sub_node, u"empty"))
3496         d.addCallback(lambda res:
3497                       self._sub_node.get_child_at_path(u"empty"))
3498         d.addCallback(lambda node:
3499                       self.failUnlessNodeHasChild(node, u"bar.txt"))
3500         d.addCallback(lambda res:
3501                       self.GET(self.public_url + "/foo/sub/empty/bar.txt"))
3502         d.addCallback(self.failUnlessIsBarDotTxt)
3503         return d
3504
3505     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
3506         """ If target is not None then the redirection has to go to target.  If
3507         statuscode is not None then the redirection has to be accomplished with
3508         that HTTP status code."""
3509         if not isinstance(res, failure.Failure):
3510             to_where = (target is None) and "somewhere" or ("to " + target)
3511             self.fail("%s: we were expecting to get redirected %s, not get an"
3512                       " actual page: %s" % (which, to_where, res))
3513         res.trap(error.PageRedirect)
3514         if statuscode is not None:
3515             self.failUnlessReallyEqual(res.value.status, statuscode,
3516                                        "%s: not a redirect" % which)
3517         if target is not None:
3518             # the PageRedirect does not seem to capture the uri= query arg
3519             # properly, so we can't check for it.
3520             realtarget = self.webish_url + target
3521             self.failUnlessReallyEqual(res.value.location, realtarget,
3522                                        "%s: wrong target" % which)
3523         return res.value.location
3524
3525     def test_GET_URI_form(self):
3526         base = "/uri?uri=%s" % self._bar_txt_uri
3527         # this is supposed to give us a redirect to /uri/$URI, plus arguments
3528         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
3529         d = self.GET(base)
3530         d.addBoth(self.shouldRedirect, targetbase)
3531         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
3532         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
3533         d.addCallback(lambda res: self.GET(base+"&t=json"))
3534         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
3535         d.addCallback(self.log, "about to get file by uri")
3536         d.addCallback(lambda res: self.GET(base, followRedirect=True))
3537         d.addCallback(self.failUnlessIsBarDotTxt)
3538         d.addCallback(self.log, "got file by uri, about to get dir by uri")
3539         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
3540                                            followRedirect=True))
3541         d.addCallback(self.failUnlessIsFooJSON)
3542         d.addCallback(self.log, "got dir by uri")
3543
3544         return d
3545
3546     def test_GET_URI_form_bad(self):
3547         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
3548                              "400 Bad Request", "GET /uri requires uri=",
3549                              self.GET, "/uri")
3550         return d
3551
3552     def test_GET_rename_form(self):
3553         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
3554                      followRedirect=True)
3555         def _check(res):
3556             self.failUnlessIn('name="when_done" value="."', res)
3557             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
3558             self.failUnlessIn(FAVICON_MARKUP, res)
3559         d.addCallback(_check)
3560         return d
3561
3562     def log(self, res, msg):
3563         #print "MSG: %s  RES: %s" % (msg, res)
3564         log.msg(msg)
3565         return res
3566
3567     def test_GET_URI_URL(self):
3568         base = "/uri/%s" % self._bar_txt_uri
3569         d = self.GET(base)
3570         d.addCallback(self.failUnlessIsBarDotTxt)
3571         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
3572         d.addCallback(self.failUnlessIsBarDotTxt)
3573         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
3574         d.addCallback(self.failUnlessIsBarDotTxt)
3575         return d
3576
3577     def test_GET_URI_URL_dir(self):
3578         base = "/uri/%s?t=json" % self._foo_uri
3579         d = self.GET(base)
3580         d.addCallback(self.failUnlessIsFooJSON)
3581         return d
3582
3583     def test_GET_URI_URL_missing(self):
3584         base = "/uri/%s" % self._bad_file_uri
3585         d = self.shouldHTTPError("test_GET_URI_URL_missing",
3586                                  http.GONE, None, "NotEnoughSharesError",
3587                                  self.GET, base)
3588         # TODO: how can we exercise both sides of WebDownloadTarget.fail
3589         # here? we must arrange for a download to fail after target.open()
3590         # has been called, and then inspect the response to see that it is
3591         # shorter than we expected.
3592         return d
3593
3594     def test_PUT_DIRURL_uri(self):
3595         d = self.s.create_dirnode()
3596         def _made_dir(dn):
3597             new_uri = dn.get_uri()
3598             # replace /foo with a new (empty) directory
3599             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
3600             d.addCallback(lambda res:
3601                           self.failUnlessReallyEqual(res.strip(), new_uri))
3602             d.addCallback(lambda res:
3603                           self.failUnlessRWChildURIIs(self.public_root,
3604                                                       u"foo",
3605                                                       new_uri))
3606             return d
3607         d.addCallback(_made_dir)
3608         return d
3609
3610     def test_PUT_DIRURL_uri_noreplace(self):
3611         d = self.s.create_dirnode()
3612         def _made_dir(dn):
3613             new_uri = dn.get_uri()
3614             # replace /foo with a new (empty) directory, but ask that
3615             # replace=false, so it should fail
3616             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
3617                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
3618                                  self.PUT,
3619                                  self.public_url + "/foo?t=uri&replace=false",
3620                                  new_uri)
3621             d.addCallback(lambda res:
3622                           self.failUnlessRWChildURIIs(self.public_root,
3623                                                       u"foo",
3624                                                       self._foo_uri))
3625             return d
3626         d.addCallback(_made_dir)
3627         return d
3628
3629     def test_PUT_DIRURL_bad_t(self):
3630         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
3631                              "400 Bad Request", "PUT to a directory",
3632                              self.PUT, self.public_url + "/foo?t=BOGUS", "")
3633         d.addCallback(lambda res:
3634                       self.failUnlessRWChildURIIs(self.public_root,
3635                                                   u"foo",
3636                                                   self._foo_uri))
3637         return d
3638
3639     def test_PUT_NEWFILEURL_uri(self):
3640         contents, n, new_uri = self.makefile(8)
3641         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
3642         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3643         d.addCallback(lambda res:
3644                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3645                                                       contents))
3646         return d
3647
3648     def test_PUT_NEWFILEURL_mdmf(self):
3649         new_contents = self.NEWFILE_CONTENTS * 300000
3650         d = self.PUT(self.public_url + \
3651                      "/foo/mdmf.txt?format=mdmf",
3652                      new_contents)
3653         d.addCallback(lambda ignored:
3654             self.GET(self.public_url + "/foo/mdmf.txt?t=json"))
3655         def _got_json(json):
3656             data = simplejson.loads(json)
3657             data = data[1]
3658             self.failUnlessIn("format", data)
3659             self.failUnlessEqual(data["format"], "MDMF")
3660             self.failUnless(data['rw_uri'].startswith("URI:MDMF"))
3661             self.failUnless(data['ro_uri'].startswith("URI:MDMF"))
3662         d.addCallback(_got_json)
3663         return d
3664
3665     def test_PUT_NEWFILEURL_sdmf(self):
3666         new_contents = self.NEWFILE_CONTENTS * 300000
3667         d = self.PUT(self.public_url + \
3668                      "/foo/sdmf.txt?format=sdmf",
3669                      new_contents)
3670         d.addCallback(lambda ignored:
3671             self.GET(self.public_url + "/foo/sdmf.txt?t=json"))
3672         def _got_json(json):
3673             data = simplejson.loads(json)
3674             data = data[1]
3675             self.failUnlessIn("format", data)
3676             self.failUnlessEqual(data["format"], "SDMF")
3677         d.addCallback(_got_json)
3678         return d
3679
3680     def test_PUT_NEWFILEURL_bad_format(self):
3681        new_contents = self.NEWFILE_CONTENTS * 300000
3682        return self.shouldHTTPError("PUT_NEWFILEURL_bad_format",
3683                                    400, "Bad Request", "Unknown format: foo",
3684                                    self.PUT, self.public_url + \
3685                                    "/foo/foo.txt?format=foo",
3686                                    new_contents)
3687
3688     def test_PUT_NEWFILEURL_uri_replace(self):
3689         contents, n, new_uri = self.makefile(8)
3690         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
3691         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3692         d.addCallback(lambda res:
3693                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3694                                                       contents))
3695         return d
3696
3697     def test_PUT_NEWFILEURL_uri_no_replace(self):
3698         contents, n, new_uri = self.makefile(8)
3699         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
3700         d.addBoth(self.shouldFail, error.Error,
3701                   "PUT_NEWFILEURL_uri_no_replace",
3702                   "409 Conflict",
3703                   "There was already a child by that name, and you asked me "
3704                   "to not replace it")
3705         return d
3706
3707     def test_PUT_NEWFILEURL_uri_unknown_bad(self):
3708         d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
3709         d.addBoth(self.shouldFail, error.Error,
3710                   "POST_put_uri_unknown_bad",
3711                   "400 Bad Request",
3712                   "unknown cap in a write slot")
3713         return d
3714
3715     def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
3716         d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
3717         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3718                       u"put-future-ro.txt")
3719         return d
3720
3721     def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
3722         d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
3723         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3724                       u"put-future-imm.txt")
3725         return d
3726
3727     def test_PUT_NEWFILE_URI(self):
3728         file_contents = "New file contents here\n"
3729         d = self.PUT("/uri", file_contents)
3730         def _check(uri):
3731             assert isinstance(uri, str), uri
3732             self.failUnlessIn(uri, self.get_all_contents())
3733             self.failUnlessReallyEqual(self.get_all_contents()[uri],
3734                                        file_contents)
3735             return self.GET("/uri/%s" % uri)
3736         d.addCallback(_check)
3737         def _check2(res):
3738             self.failUnlessReallyEqual(res, file_contents)
3739         d.addCallback(_check2)
3740         return d
3741
3742     def test_PUT_NEWFILE_URI_not_mutable(self):
3743         file_contents = "New file contents here\n"
3744         d = self.PUT("/uri?mutable=false", file_contents)
3745         def _check(uri):
3746             assert isinstance(uri, str), uri
3747             self.failUnlessIn(uri, self.get_all_contents())
3748             self.failUnlessReallyEqual(self.get_all_contents()[uri],
3749                                        file_contents)
3750             return self.GET("/uri/%s" % uri)
3751         d.addCallback(_check)
3752         def _check2(res):
3753             self.failUnlessReallyEqual(res, file_contents)
3754         d.addCallback(_check2)
3755         return d
3756
3757     def test_PUT_NEWFILE_URI_only_PUT(self):
3758         d = self.PUT("/uri?t=bogus", "")
3759         d.addBoth(self.shouldFail, error.Error,
3760                   "PUT_NEWFILE_URI_only_PUT",
3761                   "400 Bad Request",
3762                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
3763         return d
3764
3765     def test_PUT_NEWFILE_URI_mutable(self):
3766         file_contents = "New file contents here\n"
3767         d = self.PUT("/uri?mutable=true", file_contents)
3768         def _check1(filecap):
3769             filecap = filecap.strip()
3770             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
3771             self.filecap = filecap
3772             u = uri.WriteableSSKFileURI.init_from_string(filecap)
3773             self.failUnlessIn(u.get_storage_index(), self.get_all_contents())
3774             n = self.s.create_node_from_uri(filecap)
3775             return n.download_best_version()
3776         d.addCallback(_check1)
3777         def _check2(data):
3778             self.failUnlessReallyEqual(data, file_contents)
3779             return self.GET("/uri/%s" % urllib.quote(self.filecap))
3780         d.addCallback(_check2)
3781         def _check3(res):
3782             self.failUnlessReallyEqual(res, file_contents)
3783         d.addCallback(_check3)
3784         return d
3785
3786     def test_PUT_mkdir(self):
3787         d = self.PUT("/uri?t=mkdir", "")
3788         def _check(uri):
3789             n = self.s.create_node_from_uri(uri.strip())
3790             d2 = self.failUnlessNodeKeysAre(n, [])
3791             d2.addCallback(lambda res:
3792                            self.GET("/uri/%s?t=json" % uri))
3793             return d2
3794         d.addCallback(_check)
3795         d.addCallback(self.failUnlessIsEmptyJSON)
3796         return d
3797
3798     def test_PUT_mkdir_mdmf(self):
3799         d = self.PUT("/uri?t=mkdir&format=mdmf", "")
3800         def _got(res):
3801             u = uri.from_string(res)
3802             # Check that this is an MDMF writecap
3803             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
3804         d.addCallback(_got)
3805         return d
3806
3807     def test_PUT_mkdir_sdmf(self):
3808         d = self.PUT("/uri?t=mkdir&format=sdmf", "")
3809         def _got(res):
3810             u = uri.from_string(res)
3811             self.failUnlessIsInstance(u, uri.DirectoryURI)
3812         d.addCallback(_got)
3813         return d
3814
3815     def test_PUT_mkdir_bad_format(self):
3816         return self.shouldHTTPError("PUT_mkdir_bad_format",
3817                                     400, "Bad Request", "Unknown format: foo",
3818                                     self.PUT, "/uri?t=mkdir&format=foo",
3819                                     "")
3820
3821     def test_POST_check(self):
3822         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
3823         def _done(res):
3824             # this returns a string form of the results, which are probably
3825             # None since we're using fake filenodes.
3826             # TODO: verify that the check actually happened, by changing
3827             # FakeCHKFileNode to count how many times .check() has been
3828             # called.
3829             pass
3830         d.addCallback(_done)
3831         return d
3832
3833
3834     def test_PUT_update_at_offset(self):
3835         file_contents = "test file" * 100000 # about 900 KiB
3836         d = self.PUT("/uri?mutable=true", file_contents)
3837         def _then(filecap):
3838             self.filecap = filecap
3839             new_data = file_contents[:100]
3840             new = "replaced and so on"
3841             new_data += new
3842             new_data += file_contents[len(new_data):]
3843             assert len(new_data) == len(file_contents)
3844             self.new_data = new_data
3845         d.addCallback(_then)
3846         d.addCallback(lambda ignored:
3847             self.PUT("/uri/%s?replace=True&offset=100" % self.filecap,
3848                      "replaced and so on"))
3849         def _get_data(filecap):
3850             n = self.s.create_node_from_uri(filecap)
3851             return n.download_best_version()
3852         d.addCallback(_get_data)
3853         d.addCallback(lambda results:
3854             self.failUnlessEqual(results, self.new_data))
3855         # Now try appending things to the file
3856         d.addCallback(lambda ignored:
3857             self.PUT("/uri/%s?offset=%d" % (self.filecap, len(self.new_data)),
3858                      "puppies" * 100))
3859         d.addCallback(_get_data)
3860         d.addCallback(lambda results:
3861             self.failUnlessEqual(results, self.new_data + ("puppies" * 100)))
3862         # and try replacing the beginning of the file
3863         d.addCallback(lambda ignored:
3864             self.PUT("/uri/%s?offset=0" % self.filecap, "begin"))
3865         d.addCallback(_get_data)
3866         d.addCallback(lambda results:
3867             self.failUnlessEqual(results, "begin"+self.new_data[len("begin"):]+("puppies"*100)))
3868         return d
3869
3870     def test_PUT_update_at_invalid_offset(self):
3871         file_contents = "test file" * 100000 # about 900 KiB
3872         d = self.PUT("/uri?mutable=true", file_contents)
3873         def _then(filecap):
3874             self.filecap = filecap
3875         d.addCallback(_then)
3876         # Negative offsets should cause an error.
3877         d.addCallback(lambda ignored:
3878             self.shouldHTTPError("PUT_update_at_invalid_offset",
3879                                  400, "Bad Request",
3880                                  "Invalid offset",
3881                                  self.PUT,
3882                                  "/uri/%s?offset=-1" % self.filecap,
3883                                  "foo"))
3884         return d
3885
3886     def test_PUT_update_at_offset_immutable(self):
3887         file_contents = "Test file" * 100000
3888         d = self.PUT("/uri", file_contents)
3889         def _then(filecap):
3890             self.filecap = filecap
3891         d.addCallback(_then)
3892         d.addCallback(lambda ignored:
3893             self.shouldHTTPError("PUT_update_at_offset_immutable",
3894                                  400, "Bad Request",
3895                                  "immutable",
3896                                  self.PUT,
3897                                  "/uri/%s?offset=50" % self.filecap,
3898                                  "foo"))
3899         return d
3900
3901
3902     def test_bad_method(self):
3903         url = self.webish_url + self.public_url + "/foo/bar.txt"
3904         d = self.shouldHTTPError("bad_method",
3905                                  501, "Not Implemented",
3906                                  "I don't know how to treat a BOGUS request.",
3907                                  client.getPage, url, method="BOGUS")
3908         return d
3909
3910     def test_short_url(self):
3911         url = self.webish_url + "/uri"
3912         d = self.shouldHTTPError("short_url", 501, "Not Implemented",
3913                                  "I don't know how to treat a DELETE request.",
3914                                  client.getPage, url, method="DELETE")
3915         return d
3916
3917     def test_ophandle_bad(self):
3918         url = self.webish_url + "/operations/bogus?t=status"
3919         d = self.shouldHTTPError("ophandle_bad", 404, "404 Not Found",
3920                                  "unknown/expired handle 'bogus'",
3921                                  client.getPage, url)
3922         return d
3923
3924     def test_ophandle_cancel(self):
3925         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
3926                       followRedirect=True)
3927         d.addCallback(lambda ignored:
3928                       self.GET("/operations/128?t=status&output=JSON"))
3929         def _check1(res):
3930             data = simplejson.loads(res)
3931             self.failUnless("finished" in data, res)
3932             monitor = self.ws.root.child_operations.handles["128"][0]
3933             d = self.POST("/operations/128?t=cancel&output=JSON")
3934             def _check2(res):
3935                 data = simplejson.loads(res)
3936                 self.failUnless("finished" in data, res)
3937                 # t=cancel causes the handle to be forgotten
3938                 self.failUnless(monitor.is_cancelled())
3939             d.addCallback(_check2)
3940             return d
3941         d.addCallback(_check1)
3942         d.addCallback(lambda ignored:
3943                       self.shouldHTTPError("ophandle_cancel",
3944                                            404, "404 Not Found",
3945                                            "unknown/expired handle '128'",
3946                                            self.GET,
3947                                            "/operations/128?t=status&output=JSON"))
3948         return d
3949
3950     def test_ophandle_retainfor(self):
3951         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
3952                       followRedirect=True)
3953         d.addCallback(lambda ignored:
3954                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
3955         def _check1(res):
3956             data = simplejson.loads(res)
3957             self.failUnless("finished" in data, res)
3958         d.addCallback(_check1)
3959         # the retain-for=0 will cause the handle to be expired very soon
3960         d.addCallback(lambda ign:
3961             self.clock.advance(2.0))
3962         d.addCallback(lambda ignored:
3963                       self.shouldHTTPError("ophandle_retainfor",
3964                                            404, "404 Not Found",
3965                                            "unknown/expired handle '129'",
3966                                            self.GET,
3967                                            "/operations/129?t=status&output=JSON"))
3968         return d
3969
3970     def test_ophandle_release_after_complete(self):
3971         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
3972                       followRedirect=True)
3973         d.addCallback(self.wait_for_operation, "130")
3974         d.addCallback(lambda ignored:
3975                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
3976         # the release-after-complete=true will cause the handle to be expired
3977         d.addCallback(lambda ignored:
3978                       self.shouldHTTPError("ophandle_release_after_complete",
3979                                            404, "404 Not Found",
3980                                            "unknown/expired handle '130'",
3981                                            self.GET,
3982                                            "/operations/130?t=status&output=JSON"))
3983         return d
3984
3985     def test_uncollected_ophandle_expiration(self):
3986         # uncollected ophandles should expire after 4 days
3987         def _make_uncollected_ophandle(ophandle):
3988             d = self.POST(self.public_url +
3989                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3990                           followRedirect=False)
3991             # When we start the operation, the webapi server will want
3992             # to redirect us to the page for the ophandle, so we get
3993             # confirmation that the operation has started. If the
3994             # manifest operation has finished by the time we get there,
3995             # following that redirect (by setting followRedirect=True
3996             # above) has the side effect of collecting the ophandle that
3997             # we've just created, which means that we can't use the
3998             # ophandle to test the uncollected timeout anymore. So,
3999             # instead, catch the 302 here and don't follow it.
4000             d.addBoth(self.should302, "uncollected_ophandle_creation")
4001             return d
4002         # Create an ophandle, don't collect it, then advance the clock by
4003         # 4 days - 1 second and make sure that the ophandle is still there.
4004         d = _make_uncollected_ophandle(131)
4005         d.addCallback(lambda ign:
4006             self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
4007         d.addCallback(lambda ign:
4008             self.GET("/operations/131?t=status&output=JSON"))
4009         def _check1(res):
4010             data = simplejson.loads(res)
4011             self.failUnless("finished" in data, res)
4012         d.addCallback(_check1)
4013         # Create an ophandle, don't collect it, then try to collect it
4014         # after 4 days. It should be gone.
4015         d.addCallback(lambda ign:
4016             _make_uncollected_ophandle(132))
4017         d.addCallback(lambda ign:
4018             self.clock.advance(96*60*60))
4019         d.addCallback(lambda ign:
4020             self.shouldHTTPError("uncollected_ophandle_expired_after_100_hours",
4021                                  404, "404 Not Found",
4022                                  "unknown/expired handle '132'",
4023                                  self.GET,
4024                                  "/operations/132?t=status&output=JSON"))
4025         return d
4026
4027     def test_collected_ophandle_expiration(self):
4028         # collected ophandles should expire after 1 day
4029         def _make_collected_ophandle(ophandle):
4030             d = self.POST(self.public_url +
4031                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
4032                           followRedirect=True)
4033             # By following the initial redirect, we collect the ophandle
4034             # we've just created.
4035             return d
4036         # Create a collected ophandle, then collect it after 23 hours
4037         # and 59 seconds to make sure that it is still there.
4038         d = _make_collected_ophandle(133)
4039         d.addCallback(lambda ign:
4040             self.clock.advance((24*60*60) - 1))
4041         d.addCallback(lambda ign:
4042             self.GET("/operations/133?t=status&output=JSON"))
4043         def _check1(res):
4044             data = simplejson.loads(res)
4045             self.failUnless("finished" in data, res)
4046         d.addCallback(_check1)
4047         # Create another uncollected ophandle, then try to collect it
4048         # after 24 hours to make sure that it is gone.
4049         d.addCallback(lambda ign:
4050             _make_collected_ophandle(134))
4051         d.addCallback(lambda ign:
4052             self.clock.advance(24*60*60))
4053         d.addCallback(lambda ign:
4054             self.shouldHTTPError("collected_ophandle_expired_after_1_day",
4055                                  404, "404 Not Found",
4056                                  "unknown/expired handle '134'",
4057                                  self.GET,
4058                                  "/operations/134?t=status&output=JSON"))
4059         return d
4060
4061     def test_incident(self):
4062         d = self.POST("/report_incident", details="eek")
4063         def _done(res):
4064             self.failIfIn("<html>", res)
4065             self.failUnlessIn("Thank you for your report!", res)
4066         d.addCallback(_done)
4067         return d
4068
4069     def test_static(self):
4070         webdir = os.path.join(self.staticdir, "subdir")
4071         fileutil.make_dirs(webdir)
4072         f = open(os.path.join(webdir, "hello.txt"), "wb")
4073         f.write("hello")
4074         f.close()
4075
4076         d = self.GET("/static/subdir/hello.txt")
4077         def _check(res):
4078             self.failUnlessReallyEqual(res, "hello")
4079         d.addCallback(_check)
4080         return d
4081
4082
4083 class IntroducerWeb(unittest.TestCase):
4084     def setUp(self):
4085         self.node = None
4086
4087     def tearDown(self):
4088         d = defer.succeed(None)
4089         if self.node:
4090             d.addCallback(lambda ign: self.node.stopService())
4091         d.addCallback(flushEventualQueue)
4092         return d
4093
4094     def test_welcome(self):
4095         basedir = "web.IntroducerWeb.test_welcome"
4096         os.mkdir(basedir)
4097         fileutil.write(os.path.join(basedir, "tahoe.cfg"), "[node]\nweb.port = tcp:0\n")
4098         self.node = IntroducerNode(basedir)
4099         self.ws = self.node.getServiceNamed("webish")
4100
4101         d = fireEventually(None)
4102         d.addCallback(lambda ign: self.node.startService())
4103         d.addCallback(lambda ign: self.node.when_tub_ready())
4104
4105         d.addCallback(lambda ign: self.GET("/"))
4106         def _check(res):
4107             self.failUnlessIn('Welcome to the Tahoe-LAFS Introducer', res)
4108             self.failUnlessIn(FAVICON_MARKUP, res)
4109         d.addCallback(_check)
4110         return d
4111
4112     def GET(self, urlpath, followRedirect=False, return_response=False,
4113             **kwargs):
4114         # if return_response=True, this fires with (data, statuscode,
4115         # respheaders) instead of just data.
4116         assert not isinstance(urlpath, unicode)
4117         url = self.ws.getURL().rstrip('/') + urlpath
4118         factory = HTTPClientGETFactory(url, method="GET",
4119                                        followRedirect=followRedirect, **kwargs)
4120         reactor.connectTCP("localhost", self.ws.getPortnum(), factory)
4121         d = factory.deferred
4122         def _got_data(data):
4123             return (data, factory.status, factory.response_headers)
4124         if return_response:
4125             d.addCallback(_got_data)
4126         return factory.deferred
4127
4128
4129 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4130     def test_load_file(self):
4131         # This will raise an exception unless a well-formed XML file is found under that name.
4132         common.getxmlfile('directory.xhtml').load()
4133
4134     def test_parse_replace_arg(self):
4135         self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
4136         self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
4137         self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
4138                                    "only-files")
4139         self.shouldFail(AssertionError, "test_parse_replace_arg", "",
4140                         common.parse_replace_arg, "only_fles")
4141
4142     def test_abbreviate_time(self):
4143         self.failUnlessReallyEqual(common.abbreviate_time(None), "")
4144         self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
4145         self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
4146         self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
4147         self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
4148         self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
4149
4150     def test_compute_rate(self):
4151         self.failUnlessReallyEqual(common.compute_rate(None, None), None)
4152         self.failUnlessReallyEqual(common.compute_rate(None, 1), None)
4153         self.failUnlessReallyEqual(common.compute_rate(250000, None), None)
4154         self.failUnlessReallyEqual(common.compute_rate(250000, 0), None)
4155         self.failUnlessReallyEqual(common.compute_rate(250000, 10), 25000.0)
4156         self.failUnlessReallyEqual(common.compute_rate(0, 10), 0.0)
4157         self.shouldFail(AssertionError, "test_compute_rate", "",
4158                         common.compute_rate, -100, 10)
4159         self.shouldFail(AssertionError, "test_compute_rate", "",
4160                         common.compute_rate, 100, -10)
4161
4162         # Sanity check
4163         rate = common.compute_rate(10*1000*1000, 1)
4164         self.failUnlessReallyEqual(common.abbreviate_rate(rate), "10.00MBps")
4165
4166     def test_abbreviate_rate(self):
4167         self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
4168         self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
4169         self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
4170         self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
4171
4172     def test_abbreviate_size(self):
4173         self.failUnlessReallyEqual(common.abbreviate_size(None), "")
4174         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
4175         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
4176         self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
4177         self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
4178
4179     def test_plural(self):
4180         def convert(s):
4181             return "%d second%s" % (s, status.plural(s))
4182         self.failUnlessReallyEqual(convert(0), "0 seconds")
4183         self.failUnlessReallyEqual(convert(1), "1 second")
4184         self.failUnlessReallyEqual(convert(2), "2 seconds")
4185         def convert2(s):
4186             return "has share%s: %s" % (status.plural(s), ",".join(s))
4187         self.failUnlessReallyEqual(convert2([]), "has shares: ")
4188         self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
4189         self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
4190
4191
4192 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4193
4194     def CHECK(self, ign, which, args, clientnum=0):
4195         fileurl = self.fileurls[which]
4196         url = fileurl + "?" + args
4197         return self.GET(url, method="POST", clientnum=clientnum)
4198
4199     def test_filecheck(self):
4200         self.basedir = "web/Grid/filecheck"
4201         self.set_up_grid()
4202         c0 = self.g.clients[0]
4203         self.uris = {}
4204         DATA = "data" * 100
4205         d = c0.upload(upload.Data(DATA, convergence=""))
4206         def _stash_uri(ur, which):
4207             self.uris[which] = ur.get_uri()
4208         d.addCallback(_stash_uri, "good")
4209         d.addCallback(lambda ign:
4210                       c0.upload(upload.Data(DATA+"1", convergence="")))
4211         d.addCallback(_stash_uri, "sick")
4212         d.addCallback(lambda ign:
4213                       c0.upload(upload.Data(DATA+"2", convergence="")))
4214         d.addCallback(_stash_uri, "dead")
4215         def _stash_mutable_uri(n, which):
4216             self.uris[which] = n.get_uri()
4217             assert isinstance(self.uris[which], str)
4218         d.addCallback(lambda ign:
4219             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4220         d.addCallback(_stash_mutable_uri, "corrupt")
4221         d.addCallback(lambda ign:
4222                       c0.upload(upload.Data("literal", convergence="")))
4223         d.addCallback(_stash_uri, "small")
4224         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
4225         d.addCallback(_stash_mutable_uri, "smalldir")
4226
4227         def _compute_fileurls(ignored):
4228             self.fileurls = {}
4229             for which in self.uris:
4230                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4231         d.addCallback(_compute_fileurls)
4232
4233         def _clobber_shares(ignored):
4234             good_shares = self.find_uri_shares(self.uris["good"])
4235             self.failUnlessReallyEqual(len(good_shares), 10)
4236             sick_shares = self.find_uri_shares(self.uris["sick"])
4237             os.unlink(sick_shares[0][2])
4238             dead_shares = self.find_uri_shares(self.uris["dead"])
4239             for i in range(1, 10):
4240                 os.unlink(dead_shares[i][2])
4241             c_shares = self.find_uri_shares(self.uris["corrupt"])
4242             cso = CorruptShareOptions()
4243             cso.stdout = StringIO()
4244             cso.parseOptions([c_shares[0][2]])
4245             corrupt_share(cso)
4246         d.addCallback(_clobber_shares)
4247
4248         d.addCallback(self.CHECK, "good", "t=check")
4249         def _got_html_good(res):
4250             self.failUnlessIn("Healthy", res)
4251             self.failIfIn("Not Healthy", res)
4252             self.failUnlessIn(FAVICON_MARKUP, res)
4253         d.addCallback(_got_html_good)
4254         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
4255         def _got_html_good_return_to(res):
4256             self.failUnlessIn("Healthy", res)
4257             self.failIfIn("Not Healthy", res)
4258             self.failUnlessIn('<a href="somewhere">Return to file', res)
4259         d.addCallback(_got_html_good_return_to)
4260         d.addCallback(self.CHECK, "good", "t=check&output=json")
4261         def _got_json_good(res):
4262             r = simplejson.loads(res)
4263             self.failUnlessEqual(r["summary"], "Healthy")
4264             self.failUnless(r["results"]["healthy"])
4265             self.failIf(r["results"]["needs-rebalancing"])
4266             self.failUnless(r["results"]["recoverable"])
4267         d.addCallback(_got_json_good)
4268
4269         d.addCallback(self.CHECK, "small", "t=check")
4270         def _got_html_small(res):
4271             self.failUnlessIn("Literal files are always healthy", res)
4272             self.failIfIn("Not Healthy", res)
4273         d.addCallback(_got_html_small)
4274         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
4275         def _got_html_small_return_to(res):
4276             self.failUnlessIn("Literal files are always healthy", res)
4277             self.failIfIn("Not Healthy", res)
4278             self.failUnlessIn('<a href="somewhere">Return to file', res)
4279         d.addCallback(_got_html_small_return_to)
4280         d.addCallback(self.CHECK, "small", "t=check&output=json")
4281         def _got_json_small(res):
4282             r = simplejson.loads(res)
4283             self.failUnlessEqual(r["storage-index"], "")
4284             self.failUnless(r["results"]["healthy"])
4285         d.addCallback(_got_json_small)
4286
4287         d.addCallback(self.CHECK, "smalldir", "t=check")
4288         def _got_html_smalldir(res):
4289             self.failUnlessIn("Literal files are always healthy", res)
4290             self.failIfIn("Not Healthy", res)
4291         d.addCallback(_got_html_smalldir)
4292         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
4293         def _got_json_smalldir(res):
4294             r = simplejson.loads(res)
4295             self.failUnlessEqual(r["storage-index"], "")
4296             self.failUnless(r["results"]["healthy"])
4297         d.addCallback(_got_json_smalldir)
4298
4299         d.addCallback(self.CHECK, "sick", "t=check")
4300         def _got_html_sick(res):
4301             self.failUnlessIn("Not Healthy", res)
4302         d.addCallback(_got_html_sick)
4303         d.addCallback(self.CHECK, "sick", "t=check&output=json")
4304         def _got_json_sick(res):
4305             r = simplejson.loads(res)
4306             self.failUnlessEqual(r["summary"],
4307                                  "Not Healthy: 9 shares (enc 3-of-10)")
4308             self.failIf(r["results"]["healthy"])
4309             self.failIf(r["results"]["needs-rebalancing"])
4310             self.failUnless(r["results"]["recoverable"])
4311         d.addCallback(_got_json_sick)
4312
4313         d.addCallback(self.CHECK, "dead", "t=check")
4314         def _got_html_dead(res):
4315             self.failUnlessIn("Not Healthy", res)
4316         d.addCallback(_got_html_dead)
4317         d.addCallback(self.CHECK, "dead", "t=check&output=json")
4318         def _got_json_dead(res):
4319             r = simplejson.loads(res)
4320             self.failUnlessEqual(r["summary"],
4321                                  "Not Healthy: 1 shares (enc 3-of-10)")
4322             self.failIf(r["results"]["healthy"])
4323             self.failIf(r["results"]["needs-rebalancing"])
4324             self.failIf(r["results"]["recoverable"])
4325         d.addCallback(_got_json_dead)
4326
4327         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
4328         def _got_html_corrupt(res):
4329             self.failUnlessIn("Not Healthy! : Unhealthy", res)
4330         d.addCallback(_got_html_corrupt)
4331         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
4332         def _got_json_corrupt(res):
4333             r = simplejson.loads(res)
4334             self.failUnlessIn("Unhealthy: 9 shares (enc 3-of-10)", r["summary"])
4335             self.failIf(r["results"]["healthy"])
4336             self.failUnless(r["results"]["recoverable"])
4337             self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
4338             self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
4339         d.addCallback(_got_json_corrupt)
4340
4341         d.addErrback(self.explain_web_error)
4342         return d
4343
4344     def test_repair_html(self):
4345         self.basedir = "web/Grid/repair_html"
4346         self.set_up_grid()
4347         c0 = self.g.clients[0]
4348         self.uris = {}
4349         DATA = "data" * 100
4350         d = c0.upload(upload.Data(DATA, convergence=""))
4351         def _stash_uri(ur, which):
4352             self.uris[which] = ur.get_uri()
4353         d.addCallback(_stash_uri, "good")
4354         d.addCallback(lambda ign:
4355                       c0.upload(upload.Data(DATA+"1", convergence="")))
4356         d.addCallback(_stash_uri, "sick")
4357         d.addCallback(lambda ign:
4358                       c0.upload(upload.Data(DATA+"2", convergence="")))
4359         d.addCallback(_stash_uri, "dead")
4360         def _stash_mutable_uri(n, which):
4361             self.uris[which] = n.get_uri()
4362             assert isinstance(self.uris[which], str)
4363         d.addCallback(lambda ign:
4364             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4365         d.addCallback(_stash_mutable_uri, "corrupt")
4366
4367         def _compute_fileurls(ignored):
4368             self.fileurls = {}
4369             for which in self.uris:
4370                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4371         d.addCallback(_compute_fileurls)
4372
4373         def _clobber_shares(ignored):
4374             good_shares = self.find_uri_shares(self.uris["good"])
4375             self.failUnlessReallyEqual(len(good_shares), 10)
4376             sick_shares = self.find_uri_shares(self.uris["sick"])
4377             os.unlink(sick_shares[0][2])
4378             dead_shares = self.find_uri_shares(self.uris["dead"])
4379             for i in range(1, 10):
4380                 os.unlink(dead_shares[i][2])
4381             c_shares = self.find_uri_shares(self.uris["corrupt"])
4382             cso = CorruptShareOptions()
4383             cso.stdout = StringIO()
4384             cso.parseOptions([c_shares[0][2]])
4385             corrupt_share(cso)
4386         d.addCallback(_clobber_shares)
4387
4388         d.addCallback(self.CHECK, "good", "t=check&repair=true")
4389         def _got_html_good(res):
4390             self.failUnlessIn("Healthy", res)
4391             self.failIfIn("Not Healthy", res)
4392             self.failUnlessIn("No repair necessary", res)
4393             self.failUnlessIn(FAVICON_MARKUP, res)
4394         d.addCallback(_got_html_good)
4395
4396         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
4397         def _got_html_sick(res):
4398             self.failUnlessIn("Healthy : healthy", res)
4399             self.failIfIn("Not Healthy", res)
4400             self.failUnlessIn("Repair successful", res)
4401         d.addCallback(_got_html_sick)
4402
4403         # repair of a dead file will fail, of course, but it isn't yet
4404         # clear how this should be reported. Right now it shows up as
4405         # a "410 Gone".
4406         #
4407         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
4408         #def _got_html_dead(res):
4409         #    print res
4410         #    self.failUnlessIn("Healthy : healthy", res)
4411         #    self.failIfIn("Not Healthy", res)
4412         #    self.failUnlessIn("No repair necessary", res)
4413         #d.addCallback(_got_html_dead)
4414
4415         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
4416         def _got_html_corrupt(res):
4417             self.failUnlessIn("Healthy : Healthy", res)
4418             self.failIfIn("Not Healthy", res)
4419             self.failUnlessIn("Repair successful", res)
4420         d.addCallback(_got_html_corrupt)
4421
4422         d.addErrback(self.explain_web_error)
4423         return d
4424
4425     def test_repair_json(self):
4426         self.basedir = "web/Grid/repair_json"
4427         self.set_up_grid()
4428         c0 = self.g.clients[0]
4429         self.uris = {}
4430         DATA = "data" * 100
4431         d = c0.upload(upload.Data(DATA+"1", convergence=""))
4432         def _stash_uri(ur, which):
4433             self.uris[which] = ur.get_uri()
4434         d.addCallback(_stash_uri, "sick")
4435
4436         def _compute_fileurls(ignored):
4437             self.fileurls = {}
4438             for which in self.uris:
4439                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4440         d.addCallback(_compute_fileurls)
4441
4442         def _clobber_shares(ignored):
4443             sick_shares = self.find_uri_shares(self.uris["sick"])
4444             os.unlink(sick_shares[0][2])
4445         d.addCallback(_clobber_shares)
4446
4447         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
4448         def _got_json_sick(res):
4449             r = simplejson.loads(res)
4450             self.failUnlessReallyEqual(r["repair-attempted"], True)
4451             self.failUnlessReallyEqual(r["repair-successful"], True)
4452             self.failUnlessEqual(r["pre-repair-results"]["summary"],
4453                                  "Not Healthy: 9 shares (enc 3-of-10)")
4454             self.failIf(r["pre-repair-results"]["results"]["healthy"])
4455             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
4456             self.failUnless(r["post-repair-results"]["results"]["healthy"])
4457         d.addCallback(_got_json_sick)
4458
4459         d.addErrback(self.explain_web_error)
4460         return d
4461
4462     def test_unknown(self, immutable=False):
4463         self.basedir = "web/Grid/unknown"
4464         if immutable:
4465             self.basedir = "web/Grid/unknown-immutable"
4466
4467         self.set_up_grid()
4468         c0 = self.g.clients[0]
4469         self.uris = {}
4470         self.fileurls = {}
4471
4472         # the future cap format may contain slashes, which must be tolerated
4473         expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
4474                                                            safe="")
4475
4476         if immutable:
4477             name = u"future-imm"
4478             future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
4479             d = c0.create_immutable_dirnode({name: (future_node, {})})
4480         else:
4481             name = u"future"
4482             future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4483             d = c0.create_dirnode()
4484
4485         def _stash_root_and_create_file(n):
4486             self.rootnode = n
4487             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
4488             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
4489             if not immutable:
4490                 return self.rootnode.set_node(name, future_node)
4491         d.addCallback(_stash_root_and_create_file)
4492
4493         # make sure directory listing tolerates unknown nodes
4494         d.addCallback(lambda ign: self.GET(self.rooturl))
4495         def _check_directory_html(res, expected_type_suffix):
4496             pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
4497                                   '<td>%s</td>' % (expected_type_suffix, str(name)),
4498                                  re.DOTALL)
4499             self.failUnless(re.search(pattern, res), res)
4500             # find the More Info link for name, should be relative
4501             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4502             info_url = mo.group(1)
4503             self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
4504         if immutable:
4505             d.addCallback(_check_directory_html, "-IMM")
4506         else:
4507             d.addCallback(_check_directory_html, "")
4508
4509         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4510         def _check_directory_json(res, expect_rw_uri):
4511             data = simplejson.loads(res)
4512             self.failUnlessEqual(data[0], "dirnode")
4513             f = data[1]["children"][name]
4514             self.failUnlessEqual(f[0], "unknown")
4515             if expect_rw_uri:
4516                 self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data)
4517             else:
4518                 self.failIfIn("rw_uri", f[1])
4519             if immutable:
4520                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
4521             else:
4522                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
4523             self.failUnlessIn("metadata", f[1])
4524         d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
4525
4526         def _check_info(res, expect_rw_uri, expect_ro_uri):
4527             self.failUnlessIn("Object Type: <span>unknown</span>", res)
4528             if expect_rw_uri:
4529                 self.failUnlessIn(unknown_rwcap, res)
4530             if expect_ro_uri:
4531                 if immutable:
4532                     self.failUnlessIn(unknown_immcap, res)
4533                 else:
4534                     self.failUnlessIn(unknown_rocap, res)
4535             else:
4536                 self.failIfIn(unknown_rocap, res)
4537             self.failIfIn("Raw data as", res)
4538             self.failIfIn("Directory writecap", res)
4539             self.failIfIn("Checker Operations", res)
4540             self.failIfIn("Mutable File Operations", res)
4541             self.failIfIn("Directory Operations", res)
4542
4543         # FIXME: these should have expect_rw_uri=not immutable; I don't know
4544         # why they fail. Possibly related to ticket #922.
4545
4546         d.addCallback(lambda ign: self.GET(expected_info_url))
4547         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
4548         d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
4549         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
4550
4551         def _check_json(res, expect_rw_uri):
4552             data = simplejson.loads(res)
4553             self.failUnlessEqual(data[0], "unknown")
4554             if expect_rw_uri:
4555                 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data)
4556             else:
4557                 self.failIfIn("rw_uri", data[1])
4558
4559             if immutable:
4560                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data)
4561                 self.failUnlessReallyEqual(data[1]["mutable"], False)
4562             elif expect_rw_uri:
4563                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4564                 self.failUnlessReallyEqual(data[1]["mutable"], True)
4565             else:
4566                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4567                 self.failIfIn("mutable", data[1])
4568
4569             # TODO: check metadata contents
4570             self.failUnlessIn("metadata", data[1])
4571
4572         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
4573         d.addCallback(_check_json, expect_rw_uri=not immutable)
4574
4575         # and make sure that a read-only version of the directory can be
4576         # rendered too. This version will not have unknown_rwcap, whether
4577         # or not future_node was immutable.
4578         d.addCallback(lambda ign: self.GET(self.rourl))
4579         if immutable:
4580             d.addCallback(_check_directory_html, "-IMM")
4581         else:
4582             d.addCallback(_check_directory_html, "-RO")
4583
4584         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
4585         d.addCallback(_check_directory_json, expect_rw_uri=False)
4586
4587         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
4588         d.addCallback(_check_json, expect_rw_uri=False)
4589
4590         # TODO: check that getting t=info from the Info link in the ro directory
4591         # works, and does not include the writecap URI.
4592         return d
4593
4594     def test_immutable_unknown(self):
4595         return self.test_unknown(immutable=True)
4596
4597     def test_mutant_dirnodes_are_omitted(self):
4598         self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
4599
4600         self.set_up_grid()
4601         c = self.g.clients[0]
4602         nm = c.nodemaker
4603         self.uris = {}
4604         self.fileurls = {}
4605
4606         lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
4607         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
4608         mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
4609
4610         # This method tests mainly dirnode, but we'd have to duplicate code in order to
4611         # test the dirnode and web layers separately.
4612
4613         # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
4614         # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
4615         # When the directory is read, the mutants should be silently disposed of, leaving
4616         # their lonely sibling.
4617         # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
4618         # because immutable directories don't have a writecap and therefore that field
4619         # isn't (and can't be) decrypted.
4620         # TODO: The field still exists in the netstring. Technically we should check what
4621         # happens if something is put there (_unpack_contents should raise ValueError),
4622         # but that can wait.
4623
4624         lonely_child = nm.create_from_cap(lonely_uri)
4625         mutant_ro_child = nm.create_from_cap(mut_read_uri)
4626         mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
4627
4628         def _by_hook_or_by_crook():
4629             return True
4630         for n in [mutant_ro_child, mutant_write_in_ro_child]:
4631             n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
4632
4633         mutant_write_in_ro_child.get_write_uri    = lambda: None
4634         mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
4635
4636         kids = {u"lonely":      (lonely_child, {}),
4637                 u"ro":          (mutant_ro_child, {}),
4638                 u"write-in-ro": (mutant_write_in_ro_child, {}),
4639                 }
4640         d = c.create_immutable_dirnode(kids)
4641
4642         def _created(dn):
4643             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
4644             self.failIf(dn.is_mutable())
4645             self.failUnless(dn.is_readonly())
4646             # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
4647             self.failIf(hasattr(dn._node, 'get_writekey'))
4648             rep = str(dn)
4649             self.failUnlessIn("RO-IMM", rep)
4650             cap = dn.get_cap()
4651             self.failUnlessIn("CHK", cap.to_string())
4652             self.cap = cap
4653             self.rootnode = dn
4654             self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
4655             return download_to_data(dn._node)
4656         d.addCallback(_created)
4657
4658         def _check_data(data):
4659             # Decode the netstring representation of the directory to check that all children
4660             # are present. This is a bit of an abstraction violation, but there's not really
4661             # any other way to do it given that the real DirectoryNode._unpack_contents would
4662             # strip the mutant children out (which is what we're trying to test, later).
4663             position = 0
4664             numkids = 0
4665             while position < len(data):
4666                 entries, position = split_netstring(data, 1, position)
4667                 entry = entries[0]
4668                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
4669                 name = name_utf8.decode("utf-8")
4670                 self.failUnlessEqual(rwcapdata, "")
4671                 self.failUnlessIn(name, kids)
4672                 (expected_child, ign) = kids[name]
4673                 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
4674                 numkids += 1
4675
4676             self.failUnlessReallyEqual(numkids, 3)
4677             return self.rootnode.list()
4678         d.addCallback(_check_data)
4679
4680         # Now when we use the real directory listing code, the mutants should be absent.
4681         def _check_kids(children):
4682             self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
4683             lonely_node, lonely_metadata = children[u"lonely"]
4684
4685             self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
4686             self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
4687         d.addCallback(_check_kids)
4688
4689         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
4690         d.addCallback(lambda n: n.list())
4691         d.addCallback(_check_kids)  # again with dirnode recreated from cap
4692
4693         # Make sure the lonely child can be listed in HTML...
4694         d.addCallback(lambda ign: self.GET(self.rooturl))
4695         def _check_html(res):
4696             self.failIfIn("URI:SSK", res)
4697             get_lonely = "".join([r'<td>FILE</td>',
4698                                   r'\s+<td>',
4699                                   r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
4700                                   r'</td>',
4701                                   r'\s+<td align="right">%d</td>' % len("one"),
4702                                   ])
4703             self.failUnless(re.search(get_lonely, res), res)
4704
4705             # find the More Info link for name, should be relative
4706             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4707             info_url = mo.group(1)
4708             self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
4709         d.addCallback(_check_html)
4710
4711         # ... and in JSON.
4712         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4713         def _check_json(res):
4714             data = simplejson.loads(res)
4715             self.failUnlessEqual(data[0], "dirnode")
4716             listed_children = data[1]["children"]
4717             self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
4718             ll_type, ll_data = listed_children[u"lonely"]
4719             self.failUnlessEqual(ll_type, "filenode")
4720             self.failIfIn("rw_uri", ll_data)
4721             self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
4722         d.addCallback(_check_json)
4723         return d
4724
4725     def test_deep_check(self):
4726         self.basedir = "web/Grid/deep_check"
4727         self.set_up_grid()
4728         c0 = self.g.clients[0]
4729         self.uris = {}
4730         self.fileurls = {}
4731         DATA = "data" * 100
4732         d = c0.create_dirnode()
4733         def _stash_root_and_create_file(n):
4734             self.rootnode = n
4735             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4736             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4737         d.addCallback(_stash_root_and_create_file)
4738         def _stash_uri(fn, which):
4739             self.uris[which] = fn.get_uri()
4740             return fn
4741         d.addCallback(_stash_uri, "good")
4742         d.addCallback(lambda ign:
4743                       self.rootnode.add_file(u"small",
4744                                              upload.Data("literal",
4745                                                         convergence="")))
4746         d.addCallback(_stash_uri, "small")
4747         d.addCallback(lambda ign:
4748                       self.rootnode.add_file(u"sick",
4749                                              upload.Data(DATA+"1",
4750                                                         convergence="")))
4751         d.addCallback(_stash_uri, "sick")
4752
4753         # this tests that deep-check and stream-manifest will ignore
4754         # UnknownNode instances. Hopefully this will also cover deep-stats.
4755         future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4756         d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
4757
4758         def _clobber_shares(ignored):
4759             self.delete_shares_numbered(self.uris["sick"], [0,1])
4760         d.addCallback(_clobber_shares)
4761
4762         # root
4763         # root/good
4764         # root/small
4765         # root/sick
4766         # root/future
4767
4768         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4769         def _done(res):
4770             try:
4771                 units = [simplejson.loads(line)
4772                          for line in res.splitlines()
4773                          if line]
4774             except ValueError:
4775                 print "response is:", res
4776                 print "undecodeable line was '%s'" % line
4777                 raise
4778             self.failUnlessReallyEqual(len(units), 5+1)
4779             # should be parent-first
4780             u0 = units[0]
4781             self.failUnlessEqual(u0["path"], [])
4782             self.failUnlessEqual(u0["type"], "directory")
4783             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4784             u0cr = u0["check-results"]
4785             self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
4786
4787             ugood = [u for u in units
4788                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4789             self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
4790             ugoodcr = ugood["check-results"]
4791             self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
4792
4793             stats = units[-1]
4794             self.failUnlessEqual(stats["type"], "stats")
4795             s = stats["stats"]
4796             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4797             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4798             self.failUnlessReallyEqual(s["count-directories"], 1)
4799             self.failUnlessReallyEqual(s["count-unknown"], 1)
4800         d.addCallback(_done)
4801
4802         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4803         def _check_manifest(res):
4804             self.failUnless(res.endswith("\n"))
4805             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
4806             self.failUnlessReallyEqual(len(units), 5+1)
4807             self.failUnlessEqual(units[-1]["type"], "stats")
4808             first = units[0]
4809             self.failUnlessEqual(first["path"], [])
4810             self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri())
4811             self.failUnlessEqual(first["type"], "directory")
4812             stats = units[-1]["stats"]
4813             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4814             self.failUnlessReallyEqual(stats["count-literal-files"], 1)
4815             self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
4816             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4817             self.failUnlessReallyEqual(stats["count-unknown"], 1)
4818         d.addCallback(_check_manifest)
4819
4820         # now add root/subdir and root/subdir/grandchild, then make subdir
4821         # unrecoverable, then see what happens
4822
4823         d.addCallback(lambda ign:
4824                       self.rootnode.create_subdirectory(u"subdir"))
4825         d.addCallback(_stash_uri, "subdir")
4826         d.addCallback(lambda subdir_node:
4827                       subdir_node.add_file(u"grandchild",
4828                                            upload.Data(DATA+"2",
4829                                                        convergence="")))
4830         d.addCallback(_stash_uri, "grandchild")
4831
4832         d.addCallback(lambda ign:
4833                       self.delete_shares_numbered(self.uris["subdir"],
4834                                                   range(1, 10)))
4835
4836         # root
4837         # root/good
4838         # root/small
4839         # root/sick
4840         # root/future
4841         # root/subdir [unrecoverable]
4842         # root/subdir/grandchild
4843
4844         # how should a streaming-JSON API indicate fatal error?
4845         # answer: emit ERROR: instead of a JSON string
4846
4847         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4848         def _check_broken_manifest(res):
4849             lines = res.splitlines()
4850             error_lines = [i
4851                            for (i,line) in enumerate(lines)
4852                            if line.startswith("ERROR:")]
4853             if not error_lines:
4854                 self.fail("no ERROR: in output: %s" % (res,))
4855             first_error = error_lines[0]
4856             error_line = lines[first_error]
4857             error_msg = lines[first_error+1:]
4858             error_msg_s = "\n".join(error_msg) + "\n"
4859             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4860                               error_line)
4861             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4862             units = [simplejson.loads(line) for line in lines[:first_error]]
4863             self.failUnlessReallyEqual(len(units), 6) # includes subdir
4864             last_unit = units[-1]
4865             self.failUnlessEqual(last_unit["path"], ["subdir"])
4866         d.addCallback(_check_broken_manifest)
4867
4868         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4869         def _check_broken_deepcheck(res):
4870             lines = res.splitlines()
4871             error_lines = [i
4872                            for (i,line) in enumerate(lines)
4873                            if line.startswith("ERROR:")]
4874             if not error_lines:
4875                 self.fail("no ERROR: in output: %s" % (res,))
4876             first_error = error_lines[0]
4877             error_line = lines[first_error]
4878             error_msg = lines[first_error+1:]
4879             error_msg_s = "\n".join(error_msg) + "\n"
4880             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4881                               error_line)
4882             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4883             units = [simplejson.loads(line) for line in lines[:first_error]]
4884             self.failUnlessReallyEqual(len(units), 6) # includes subdir
4885             last_unit = units[-1]
4886             self.failUnlessEqual(last_unit["path"], ["subdir"])
4887             r = last_unit["check-results"]["results"]
4888             self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
4889             self.failUnlessReallyEqual(r["count-shares-good"], 1)
4890             self.failUnlessReallyEqual(r["recoverable"], False)
4891         d.addCallback(_check_broken_deepcheck)
4892
4893         d.addErrback(self.explain_web_error)
4894         return d
4895
4896     def test_deep_check_and_repair(self):
4897         self.basedir = "web/Grid/deep_check_and_repair"
4898         self.set_up_grid()
4899         c0 = self.g.clients[0]
4900         self.uris = {}
4901         self.fileurls = {}
4902         DATA = "data" * 100
4903         d = c0.create_dirnode()
4904         def _stash_root_and_create_file(n):
4905             self.rootnode = n
4906             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4907             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4908         d.addCallback(_stash_root_and_create_file)
4909         def _stash_uri(fn, which):
4910             self.uris[which] = fn.get_uri()
4911         d.addCallback(_stash_uri, "good")
4912         d.addCallback(lambda ign:
4913                       self.rootnode.add_file(u"small",
4914                                              upload.Data("literal",
4915                                                         convergence="")))
4916         d.addCallback(_stash_uri, "small")
4917         d.addCallback(lambda ign:
4918                       self.rootnode.add_file(u"sick",
4919                                              upload.Data(DATA+"1",
4920                                                         convergence="")))
4921         d.addCallback(_stash_uri, "sick")
4922         #d.addCallback(lambda ign:
4923         #              self.rootnode.add_file(u"dead",
4924         #                                     upload.Data(DATA+"2",
4925         #                                                convergence="")))
4926         #d.addCallback(_stash_uri, "dead")
4927
4928         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
4929         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
4930         #d.addCallback(_stash_uri, "corrupt")
4931
4932         def _clobber_shares(ignored):
4933             good_shares = self.find_uri_shares(self.uris["good"])
4934             self.failUnlessReallyEqual(len(good_shares), 10)
4935             sick_shares = self.find_uri_shares(self.uris["sick"])
4936             os.unlink(sick_shares[0][2])
4937             #dead_shares = self.find_uri_shares(self.uris["dead"])
4938             #for i in range(1, 10):
4939             #    os.unlink(dead_shares[i][2])
4940
4941             #c_shares = self.find_uri_shares(self.uris["corrupt"])
4942             #cso = CorruptShareOptions()
4943             #cso.stdout = StringIO()
4944             #cso.parseOptions([c_shares[0][2]])
4945             #corrupt_share(cso)
4946         d.addCallback(_clobber_shares)
4947
4948         # root
4949         # root/good   CHK, 10 shares
4950         # root/small  LIT
4951         # root/sick   CHK, 9 shares
4952
4953         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
4954         def _done(res):
4955             units = [simplejson.loads(line)
4956                      for line in res.splitlines()
4957                      if line]
4958             self.failUnlessReallyEqual(len(units), 4+1)
4959             # should be parent-first
4960             u0 = units[0]
4961             self.failUnlessEqual(u0["path"], [])
4962             self.failUnlessEqual(u0["type"], "directory")
4963             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4964             u0crr = u0["check-and-repair-results"]
4965             self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
4966             self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
4967
4968             ugood = [u for u in units
4969                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4970             self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
4971             ugoodcrr = ugood["check-and-repair-results"]
4972             self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
4973             self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
4974
4975             usick = [u for u in units
4976                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
4977             self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"])
4978             usickcrr = usick["check-and-repair-results"]
4979             self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
4980             self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
4981             self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
4982             self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
4983
4984             stats = units[-1]
4985             self.failUnlessEqual(stats["type"], "stats")
4986             s = stats["stats"]
4987             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4988             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4989             self.failUnlessReallyEqual(s["count-directories"], 1)
4990         d.addCallback(_done)
4991
4992         d.addErrback(self.explain_web_error)
4993         return d
4994
4995     def _count_leases(self, ignored, which):
4996         u = self.uris[which]
4997         shares = self.find_uri_shares(u)
4998         lease_counts = []
4999         for shnum, serverid, fn in shares:
5000             sf = get_share_file(fn)
5001             num_leases = len(list(sf.get_leases()))
5002             lease_counts.append( (fn, num_leases) )
5003         return lease_counts
5004
5005     def _assert_leasecount(self, lease_counts, expected):
5006         for (fn, num_leases) in lease_counts:
5007             if num_leases != expected:
5008                 self.fail("expected %d leases, have %d, on %s" %
5009                           (expected, num_leases, fn))
5010
5011     def test_add_lease(self):
5012         self.basedir = "web/Grid/add_lease"
5013         self.set_up_grid(num_clients=2)
5014         c0 = self.g.clients[0]
5015         self.uris = {}
5016         DATA = "data" * 100
5017         d = c0.upload(upload.Data(DATA, convergence=""))
5018         def _stash_uri(ur, which):
5019             self.uris[which] = ur.get_uri()
5020         d.addCallback(_stash_uri, "one")
5021         d.addCallback(lambda ign:
5022                       c0.upload(upload.Data(DATA+"1", convergence="")))
5023         d.addCallback(_stash_uri, "two")
5024         def _stash_mutable_uri(n, which):
5025             self.uris[which] = n.get_uri()
5026             assert isinstance(self.uris[which], str)
5027         d.addCallback(lambda ign:
5028             c0.create_mutable_file(publish.MutableData(DATA+"2")))
5029         d.addCallback(_stash_mutable_uri, "mutable")
5030
5031         def _compute_fileurls(ignored):
5032             self.fileurls = {}
5033             for which in self.uris:
5034                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
5035         d.addCallback(_compute_fileurls)
5036
5037         d.addCallback(self._count_leases, "one")
5038         d.addCallback(self._assert_leasecount, 1)
5039         d.addCallback(self._count_leases, "two")
5040         d.addCallback(self._assert_leasecount, 1)
5041         d.addCallback(self._count_leases, "mutable")
5042         d.addCallback(self._assert_leasecount, 1)
5043
5044         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
5045         def _got_html_good(res):
5046             self.failUnlessIn("Healthy", res)
5047             self.failIfIn("Not Healthy", res)
5048         d.addCallback(_got_html_good)
5049
5050         d.addCallback(self._count_leases, "one")
5051         d.addCallback(self._assert_leasecount, 1)
5052         d.addCallback(self._count_leases, "two")
5053         d.addCallback(self._assert_leasecount, 1)
5054         d.addCallback(self._count_leases, "mutable")
5055         d.addCallback(self._assert_leasecount, 1)
5056
5057         # this CHECK uses the original client, which uses the same
5058         # lease-secrets, so it will just renew the original lease
5059         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
5060         d.addCallback(_got_html_good)
5061
5062         d.addCallback(self._count_leases, "one")
5063         d.addCallback(self._assert_leasecount, 1)
5064         d.addCallback(self._count_leases, "two")
5065         d.addCallback(self._assert_leasecount, 1)
5066         d.addCallback(self._count_leases, "mutable")
5067         d.addCallback(self._assert_leasecount, 1)
5068
5069         # this CHECK uses an alternate client, which adds a second lease
5070         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
5071         d.addCallback(_got_html_good)
5072
5073         d.addCallback(self._count_leases, "one")
5074         d.addCallback(self._assert_leasecount, 2)
5075         d.addCallback(self._count_leases, "two")
5076         d.addCallback(self._assert_leasecount, 1)
5077         d.addCallback(self._count_leases, "mutable")
5078         d.addCallback(self._assert_leasecount, 1)
5079
5080         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
5081         d.addCallback(_got_html_good)
5082
5083         d.addCallback(self._count_leases, "one")
5084         d.addCallback(self._assert_leasecount, 2)
5085         d.addCallback(self._count_leases, "two")
5086         d.addCallback(self._assert_leasecount, 1)
5087         d.addCallback(self._count_leases, "mutable")
5088         d.addCallback(self._assert_leasecount, 1)
5089
5090         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
5091                       clientnum=1)
5092         d.addCallback(_got_html_good)
5093
5094         d.addCallback(self._count_leases, "one")
5095         d.addCallback(self._assert_leasecount, 2)
5096         d.addCallback(self._count_leases, "two")
5097         d.addCallback(self._assert_leasecount, 1)
5098         d.addCallback(self._count_leases, "mutable")
5099         d.addCallback(self._assert_leasecount, 2)
5100
5101         d.addErrback(self.explain_web_error)
5102         return d
5103
5104     def test_deep_add_lease(self):
5105         self.basedir = "web/Grid/deep_add_lease"
5106         self.set_up_grid(num_clients=2)
5107         c0 = self.g.clients[0]
5108         self.uris = {}
5109         self.fileurls = {}
5110         DATA = "data" * 100
5111         d = c0.create_dirnode()
5112         def _stash_root_and_create_file(n):
5113             self.rootnode = n
5114             self.uris["root"] = n.get_uri()
5115             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5116             return n.add_file(u"one", upload.Data(DATA, convergence=""))
5117         d.addCallback(_stash_root_and_create_file)
5118         def _stash_uri(fn, which):
5119             self.uris[which] = fn.get_uri()
5120         d.addCallback(_stash_uri, "one")
5121         d.addCallback(lambda ign:
5122                       self.rootnode.add_file(u"small",
5123                                              upload.Data("literal",
5124                                                         convergence="")))
5125         d.addCallback(_stash_uri, "small")
5126
5127         d.addCallback(lambda ign:
5128             c0.create_mutable_file(publish.MutableData("mutable")))
5129         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
5130         d.addCallback(_stash_uri, "mutable")
5131
5132         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
5133         def _done(res):
5134             units = [simplejson.loads(line)
5135                      for line in res.splitlines()
5136                      if line]
5137             # root, one, small, mutable,   stats
5138             self.failUnlessReallyEqual(len(units), 4+1)
5139         d.addCallback(_done)
5140
5141         d.addCallback(self._count_leases, "root")
5142         d.addCallback(self._assert_leasecount, 1)
5143         d.addCallback(self._count_leases, "one")
5144         d.addCallback(self._assert_leasecount, 1)
5145         d.addCallback(self._count_leases, "mutable")
5146         d.addCallback(self._assert_leasecount, 1)
5147
5148         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
5149         d.addCallback(_done)
5150
5151         d.addCallback(self._count_leases, "root")
5152         d.addCallback(self._assert_leasecount, 1)
5153         d.addCallback(self._count_leases, "one")
5154         d.addCallback(self._assert_leasecount, 1)
5155         d.addCallback(self._count_leases, "mutable")
5156         d.addCallback(self._assert_leasecount, 1)
5157
5158         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
5159                       clientnum=1)
5160         d.addCallback(_done)
5161
5162         d.addCallback(self._count_leases, "root")
5163         d.addCallback(self._assert_leasecount, 2)
5164         d.addCallback(self._count_leases, "one")
5165         d.addCallback(self._assert_leasecount, 2)
5166         d.addCallback(self._count_leases, "mutable")
5167         d.addCallback(self._assert_leasecount, 2)
5168
5169         d.addErrback(self.explain_web_error)
5170         return d
5171
5172
5173     def test_exceptions(self):
5174         self.basedir = "web/Grid/exceptions"
5175         self.set_up_grid(num_clients=1, num_servers=2)
5176         c0 = self.g.clients[0]
5177         c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 2
5178         self.fileurls = {}
5179         DATA = "data" * 100
5180         d = c0.create_dirnode()
5181         def _stash_root(n):
5182             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5183             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
5184             return n
5185         d.addCallback(_stash_root)
5186         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
5187         def _stash_bad(ur):
5188             self.fileurls["1share"] = "uri/" + urllib.quote(ur.get_uri())
5189             self.delete_shares_numbered(ur.get_uri(), range(1,10))
5190
5191             u = uri.from_string(ur.get_uri())
5192             u.key = testutil.flip_bit(u.key, 0)
5193             baduri = u.to_string()
5194             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
5195         d.addCallback(_stash_bad)
5196         d.addCallback(lambda ign: c0.create_dirnode())
5197         def _mangle_dirnode_1share(n):
5198             u = n.get_uri()
5199             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
5200             self.fileurls["dir-1share-json"] = url + "?t=json"
5201             self.delete_shares_numbered(u, range(1,10))
5202         d.addCallback(_mangle_dirnode_1share)
5203         d.addCallback(lambda ign: c0.create_dirnode())
5204         def _mangle_dirnode_0share(n):
5205             u = n.get_uri()
5206             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
5207             self.fileurls["dir-0share-json"] = url + "?t=json"
5208             self.delete_shares_numbered(u, range(0,10))
5209         d.addCallback(_mangle_dirnode_0share)
5210
5211         # NotEnoughSharesError should be reported sensibly, with a
5212         # text/plain explanation of the problem, and perhaps some
5213         # information on which shares *could* be found.
5214
5215         d.addCallback(lambda ignored:
5216                       self.shouldHTTPError("GET unrecoverable",
5217                                            410, "Gone", "NoSharesError",
5218                                            self.GET, self.fileurls["0shares"]))
5219         def _check_zero_shares(body):
5220             self.failIfIn("<html>", body)
5221             body = " ".join(body.strip().split())
5222             exp = ("NoSharesError: no shares could be found. "
5223                    "Zero shares usually indicates a corrupt URI, or that "
5224                    "no servers were connected, but it might also indicate "
5225                    "severe corruption. You should perform a filecheck on "
5226                    "this object to learn more. The full error message is: "
5227                    "no shares (need 3). Last failure: None")
5228             self.failUnlessReallyEqual(exp, body)
5229         d.addCallback(_check_zero_shares)
5230
5231
5232         d.addCallback(lambda ignored:
5233                       self.shouldHTTPError("GET 1share",
5234                                            410, "Gone", "NotEnoughSharesError",
5235                                            self.GET, self.fileurls["1share"]))
5236         def _check_one_share(body):
5237             self.failIfIn("<html>", body)
5238             body = " ".join(body.strip().split())
5239             msgbase = ("NotEnoughSharesError: This indicates that some "
5240                        "servers were unavailable, or that shares have been "
5241                        "lost to server departure, hard drive failure, or disk "
5242                        "corruption. You should perform a filecheck on "
5243                        "this object to learn more. The full error message is:"
5244                        )
5245             msg1 = msgbase + (" ran out of shares:"
5246                               " complete=sh0"
5247                               " pending="
5248                               " overdue= unused= need 3. Last failure: None")
5249             msg2 = msgbase + (" ran out of shares:"
5250                               " complete="
5251                               " pending=Share(sh0-on-xgru5)"
5252                               " overdue= unused= need 3. Last failure: None")
5253             self.failUnless(body == msg1 or body == msg2, body)
5254         d.addCallback(_check_one_share)
5255
5256         d.addCallback(lambda ignored:
5257                       self.shouldHTTPError("GET imaginary",
5258                                            404, "Not Found", None,
5259                                            self.GET, self.fileurls["imaginary"]))
5260         def _missing_child(body):
5261             self.failUnlessIn("No such child: imaginary", body)
5262         d.addCallback(_missing_child)
5263
5264         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
5265         def _check_0shares_dir_html(body):
5266             self.failUnlessIn("<html>", body)
5267             # we should see the regular page, but without the child table or
5268             # the dirops forms
5269             body = " ".join(body.strip().split())
5270             self.failUnlessIn('href="?t=info">More info on this directory',
5271                               body)
5272             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5273                    "could not be retrieved, because there were insufficient "
5274                    "good shares. This might indicate that no servers were "
5275                    "connected, insufficient servers were connected, the URI "
5276                    "was corrupt, or that shares have been lost due to server "
5277                    "departure, hard drive failure, or disk corruption. You "
5278                    "should perform a filecheck on this object to learn more.")
5279             self.failUnlessIn(exp, body)
5280             self.failUnlessIn("No upload forms: directory is unreadable", body)
5281         d.addCallback(_check_0shares_dir_html)
5282
5283         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
5284         def _check_1shares_dir_html(body):
5285             # at some point, we'll split UnrecoverableFileError into 0-shares
5286             # and some-shares like we did for immutable files (since there
5287             # are different sorts of advice to offer in each case). For now,
5288             # they present the same way.
5289             self.failUnlessIn("<html>", body)
5290             body = " ".join(body.strip().split())
5291             self.failUnlessIn('href="?t=info">More info on this directory',
5292                               body)
5293             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5294                    "could not be retrieved, because there were insufficient "
5295                    "good shares. This might indicate that no servers were "
5296                    "connected, insufficient servers were connected, the URI "
5297                    "was corrupt, or that shares have been lost due to server "
5298                    "departure, hard drive failure, or disk corruption. You "
5299                    "should perform a filecheck on this object to learn more.")
5300             self.failUnlessIn(exp, body)
5301             self.failUnlessIn("No upload forms: directory is unreadable", body)
5302         d.addCallback(_check_1shares_dir_html)
5303
5304         d.addCallback(lambda ignored:
5305                       self.shouldHTTPError("GET dir-0share-json",
5306                                            410, "Gone", "UnrecoverableFileError",
5307                                            self.GET,
5308                                            self.fileurls["dir-0share-json"]))
5309         def _check_unrecoverable_file(body):
5310             self.failIfIn("<html>", body)
5311             body = " ".join(body.strip().split())
5312             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5313                    "could not be retrieved, because there were insufficient "
5314                    "good shares. This might indicate that no servers were "
5315                    "connected, insufficient servers were connected, the URI "
5316                    "was corrupt, or that shares have been lost due to server "
5317                    "departure, hard drive failure, or disk corruption. You "
5318                    "should perform a filecheck on this object to learn more.")
5319             self.failUnlessReallyEqual(exp, body)
5320         d.addCallback(_check_unrecoverable_file)
5321
5322         d.addCallback(lambda ignored:
5323                       self.shouldHTTPError("GET dir-1share-json",
5324                                            410, "Gone", "UnrecoverableFileError",
5325                                            self.GET,
5326                                            self.fileurls["dir-1share-json"]))
5327         d.addCallback(_check_unrecoverable_file)
5328
5329         d.addCallback(lambda ignored:
5330                       self.shouldHTTPError("GET imaginary",
5331                                            404, "Not Found", None,
5332                                            self.GET, self.fileurls["imaginary"]))
5333
5334         # attach a webapi child that throws a random error, to test how it
5335         # gets rendered.
5336         w = c0.getServiceNamed("webish")
5337         w.root.putChild("ERRORBOOM", ErrorBoom())
5338
5339         # "Accept: */*" :        should get a text/html stack trace
5340         # "Accept: text/plain" : should get a text/plain stack trace
5341         # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
5342         # no Accept header:      should get a text/html stack trace
5343
5344         d.addCallback(lambda ignored:
5345                       self.shouldHTTPError("GET errorboom_html",
5346                                            500, "Internal Server Error", None,
5347                                            self.GET, "ERRORBOOM",
5348                                            headers={"accept": "*/*"}))
5349         def _internal_error_html1(body):
5350             self.failUnlessIn("<html>", "expected HTML, not '%s'" % body)
5351         d.addCallback(_internal_error_html1)
5352
5353         d.addCallback(lambda ignored:
5354                       self.shouldHTTPError("GET errorboom_text",
5355                                            500, "Internal Server Error", None,
5356                                            self.GET, "ERRORBOOM",
5357                                            headers={"accept": "text/plain"}))
5358         def _internal_error_text2(body):
5359             self.failIfIn("<html>", body)
5360             self.failUnless(body.startswith("Traceback "), body)
5361         d.addCallback(_internal_error_text2)
5362
5363         CLI_accepts = "text/plain, application/octet-stream"
5364         d.addCallback(lambda ignored:
5365                       self.shouldHTTPError("GET errorboom_text",
5366                                            500, "Internal Server Error", None,
5367                                            self.GET, "ERRORBOOM",
5368                                            headers={"accept": CLI_accepts}))
5369         def _internal_error_text3(body):
5370             self.failIfIn("<html>", body)
5371             self.failUnless(body.startswith("Traceback "), body)
5372         d.addCallback(_internal_error_text3)
5373
5374         d.addCallback(lambda ignored:
5375                       self.shouldHTTPError("GET errorboom_text",
5376                                            500, "Internal Server Error", None,
5377                                            self.GET, "ERRORBOOM"))
5378         def _internal_error_html4(body):
5379             self.failUnlessIn("<html>", body)
5380         d.addCallback(_internal_error_html4)
5381
5382         def _flush_errors(res):
5383             # Trial: please ignore the CompletelyUnhandledError in the logs
5384             self.flushLoggedErrors(CompletelyUnhandledError)
5385             return res
5386         d.addBoth(_flush_errors)
5387
5388         return d
5389
5390     def test_blacklist(self):
5391         # download from a blacklisted URI, get an error
5392         self.basedir = "web/Grid/blacklist"
5393         self.set_up_grid()
5394         c0 = self.g.clients[0]
5395         c0_basedir = c0.basedir
5396         fn = os.path.join(c0_basedir, "access.blacklist")
5397         self.uris = {}
5398         DATA = "off-limits " * 50
5399
5400         d = c0.upload(upload.Data(DATA, convergence=""))
5401         def _stash_uri_and_create_dir(ur):
5402             self.uri = ur.get_uri()
5403             self.url = "uri/"+self.uri
5404             u = uri.from_string_filenode(self.uri)
5405             self.si = u.get_storage_index()
5406             childnode = c0.create_node_from_uri(self.uri, None)
5407             return c0.create_dirnode({u"blacklisted.txt": (childnode,{}) })
5408         d.addCallback(_stash_uri_and_create_dir)
5409         def _stash_dir(node):
5410             self.dir_node = node
5411             self.dir_uri = node.get_uri()
5412             self.dir_url = "uri/"+self.dir_uri
5413         d.addCallback(_stash_dir)
5414         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5415         def _check_dir_html(body):
5416             self.failUnlessIn("<html>", body)
5417             self.failUnlessIn("blacklisted.txt</a>", body)
5418         d.addCallback(_check_dir_html)
5419         d.addCallback(lambda ign: self.GET(self.url))
5420         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5421
5422         def _blacklist(ign):
5423             f = open(fn, "w")
5424             f.write(" # this is a comment\n")
5425             f.write(" \n")
5426             f.write("\n") # also exercise blank lines
5427             f.write("%s %s\n" % (base32.b2a(self.si), "off-limits to you"))
5428             f.close()
5429             # clients should be checking the blacklist each time, so we don't
5430             # need to restart the client
5431         d.addCallback(_blacklist)
5432         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_uri",
5433                                                        403, "Forbidden",
5434                                                        "Access Prohibited: off-limits",
5435                                                        self.GET, self.url))
5436
5437         # We should still be able to list the parent directory, in HTML...
5438         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5439         def _check_dir_html2(body):
5440             self.failUnlessIn("<html>", body)
5441             self.failUnlessIn("blacklisted.txt</strike>", body)
5442         d.addCallback(_check_dir_html2)
5443
5444         # ... and in JSON (used by CLI).
5445         d.addCallback(lambda ign: self.GET(self.dir_url+"?t=json", followRedirect=True))
5446         def _check_dir_json(res):
5447             data = simplejson.loads(res)
5448             self.failUnless(isinstance(data, list), data)
5449             self.failUnlessEqual(data[0], "dirnode")
5450             self.failUnless(isinstance(data[1], dict), data)
5451             self.failUnlessIn("children", data[1])
5452             self.failUnlessIn("blacklisted.txt", data[1]["children"])
5453             childdata = data[1]["children"]["blacklisted.txt"]
5454             self.failUnless(isinstance(childdata, list), data)
5455             self.failUnlessEqual(childdata[0], "filenode")
5456             self.failUnless(isinstance(childdata[1], dict), data)
5457         d.addCallback(_check_dir_json)
5458
5459         def _unblacklist(ign):
5460             open(fn, "w").close()
5461             # the Blacklist object watches mtime to tell when the file has
5462             # changed, but on windows this test will run faster than the
5463             # filesystem's mtime resolution. So we edit Blacklist.last_mtime
5464             # to force a reload.
5465             self.g.clients[0].blacklist.last_mtime -= 2.0
5466         d.addCallback(_unblacklist)
5467
5468         # now a read should work
5469         d.addCallback(lambda ign: self.GET(self.url))
5470         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5471
5472         # read again to exercise the blacklist-is-unchanged logic
5473         d.addCallback(lambda ign: self.GET(self.url))
5474         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5475
5476         # now add a blacklisted directory, and make sure files under it are
5477         # refused too
5478         def _add_dir(ign):
5479             childnode = c0.create_node_from_uri(self.uri, None)
5480             return c0.create_dirnode({u"child": (childnode,{}) })
5481         d.addCallback(_add_dir)
5482         def _get_dircap(dn):
5483             self.dir_si_b32 = base32.b2a(dn.get_storage_index())
5484             self.dir_url_base = "uri/"+dn.get_write_uri()
5485             self.dir_url_json1 = "uri/"+dn.get_write_uri()+"?t=json"
5486             self.dir_url_json2 = "uri/"+dn.get_write_uri()+"/?t=json"
5487             self.dir_url_json_ro = "uri/"+dn.get_readonly_uri()+"/?t=json"
5488             self.child_url = "uri/"+dn.get_readonly_uri()+"/child"
5489         d.addCallback(_get_dircap)
5490         d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True))
5491         d.addCallback(lambda body: self.failUnlessIn("<html>", body))
5492         d.addCallback(lambda ign: self.GET(self.dir_url_json1))
5493         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5494         d.addCallback(lambda ign: self.GET(self.dir_url_json2))
5495         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5496         d.addCallback(lambda ign: self.GET(self.dir_url_json_ro))
5497         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5498         d.addCallback(lambda ign: self.GET(self.child_url))
5499         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5500
5501         def _block_dir(ign):
5502             f = open(fn, "w")
5503             f.write("%s %s\n" % (self.dir_si_b32, "dir-off-limits to you"))
5504             f.close()
5505             self.g.clients[0].blacklist.last_mtime -= 2.0
5506         d.addCallback(_block_dir)
5507         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir base",
5508                                                        403, "Forbidden",
5509                                                        "Access Prohibited: dir-off-limits",
5510                                                        self.GET, self.dir_url_base))
5511         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json1",
5512                                                        403, "Forbidden",
5513                                                        "Access Prohibited: dir-off-limits",
5514                                                        self.GET, self.dir_url_json1))
5515         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json2",
5516                                                        403, "Forbidden",
5517                                                        "Access Prohibited: dir-off-limits",
5518                                                        self.GET, self.dir_url_json2))
5519         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json_ro",
5520                                                        403, "Forbidden",
5521                                                        "Access Prohibited: dir-off-limits",
5522                                                        self.GET, self.dir_url_json_ro))
5523         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir child",
5524                                                        403, "Forbidden",
5525                                                        "Access Prohibited: dir-off-limits",
5526                                                        self.GET, self.child_url))
5527         return d
5528
5529
5530 class CompletelyUnhandledError(Exception):
5531     pass
5532 class ErrorBoom(rend.Page):
5533     def beforeRender(self, ctx):
5534         raise CompletelyUnhandledError("whoops")