src/allmydata/test/test_web.py

   1 import os.path, re, urllib, time
   2 import simplejson
   3 from StringIO import StringIO
   4
   5 from twisted.application import service
   6 from twisted.trial import unittest
   7 from twisted.internet import defer, reactor
   8 from twisted.internet.task import Clock
   9 from twisted.web import client, error, http
  10 from twisted.python import failure, log
  11
  12 from foolscap.api import fireEventually, flushEventualQueue
  13
  14 from nevow import rend
  15
  16 from allmydata import interfaces, uri, webish, dirnode
  17 from allmydata.storage.shares import get_share_file
  18 from allmydata.storage_client import StorageFarmBroker
  19 from allmydata.immutable import upload
  20 from allmydata.immutable.downloader.status import DownloadStatus
  21 from allmydata.dirnode import DirectoryNode
  22 from allmydata.nodemaker import NodeMaker
  23 from allmydata.unknown import UnknownNode
  24 from allmydata.web import status, common
  25 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  26 from allmydata.util import fileutil, base32, hashutil
  27 from allmydata.util.consumer import download_to_data
  28 from allmydata.util.netstring import split_netstring
  29 from allmydata.util.encodingutil import to_str
  30 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  31      create_chk_filenode, WebErrorMixin, ShouldFailMixin, \
  32      make_mutable_file_uri, create_mutable_filenode
  33 from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
  34 from allmydata.mutable import servermap, publish, retrieve
  35 import allmydata.test.common_util as testutil
  36 from allmydata.test.no_network import GridTestMixin
  37 from allmydata.test.common_web import HTTPClientGETFactory, \
  38      HTTPClientHEADFactory
  39 from allmydata.client import Client, SecretHolder
  40 from allmydata.introducer import IntroducerNode
  41
  42 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  43 # create a webserver that works against them
  44
  45 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  46
  47 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
  48 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
  49 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
  50
  51 FAVICON_MARKUP = '<link href="/icon.png" rel="shortcut icon" />'
  52
  53
  54 class FakeStatsProvider:
  55     def get_stats(self):
  56         stats = {'stats': {}, 'counters': {}}
  57         return stats
  58
  59 class FakeNodeMaker(NodeMaker):
  60     encoding_params = {
  61         'k': 3,
  62         'n': 10,
  63         'happy': 7,
  64         'max_segment_size':128*1024 # 1024=KiB
  65     }
  66     def _create_lit(self, cap):
  67         return FakeCHKFileNode(cap)
  68     def _create_immutable(self, cap):
  69         return FakeCHKFileNode(cap)
  70     def _create_mutable(self, cap):
  71         return FakeMutableFileNode(None,
  72                                    None,
  73                                    self.encoding_params, None).init_from_cap(cap)
  74     def create_mutable_file(self, contents="", keysize=None,
  75                             version=SDMF_VERSION):
  76         n = FakeMutableFileNode(None, None, self.encoding_params, None)
  77         return n.create(contents, version=version)
  78
  79 class FakeUploader(service.Service):
  80     name = "uploader"
  81     def upload(self, uploadable):
  82         d = uploadable.get_size()
  83         d.addCallback(lambda size: uploadable.read(size))
  84         def _got_data(datav):
  85             data = "".join(datav)
  86             n = create_chk_filenode(data)
  87             results = upload.UploadResults()
  88             results.uri = n.get_uri()
  89             return results
  90         d.addCallback(_got_data)
  91         return d
  92     def get_helper_info(self):
  93         return (None, False)
  94
  95 class FakeIServer:
  96     def __init__(self, binaryserverid):
  97         self.binaryserverid = binaryserverid
  98     def get_name(self): return "short"
  99     def get_longname(self): return "long"
 100     def get_serverid(self): return self.binaryserverid
 101
 102 def build_one_ds():
 103     ds = DownloadStatus("storage_index", 1234)
 104     now = time.time()
 105
 106     serverA = FakeIServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
 107     serverB = FakeIServer(hashutil.tagged_hash("foo", "serverid_b")[:20])
 108     storage_index = hashutil.storage_index_hash("SI")
 109     e0 = ds.add_segment_request(0, now)
 110     e0.activate(now+0.5)
 111     e0.deliver(now+1, 0, 100, 0.5) # when, start,len, decodetime
 112     e1 = ds.add_segment_request(1, now+2)
 113     e1.error(now+3)
 114     # two outstanding requests
 115     e2 = ds.add_segment_request(2, now+4)
 116     e3 = ds.add_segment_request(3, now+5)
 117     del e2,e3 # hush pyflakes
 118
 119     # simulate a segment which gets delivered faster than a system clock tick (ticket #1166)
 120     e = ds.add_segment_request(4, now)
 121     e.activate(now)
 122     e.deliver(now, 0, 140, 0.5)
 123
 124     e = ds.add_dyhb_request(serverA, now)
 125     e.finished([1,2], now+1)
 126     e = ds.add_dyhb_request(serverB, now+2) # left unfinished
 127
 128     e = ds.add_read_event(0, 120, now)
 129     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 130     e.finished(now+1)
 131     e = ds.add_read_event(120, 30, now+2) # left unfinished
 132
 133     e = ds.add_block_request(serverA, 1, 100, 20, now)
 134     e.finished(20, now+1)
 135     e = ds.add_block_request(serverB, 1, 120, 30, now+1) # left unfinished
 136
 137     # make sure that add_read_event() can come first too
 138     ds1 = DownloadStatus(storage_index, 1234)
 139     e = ds1.add_read_event(0, 120, now)
 140     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 141     e.finished(now+1)
 142
 143     return ds
 144
 145 class FakeHistory:
 146     _all_upload_status = [upload.UploadStatus()]
 147     _all_download_status = [build_one_ds()]
 148     _all_mapupdate_statuses = [servermap.UpdateStatus()]
 149     _all_publish_statuses = [publish.PublishStatus()]
 150     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
 151
 152     def list_all_upload_statuses(self):
 153         return self._all_upload_status
 154     def list_all_download_statuses(self):
 155         return self._all_download_status
 156     def list_all_mapupdate_statuses(self):
 157         return self._all_mapupdate_statuses
 158     def list_all_publish_statuses(self):
 159         return self._all_publish_statuses
 160     def list_all_retrieve_statuses(self):
 161         return self._all_retrieve_statuses
 162     def list_all_helper_statuses(self):
 163         return []
 164
 165 class FakeClient(Client):
 166     def __init__(self):
 167         # don't upcall to Client.__init__, since we only want to initialize a
 168         # minimal subset
 169         service.MultiService.__init__(self)
 170         self.nodeid = "fake_nodeid"
 171         self.nickname = "fake_nickname"
 172         self.introducer_furl = "None"
 173         self.stats_provider = FakeStatsProvider()
 174         self._secret_holder = SecretHolder("lease secret", "convergence secret")
 175         self.helper = None
 176         self.convergence = "some random string"
 177         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 178         self.introducer_client = None
 179         self.history = FakeHistory()
 180         self.uploader = FakeUploader()
 181         self.uploader.setServiceParent(self)
 182         self.blacklist = None
 183         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 184                                        self.uploader, None,
 185                                        None, None, None)
 186         self.mutable_file_default = SDMF_VERSION
 187
 188     def startService(self):
 189         return service.MultiService.startService(self)
 190     def stopService(self):
 191         return service.MultiService.stopService(self)
 192
 193     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 194
 195 class WebMixin(object):
 196     def setUp(self):
 197         self.s = FakeClient()
 198         self.s.startService()
 199         self.staticdir = self.mktemp()
 200         self.clock = Clock()
 201         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
 202                                       clock=self.clock)
 203         self.ws.setServiceParent(self.s)
 204         self.webish_port = self.ws.getPortnum()
 205         self.webish_url = self.ws.getURL()
 206         assert self.webish_url.endswith("/")
 207         self.webish_url = self.webish_url[:-1] # these tests add their own /
 208
 209         l = [ self.s.create_dirnode() for x in range(6) ]
 210         d = defer.DeferredList(l)
 211         def _then(res):
 212             self.public_root = res[0][1]
 213             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 214             self.public_url = "/uri/" + self.public_root.get_uri()
 215             self.private_root = res[1][1]
 216
 217             foo = res[2][1]
 218             self._foo_node = foo
 219             self._foo_uri = foo.get_uri()
 220             self._foo_readonly_uri = foo.get_readonly_uri()
 221             self._foo_verifycap = foo.get_verify_cap().to_string()
 222             # NOTE: we ignore the deferred on all set_uri() calls, because we
 223             # know the fake nodes do these synchronously
 224             self.public_root.set_uri(u"foo", foo.get_uri(),
 225                                      foo.get_readonly_uri())
 226
 227             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 228             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 229             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 230
 231             # sdmf
 232             # XXX: Do we ever use this?
 233             self.BAZ_CONTENTS, n, self._baz_txt_uri, self._baz_txt_readonly_uri = self.makefile_mutable(0)
 234
 235             foo.set_uri(u"baz.txt", self._baz_txt_uri, self._baz_txt_readonly_uri)
 236
 237             # mdmf
 238             self.QUUX_CONTENTS, n, self._quux_txt_uri, self._quux_txt_readonly_uri = self.makefile_mutable(0, mdmf=True)
 239             assert self._quux_txt_uri.startswith("URI:MDMF")
 240             foo.set_uri(u"quux.txt", self._quux_txt_uri, self._quux_txt_readonly_uri)
 241
 242             foo.set_uri(u"empty", res[3][1].get_uri(),
 243                         res[3][1].get_readonly_uri())
 244             sub_uri = res[4][1].get_uri()
 245             self._sub_uri = sub_uri
 246             foo.set_uri(u"sub", sub_uri, sub_uri)
 247             sub = self.s.create_node_from_uri(sub_uri)
 248             self._sub_node = sub
 249
 250             _ign, n, blocking_uri = self.makefile(1)
 251             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 252
 253             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 254             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 255             # still think of it as an umlaut
 256             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 257
 258             self.SUBBAZ_CONTENTS, n, baz_file = self.makefile(2)
 259             self._baz_file_uri = baz_file
 260             sub.set_uri(u"baz.txt", baz_file, baz_file)
 261
 262             _ign, n, self._bad_file_uri = self.makefile(3)
 263             # this uri should not be downloadable
 264             del FakeCHKFileNode.all_contents[self._bad_file_uri]
 265
 266             rodir = res[5][1]
 267             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 268                                      rodir.get_readonly_uri())
 269             rodir.set_uri(u"nor", baz_file, baz_file)
 270
 271             # public/
 272             # public/foo/
 273             # public/foo/bar.txt
 274             # public/foo/baz.txt
 275             # public/foo/quux.txt
 276             # public/foo/blockingfile
 277             # public/foo/empty/
 278             # public/foo/sub/
 279             # public/foo/sub/baz.txt
 280             # public/reedownlee/
 281             # public/reedownlee/nor
 282             self.NEWFILE_CONTENTS = "newfile contents\n"
 283
 284             return foo.get_metadata_for(u"bar.txt")
 285         d.addCallback(_then)
 286         def _got_metadata(metadata):
 287             self._bar_txt_metadata = metadata
 288         d.addCallback(_got_metadata)
 289         return d
 290
 291     def makefile(self, number):
 292         contents = "contents of file %s\n" % number
 293         n = create_chk_filenode(contents)
 294         return contents, n, n.get_uri()
 295
 296     def makefile_mutable(self, number, mdmf=False):
 297         contents = "contents of mutable file %s\n" % number
 298         n = create_mutable_filenode(contents, mdmf)
 299         return contents, n, n.get_uri(), n.get_readonly_uri()
 300
 301     def tearDown(self):
 302         return self.s.stopService()
 303
 304     def failUnlessIsBarDotTxt(self, res):
 305         self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
 306
 307     def failUnlessIsQuuxDotTxt(self, res):
 308         self.failUnlessReallyEqual(res, self.QUUX_CONTENTS, res)
 309
 310     def failUnlessIsBazDotTxt(self, res):
 311         self.failUnlessReallyEqual(res, self.BAZ_CONTENTS, res)
 312
 313     def failUnlessIsSubBazDotTxt(self, res):
 314         self.failUnlessReallyEqual(res, self.SUBBAZ_CONTENTS, res)
 315
 316     def failUnlessIsBarJSON(self, res):
 317         data = simplejson.loads(res)
 318         self.failUnless(isinstance(data, list))
 319         self.failUnlessEqual(data[0], "filenode")
 320         self.failUnless(isinstance(data[1], dict))
 321         self.failIf(data[1]["mutable"])
 322         self.failIfIn("rw_uri", data[1]) # immutable
 323         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
 324         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
 325         self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
 326
 327     def failUnlessIsQuuxJSON(self, res, readonly=False):
 328         data = simplejson.loads(res)
 329         self.failUnless(isinstance(data, list))
 330         self.failUnlessEqual(data[0], "filenode")
 331         self.failUnless(isinstance(data[1], dict))
 332         metadata = data[1]
 333         return self.failUnlessIsQuuxDotTxtMetadata(metadata, readonly)
 334
 335     def failUnlessIsQuuxDotTxtMetadata(self, metadata, readonly):
 336         self.failUnless(metadata['mutable'])
 337         if readonly:
 338             self.failIfIn("rw_uri", metadata)
 339         else:
 340             self.failUnlessIn("rw_uri", metadata)
 341             self.failUnlessEqual(metadata['rw_uri'], self._quux_txt_uri)
 342         self.failUnlessIn("ro_uri", metadata)
 343         self.failUnlessEqual(metadata['ro_uri'], self._quux_txt_readonly_uri)
 344         self.failUnlessReallyEqual(metadata['size'], len(self.QUUX_CONTENTS))
 345
 346     def failUnlessIsFooJSON(self, res):
 347         data = simplejson.loads(res)
 348         self.failUnless(isinstance(data, list))
 349         self.failUnlessEqual(data[0], "dirnode", res)
 350         self.failUnless(isinstance(data[1], dict))
 351         self.failUnless(data[1]["mutable"])
 352         self.failUnlessIn("rw_uri", data[1]) # mutable
 353         self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
 354         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
 355         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
 356
 357         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 358         self.failUnlessEqual(kidnames,
 359                              [u"bar.txt", u"baz.txt", u"blockingfile",
 360                               u"empty", u"n\u00fc.txt", u"quux.txt", u"sub"])
 361         kids = dict( [(unicode(name),value)
 362                       for (name,value)
 363                       in data[1]["children"].iteritems()] )
 364         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 365         self.failUnlessIn("metadata", kids[u"sub"][1])
 366         self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
 367         tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
 368         self.failUnlessIn("linkcrtime", tahoe_md)
 369         self.failUnlessIn("linkmotime", tahoe_md)
 370         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 371         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 372         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri)
 373         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]),
 374                                    self._bar_txt_verifycap)
 375         self.failUnlessIn("metadata", kids[u"bar.txt"][1])
 376         self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
 377         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
 378                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
 379         self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
 380                                    self._bar_txt_uri)
 381         self.failUnlessIn("quux.txt", kids)
 382         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["rw_uri"]),
 383                                    self._quux_txt_uri)
 384         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["ro_uri"]),
 385                                    self._quux_txt_readonly_uri)
 386
 387     def GET(self, urlpath, followRedirect=False, return_response=False,
 388             **kwargs):
 389         # if return_response=True, this fires with (data, statuscode,
 390         # respheaders) instead of just data.
 391         assert not isinstance(urlpath, unicode)
 392         url = self.webish_url + urlpath
 393         factory = HTTPClientGETFactory(url, method="GET",
 394                                        followRedirect=followRedirect, **kwargs)
 395         reactor.connectTCP("localhost", self.webish_port, factory)
 396         d = factory.deferred
 397         def _got_data(data):
 398             return (data, factory.status, factory.response_headers)
 399         if return_response:
 400             d.addCallback(_got_data)
 401         return factory.deferred
 402
 403     def HEAD(self, urlpath, return_response=False, **kwargs):
 404         # this requires some surgery, because twisted.web.client doesn't want
 405         # to give us back the response headers.
 406         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 407         reactor.connectTCP("localhost", self.webish_port, factory)
 408         d = factory.deferred
 409         def _got_data(data):
 410             return (data, factory.status, factory.response_headers)
 411         if return_response:
 412             d.addCallback(_got_data)
 413         return factory.deferred
 414
 415     def PUT(self, urlpath, data, **kwargs):
 416         url = self.webish_url + urlpath
 417         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 418
 419     def DELETE(self, urlpath):
 420         url = self.webish_url + urlpath
 421         return client.getPage(url, method="DELETE")
 422
 423     def POST(self, urlpath, followRedirect=False, **fields):
 424         sepbase = "boogabooga"
 425         sep = "--" + sepbase
 426         form = []
 427         form.append(sep)
 428         form.append('Content-Disposition: form-data; name="_charset"')
 429         form.append('')
 430         form.append('UTF-8')
 431         form.append(sep)
 432         for name, value in fields.iteritems():
 433             if isinstance(value, tuple):
 434                 filename, value = value
 435                 form.append('Content-Disposition: form-data; name="%s"; '
 436                             'filename="%s"' % (name, filename.encode("utf-8")))
 437             else:
 438                 form.append('Content-Disposition: form-data; name="%s"' % name)
 439             form.append('')
 440             if isinstance(value, unicode):
 441                 value = value.encode("utf-8")
 442             else:
 443                 value = str(value)
 444             assert isinstance(value, str)
 445             form.append(value)
 446             form.append(sep)
 447         form[-1] += "--"
 448         body = ""
 449         headers = {}
 450         if fields:
 451             body = "\r\n".join(form) + "\r\n"
 452             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 453         return self.POST2(urlpath, body, headers, followRedirect)
 454
 455     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 456         url = self.webish_url + urlpath
 457         return client.getPage(url, method="POST", postdata=body,
 458                               headers=headers, followRedirect=followRedirect)
 459
 460     def shouldFail(self, res, expected_failure, which,
 461                    substring=None, response_substring=None):
 462         if isinstance(res, failure.Failure):
 463             res.trap(expected_failure)
 464             if substring:
 465                 self.failUnlessIn(substring, str(res), which)
 466             if response_substring:
 467                 self.failUnlessIn(response_substring, res.value.response, which)
 468         else:
 469             self.fail("%s was supposed to raise %s, not get '%s'" %
 470                       (which, expected_failure, res))
 471
 472     def shouldFail2(self, expected_failure, which, substring,
 473                     response_substring,
 474                     callable, *args, **kwargs):
 475         assert substring is None or isinstance(substring, str)
 476         assert response_substring is None or isinstance(response_substring, str)
 477         d = defer.maybeDeferred(callable, *args, **kwargs)
 478         def done(res):
 479             if isinstance(res, failure.Failure):
 480                 res.trap(expected_failure)
 481                 if substring:
 482                     self.failUnlessIn(substring, str(res),
 483                                       "'%s' not in '%s' for test '%s'" % \
 484                                       (substring, str(res), which))
 485                 if response_substring:
 486                     self.failUnlessIn(response_substring, res.value.response,
 487                                       "'%s' not in '%s' for test '%s'" % \
 488                                       (response_substring, res.value.response,
 489                                        which))
 490             else:
 491                 self.fail("%s was supposed to raise %s, not get '%s'" %
 492                           (which, expected_failure, res))
 493         d.addBoth(done)
 494         return d
 495
 496     def should404(self, res, which):
 497         if isinstance(res, failure.Failure):
 498             res.trap(error.Error)
 499             self.failUnlessReallyEqual(res.value.status, "404")
 500         else:
 501             self.fail("%s was supposed to Error(404), not get '%s'" %
 502                       (which, res))
 503
 504     def should302(self, res, which):
 505         if isinstance(res, failure.Failure):
 506             res.trap(error.Error)
 507             self.failUnlessReallyEqual(res.value.status, "302")
 508         else:
 509             self.fail("%s was supposed to Error(302), not get '%s'" %
 510                       (which, res))
 511
 512
 513 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
 514     def test_create(self):
 515         pass
 516
 517     def test_welcome(self):
 518         d = self.GET("/")
 519         def _check(res):
 520             self.failUnlessIn('Welcome to Tahoe-LAFS', res)
 521             self.failUnlessIn(FAVICON_MARKUP, res)
 522             self.failUnlessIn('href="https://tahoe-lafs.org/"', res)
 523
 524             self.s.basedir = 'web/test_welcome'
 525             fileutil.make_dirs("web/test_welcome")
 526             fileutil.make_dirs("web/test_welcome/private")
 527             return self.GET("/")
 528         d.addCallback(_check)
 529         return d
 530
 531     def test_status(self):
 532         h = self.s.get_history()
 533         dl_num = h.list_all_download_statuses()[0].get_counter()
 534         ul_num = h.list_all_upload_statuses()[0].get_counter()
 535         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 536         pub_num = h.list_all_publish_statuses()[0].get_counter()
 537         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 538         d = self.GET("/status", followRedirect=True)
 539         def _check(res):
 540             self.failUnlessIn('Upload and Download Status', res)
 541             self.failUnlessIn('"down-%d"' % dl_num, res)
 542             self.failUnlessIn('"up-%d"' % ul_num, res)
 543             self.failUnlessIn('"mapupdate-%d"' % mu_num, res)
 544             self.failUnlessIn('"publish-%d"' % pub_num, res)
 545             self.failUnlessIn('"retrieve-%d"' % ret_num, res)
 546         d.addCallback(_check)
 547         d.addCallback(lambda res: self.GET("/status/?t=json"))
 548         def _check_json(res):
 549             data = simplejson.loads(res)
 550             self.failUnless(isinstance(data, dict))
 551             #active = data["active"]
 552             # TODO: test more. We need a way to fake an active operation
 553             # here.
 554         d.addCallback(_check_json)
 555
 556         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 557         def _check_dl(res):
 558             self.failUnlessIn("File Download Status", res)
 559         d.addCallback(_check_dl)
 560         d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
 561         def _check_dl_json(res):
 562             data = simplejson.loads(res)
 563             self.failUnless(isinstance(data, dict))
 564             self.failUnlessIn("read", data)
 565             self.failUnlessEqual(data["read"][0]["length"], 120)
 566             self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
 567             self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
 568             self.failUnlessEqual(data["segment"][2]["finish_time"], None)
 569             phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
 570             cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
 571             # serverids[] keys are strings, since that's what JSON does, but
 572             # we'd really like them to be ints
 573             self.failUnlessEqual(data["serverids"]["0"], "phwr")
 574             self.failUnless(data["serverids"].has_key("1"), data["serverids"])
 575             self.failUnlessEqual(data["serverids"]["1"], "cmpu", data["serverids"])
 576             self.failUnlessEqual(data["server_info"][phwr_id]["short"], "phwr")
 577             self.failUnlessEqual(data["server_info"][cmpu_id]["short"], "cmpu")
 578             self.failUnlessIn("dyhb", data)
 579             self.failUnlessIn("misc", data)
 580         d.addCallback(_check_dl_json)
 581         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 582         def _check_ul(res):
 583             self.failUnlessIn("File Upload Status", res)
 584         d.addCallback(_check_ul)
 585         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 586         def _check_mapupdate(res):
 587             self.failUnlessIn("Mutable File Servermap Update Status", res)
 588         d.addCallback(_check_mapupdate)
 589         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 590         def _check_publish(res):
 591             self.failUnlessIn("Mutable File Publish Status", res)
 592         d.addCallback(_check_publish)
 593         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 594         def _check_retrieve(res):
 595             self.failUnlessIn("Mutable File Retrieve Status", res)
 596         d.addCallback(_check_retrieve)
 597
 598         return d
 599
 600     def test_status_numbers(self):
 601         drrm = status.DownloadResultsRendererMixin()
 602         self.failUnlessReallyEqual(drrm.render_time(None, None), "")
 603         self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
 604         self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
 605         self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
 606         self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
 607         self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
 608         self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 609         self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
 610         self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
 611
 612         urrm = status.UploadResultsRendererMixin()
 613         self.failUnlessReallyEqual(urrm.render_time(None, None), "")
 614         self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
 615         self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
 616         self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
 617         self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
 618         self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
 619         self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 620         self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
 621         self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
 622
 623     def test_GET_FILEURL(self):
 624         d = self.GET(self.public_url + "/foo/bar.txt")
 625         d.addCallback(self.failUnlessIsBarDotTxt)
 626         return d
 627
 628     def test_GET_FILEURL_range(self):
 629         headers = {"range": "bytes=1-10"}
 630         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 631                      return_response=True)
 632         def _got((res, status, headers)):
 633             self.failUnlessReallyEqual(int(status), 206)
 634             self.failUnless(headers.has_key("content-range"))
 635             self.failUnlessReallyEqual(headers["content-range"][0],
 636                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 637             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
 638         d.addCallback(_got)
 639         return d
 640
 641     def test_GET_FILEURL_partial_range(self):
 642         headers = {"range": "bytes=5-"}
 643         length  = len(self.BAR_CONTENTS)
 644         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 645                      return_response=True)
 646         def _got((res, status, headers)):
 647             self.failUnlessReallyEqual(int(status), 206)
 648             self.failUnless(headers.has_key("content-range"))
 649             self.failUnlessReallyEqual(headers["content-range"][0],
 650                                        "bytes 5-%d/%d" % (length-1, length))
 651             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
 652         d.addCallback(_got)
 653         return d
 654
 655     def test_GET_FILEURL_partial_end_range(self):
 656         headers = {"range": "bytes=-5"}
 657         length  = len(self.BAR_CONTENTS)
 658         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 659                      return_response=True)
 660         def _got((res, status, headers)):
 661             self.failUnlessReallyEqual(int(status), 206)
 662             self.failUnless(headers.has_key("content-range"))
 663             self.failUnlessReallyEqual(headers["content-range"][0],
 664                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 665             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
 666         d.addCallback(_got)
 667         return d
 668
 669     def test_GET_FILEURL_partial_range_overrun(self):
 670         headers = {"range": "bytes=100-200"}
 671         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
 672                              "416 Requested Range not satisfiable",
 673                              "First beyond end of file",
 674                              self.GET, self.public_url + "/foo/bar.txt",
 675                              headers=headers)
 676         return d
 677
 678     def test_HEAD_FILEURL_range(self):
 679         headers = {"range": "bytes=1-10"}
 680         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 681                      return_response=True)
 682         def _got((res, status, headers)):
 683             self.failUnlessReallyEqual(res, "")
 684             self.failUnlessReallyEqual(int(status), 206)
 685             self.failUnless(headers.has_key("content-range"))
 686             self.failUnlessReallyEqual(headers["content-range"][0],
 687                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 688         d.addCallback(_got)
 689         return d
 690
 691     def test_HEAD_FILEURL_partial_range(self):
 692         headers = {"range": "bytes=5-"}
 693         length  = len(self.BAR_CONTENTS)
 694         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 695                      return_response=True)
 696         def _got((res, status, headers)):
 697             self.failUnlessReallyEqual(int(status), 206)
 698             self.failUnless(headers.has_key("content-range"))
 699             self.failUnlessReallyEqual(headers["content-range"][0],
 700                                        "bytes 5-%d/%d" % (length-1, length))
 701         d.addCallback(_got)
 702         return d
 703
 704     def test_HEAD_FILEURL_partial_end_range(self):
 705         headers = {"range": "bytes=-5"}
 706         length  = len(self.BAR_CONTENTS)
 707         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 708                      return_response=True)
 709         def _got((res, status, headers)):
 710             self.failUnlessReallyEqual(int(status), 206)
 711             self.failUnless(headers.has_key("content-range"))
 712             self.failUnlessReallyEqual(headers["content-range"][0],
 713                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 714         d.addCallback(_got)
 715         return d
 716
 717     def test_HEAD_FILEURL_partial_range_overrun(self):
 718         headers = {"range": "bytes=100-200"}
 719         d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
 720                              "416 Requested Range not satisfiable",
 721                              "",
 722                              self.HEAD, self.public_url + "/foo/bar.txt",
 723                              headers=headers)
 724         return d
 725
 726     def test_GET_FILEURL_range_bad(self):
 727         headers = {"range": "BOGUS=fizbop-quarnak"}
 728         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 729                      return_response=True)
 730         def _got((res, status, headers)):
 731             self.failUnlessReallyEqual(int(status), 200)
 732             self.failUnless(not headers.has_key("content-range"))
 733             self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
 734         d.addCallback(_got)
 735         return d
 736
 737     def test_HEAD_FILEURL(self):
 738         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 739         def _got((res, status, headers)):
 740             self.failUnlessReallyEqual(res, "")
 741             self.failUnlessReallyEqual(headers["content-length"][0],
 742                                        str(len(self.BAR_CONTENTS)))
 743             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
 744         d.addCallback(_got)
 745         return d
 746
 747     def test_GET_FILEURL_named(self):
 748         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 749         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 750         d = self.GET(base + "/@@name=/blah.txt")
 751         d.addCallback(self.failUnlessIsBarDotTxt)
 752         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 753         d.addCallback(self.failUnlessIsBarDotTxt)
 754         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 755         d.addCallback(self.failUnlessIsBarDotTxt)
 756         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 757         d.addCallback(self.failUnlessIsBarDotTxt)
 758         save_url = base + "?save=true&filename=blah.txt"
 759         d.addCallback(lambda res: self.GET(save_url))
 760         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 761         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 762         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 763         u_url = base + "?save=true&filename=" + u_fn_e
 764         d.addCallback(lambda res: self.GET(u_url))
 765         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 766         return d
 767
 768     def test_PUT_FILEURL_named_bad(self):
 769         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 770         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 771                              "400 Bad Request",
 772                              "/file can only be used with GET or HEAD",
 773                              self.PUT, base + "/@@name=/blah.txt", "")
 774         return d
 775
 776
 777     def test_GET_DIRURL_named_bad(self):
 778         base = "/file/%s" % urllib.quote(self._foo_uri)
 779         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 780                              "400 Bad Request",
 781                              "is not a file-cap",
 782                              self.GET, base + "/@@name=/blah.txt")
 783         return d
 784
 785     def test_GET_slash_file_bad(self):
 786         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 787                              "404 Not Found",
 788                              "/file must be followed by a file-cap and a name",
 789                              self.GET, "/file")
 790         return d
 791
 792     def test_GET_unhandled_URI_named(self):
 793         contents, n, newuri = self.makefile(12)
 794         verifier_cap = n.get_verify_cap().to_string()
 795         base = "/file/%s" % urllib.quote(verifier_cap)
 796         # client.create_node_from_uri() can't handle verify-caps
 797         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 798                              "400 Bad Request", "is not a file-cap",
 799                              self.GET, base)
 800         return d
 801
 802     def test_GET_unhandled_URI(self):
 803         contents, n, newuri = self.makefile(12)
 804         verifier_cap = n.get_verify_cap().to_string()
 805         base = "/uri/%s" % urllib.quote(verifier_cap)
 806         # client.create_node_from_uri() can't handle verify-caps
 807         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 808                              "400 Bad Request",
 809                              "GET unknown URI type: can only do t=info",
 810                              self.GET, base)
 811         return d
 812
 813     def test_GET_FILE_URI(self):
 814         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 815         d = self.GET(base)
 816         d.addCallback(self.failUnlessIsBarDotTxt)
 817         return d
 818
 819     def test_GET_FILE_URI_mdmf(self):
 820         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 821         d = self.GET(base)
 822         d.addCallback(self.failUnlessIsQuuxDotTxt)
 823         return d
 824
 825     def test_GET_FILE_URI_mdmf_extensions(self):
 826         base = "/uri/%s" % urllib.quote("%s:RANDOMSTUFF" % self._quux_txt_uri)
 827         d = self.GET(base)
 828         d.addCallback(self.failUnlessIsQuuxDotTxt)
 829         return d
 830
 831     def test_GET_FILE_URI_mdmf_readonly(self):
 832         base = "/uri/%s" % urllib.quote(self._quux_txt_readonly_uri)
 833         d = self.GET(base)
 834         d.addCallback(self.failUnlessIsQuuxDotTxt)
 835         return d
 836
 837     def test_GET_FILE_URI_badchild(self):
 838         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 839         errmsg = "Files have no children, certainly not named 'boguschild'"
 840         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 841                              "400 Bad Request", errmsg,
 842                              self.GET, base)
 843         return d
 844
 845     def test_PUT_FILE_URI_badchild(self):
 846         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 847         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 848         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 849                              "400 Bad Request", errmsg,
 850                              self.PUT, base, "")
 851         return d
 852
 853     def test_PUT_FILE_URI_mdmf(self):
 854         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 855         self._quux_new_contents = "new_contents"
 856         d = self.GET(base)
 857         d.addCallback(lambda res:
 858             self.failUnlessIsQuuxDotTxt(res))
 859         d.addCallback(lambda ignored:
 860             self.PUT(base, self._quux_new_contents))
 861         d.addCallback(lambda ignored:
 862             self.GET(base))
 863         d.addCallback(lambda res:
 864             self.failUnlessReallyEqual(res, self._quux_new_contents))
 865         return d
 866
 867     def test_PUT_FILE_URI_mdmf_extensions(self):
 868         base = "/uri/%s" % urllib.quote("%s:EXTENSIONSTUFF" % self._quux_txt_uri)
 869         self._quux_new_contents = "new_contents"
 870         d = self.GET(base)
 871         d.addCallback(lambda res: self.failUnlessIsQuuxDotTxt(res))
 872         d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
 873         d.addCallback(lambda ignored: self.GET(base))
 874         d.addCallback(lambda res: self.failUnlessEqual(self._quux_new_contents,
 875                                                        res))
 876         return d
 877
 878     def test_PUT_FILE_URI_mdmf_readonly(self):
 879         # We're not allowed to PUT things to a readonly cap.
 880         base = "/uri/%s" % self._quux_txt_readonly_uri
 881         d = self.GET(base)
 882         d.addCallback(lambda res:
 883             self.failUnlessIsQuuxDotTxt(res))
 884         # What should we get here? We get a 500 error now; that's not right.
 885         d.addCallback(lambda ignored:
 886             self.shouldFail2(error.Error, "test_PUT_FILE_URI_mdmf_readonly",
 887                              "400 Bad Request", "read-only cap",
 888                              self.PUT, base, "new data"))
 889         return d
 890
 891     def test_PUT_FILE_URI_sdmf_readonly(self):
 892         # We're not allowed to put things to a readonly cap.
 893         base = "/uri/%s" % self._baz_txt_readonly_uri
 894         d = self.GET(base)
 895         d.addCallback(lambda res:
 896             self.failUnlessIsBazDotTxt(res))
 897         d.addCallback(lambda ignored:
 898             self.shouldFail2(error.Error, "test_PUT_FILE_URI_sdmf_readonly",
 899                              "400 Bad Request", "read-only cap",
 900                              self.PUT, base, "new_data"))
 901         return d
 902
 903     def test_GET_etags(self):
 904
 905         def _check_etags(uri):
 906             d1 = _get_etag(uri)
 907             d2 = _get_etag(uri, 'json')
 908             d = defer.DeferredList([d1, d2], consumeErrors=True)
 909             def _check(results):
 910                 # All deferred must succeed
 911                 self.failUnless(all([r[0] for r in results]))
 912                 # the etag for the t=json form should be just like the etag
 913                 # fo the default t='' form, but with a 'json' suffix
 914                 self.failUnlessEqual(results[0][1] + 'json', results[1][1])
 915             d.addCallback(_check)
 916             return d
 917
 918         def _get_etag(uri, t=''):
 919             targetbase = "/uri/%s?t=%s" % (urllib.quote(uri.strip()), t)
 920             d = self.GET(targetbase, return_response=True, followRedirect=True)
 921             def _just_the_etag(result):
 922                 data, response, headers = result
 923                 etag = headers['etag'][0]
 924                 if uri.startswith('URI:DIR'):
 925                     self.failUnless(etag.startswith('DIR:'), etag)
 926                 return etag
 927             return d.addCallback(_just_the_etag)
 928
 929         # Check that etags work with immutable directories
 930         (newkids, caps) = self._create_immutable_children()
 931         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
 932                       simplejson.dumps(newkids))
 933         def _stash_immdir_uri(uri):
 934             self._immdir_uri = uri
 935             return uri
 936         d.addCallback(_stash_immdir_uri)
 937         d.addCallback(_check_etags)
 938
 939         # Check that etags work with immutable files
 940         d.addCallback(lambda _: _check_etags(self._bar_txt_uri))
 941
 942         # use the ETag on GET
 943         def _check_match(ign):
 944             uri = "/uri/%s" % self._bar_txt_uri
 945             d = self.GET(uri, return_response=True)
 946             # extract the ETag
 947             d.addCallback(lambda (data, code, headers):
 948                           headers['etag'][0])
 949             # do a GET that's supposed to match the ETag
 950             d.addCallback(lambda etag:
 951                           self.GET(uri, return_response=True,
 952                                    headers={"If-None-Match": etag}))
 953             # make sure it short-circuited (304 instead of 200)
 954             d.addCallback(lambda (data, code, headers):
 955                           self.failUnlessEqual(int(code), http.NOT_MODIFIED))
 956             return d
 957         d.addCallback(_check_match)
 958
 959         def _no_etag(uri, t):
 960             target = "/uri/%s?t=%s" % (uri, t)
 961             d = self.GET(target, return_response=True, followRedirect=True)
 962             d.addCallback(lambda (data, code, headers):
 963                           self.failIf("etag" in headers, target))
 964             return d
 965         def _yes_etag(uri, t):
 966             target = "/uri/%s?t=%s" % (uri, t)
 967             d = self.GET(target, return_response=True, followRedirect=True)
 968             d.addCallback(lambda (data, code, headers):
 969                           self.failUnless("etag" in headers, target))
 970             return d
 971
 972         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, ""))
 973         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "json"))
 974         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "uri"))
 975         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "readonly-uri"))
 976         d.addCallback(lambda ign: _no_etag(self._bar_txt_uri, "info"))
 977
 978         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, ""))
 979         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "json"))
 980         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "uri"))
 981         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "readonly-uri"))
 982         d.addCallback(lambda ign: _no_etag(self._immdir_uri, "info"))
 983         d.addCallback(lambda ign: _no_etag(self._immdir_uri, "rename-form"))
 984
 985         return d
 986
 987     # TODO: version of this with a Unicode filename
 988     def test_GET_FILEURL_save(self):
 989         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
 990                      return_response=True)
 991         def _got((res, statuscode, headers)):
 992             content_disposition = headers["content-disposition"][0]
 993             self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
 994             self.failUnlessIsBarDotTxt(res)
 995         d.addCallback(_got)
 996         return d
 997
 998     def test_GET_FILEURL_missing(self):
 999         d = self.GET(self.public_url + "/foo/missing")
1000         d.addBoth(self.should404, "test_GET_FILEURL_missing")
1001         return d
1002
1003     def test_GET_FILEURL_info_mdmf(self):
1004         d = self.GET("/uri/%s?t=info" % self._quux_txt_uri)
1005         def _got(res):
1006             self.failUnlessIn("mutable file (mdmf)", res)
1007             self.failUnlessIn(self._quux_txt_uri, res)
1008             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1009         d.addCallback(_got)
1010         return d
1011
1012     def test_GET_FILEURL_info_mdmf_readonly(self):
1013         d = self.GET("/uri/%s?t=info" % self._quux_txt_readonly_uri)
1014         def _got(res):
1015             self.failUnlessIn("mutable file (mdmf)", res)
1016             self.failIfIn(self._quux_txt_uri, res)
1017             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1018         d.addCallback(_got)
1019         return d
1020
1021     def test_GET_FILEURL_info_sdmf(self):
1022         d = self.GET("/uri/%s?t=info" % self._baz_txt_uri)
1023         def _got(res):
1024             self.failUnlessIn("mutable file (sdmf)", res)
1025             self.failUnlessIn(self._baz_txt_uri, res)
1026         d.addCallback(_got)
1027         return d
1028
1029     def test_GET_FILEURL_info_mdmf_extensions(self):
1030         d = self.GET("/uri/%s:STUFF?t=info" % self._quux_txt_uri)
1031         def _got(res):
1032             self.failUnlessIn("mutable file (mdmf)", res)
1033             self.failUnlessIn(self._quux_txt_uri, res)
1034             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1035         d.addCallback(_got)
1036         return d
1037
1038     def test_PUT_overwrite_only_files(self):
1039         # create a directory, put a file in that directory.
1040         contents, n, filecap = self.makefile(8)
1041         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
1042         d.addCallback(lambda res:
1043             self.PUT(self.public_url + "/foo/dir/file1.txt",
1044                      self.NEWFILE_CONTENTS))
1045         # try to overwrite the file with replace=only-files
1046         # (this should work)
1047         d.addCallback(lambda res:
1048             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
1049                      filecap))
1050         d.addCallback(lambda res:
1051             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
1052                  "There was already a child by that name, and you asked me "
1053                  "to not replace it",
1054                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
1055                  filecap))
1056         return d
1057
1058     def test_PUT_NEWFILEURL(self):
1059         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
1060         # TODO: we lose the response code, so we can't check this
1061         #self.failUnlessReallyEqual(responsecode, 201)
1062         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1063         d.addCallback(lambda res:
1064                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1065                                                       self.NEWFILE_CONTENTS))
1066         return d
1067
1068     def test_PUT_NEWFILEURL_not_mutable(self):
1069         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
1070                      self.NEWFILE_CONTENTS)
1071         # TODO: we lose the response code, so we can't check this
1072         #self.failUnlessReallyEqual(responsecode, 201)
1073         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1074         d.addCallback(lambda res:
1075                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1076                                                       self.NEWFILE_CONTENTS))
1077         return d
1078
1079     def test_PUT_NEWFILEURL_unlinked_mdmf(self):
1080         # this should get us a few segments of an MDMF mutable file,
1081         # which we can then test for.
1082         contents = self.NEWFILE_CONTENTS * 300000
1083         d = self.PUT("/uri?format=mdmf",
1084                      contents)
1085         def _got_filecap(filecap):
1086             self.failUnless(filecap.startswith("URI:MDMF"))
1087             return filecap
1088         d.addCallback(_got_filecap)
1089         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1090         d.addCallback(lambda json: self.failUnlessIn("MDMF", json))
1091         return d
1092
1093     def test_PUT_NEWFILEURL_unlinked_sdmf(self):
1094         contents = self.NEWFILE_CONTENTS * 300000
1095         d = self.PUT("/uri?format=sdmf",
1096                      contents)
1097         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1098         d.addCallback(lambda json: self.failUnlessIn("SDMF", json))
1099         return d
1100
1101     def test_PUT_NEWFILEURL_unlinked_bad_format(self):
1102         contents = self.NEWFILE_CONTENTS * 300000
1103         return self.shouldHTTPError("PUT_NEWFILEURL_unlinked_bad_format",
1104                                     400, "Bad Request", "Unknown format: foo",
1105                                     self.PUT, "/uri?format=foo",
1106                                     contents)
1107
1108     def test_PUT_NEWFILEURL_range_bad(self):
1109         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
1110         target = self.public_url + "/foo/new.txt"
1111         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
1112                              "501 Not Implemented",
1113                              "Content-Range in PUT not yet supported",
1114                              # (and certainly not for immutable files)
1115                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
1116                              headers=headers)
1117         d.addCallback(lambda res:
1118                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
1119         return d
1120
1121     def test_PUT_NEWFILEURL_mutable(self):
1122         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1123                      self.NEWFILE_CONTENTS)
1124         # TODO: we lose the response code, so we can't check this
1125         #self.failUnlessReallyEqual(responsecode, 201)
1126         def _check_uri(res):
1127             u = uri.from_string_mutable_filenode(res)
1128             self.failUnless(u.is_mutable())
1129             self.failIf(u.is_readonly())
1130             return res
1131         d.addCallback(_check_uri)
1132         d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
1133         d.addCallback(lambda res:
1134                       self.failUnlessMutableChildContentsAre(self._foo_node,
1135                                                              u"new.txt",
1136                                                              self.NEWFILE_CONTENTS))
1137         return d
1138
1139     def test_PUT_NEWFILEURL_mutable_toobig(self):
1140         # It is okay to upload large mutable files, so we should be able
1141         # to do that.
1142         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1143                      "b" * (self.s.MUTABLE_SIZELIMIT + 1))
1144         return d
1145
1146     def test_PUT_NEWFILEURL_replace(self):
1147         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
1148         # TODO: we lose the response code, so we can't check this
1149         #self.failUnlessReallyEqual(responsecode, 200)
1150         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
1151         d.addCallback(lambda res:
1152                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1153                                                       self.NEWFILE_CONTENTS))
1154         return d
1155
1156     def test_PUT_NEWFILEURL_bad_t(self):
1157         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
1158                              "PUT to a file: bad t=bogus",
1159                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
1160                              "contents")
1161         return d
1162
1163     def test_PUT_NEWFILEURL_no_replace(self):
1164         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
1165                      self.NEWFILE_CONTENTS)
1166         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
1167                   "409 Conflict",
1168                   "There was already a child by that name, and you asked me "
1169                   "to not replace it")
1170         return d
1171
1172     def test_PUT_NEWFILEURL_mkdirs(self):
1173         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
1174         fn = self._foo_node
1175         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
1176         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
1177         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
1178         d.addCallback(lambda res:
1179                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
1180                                                       self.NEWFILE_CONTENTS))
1181         return d
1182
1183     def test_PUT_NEWFILEURL_blocked(self):
1184         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
1185                      self.NEWFILE_CONTENTS)
1186         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
1187                   "409 Conflict",
1188                   "Unable to create directory 'blockingfile': a file was in the way")
1189         return d
1190
1191     def test_PUT_NEWFILEURL_emptyname(self):
1192         # an empty pathname component (i.e. a double-slash) is disallowed
1193         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
1194                              "400 Bad Request",
1195                              "The webapi does not allow empty pathname components",
1196                              self.PUT, self.public_url + "/foo//new.txt", "")
1197         return d
1198
1199     def test_DELETE_FILEURL(self):
1200         d = self.DELETE(self.public_url + "/foo/bar.txt")
1201         d.addCallback(lambda res:
1202                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1203         return d
1204
1205     def test_DELETE_FILEURL_missing(self):
1206         d = self.DELETE(self.public_url + "/foo/missing")
1207         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
1208         return d
1209
1210     def test_DELETE_FILEURL_missing2(self):
1211         d = self.DELETE(self.public_url + "/missing/missing")
1212         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
1213         return d
1214
1215     def failUnlessHasBarDotTxtMetadata(self, res):
1216         data = simplejson.loads(res)
1217         self.failUnless(isinstance(data, list))
1218         self.failUnlessIn("metadata", data[1])
1219         self.failUnlessIn("tahoe", data[1]["metadata"])
1220         self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
1221         self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
1222         self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
1223                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
1224
1225     def test_GET_FILEURL_json(self):
1226         # twisted.web.http.parse_qs ignores any query args without an '=', so
1227         # I can't do "GET /path?json", I have to do "GET /path/t=json"
1228         # instead. This may make it tricky to emulate the S3 interface
1229         # completely.
1230         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
1231         def _check1(data):
1232             self.failUnlessIsBarJSON(data)
1233             self.failUnlessHasBarDotTxtMetadata(data)
1234             return
1235         d.addCallback(_check1)
1236         return d
1237
1238     def test_GET_FILEURL_json_mutable_type(self):
1239         # The JSON should include format, which says whether the
1240         # file is SDMF or MDMF
1241         d = self.PUT("/uri?format=mdmf",
1242                      self.NEWFILE_CONTENTS * 300000)
1243         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1244         def _got_json(json, version):
1245             data = simplejson.loads(json)
1246             assert "filenode" == data[0]
1247             data = data[1]
1248             assert isinstance(data, dict)
1249
1250             self.failUnlessIn("format", data)
1251             self.failUnlessEqual(data["format"], version)
1252
1253         d.addCallback(_got_json, "MDMF")
1254         # Now make an SDMF file and check that it is reported correctly.
1255         d.addCallback(lambda ignored:
1256             self.PUT("/uri?format=sdmf",
1257                       self.NEWFILE_CONTENTS * 300000))
1258         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1259         d.addCallback(_got_json, "SDMF")
1260         return d
1261
1262     def test_GET_FILEURL_json_mdmf(self):
1263         d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
1264         d.addCallback(self.failUnlessIsQuuxJSON)
1265         return d
1266
1267     def test_GET_FILEURL_json_missing(self):
1268         d = self.GET(self.public_url + "/foo/missing?json")
1269         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
1270         return d
1271
1272     def test_GET_FILEURL_uri(self):
1273         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
1274         def _check(res):
1275             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1276         d.addCallback(_check)
1277         d.addCallback(lambda res:
1278                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
1279         def _check2(res):
1280             # for now, for files, uris and readonly-uris are the same
1281             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1282         d.addCallback(_check2)
1283         return d
1284
1285     def test_GET_FILEURL_badtype(self):
1286         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
1287                                  "bad t=bogus",
1288                                  self.GET,
1289                                  self.public_url + "/foo/bar.txt?t=bogus")
1290         return d
1291
1292     def test_CSS_FILE(self):
1293         d = self.GET("/tahoe.css", followRedirect=True)
1294         def _check(res):
1295             CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
1296             self.failUnless(CSS_STYLE.search(res), res)
1297         d.addCallback(_check)
1298         return d
1299
1300     def test_GET_FILEURL_uri_missing(self):
1301         d = self.GET(self.public_url + "/foo/missing?t=uri")
1302         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
1303         return d
1304
1305     def _check_upload_and_mkdir_forms(self, html):
1306         # We should have a form to create a file, with radio buttons that allow
1307         # the user to toggle whether it is a CHK/LIT (default), SDMF, or MDMF file.
1308         self.failUnlessIn('name="t" value="upload"', html)
1309         self.failUnlessIn('input checked="checked" type="radio" id="upload-chk" value="chk" name="format"', html)
1310         self.failUnlessIn('input type="radio" id="upload-sdmf" value="sdmf" name="format"', html)
1311         self.failUnlessIn('input type="radio" id="upload-mdmf" value="mdmf" name="format"', html)
1312
1313         # We should also have the ability to create a mutable directory, with
1314         # radio buttons that allow the user to toggle whether it is an SDMF (default)
1315         # or MDMF directory.
1316         self.failUnlessIn('name="t" value="mkdir"', html)
1317         self.failUnlessIn('input checked="checked" type="radio" id="mkdir-sdmf" value="sdmf" name="format"', html)
1318         self.failUnlessIn('input type="radio" id="mkdir-mdmf" value="mdmf" name="format"', html)
1319
1320         self.failUnlessIn(FAVICON_MARKUP, html)
1321
1322     def test_GET_DIRECTORY_html(self):
1323         d = self.GET(self.public_url + "/foo", followRedirect=True)
1324         def _check(html):
1325             self.failUnlessIn('<div class="toolbar-item"><a href="../../..">Return to Welcome page</a></div>', html)
1326             self._check_upload_and_mkdir_forms(html)
1327             self.failUnlessIn("quux", html)
1328         d.addCallback(_check)
1329         return d
1330
1331     def test_GET_root_html(self):
1332         d = self.GET("/")
1333         d.addCallback(self._check_upload_and_mkdir_forms)
1334         return d
1335
1336     def test_GET_DIRURL(self):
1337         # the addSlash means we get a redirect here
1338         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1339         ROOT = "../../.."
1340         d = self.GET(self.public_url + "/foo", followRedirect=True)
1341         def _check(res):
1342             self.failUnlessIn('<a href="%s">Return to Welcome page' % ROOT, res)
1343
1344             # the FILE reference points to a URI, but it should end in bar.txt
1345             bar_url = ("%s/file/%s/@@named=/bar.txt" %
1346                        (ROOT, urllib.quote(self._bar_txt_uri)))
1347             get_bar = "".join([r'<td>FILE</td>',
1348                                r'\s+<td>',
1349                                r'<a href="%s">bar.txt</a>' % bar_url,
1350                                r'</td>',
1351                                r'\s+<td align="right">%d</td>' % len(self.BAR_CONTENTS),
1352                                ])
1353             self.failUnless(re.search(get_bar, res), res)
1354             for label in ['unlink', 'rename/move']:
1355                 for line in res.split("\n"):
1356                     # find the line that contains the relevant button for bar.txt
1357                     if ("form action" in line and
1358                         ('value="%s"' % (label,)) in line and
1359                         'value="bar.txt"' in line):
1360                         # the form target should use a relative URL
1361                         foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1362                         self.failUnlessIn('action="%s"' % foo_url, line)
1363                         # and the when_done= should too
1364                         #done_url = urllib.quote(???)
1365                         #self.failUnlessIn('name="when_done" value="%s"' % done_url, line)
1366
1367                         # 'unlink' needs to use POST because it directly has a side effect
1368                         if label == 'unlink':
1369                             self.failUnlessIn('method="post"', line)
1370                         break
1371                 else:
1372                     self.fail("unable to find '%s bar.txt' line" % (label,))
1373
1374             # the DIR reference just points to a URI
1375             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1376             get_sub = ((r'<td>DIR</td>')
1377                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1378             self.failUnless(re.search(get_sub, res), res)
1379         d.addCallback(_check)
1380
1381         # look at a readonly directory
1382         d.addCallback(lambda res:
1383                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
1384         def _check2(res):
1385             self.failUnlessIn("(read-only)", res)
1386             self.failIfIn("Upload a file", res)
1387         d.addCallback(_check2)
1388
1389         # and at a directory that contains a readonly directory
1390         d.addCallback(lambda res:
1391                       self.GET(self.public_url, followRedirect=True))
1392         def _check3(res):
1393             self.failUnless(re.search('<td>DIR-RO</td>'
1394                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1395         d.addCallback(_check3)
1396
1397         # and an empty directory
1398         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1399         def _check4(res):
1400             self.failUnlessIn("directory is empty", res)
1401             MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
1402             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1403         d.addCallback(_check4)
1404
1405         # and at a literal directory
1406         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1407         d.addCallback(lambda res:
1408                       self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1409         def _check5(res):
1410             self.failUnlessIn('(immutable)', res)
1411             self.failUnless(re.search('<td>FILE</td>'
1412                                       r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1413         d.addCallback(_check5)
1414         return d
1415
1416     def test_GET_DIRURL_badtype(self):
1417         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1418                                  400, "Bad Request",
1419                                  "bad t=bogus",
1420                                  self.GET,
1421                                  self.public_url + "/foo?t=bogus")
1422         return d
1423
1424     def test_GET_DIRURL_json(self):
1425         d = self.GET(self.public_url + "/foo?t=json")
1426         d.addCallback(self.failUnlessIsFooJSON)
1427         return d
1428
1429     def test_GET_DIRURL_json_format(self):
1430         d = self.PUT(self.public_url + \
1431                      "/foo/sdmf.txt?format=sdmf",
1432                      self.NEWFILE_CONTENTS * 300000)
1433         d.addCallback(lambda ignored:
1434             self.PUT(self.public_url + \
1435                      "/foo/mdmf.txt?format=mdmf",
1436                      self.NEWFILE_CONTENTS * 300000))
1437         # Now we have an MDMF and SDMF file in the directory. If we GET
1438         # its JSON, we should see their encodings.
1439         d.addCallback(lambda ignored:
1440             self.GET(self.public_url + "/foo?t=json"))
1441         def _got_json(json):
1442             data = simplejson.loads(json)
1443             assert data[0] == "dirnode"
1444
1445             data = data[1]
1446             kids = data['children']
1447
1448             mdmf_data = kids['mdmf.txt'][1]
1449             self.failUnlessIn("format", mdmf_data)
1450             self.failUnlessEqual(mdmf_data["format"], "MDMF")
1451
1452             sdmf_data = kids['sdmf.txt'][1]
1453             self.failUnlessIn("format", sdmf_data)
1454             self.failUnlessEqual(sdmf_data["format"], "SDMF")
1455         d.addCallback(_got_json)
1456         return d
1457
1458
1459     def test_POST_DIRURL_manifest_no_ophandle(self):
1460         d = self.shouldFail2(error.Error,
1461                              "test_POST_DIRURL_manifest_no_ophandle",
1462                              "400 Bad Request",
1463                              "slow operation requires ophandle=",
1464                              self.POST, self.public_url, t="start-manifest")
1465         return d
1466
1467     def test_POST_DIRURL_manifest(self):
1468         d = defer.succeed(None)
1469         def getman(ignored, output):
1470             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1471                           followRedirect=True)
1472             d.addCallback(self.wait_for_operation, "125")
1473             d.addCallback(self.get_operation_results, "125", output)
1474             return d
1475         d.addCallback(getman, None)
1476         def _got_html(manifest):
1477             self.failUnlessIn("Manifest of SI=", manifest)
1478             self.failUnlessIn("<td>sub</td>", manifest)
1479             self.failUnlessIn(self._sub_uri, manifest)
1480             self.failUnlessIn("<td>sub/baz.txt</td>", manifest)
1481             self.failUnlessIn(FAVICON_MARKUP, manifest)
1482         d.addCallback(_got_html)
1483
1484         # both t=status and unadorned GET should be identical
1485         d.addCallback(lambda res: self.GET("/operations/125"))
1486         d.addCallback(_got_html)
1487
1488         d.addCallback(getman, "html")
1489         d.addCallback(_got_html)
1490         d.addCallback(getman, "text")
1491         def _got_text(manifest):
1492             self.failUnlessIn("\nsub " + self._sub_uri + "\n", manifest)
1493             self.failUnlessIn("\nsub/baz.txt URI:CHK:", manifest)
1494         d.addCallback(_got_text)
1495         d.addCallback(getman, "JSON")
1496         def _got_json(res):
1497             data = res["manifest"]
1498             got = {}
1499             for (path_list, cap) in data:
1500                 got[tuple(path_list)] = cap
1501             self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
1502             self.failUnlessIn((u"sub", u"baz.txt"), got)
1503             self.failUnlessIn("finished", res)
1504             self.failUnlessIn("origin", res)
1505             self.failUnlessIn("storage-index", res)
1506             self.failUnlessIn("verifycaps", res)
1507             self.failUnlessIn("stats", res)
1508         d.addCallback(_got_json)
1509         return d
1510
1511     def test_POST_DIRURL_deepsize_no_ophandle(self):
1512         d = self.shouldFail2(error.Error,
1513                              "test_POST_DIRURL_deepsize_no_ophandle",
1514                              "400 Bad Request",
1515                              "slow operation requires ophandle=",
1516                              self.POST, self.public_url, t="start-deep-size")
1517         return d
1518
1519     def test_POST_DIRURL_deepsize(self):
1520         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1521                       followRedirect=True)
1522         d.addCallback(self.wait_for_operation, "126")
1523         d.addCallback(self.get_operation_results, "126", "json")
1524         def _got_json(data):
1525             self.failUnlessReallyEqual(data["finished"], True)
1526             size = data["size"]
1527             self.failUnless(size > 1000)
1528         d.addCallback(_got_json)
1529         d.addCallback(self.get_operation_results, "126", "text")
1530         def _got_text(res):
1531             mo = re.search(r'^size: (\d+)$', res, re.M)
1532             self.failUnless(mo, res)
1533             size = int(mo.group(1))
1534             # with directories, the size varies.
1535             self.failUnless(size > 1000)
1536         d.addCallback(_got_text)
1537         return d
1538
1539     def test_POST_DIRURL_deepstats_no_ophandle(self):
1540         d = self.shouldFail2(error.Error,
1541                              "test_POST_DIRURL_deepstats_no_ophandle",
1542                              "400 Bad Request",
1543                              "slow operation requires ophandle=",
1544                              self.POST, self.public_url, t="start-deep-stats")
1545         return d
1546
1547     def test_POST_DIRURL_deepstats(self):
1548         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1549                       followRedirect=True)
1550         d.addCallback(self.wait_for_operation, "127")
1551         d.addCallback(self.get_operation_results, "127", "json")
1552         def _got_json(stats):
1553             expected = {"count-immutable-files": 3,
1554                         "count-mutable-files": 2,
1555                         "count-literal-files": 0,
1556                         "count-files": 5,
1557                         "count-directories": 3,
1558                         "size-immutable-files": 57,
1559                         "size-literal-files": 0,
1560                         #"size-directories": 1912, # varies
1561                         #"largest-directory": 1590,
1562                         "largest-directory-children": 7,
1563                         "largest-immutable-file": 19,
1564                         }
1565             for k,v in expected.iteritems():
1566                 self.failUnlessReallyEqual(stats[k], v,
1567                                            "stats[%s] was %s, not %s" %
1568                                            (k, stats[k], v))
1569             self.failUnlessReallyEqual(stats["size-files-histogram"],
1570                                        [ [11, 31, 3] ])
1571         d.addCallback(_got_json)
1572         return d
1573
1574     def test_POST_DIRURL_stream_manifest(self):
1575         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1576         def _check(res):
1577             self.failUnless(res.endswith("\n"))
1578             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1579             self.failUnlessReallyEqual(len(units), 9)
1580             self.failUnlessEqual(units[-1]["type"], "stats")
1581             first = units[0]
1582             self.failUnlessEqual(first["path"], [])
1583             self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri)
1584             self.failUnlessEqual(first["type"], "directory")
1585             baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0]
1586             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1587             self.failIfEqual(baz["storage-index"], None)
1588             self.failIfEqual(baz["verifycap"], None)
1589             self.failIfEqual(baz["repaircap"], None)
1590             # XXX: Add quux and baz to this test.
1591             return
1592         d.addCallback(_check)
1593         return d
1594
1595     def test_GET_DIRURL_uri(self):
1596         d = self.GET(self.public_url + "/foo?t=uri")
1597         def _check(res):
1598             self.failUnlessReallyEqual(to_str(res), self._foo_uri)
1599         d.addCallback(_check)
1600         return d
1601
1602     def test_GET_DIRURL_readonly_uri(self):
1603         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1604         def _check(res):
1605             self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri)
1606         d.addCallback(_check)
1607         return d
1608
1609     def test_PUT_NEWDIRURL(self):
1610         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1611         d.addCallback(lambda res:
1612                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1613         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1614         d.addCallback(self.failUnlessNodeKeysAre, [])
1615         return d
1616
1617     def test_PUT_NEWDIRURL_mdmf(self):
1618         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1619         d.addCallback(lambda res:
1620                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1621         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1622         d.addCallback(lambda node:
1623             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1624         return d
1625
1626     def test_PUT_NEWDIRURL_sdmf(self):
1627         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=sdmf",
1628                      "")
1629         d.addCallback(lambda res:
1630                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1631         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1632         d.addCallback(lambda node:
1633             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1634         return d
1635
1636     def test_PUT_NEWDIRURL_bad_format(self):
1637         return self.shouldHTTPError("PUT_NEWDIRURL_bad_format",
1638                                     400, "Bad Request", "Unknown format: foo",
1639                                     self.PUT, self.public_url +
1640                                     "/foo/newdir=?t=mkdir&format=foo", "")
1641
1642     def test_POST_NEWDIRURL(self):
1643         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1644         d.addCallback(lambda res:
1645                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1646         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1647         d.addCallback(self.failUnlessNodeKeysAre, [])
1648         return d
1649
1650     def test_POST_NEWDIRURL_mdmf(self):
1651         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1652         d.addCallback(lambda res:
1653                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1654         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1655         d.addCallback(lambda node:
1656             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1657         return d
1658
1659     def test_POST_NEWDIRURL_sdmf(self):
1660         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=sdmf", "")
1661         d.addCallback(lambda res:
1662             self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1663         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1664         d.addCallback(lambda node:
1665             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1666         return d
1667
1668     def test_POST_NEWDIRURL_bad_format(self):
1669         return self.shouldHTTPError("POST_NEWDIRURL_bad_format",
1670                                     400, "Bad Request", "Unknown format: foo",
1671                                     self.POST2, self.public_url + \
1672                                     "/foo/newdir?t=mkdir&format=foo", "")
1673
1674     def test_POST_NEWDIRURL_emptyname(self):
1675         # an empty pathname component (i.e. a double-slash) is disallowed
1676         d = self.shouldFail2(error.Error, "POST_NEWDIRURL_emptyname",
1677                              "400 Bad Request",
1678                              "The webapi does not allow empty pathname components, i.e. a double slash",
1679                              self.POST, self.public_url + "//?t=mkdir")
1680         return d
1681
1682     def _do_POST_NEWDIRURL_initial_children_test(self, version=None):
1683         (newkids, caps) = self._create_initial_children()
1684         query = "/foo/newdir?t=mkdir-with-children"
1685         if version == MDMF_VERSION:
1686             query += "&format=mdmf"
1687         elif version == SDMF_VERSION:
1688             query += "&format=sdmf"
1689         else:
1690             version = SDMF_VERSION # for later
1691         d = self.POST2(self.public_url + query,
1692                        simplejson.dumps(newkids))
1693         def _check(uri):
1694             n = self.s.create_node_from_uri(uri.strip())
1695             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1696             self.failUnlessEqual(n._node.get_version(), version)
1697             d2.addCallback(lambda ign:
1698                            self.failUnlessROChildURIIs(n, u"child-imm",
1699                                                        caps['filecap1']))
1700             d2.addCallback(lambda ign:
1701                            self.failUnlessRWChildURIIs(n, u"child-mutable",
1702                                                        caps['filecap2']))
1703             d2.addCallback(lambda ign:
1704                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1705                                                        caps['filecap3']))
1706             d2.addCallback(lambda ign:
1707                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1708                                                        caps['unknown_rocap']))
1709             d2.addCallback(lambda ign:
1710                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1711                                                        caps['unknown_rwcap']))
1712             d2.addCallback(lambda ign:
1713                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1714                                                        caps['unknown_immcap']))
1715             d2.addCallback(lambda ign:
1716                            self.failUnlessRWChildURIIs(n, u"dirchild",
1717                                                        caps['dircap']))
1718             d2.addCallback(lambda ign:
1719                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1720                                                        caps['litdircap']))
1721             d2.addCallback(lambda ign:
1722                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1723                                                        caps['emptydircap']))
1724             return d2
1725         d.addCallback(_check)
1726         d.addCallback(lambda res:
1727                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1728         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1729         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1730         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1731         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1732         return d
1733
1734     def test_POST_NEWDIRURL_initial_children(self):
1735         return self._do_POST_NEWDIRURL_initial_children_test()
1736
1737     def test_POST_NEWDIRURL_initial_children_mdmf(self):
1738         return self._do_POST_NEWDIRURL_initial_children_test(MDMF_VERSION)
1739
1740     def test_POST_NEWDIRURL_initial_children_sdmf(self):
1741         return self._do_POST_NEWDIRURL_initial_children_test(SDMF_VERSION)
1742
1743     def test_POST_NEWDIRURL_initial_children_bad_format(self):
1744         (newkids, caps) = self._create_initial_children()
1745         return self.shouldHTTPError("POST_NEWDIRURL_initial_children_bad_format",
1746                                     400, "Bad Request", "Unknown format: foo",
1747                                     self.POST2, self.public_url + \
1748                                     "/foo/newdir?t=mkdir-with-children&format=foo",
1749                                     simplejson.dumps(newkids))
1750
1751     def test_POST_NEWDIRURL_immutable(self):
1752         (newkids, caps) = self._create_immutable_children()
1753         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1754                        simplejson.dumps(newkids))
1755         def _check(uri):
1756             n = self.s.create_node_from_uri(uri.strip())
1757             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1758             d2.addCallback(lambda ign:
1759                            self.failUnlessROChildURIIs(n, u"child-imm",
1760                                                        caps['filecap1']))
1761             d2.addCallback(lambda ign:
1762                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1763                                                        caps['unknown_immcap']))
1764             d2.addCallback(lambda ign:
1765                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
1766                                                        caps['immdircap']))
1767             d2.addCallback(lambda ign:
1768                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1769                                                        caps['litdircap']))
1770             d2.addCallback(lambda ign:
1771                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1772                                                        caps['emptydircap']))
1773             return d2
1774         d.addCallback(_check)
1775         d.addCallback(lambda res:
1776                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1777         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1778         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1779         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1780         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1781         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1782         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1783         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1784         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
1785         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1786         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
1787         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1788         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
1789         d.addErrback(self.explain_web_error)
1790         return d
1791
1792     def test_POST_NEWDIRURL_immutable_bad(self):
1793         (newkids, caps) = self._create_initial_children()
1794         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
1795                              "400 Bad Request",
1796                              "needed to be immutable but was not",
1797                              self.POST2,
1798                              self.public_url + "/foo/newdir?t=mkdir-immutable",
1799                              simplejson.dumps(newkids))
1800         return d
1801
1802     def test_PUT_NEWDIRURL_exists(self):
1803         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1804         d.addCallback(lambda res:
1805                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1806         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1807         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1808         return d
1809
1810     def test_PUT_NEWDIRURL_blocked(self):
1811         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1812                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1813                              self.PUT,
1814                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1815         d.addCallback(lambda res:
1816                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1817         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1818         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1819         return d
1820
1821     def test_PUT_NEWDIRURL_mkdirs(self):
1822         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1823         d.addCallback(lambda res:
1824                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1825         d.addCallback(lambda res:
1826                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1827         d.addCallback(lambda res:
1828                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1829         d.addCallback(self.failUnlessNodeKeysAre, [])
1830         return d
1831
1832     def test_PUT_NEWDIRURL_mkdirs_mdmf(self):
1833         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=mdmf", "")
1834         d.addCallback(lambda ignored:
1835             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1836         d.addCallback(lambda ignored:
1837             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1838         d.addCallback(lambda ignored:
1839             self._foo_node.get_child_at_path(u"subdir"))
1840         def _got_subdir(subdir):
1841             # XXX: What we want?
1842             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1843             self.failUnlessNodeHasChild(subdir, u"newdir")
1844             return subdir.get_child_at_path(u"newdir")
1845         d.addCallback(_got_subdir)
1846         d.addCallback(lambda newdir:
1847             self.failUnlessEqual(newdir._node.get_version(), MDMF_VERSION))
1848         return d
1849
1850     def test_PUT_NEWDIRURL_mkdirs_sdmf(self):
1851         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=sdmf", "")
1852         d.addCallback(lambda ignored:
1853             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1854         d.addCallback(lambda ignored:
1855             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1856         d.addCallback(lambda ignored:
1857             self._foo_node.get_child_at_path(u"subdir"))
1858         def _got_subdir(subdir):
1859             # XXX: What we want?
1860             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1861             self.failUnlessNodeHasChild(subdir, u"newdir")
1862             return subdir.get_child_at_path(u"newdir")
1863         d.addCallback(_got_subdir)
1864         d.addCallback(lambda newdir:
1865             self.failUnlessEqual(newdir._node.get_version(), SDMF_VERSION))
1866         return d
1867
1868     def test_PUT_NEWDIRURL_mkdirs_bad_format(self):
1869         return self.shouldHTTPError("PUT_NEWDIRURL_mkdirs_bad_format",
1870                                     400, "Bad Request", "Unknown format: foo",
1871                                     self.PUT, self.public_url + \
1872                                     "/foo/subdir/newdir?t=mkdir&format=foo",
1873                                     "")
1874
1875     def test_DELETE_DIRURL(self):
1876         d = self.DELETE(self.public_url + "/foo")
1877         d.addCallback(lambda res:
1878                       self.failIfNodeHasChild(self.public_root, u"foo"))
1879         return d
1880
1881     def test_DELETE_DIRURL_missing(self):
1882         d = self.DELETE(self.public_url + "/foo/missing")
1883         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1884         d.addCallback(lambda res:
1885                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1886         return d
1887
1888     def test_DELETE_DIRURL_missing2(self):
1889         d = self.DELETE(self.public_url + "/missing")
1890         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1891         return d
1892
1893     def dump_root(self):
1894         print "NODEWALK"
1895         w = webish.DirnodeWalkerMixin()
1896         def visitor(childpath, childnode, metadata):
1897             print childpath
1898         d = w.walk(self.public_root, visitor)
1899         return d
1900
1901     def failUnlessNodeKeysAre(self, node, expected_keys):
1902         for k in expected_keys:
1903             assert isinstance(k, unicode)
1904         d = node.list()
1905         def _check(children):
1906             self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
1907         d.addCallback(_check)
1908         return d
1909     def failUnlessNodeHasChild(self, node, name):
1910         assert isinstance(name, unicode)
1911         d = node.list()
1912         def _check(children):
1913             self.failUnlessIn(name, children)
1914         d.addCallback(_check)
1915         return d
1916     def failIfNodeHasChild(self, node, name):
1917         assert isinstance(name, unicode)
1918         d = node.list()
1919         def _check(children):
1920             self.failIfIn(name, children)
1921         d.addCallback(_check)
1922         return d
1923
1924     def failUnlessChildContentsAre(self, node, name, expected_contents):
1925         assert isinstance(name, unicode)
1926         d = node.get_child_at_path(name)
1927         d.addCallback(lambda node: download_to_data(node))
1928         def _check(contents):
1929             self.failUnlessReallyEqual(contents, expected_contents)
1930         d.addCallback(_check)
1931         return d
1932
1933     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1934         assert isinstance(name, unicode)
1935         d = node.get_child_at_path(name)
1936         d.addCallback(lambda node: node.download_best_version())
1937         def _check(contents):
1938             self.failUnlessReallyEqual(contents, expected_contents)
1939         d.addCallback(_check)
1940         return d
1941
1942     def failUnlessRWChildURIIs(self, node, name, expected_uri):
1943         assert isinstance(name, unicode)
1944         d = node.get_child_at_path(name)
1945         def _check(child):
1946             self.failUnless(child.is_unknown() or not child.is_readonly())
1947             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1948             self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
1949             expected_ro_uri = self._make_readonly(expected_uri)
1950             if expected_ro_uri:
1951                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1952         d.addCallback(_check)
1953         return d
1954
1955     def failUnlessROChildURIIs(self, node, name, expected_uri):
1956         assert isinstance(name, unicode)
1957         d = node.get_child_at_path(name)
1958         def _check(child):
1959             self.failUnless(child.is_unknown() or child.is_readonly())
1960             self.failUnlessReallyEqual(child.get_write_uri(), None)
1961             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1962             self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
1963         d.addCallback(_check)
1964         return d
1965
1966     def failUnlessURIMatchesRWChild(self, got_uri, node, name):
1967         assert isinstance(name, unicode)
1968         d = node.get_child_at_path(name)
1969         def _check(child):
1970             self.failUnless(child.is_unknown() or not child.is_readonly())
1971             self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
1972             self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
1973             expected_ro_uri = self._make_readonly(got_uri)
1974             if expected_ro_uri:
1975                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1976         d.addCallback(_check)
1977         return d
1978
1979     def failUnlessURIMatchesROChild(self, got_uri, node, name):
1980         assert isinstance(name, unicode)
1981         d = node.get_child_at_path(name)
1982         def _check(child):
1983             self.failUnless(child.is_unknown() or child.is_readonly())
1984             self.failUnlessReallyEqual(child.get_write_uri(), None)
1985             self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
1986             self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
1987         d.addCallback(_check)
1988         return d
1989
1990     def failUnlessCHKURIHasContents(self, got_uri, contents):
1991         self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
1992
1993     def test_POST_upload(self):
1994         d = self.POST(self.public_url + "/foo", t="upload",
1995                       file=("new.txt", self.NEWFILE_CONTENTS))
1996         fn = self._foo_node
1997         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
1998         d.addCallback(lambda res:
1999                       self.failUnlessChildContentsAre(fn, u"new.txt",
2000                                                       self.NEWFILE_CONTENTS))
2001         return d
2002
2003     def test_POST_upload_unicode(self):
2004         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2005         d = self.POST(self.public_url + "/foo", t="upload",
2006                       file=(filename, self.NEWFILE_CONTENTS))
2007         fn = self._foo_node
2008         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2009         d.addCallback(lambda res:
2010                       self.failUnlessChildContentsAre(fn, filename,
2011                                                       self.NEWFILE_CONTENTS))
2012         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2013         d.addCallback(lambda res: self.GET(target_url))
2014         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2015                                                                   self.NEWFILE_CONTENTS,
2016                                                                   contents))
2017         return d
2018
2019     def test_POST_upload_unicode_named(self):
2020         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2021         d = self.POST(self.public_url + "/foo", t="upload",
2022                       name=filename,
2023                       file=("overridden", self.NEWFILE_CONTENTS))
2024         fn = self._foo_node
2025         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2026         d.addCallback(lambda res:
2027                       self.failUnlessChildContentsAre(fn, filename,
2028                                                       self.NEWFILE_CONTENTS))
2029         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2030         d.addCallback(lambda res: self.GET(target_url))
2031         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2032                                                                   self.NEWFILE_CONTENTS,
2033                                                                   contents))
2034         return d
2035
2036     def test_POST_upload_no_link(self):
2037         d = self.POST("/uri", t="upload",
2038                       file=("new.txt", self.NEWFILE_CONTENTS))
2039         def _check_upload_results(page):
2040             # this should be a page which describes the results of the upload
2041             # that just finished.
2042             self.failUnlessIn("Upload Results:", page)
2043             self.failUnlessIn("URI:", page)
2044             uri_re = re.compile("URI: <tt><span>(.*)</span>")
2045             mo = uri_re.search(page)
2046             self.failUnless(mo, page)
2047             new_uri = mo.group(1)
2048             return new_uri
2049         d.addCallback(_check_upload_results)
2050         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
2051         return d
2052
2053     def test_POST_upload_no_link_whendone(self):
2054         d = self.POST("/uri", t="upload", when_done="/",
2055                       file=("new.txt", self.NEWFILE_CONTENTS))
2056         d.addBoth(self.shouldRedirect, "/")
2057         return d
2058
2059     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
2060         d = defer.maybeDeferred(callable, *args, **kwargs)
2061         def done(res):
2062             if isinstance(res, failure.Failure):
2063                 res.trap(error.PageRedirect)
2064                 statuscode = res.value.status
2065                 target = res.value.location
2066                 return checker(statuscode, target)
2067             self.fail("%s: callable was supposed to redirect, not return '%s'"
2068                       % (which, res))
2069         d.addBoth(done)
2070         return d
2071
2072     def test_POST_upload_no_link_whendone_results(self):
2073         def check(statuscode, target):
2074             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2075             self.failUnless(target.startswith(self.webish_url), target)
2076             return client.getPage(target, method="GET")
2077         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
2078                                  check,
2079                                  self.POST, "/uri", t="upload",
2080                                  when_done="/uri/%(uri)s",
2081                                  file=("new.txt", self.NEWFILE_CONTENTS))
2082         d.addCallback(lambda res:
2083                       self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
2084         return d
2085
2086     def test_POST_upload_no_link_mutable(self):
2087         d = self.POST("/uri", t="upload", mutable="true",
2088                       file=("new.txt", self.NEWFILE_CONTENTS))
2089         def _check(filecap):
2090             filecap = filecap.strip()
2091             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2092             self.filecap = filecap
2093             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2094             self.failUnlessIn(u.get_storage_index(), FakeMutableFileNode.all_contents)
2095             n = self.s.create_node_from_uri(filecap)
2096             return n.download_best_version()
2097         d.addCallback(_check)
2098         def _check2(data):
2099             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2100             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2101         d.addCallback(_check2)
2102         def _check3(data):
2103             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2104             return self.GET("/file/%s" % urllib.quote(self.filecap))
2105         d.addCallback(_check3)
2106         def _check4(data):
2107             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2108         d.addCallback(_check4)
2109         return d
2110
2111     def test_POST_upload_no_link_mutable_toobig(self):
2112         # The SDMF size limit is no longer in place, so we should be
2113         # able to upload mutable files that are as large as we want them
2114         # to be.
2115         d = self.POST("/uri", t="upload", mutable="true",
2116                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2117         return d
2118
2119
2120     def test_POST_upload_format_unlinked(self):
2121         def _check_upload_unlinked(ign, format, uri_prefix):
2122             filename = format + ".txt"
2123             d = self.POST("/uri?t=upload&format=" + format,
2124                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2125             def _got_results(results):
2126                 if format.upper() in ("SDMF", "MDMF"):
2127                     # webapi.rst says this returns a filecap
2128                     filecap = results
2129                 else:
2130                     # for immutable, it returns an "upload results page", and
2131                     # the filecap is buried inside
2132                     line = [l for l in results.split("\n") if "URI: " in l][0]
2133                     mo = re.search(r'<span>([^<]+)</span>', line)
2134                     filecap = mo.group(1)
2135                 self.failUnless(filecap.startswith(uri_prefix),
2136                                 (uri_prefix, filecap))
2137                 return self.GET("/uri/%s?t=json" % filecap)
2138             d.addCallback(_got_results)
2139             def _got_json(json):
2140                 data = simplejson.loads(json)
2141                 data = data[1]
2142                 self.failUnlessIn("format", data)
2143                 self.failUnlessEqual(data["format"], format.upper())
2144             d.addCallback(_got_json)
2145             return d
2146         d = defer.succeed(None)
2147         d.addCallback(_check_upload_unlinked, "chk", "URI:CHK")
2148         d.addCallback(_check_upload_unlinked, "CHK", "URI:CHK")
2149         d.addCallback(_check_upload_unlinked, "sdmf", "URI:SSK")
2150         d.addCallback(_check_upload_unlinked, "mdmf", "URI:MDMF")
2151         return d
2152
2153     def test_POST_upload_bad_format_unlinked(self):
2154         return self.shouldHTTPError("POST_upload_bad_format_unlinked",
2155                                     400, "Bad Request", "Unknown format: foo",
2156                                     self.POST,
2157                                     "/uri?t=upload&format=foo",
2158                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2159
2160     def test_POST_upload_format(self):
2161         def _check_upload(ign, format, uri_prefix, fn=None):
2162             filename = format + ".txt"
2163             d = self.POST(self.public_url +
2164                           "/foo?t=upload&format=" + format,
2165                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2166             def _got_filecap(filecap):
2167                 if fn is not None:
2168                     filenameu = unicode(filename)
2169                     self.failUnlessURIMatchesRWChild(filecap, fn, filenameu)
2170                 self.failUnless(filecap.startswith(uri_prefix))
2171                 return self.GET(self.public_url + "/foo/%s?t=json" % filename)
2172             d.addCallback(_got_filecap)
2173             def _got_json(json):
2174                 data = simplejson.loads(json)
2175                 data = data[1]
2176                 self.failUnlessIn("format", data)
2177                 self.failUnlessEqual(data["format"], format.upper())
2178             d.addCallback(_got_json)
2179             return d
2180
2181         d = defer.succeed(None)
2182         d.addCallback(_check_upload, "chk", "URI:CHK")
2183         d.addCallback(_check_upload, "sdmf", "URI:SSK", self._foo_node)
2184         d.addCallback(_check_upload, "mdmf", "URI:MDMF")
2185         d.addCallback(_check_upload, "MDMF", "URI:MDMF")
2186         return d
2187
2188     def test_POST_upload_bad_format(self):
2189         return self.shouldHTTPError("POST_upload_bad_format",
2190                                     400, "Bad Request", "Unknown format: foo",
2191                                     self.POST, self.public_url + \
2192                                     "/foo?t=upload&format=foo",
2193                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2194
2195     def test_POST_upload_mutable(self):
2196         # this creates a mutable file
2197         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
2198                       file=("new.txt", self.NEWFILE_CONTENTS))
2199         fn = self._foo_node
2200         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2201         d.addCallback(lambda res:
2202                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2203                                                              self.NEWFILE_CONTENTS))
2204         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2205         def _got(newnode):
2206             self.failUnless(IMutableFileNode.providedBy(newnode))
2207             self.failUnless(newnode.is_mutable())
2208             self.failIf(newnode.is_readonly())
2209             self._mutable_node = newnode
2210             self._mutable_uri = newnode.get_uri()
2211         d.addCallback(_got)
2212
2213         # now upload it again and make sure that the URI doesn't change
2214         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
2215         d.addCallback(lambda res:
2216                       self.POST(self.public_url + "/foo", t="upload",
2217                                 mutable="true",
2218                                 file=("new.txt", NEWER_CONTENTS)))
2219         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2220         d.addCallback(lambda res:
2221                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2222                                                              NEWER_CONTENTS))
2223         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2224         def _got2(newnode):
2225             self.failUnless(IMutableFileNode.providedBy(newnode))
2226             self.failUnless(newnode.is_mutable())
2227             self.failIf(newnode.is_readonly())
2228             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2229         d.addCallback(_got2)
2230
2231         # upload a second time, using PUT instead of POST
2232         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
2233         d.addCallback(lambda res:
2234                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
2235         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2236         d.addCallback(lambda res:
2237                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2238                                                              NEW2_CONTENTS))
2239
2240         # finally list the directory, since mutable files are displayed
2241         # slightly differently
2242
2243         d.addCallback(lambda res:
2244                       self.GET(self.public_url + "/foo/",
2245                                followRedirect=True))
2246         def _check_page(res):
2247             # TODO: assert more about the contents
2248             self.failUnlessIn("SSK", res)
2249             return res
2250         d.addCallback(_check_page)
2251
2252         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2253         def _got3(newnode):
2254             self.failUnless(IMutableFileNode.providedBy(newnode))
2255             self.failUnless(newnode.is_mutable())
2256             self.failIf(newnode.is_readonly())
2257             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2258         d.addCallback(_got3)
2259
2260         # look at the JSON form of the enclosing directory
2261         d.addCallback(lambda res:
2262                       self.GET(self.public_url + "/foo/?t=json",
2263                                followRedirect=True))
2264         def _check_page_json(res):
2265             parsed = simplejson.loads(res)
2266             self.failUnlessEqual(parsed[0], "dirnode")
2267             children = dict( [(unicode(name),value)
2268                               for (name,value)
2269                               in parsed[1]["children"].iteritems()] )
2270             self.failUnlessIn(u"new.txt", children)
2271             new_json = children[u"new.txt"]
2272             self.failUnlessEqual(new_json[0], "filenode")
2273             self.failUnless(new_json[1]["mutable"])
2274             self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri)
2275             ro_uri = self._mutable_node.get_readonly().to_string()
2276             self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri)
2277         d.addCallback(_check_page_json)
2278
2279         # and the JSON form of the file
2280         d.addCallback(lambda res:
2281                       self.GET(self.public_url + "/foo/new.txt?t=json"))
2282         def _check_file_json(res):
2283             parsed = simplejson.loads(res)
2284             self.failUnlessEqual(parsed[0], "filenode")
2285             self.failUnless(parsed[1]["mutable"])
2286             self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri)
2287             ro_uri = self._mutable_node.get_readonly().to_string()
2288             self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri)
2289         d.addCallback(_check_file_json)
2290
2291         # and look at t=uri and t=readonly-uri
2292         d.addCallback(lambda res:
2293                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
2294         d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
2295         d.addCallback(lambda res:
2296                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
2297         def _check_ro_uri(res):
2298             ro_uri = self._mutable_node.get_readonly().to_string()
2299             self.failUnlessReallyEqual(res, ro_uri)
2300         d.addCallback(_check_ro_uri)
2301
2302         # make sure we can get to it from /uri/URI
2303         d.addCallback(lambda res:
2304                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
2305         d.addCallback(lambda res:
2306                       self.failUnlessReallyEqual(res, NEW2_CONTENTS))
2307
2308         # and that HEAD computes the size correctly
2309         d.addCallback(lambda res:
2310                       self.HEAD(self.public_url + "/foo/new.txt",
2311                                 return_response=True))
2312         def _got_headers((res, status, headers)):
2313             self.failUnlessReallyEqual(res, "")
2314             self.failUnlessReallyEqual(headers["content-length"][0],
2315                                        str(len(NEW2_CONTENTS)))
2316             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
2317         d.addCallback(_got_headers)
2318
2319         # make sure that outdated size limits aren't enforced anymore.
2320         d.addCallback(lambda ignored:
2321             self.POST(self.public_url + "/foo", t="upload",
2322                       mutable="true",
2323                       file=("new.txt",
2324                             "b" * (self.s.MUTABLE_SIZELIMIT+1))))
2325         d.addErrback(self.dump_error)
2326         return d
2327
2328     def test_POST_upload_mutable_toobig(self):
2329         # SDMF had a size limti that was removed a while ago. MDMF has
2330         # never had a size limit. Test to make sure that we do not
2331         # encounter errors when trying to upload large mutable files,
2332         # since there should be no coded prohibitions regarding large
2333         # mutable files.
2334         d = self.POST(self.public_url + "/foo",
2335                       t="upload", mutable="true",
2336                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2337         return d
2338
2339     def dump_error(self, f):
2340         # if the web server returns an error code (like 400 Bad Request),
2341         # web.client.getPage puts the HTTP response body into the .response
2342         # attribute of the exception object that it gives back. It does not
2343         # appear in the Failure's repr(), so the ERROR that trial displays
2344         # will be rather terse and unhelpful. addErrback this method to the
2345         # end of your chain to get more information out of these errors.
2346         if f.check(error.Error):
2347             print "web.error.Error:"
2348             print f
2349             print f.value.response
2350         return f
2351
2352     def test_POST_upload_replace(self):
2353         d = self.POST(self.public_url + "/foo", t="upload",
2354                       file=("bar.txt", self.NEWFILE_CONTENTS))
2355         fn = self._foo_node
2356         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
2357         d.addCallback(lambda res:
2358                       self.failUnlessChildContentsAre(fn, u"bar.txt",
2359                                                       self.NEWFILE_CONTENTS))
2360         return d
2361
2362     def test_POST_upload_no_replace_ok(self):
2363         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2364                       file=("new.txt", self.NEWFILE_CONTENTS))
2365         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
2366         d.addCallback(lambda res: self.failUnlessReallyEqual(res,
2367                                                              self.NEWFILE_CONTENTS))
2368         return d
2369
2370     def test_POST_upload_no_replace_queryarg(self):
2371         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2372                       file=("bar.txt", self.NEWFILE_CONTENTS))
2373         d.addBoth(self.shouldFail, error.Error,
2374                   "POST_upload_no_replace_queryarg",
2375                   "409 Conflict",
2376                   "There was already a child by that name, and you asked me "
2377                   "to not replace it")
2378         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2379         d.addCallback(self.failUnlessIsBarDotTxt)
2380         return d
2381
2382     def test_POST_upload_no_replace_field(self):
2383         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
2384                       file=("bar.txt", self.NEWFILE_CONTENTS))
2385         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
2386                   "409 Conflict",
2387                   "There was already a child by that name, and you asked me "
2388                   "to not replace it")
2389         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2390         d.addCallback(self.failUnlessIsBarDotTxt)
2391         return d
2392
2393     def test_POST_upload_whendone(self):
2394         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
2395                       file=("new.txt", self.NEWFILE_CONTENTS))
2396         d.addBoth(self.shouldRedirect, "/THERE")
2397         fn = self._foo_node
2398         d.addCallback(lambda res:
2399                       self.failUnlessChildContentsAre(fn, u"new.txt",
2400                                                       self.NEWFILE_CONTENTS))
2401         return d
2402
2403     def test_POST_upload_named(self):
2404         fn = self._foo_node
2405         d = self.POST(self.public_url + "/foo", t="upload",
2406                       name="new.txt", file=self.NEWFILE_CONTENTS)
2407         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2408         d.addCallback(lambda res:
2409                       self.failUnlessChildContentsAre(fn, u"new.txt",
2410                                                       self.NEWFILE_CONTENTS))
2411         return d
2412
2413     def test_POST_upload_named_badfilename(self):
2414         d = self.POST(self.public_url + "/foo", t="upload",
2415                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
2416         d.addBoth(self.shouldFail, error.Error,
2417                   "test_POST_upload_named_badfilename",
2418                   "400 Bad Request",
2419                   "name= may not contain a slash",
2420                   )
2421         # make sure that nothing was added
2422         d.addCallback(lambda res:
2423                       self.failUnlessNodeKeysAre(self._foo_node,
2424                                                  [u"bar.txt", u"baz.txt", u"blockingfile",
2425                                                   u"empty", u"n\u00fc.txt", u"quux.txt",
2426                                                   u"sub"]))
2427         return d
2428
2429     def test_POST_FILEURL_check(self):
2430         bar_url = self.public_url + "/foo/bar.txt"
2431         d = self.POST(bar_url, t="check")
2432         def _check(res):
2433             self.failUnlessIn("Healthy :", res)
2434         d.addCallback(_check)
2435         redir_url = "http://allmydata.org/TARGET"
2436         def _check2(statuscode, target):
2437             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2438             self.failUnlessReallyEqual(target, redir_url)
2439         d.addCallback(lambda res:
2440                       self.shouldRedirect2("test_POST_FILEURL_check",
2441                                            _check2,
2442                                            self.POST, bar_url,
2443                                            t="check",
2444                                            when_done=redir_url))
2445         d.addCallback(lambda res:
2446                       self.POST(bar_url, t="check", return_to=redir_url))
2447         def _check3(res):
2448             self.failUnlessIn("Healthy :", res)
2449             self.failUnlessIn("Return to file", res)
2450             self.failUnlessIn(redir_url, res)
2451         d.addCallback(_check3)
2452
2453         d.addCallback(lambda res:
2454                       self.POST(bar_url, t="check", output="JSON"))
2455         def _check_json(res):
2456             data = simplejson.loads(res)
2457             self.failUnlessIn("storage-index", data)
2458             self.failUnless(data["results"]["healthy"])
2459         d.addCallback(_check_json)
2460
2461         return d
2462
2463     def test_POST_FILEURL_check_and_repair(self):
2464         bar_url = self.public_url + "/foo/bar.txt"
2465         d = self.POST(bar_url, t="check", repair="true")
2466         def _check(res):
2467             self.failUnlessIn("Healthy :", res)
2468         d.addCallback(_check)
2469         redir_url = "http://allmydata.org/TARGET"
2470         def _check2(statuscode, target):
2471             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2472             self.failUnlessReallyEqual(target, redir_url)
2473         d.addCallback(lambda res:
2474                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
2475                                            _check2,
2476                                            self.POST, bar_url,
2477                                            t="check", repair="true",
2478                                            when_done=redir_url))
2479         d.addCallback(lambda res:
2480                       self.POST(bar_url, t="check", return_to=redir_url))
2481         def _check3(res):
2482             self.failUnlessIn("Healthy :", res)
2483             self.failUnlessIn("Return to file", res)
2484             self.failUnlessIn(redir_url, res)
2485         d.addCallback(_check3)
2486         return d
2487
2488     def test_POST_DIRURL_check(self):
2489         foo_url = self.public_url + "/foo/"
2490         d = self.POST(foo_url, t="check")
2491         def _check(res):
2492             self.failUnlessIn("Healthy :", res)
2493         d.addCallback(_check)
2494         redir_url = "http://allmydata.org/TARGET"
2495         def _check2(statuscode, target):
2496             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2497             self.failUnlessReallyEqual(target, redir_url)
2498         d.addCallback(lambda res:
2499                       self.shouldRedirect2("test_POST_DIRURL_check",
2500                                            _check2,
2501                                            self.POST, foo_url,
2502                                            t="check",
2503                                            when_done=redir_url))
2504         d.addCallback(lambda res:
2505                       self.POST(foo_url, t="check", return_to=redir_url))
2506         def _check3(res):
2507             self.failUnlessIn("Healthy :", res)
2508             self.failUnlessIn("Return to file/directory", res)
2509             self.failUnlessIn(redir_url, res)
2510         d.addCallback(_check3)
2511
2512         d.addCallback(lambda res:
2513                       self.POST(foo_url, t="check", output="JSON"))
2514         def _check_json(res):
2515             data = simplejson.loads(res)
2516             self.failUnlessIn("storage-index", data)
2517             self.failUnless(data["results"]["healthy"])
2518         d.addCallback(_check_json)
2519
2520         return d
2521
2522     def test_POST_DIRURL_check_and_repair(self):
2523         foo_url = self.public_url + "/foo/"
2524         d = self.POST(foo_url, t="check", repair="true")
2525         def _check(res):
2526             self.failUnlessIn("Healthy :", res)
2527         d.addCallback(_check)
2528         redir_url = "http://allmydata.org/TARGET"
2529         def _check2(statuscode, target):
2530             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2531             self.failUnlessReallyEqual(target, redir_url)
2532         d.addCallback(lambda res:
2533                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2534                                            _check2,
2535                                            self.POST, foo_url,
2536                                            t="check", repair="true",
2537                                            when_done=redir_url))
2538         d.addCallback(lambda res:
2539                       self.POST(foo_url, t="check", return_to=redir_url))
2540         def _check3(res):
2541             self.failUnlessIn("Healthy :", res)
2542             self.failUnlessIn("Return to file/directory", res)
2543             self.failUnlessIn(redir_url, res)
2544         d.addCallback(_check3)
2545         return d
2546
2547     def test_POST_FILEURL_mdmf_check(self):
2548         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2549         d = self.POST(quux_url, t="check")
2550         def _check(res):
2551             self.failUnlessIn("Healthy", res)
2552         d.addCallback(_check)
2553         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2554         d.addCallback(lambda ignored:
2555                       self.POST(quux_extension_url, t="check"))
2556         d.addCallback(_check)
2557         return d
2558
2559     def test_POST_FILEURL_mdmf_check_and_repair(self):
2560         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2561         d = self.POST(quux_url, t="check", repair="true")
2562         def _check(res):
2563             self.failUnlessIn("Healthy", res)
2564         d.addCallback(_check)
2565         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2566         d.addCallback(lambda ignored:
2567                       self.POST(quux_extension_url, t="check", repair="true"))
2568         d.addCallback(_check)
2569         return d
2570
2571     def wait_for_operation(self, ignored, ophandle):
2572         url = "/operations/" + ophandle
2573         url += "?t=status&output=JSON"
2574         d = self.GET(url)
2575         def _got(res):
2576             data = simplejson.loads(res)
2577             if not data["finished"]:
2578                 d = self.stall(delay=1.0)
2579                 d.addCallback(self.wait_for_operation, ophandle)
2580                 return d
2581             return data
2582         d.addCallback(_got)
2583         return d
2584
2585     def get_operation_results(self, ignored, ophandle, output=None):
2586         url = "/operations/" + ophandle
2587         url += "?t=status"
2588         if output:
2589             url += "&output=" + output
2590         d = self.GET(url)
2591         def _got(res):
2592             if output and output.lower() == "json":
2593                 return simplejson.loads(res)
2594             return res
2595         d.addCallback(_got)
2596         return d
2597
2598     def test_POST_DIRURL_deepcheck_no_ophandle(self):
2599         d = self.shouldFail2(error.Error,
2600                              "test_POST_DIRURL_deepcheck_no_ophandle",
2601                              "400 Bad Request",
2602                              "slow operation requires ophandle=",
2603                              self.POST, self.public_url, t="start-deep-check")
2604         return d
2605
2606     def test_POST_DIRURL_deepcheck(self):
2607         def _check_redirect(statuscode, target):
2608             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2609             self.failUnless(target.endswith("/operations/123"))
2610         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2611                                  self.POST, self.public_url,
2612                                  t="start-deep-check", ophandle="123")
2613         d.addCallback(self.wait_for_operation, "123")
2614         def _check_json(data):
2615             self.failUnlessReallyEqual(data["finished"], True)
2616             self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2617             self.failUnlessReallyEqual(data["count-objects-healthy"], 10)
2618         d.addCallback(_check_json)
2619         d.addCallback(self.get_operation_results, "123", "html")
2620         def _check_html(res):
2621             self.failUnlessIn("Objects Checked: <span>10</span>", res)
2622             self.failUnlessIn("Objects Healthy: <span>10</span>", res)
2623             self.failUnlessIn(FAVICON_MARKUP, res)
2624         d.addCallback(_check_html)
2625
2626         d.addCallback(lambda res:
2627                       self.GET("/operations/123/"))
2628         d.addCallback(_check_html) # should be the same as without the slash
2629
2630         d.addCallback(lambda res:
2631                       self.shouldFail2(error.Error, "one", "404 Not Found",
2632                                        "No detailed results for SI bogus",
2633                                        self.GET, "/operations/123/bogus"))
2634
2635         foo_si = self._foo_node.get_storage_index()
2636         foo_si_s = base32.b2a(foo_si)
2637         d.addCallback(lambda res:
2638                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2639         def _check_foo_json(res):
2640             data = simplejson.loads(res)
2641             self.failUnlessEqual(data["storage-index"], foo_si_s)
2642             self.failUnless(data["results"]["healthy"])
2643         d.addCallback(_check_foo_json)
2644         return d
2645
2646     def test_POST_DIRURL_deepcheck_and_repair(self):
2647         d = self.POST(self.public_url, t="start-deep-check", repair="true",
2648                       ophandle="124", output="json", followRedirect=True)
2649         d.addCallback(self.wait_for_operation, "124")
2650         def _check_json(data):
2651             self.failUnlessReallyEqual(data["finished"], True)
2652             self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2653             self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 10)
2654             self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2655             self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2656             self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2657             self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2658             self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2659             self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 10)
2660             self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2661             self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2662         d.addCallback(_check_json)
2663         d.addCallback(self.get_operation_results, "124", "html")
2664         def _check_html(res):
2665             self.failUnlessIn("Objects Checked: <span>10</span>", res)
2666
2667             self.failUnlessIn("Objects Healthy (before repair): <span>10</span>", res)
2668             self.failUnlessIn("Objects Unhealthy (before repair): <span>0</span>", res)
2669             self.failUnlessIn("Corrupt Shares (before repair): <span>0</span>", res)
2670
2671             self.failUnlessIn("Repairs Attempted: <span>0</span>", res)
2672             self.failUnlessIn("Repairs Successful: <span>0</span>", res)
2673             self.failUnlessIn("Repairs Unsuccessful: <span>0</span>", res)
2674
2675             self.failUnlessIn("Objects Healthy (after repair): <span>10</span>", res)
2676             self.failUnlessIn("Objects Unhealthy (after repair): <span>0</span>", res)
2677             self.failUnlessIn("Corrupt Shares (after repair): <span>0</span>", res)
2678
2679             self.failUnlessIn(FAVICON_MARKUP, res)
2680         d.addCallback(_check_html)
2681         return d
2682
2683     def test_POST_FILEURL_bad_t(self):
2684         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2685                              "POST to file: bad t=bogus",
2686                              self.POST, self.public_url + "/foo/bar.txt",
2687                              t="bogus")
2688         return d
2689
2690     def test_POST_mkdir(self): # return value?
2691         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2692         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2693         d.addCallback(self.failUnlessNodeKeysAre, [])
2694         return d
2695
2696     def test_POST_mkdir_mdmf(self):
2697         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=mdmf")
2698         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2699         d.addCallback(lambda node:
2700             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2701         return d
2702
2703     def test_POST_mkdir_sdmf(self):
2704         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=sdmf")
2705         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2706         d.addCallback(lambda node:
2707             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2708         return d
2709
2710     def test_POST_mkdir_bad_format(self):
2711         return self.shouldHTTPError("POST_mkdir_bad_format",
2712                                     400, "Bad Request", "Unknown format: foo",
2713                                     self.POST, self.public_url +
2714                                     "/foo?t=mkdir&name=newdir&format=foo")
2715
2716     def test_POST_mkdir_initial_children(self):
2717         (newkids, caps) = self._create_initial_children()
2718         d = self.POST2(self.public_url +
2719                        "/foo?t=mkdir-with-children&name=newdir",
2720                        simplejson.dumps(newkids))
2721         d.addCallback(lambda res:
2722                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2723         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2724         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2725         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2726         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2727         return d
2728
2729     def test_POST_mkdir_initial_children_mdmf(self):
2730         (newkids, caps) = self._create_initial_children()
2731         d = self.POST2(self.public_url +
2732                        "/foo?t=mkdir-with-children&name=newdir&format=mdmf",
2733                        simplejson.dumps(newkids))
2734         d.addCallback(lambda res:
2735                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2736         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2737         d.addCallback(lambda node:
2738             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2739         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2740         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2741                        caps['filecap1'])
2742         return d
2743
2744     # XXX: Duplication.
2745     def test_POST_mkdir_initial_children_sdmf(self):
2746         (newkids, caps) = self._create_initial_children()
2747         d = self.POST2(self.public_url +
2748                        "/foo?t=mkdir-with-children&name=newdir&format=sdmf",
2749                        simplejson.dumps(newkids))
2750         d.addCallback(lambda res:
2751                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2752         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2753         d.addCallback(lambda node:
2754             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2755         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2756         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2757                        caps['filecap1'])
2758         return d
2759
2760     def test_POST_mkdir_initial_children_bad_format(self):
2761         (newkids, caps) = self._create_initial_children()
2762         return self.shouldHTTPError("POST_mkdir_initial_children_bad_format",
2763                                     400, "Bad Request", "Unknown format: foo",
2764                                     self.POST, self.public_url + \
2765                                     "/foo?t=mkdir-with-children&name=newdir&format=foo",
2766                                     simplejson.dumps(newkids))
2767
2768     def test_POST_mkdir_immutable(self):
2769         (newkids, caps) = self._create_immutable_children()
2770         d = self.POST2(self.public_url +
2771                        "/foo?t=mkdir-immutable&name=newdir",
2772                        simplejson.dumps(newkids))
2773         d.addCallback(lambda res:
2774                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2775         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2776         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2777         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2778         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2779         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2780         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2781         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2782         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2783         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2784         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2785         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2786         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2787         return d
2788
2789     def test_POST_mkdir_immutable_bad(self):
2790         (newkids, caps) = self._create_initial_children()
2791         d = self.shouldFail2(error.Error, "POST_mkdir_immutable_bad",
2792                              "400 Bad Request",
2793                              "needed to be immutable but was not",
2794                              self.POST2,
2795                              self.public_url +
2796                              "/foo?t=mkdir-immutable&name=newdir",
2797                              simplejson.dumps(newkids))
2798         return d
2799
2800     def test_POST_mkdir_2(self):
2801         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
2802         d.addCallback(lambda res:
2803                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2804         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2805         d.addCallback(self.failUnlessNodeKeysAre, [])
2806         return d
2807
2808     def test_POST_mkdirs_2(self):
2809         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
2810         d.addCallback(lambda res:
2811                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
2812         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
2813         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
2814         d.addCallback(self.failUnlessNodeKeysAre, [])
2815         return d
2816
2817     def test_POST_mkdir_no_parentdir_noredirect(self):
2818         d = self.POST("/uri?t=mkdir")
2819         def _after_mkdir(res):
2820             uri.DirectoryURI.init_from_string(res)
2821         d.addCallback(_after_mkdir)
2822         return d
2823
2824     def test_POST_mkdir_no_parentdir_noredirect_mdmf(self):
2825         d = self.POST("/uri?t=mkdir&format=mdmf")
2826         def _after_mkdir(res):
2827             u = uri.from_string(res)
2828             # Check that this is an MDMF writecap
2829             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
2830         d.addCallback(_after_mkdir)
2831         return d
2832
2833     def test_POST_mkdir_no_parentdir_noredirect_sdmf(self):
2834         d = self.POST("/uri?t=mkdir&format=sdmf")
2835         def _after_mkdir(res):
2836             u = uri.from_string(res)
2837             self.failUnlessIsInstance(u, uri.DirectoryURI)
2838         d.addCallback(_after_mkdir)
2839         return d
2840
2841     def test_POST_mkdir_no_parentdir_noredirect_bad_format(self):
2842         return self.shouldHTTPError("POST_mkdir_no_parentdir_noredirect_bad_format",
2843                                     400, "Bad Request", "Unknown format: foo",
2844                                     self.POST, self.public_url +
2845                                     "/uri?t=mkdir&format=foo")
2846
2847     def test_POST_mkdir_no_parentdir_noredirect2(self):
2848         # make sure form-based arguments (as on the welcome page) still work
2849         d = self.POST("/uri", t="mkdir")
2850         def _after_mkdir(res):
2851             uri.DirectoryURI.init_from_string(res)
2852         d.addCallback(_after_mkdir)
2853         d.addErrback(self.explain_web_error)
2854         return d
2855
2856     def test_POST_mkdir_no_parentdir_redirect(self):
2857         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
2858         d.addBoth(self.shouldRedirect, None, statuscode='303')
2859         def _check_target(target):
2860             target = urllib.unquote(target)
2861             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2862         d.addCallback(_check_target)
2863         return d
2864
2865     def test_POST_mkdir_no_parentdir_redirect2(self):
2866         d = self.POST("/uri", t="mkdir", redirect_to_result="true")
2867         d.addBoth(self.shouldRedirect, None, statuscode='303')
2868         def _check_target(target):
2869             target = urllib.unquote(target)
2870             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2871         d.addCallback(_check_target)
2872         d.addErrback(self.explain_web_error)
2873         return d
2874
2875     def _make_readonly(self, u):
2876         ro_uri = uri.from_string(u).get_readonly()
2877         if ro_uri is None:
2878             return None
2879         return ro_uri.to_string()
2880
2881     def _create_initial_children(self):
2882         contents, n, filecap1 = self.makefile(12)
2883         md1 = {"metakey1": "metavalue1"}
2884         filecap2 = make_mutable_file_uri()
2885         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
2886         filecap3 = node3.get_readonly_uri()
2887         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
2888         dircap = DirectoryNode(node4, None, None).get_uri()
2889         mdmfcap = make_mutable_file_uri(mdmf=True)
2890         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2891         emptydircap = "URI:DIR2-LIT:"
2892         newkids = {u"child-imm":        ["filenode", {"rw_uri": filecap1,
2893                                                       "ro_uri": self._make_readonly(filecap1),
2894                                                       "metadata": md1, }],
2895                    u"child-mutable":    ["filenode", {"rw_uri": filecap2,
2896                                                       "ro_uri": self._make_readonly(filecap2)}],
2897                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
2898                    u"unknownchild-rw":  ["unknown",  {"rw_uri": unknown_rwcap,
2899                                                       "ro_uri": unknown_rocap}],
2900                    u"unknownchild-ro":  ["unknown",  {"ro_uri": unknown_rocap}],
2901                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2902                    u"dirchild":         ["dirnode",  {"rw_uri": dircap,
2903                                                       "ro_uri": self._make_readonly(dircap)}],
2904                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2905                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2906                    u"child-mutable-mdmf": ["filenode", {"rw_uri": mdmfcap,
2907                                                         "ro_uri": self._make_readonly(mdmfcap)}],
2908                    }
2909         return newkids, {'filecap1': filecap1,
2910                          'filecap2': filecap2,
2911                          'filecap3': filecap3,
2912                          'unknown_rwcap': unknown_rwcap,
2913                          'unknown_rocap': unknown_rocap,
2914                          'unknown_immcap': unknown_immcap,
2915                          'dircap': dircap,
2916                          'litdircap': litdircap,
2917                          'emptydircap': emptydircap,
2918                          'mdmfcap': mdmfcap}
2919
2920     def _create_immutable_children(self):
2921         contents, n, filecap1 = self.makefile(12)
2922         md1 = {"metakey1": "metavalue1"}
2923         tnode = create_chk_filenode("immutable directory contents\n"*10)
2924         dnode = DirectoryNode(tnode, None, None)
2925         assert not dnode.is_mutable()
2926         immdircap = dnode.get_uri()
2927         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2928         emptydircap = "URI:DIR2-LIT:"
2929         newkids = {u"child-imm":        ["filenode", {"ro_uri": filecap1,
2930                                                       "metadata": md1, }],
2931                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2932                    u"dirchild-imm":     ["dirnode",  {"ro_uri": immdircap}],
2933                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2934                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2935                    }
2936         return newkids, {'filecap1': filecap1,
2937                          'unknown_immcap': unknown_immcap,
2938                          'immdircap': immdircap,
2939                          'litdircap': litdircap,
2940                          'emptydircap': emptydircap}
2941
2942     def test_POST_mkdir_no_parentdir_initial_children(self):
2943         (newkids, caps) = self._create_initial_children()
2944         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
2945         def _after_mkdir(res):
2946             self.failUnless(res.startswith("URI:DIR"), res)
2947             n = self.s.create_node_from_uri(res)
2948             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2949             d2.addCallback(lambda ign:
2950                            self.failUnlessROChildURIIs(n, u"child-imm",
2951                                                        caps['filecap1']))
2952             d2.addCallback(lambda ign:
2953                            self.failUnlessRWChildURIIs(n, u"child-mutable",
2954                                                        caps['filecap2']))
2955             d2.addCallback(lambda ign:
2956                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
2957                                                        caps['filecap3']))
2958             d2.addCallback(lambda ign:
2959                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
2960                                                        caps['unknown_rwcap']))
2961             d2.addCallback(lambda ign:
2962                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
2963                                                        caps['unknown_rocap']))
2964             d2.addCallback(lambda ign:
2965                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2966                                                        caps['unknown_immcap']))
2967             d2.addCallback(lambda ign:
2968                            self.failUnlessRWChildURIIs(n, u"dirchild",
2969                                                        caps['dircap']))
2970             return d2
2971         d.addCallback(_after_mkdir)
2972         return d
2973
2974     def test_POST_mkdir_no_parentdir_unexpected_children(self):
2975         # the regular /uri?t=mkdir operation is specified to ignore its body.
2976         # Only t=mkdir-with-children pays attention to it.
2977         (newkids, caps) = self._create_initial_children()
2978         d = self.shouldHTTPError("POST_mkdir_no_parentdir_unexpected_children",
2979                                  400, "Bad Request",
2980                                  "t=mkdir does not accept children=, "
2981                                  "try t=mkdir-with-children instead",
2982                                  self.POST2, "/uri?t=mkdir", # without children
2983                                  simplejson.dumps(newkids))
2984         return d
2985
2986     def test_POST_noparent_bad(self):
2987         d = self.shouldHTTPError("POST_noparent_bad",
2988                                  400, "Bad Request",
2989                                  "/uri accepts only PUT, PUT?t=mkdir, "
2990                                  "POST?t=upload, and POST?t=mkdir",
2991                                  self.POST, "/uri?t=bogus")
2992         return d
2993
2994     def test_POST_mkdir_no_parentdir_immutable(self):
2995         (newkids, caps) = self._create_immutable_children()
2996         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
2997         def _after_mkdir(res):
2998             self.failUnless(res.startswith("URI:DIR"), res)
2999             n = self.s.create_node_from_uri(res)
3000             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3001             d2.addCallback(lambda ign:
3002                            self.failUnlessROChildURIIs(n, u"child-imm",
3003                                                           caps['filecap1']))
3004             d2.addCallback(lambda ign:
3005                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3006                                                           caps['unknown_immcap']))
3007             d2.addCallback(lambda ign:
3008                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
3009                                                           caps['immdircap']))
3010             d2.addCallback(lambda ign:
3011                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
3012                                                           caps['litdircap']))
3013             d2.addCallback(lambda ign:
3014                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
3015                                                           caps['emptydircap']))
3016             return d2
3017         d.addCallback(_after_mkdir)
3018         return d
3019
3020     def test_POST_mkdir_no_parentdir_immutable_bad(self):
3021         (newkids, caps) = self._create_initial_children()
3022         d = self.shouldFail2(error.Error,
3023                              "test_POST_mkdir_no_parentdir_immutable_bad",
3024                              "400 Bad Request",
3025                              "needed to be immutable but was not",
3026                              self.POST2,
3027                              "/uri?t=mkdir-immutable",
3028                              simplejson.dumps(newkids))
3029         return d
3030
3031     def test_welcome_page_mkdir_button(self):
3032         # Fetch the welcome page.
3033         d = self.GET("/")
3034         def _after_get_welcome_page(res):
3035             MKDIR_BUTTON_RE = re.compile(
3036                 '<form action="([^"]*)" method="post".*?'
3037                 '<input type="hidden" name="t" value="([^"]*)" />'
3038                 '<input type="hidden" name="([^"]*)" value="([^"]*)" />'
3039                 '<input type="submit" value="Create a directory" />',
3040                 re.I)
3041             mo = MKDIR_BUTTON_RE.search(res)
3042             formaction = mo.group(1)
3043             formt = mo.group(2)
3044             formaname = mo.group(3)
3045             formavalue = mo.group(4)
3046             return (formaction, formt, formaname, formavalue)
3047         d.addCallback(_after_get_welcome_page)
3048         def _after_parse_form(res):
3049             (formaction, formt, formaname, formavalue) = res
3050             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
3051         d.addCallback(_after_parse_form)
3052         d.addBoth(self.shouldRedirect, None, statuscode='303')
3053         return d
3054
3055     def test_POST_mkdir_replace(self): # return value?
3056         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
3057         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3058         d.addCallback(self.failUnlessNodeKeysAre, [])
3059         return d
3060
3061     def test_POST_mkdir_no_replace_queryarg(self): # return value?
3062         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
3063         d.addBoth(self.shouldFail, error.Error,
3064                   "POST_mkdir_no_replace_queryarg",
3065                   "409 Conflict",
3066                   "There was already a child by that name, and you asked me "
3067                   "to not replace it")
3068         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3069         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3070         return d
3071
3072     def test_POST_mkdir_no_replace_field(self): # return value?
3073         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
3074                       replace="false")
3075         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
3076                   "409 Conflict",
3077                   "There was already a child by that name, and you asked me "
3078                   "to not replace it")
3079         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3080         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3081         return d
3082
3083     def test_POST_mkdir_whendone_field(self):
3084         d = self.POST(self.public_url + "/foo",
3085                       t="mkdir", name="newdir", when_done="/THERE")
3086         d.addBoth(self.shouldRedirect, "/THERE")
3087         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3088         d.addCallback(self.failUnlessNodeKeysAre, [])
3089         return d
3090
3091     def test_POST_mkdir_whendone_queryarg(self):
3092         d = self.POST(self.public_url + "/foo?when_done=/THERE",
3093                       t="mkdir", name="newdir")
3094         d.addBoth(self.shouldRedirect, "/THERE")
3095         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3096         d.addCallback(self.failUnlessNodeKeysAre, [])
3097         return d
3098
3099     def test_POST_bad_t(self):
3100         d = self.shouldFail2(error.Error, "POST_bad_t",
3101                              "400 Bad Request",
3102                              "POST to a directory with bad t=BOGUS",
3103                              self.POST, self.public_url + "/foo", t="BOGUS")
3104         return d
3105
3106     def test_POST_set_children(self, command_name="set_children"):
3107         contents9, n9, newuri9 = self.makefile(9)
3108         contents10, n10, newuri10 = self.makefile(10)
3109         contents11, n11, newuri11 = self.makefile(11)
3110
3111         reqbody = """{
3112                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
3113                                                 "size": 0,
3114                                                 "metadata": {
3115                                                   "ctime": 1002777696.7564139,
3116                                                   "mtime": 1002777696.7564139
3117                                                  }
3118                                                } ],
3119                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
3120                                                 "size": 1,
3121                                                 "metadata": {
3122                                                   "ctime": 1002777696.7564139,
3123                                                   "mtime": 1002777696.7564139
3124                                                  }
3125                                                } ],
3126                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
3127                                                 "size": 2,
3128                                                 "metadata": {
3129                                                   "ctime": 1002777696.7564139,
3130                                                   "mtime": 1002777696.7564139
3131                                                  }
3132                                                } ]
3133                     }""" % (newuri9, newuri10, newuri11)
3134
3135         url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
3136
3137         d = client.getPage(url, method="POST", postdata=reqbody)
3138         def _then(res):
3139             self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
3140             self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
3141             self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
3142
3143         d.addCallback(_then)
3144         d.addErrback(self.dump_error)
3145         return d
3146
3147     def test_POST_set_children_with_hyphen(self):
3148         return self.test_POST_set_children(command_name="set-children")
3149
3150     def test_POST_link_uri(self):
3151         contents, n, newuri = self.makefile(8)
3152         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
3153         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
3154         d.addCallback(lambda res:
3155                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3156                                                       contents))
3157         return d
3158
3159     def test_POST_link_uri_replace(self):
3160         contents, n, newuri = self.makefile(8)
3161         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
3162         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
3163         d.addCallback(lambda res:
3164                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3165                                                       contents))
3166         return d
3167
3168     def test_POST_link_uri_unknown_bad(self):
3169         d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
3170         d.addBoth(self.shouldFail, error.Error,
3171                   "POST_link_uri_unknown_bad",
3172                   "400 Bad Request",
3173                   "unknown cap in a write slot")
3174         return d
3175
3176     def test_POST_link_uri_unknown_ro_good(self):
3177         d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
3178         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
3179         return d
3180
3181     def test_POST_link_uri_unknown_imm_good(self):
3182         d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
3183         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
3184         return d
3185
3186     def test_POST_link_uri_no_replace_queryarg(self):
3187         contents, n, newuri = self.makefile(8)
3188         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
3189                       name="bar.txt", uri=newuri)
3190         d.addBoth(self.shouldFail, error.Error,
3191                   "POST_link_uri_no_replace_queryarg",
3192                   "409 Conflict",
3193                   "There was already a child by that name, and you asked me "
3194                   "to not replace it")
3195         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3196         d.addCallback(self.failUnlessIsBarDotTxt)
3197         return d
3198
3199     def test_POST_link_uri_no_replace_field(self):
3200         contents, n, newuri = self.makefile(8)
3201         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
3202                       name="bar.txt", uri=newuri)
3203         d.addBoth(self.shouldFail, error.Error,
3204                   "POST_link_uri_no_replace_field",
3205                   "409 Conflict",
3206                   "There was already a child by that name, and you asked me "
3207                   "to not replace it")
3208         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3209         d.addCallback(self.failUnlessIsBarDotTxt)
3210         return d
3211
3212     def test_POST_delete(self, command_name='delete'):
3213         d = self._foo_node.list()
3214         def _check_before(children):
3215             self.failUnlessIn(u"bar.txt", children)
3216         d.addCallback(_check_before)
3217         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t=command_name, name="bar.txt"))
3218         d.addCallback(lambda res: self._foo_node.list())
3219         def _check_after(children):
3220             self.failIfIn(u"bar.txt", children)
3221         d.addCallback(_check_after)
3222         return d
3223
3224     def test_POST_unlink(self):
3225         return self.test_POST_delete(command_name='unlink')
3226
3227     def test_POST_rename_file(self):
3228         d = self.POST(self.public_url + "/foo", t="rename",
3229                       from_name="bar.txt", to_name='wibble.txt')
3230         d.addCallback(lambda res:
3231                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3232         d.addCallback(lambda res:
3233                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
3234         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
3235         d.addCallback(self.failUnlessIsBarDotTxt)
3236         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
3237         d.addCallback(self.failUnlessIsBarJSON)
3238         return d
3239
3240     def test_POST_rename_file_redundant(self):
3241         d = self.POST(self.public_url + "/foo", t="rename",
3242                       from_name="bar.txt", to_name='bar.txt')
3243         d.addCallback(lambda res:
3244                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3245         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3246         d.addCallback(self.failUnlessIsBarDotTxt)
3247         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3248         d.addCallback(self.failUnlessIsBarJSON)
3249         return d
3250
3251     def test_POST_rename_file_replace(self):
3252         # rename a file and replace a directory with it
3253         d = self.POST(self.public_url + "/foo", t="rename",
3254                       from_name="bar.txt", to_name='empty')
3255         d.addCallback(lambda res:
3256                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3257         d.addCallback(lambda res:
3258                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
3259         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
3260         d.addCallback(self.failUnlessIsBarDotTxt)
3261         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3262         d.addCallback(self.failUnlessIsBarJSON)
3263         return d
3264
3265     def test_POST_rename_file_no_replace_queryarg(self):
3266         # rename a file and replace a directory with it
3267         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
3268                       from_name="bar.txt", to_name='empty')
3269         d.addBoth(self.shouldFail, error.Error,
3270                   "POST_rename_file_no_replace_queryarg",
3271                   "409 Conflict",
3272                   "There was already a child by that name, and you asked me "
3273                   "to not replace it")
3274         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3275         d.addCallback(self.failUnlessIsEmptyJSON)
3276         return d
3277
3278     def test_POST_rename_file_no_replace_field(self):
3279         # rename a file and replace a directory with it
3280         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
3281                       from_name="bar.txt", to_name='empty')
3282         d.addBoth(self.shouldFail, error.Error,
3283                   "POST_rename_file_no_replace_field",
3284                   "409 Conflict",
3285                   "There was already a child by that name, and you asked me "
3286                   "to not replace it")
3287         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3288         d.addCallback(self.failUnlessIsEmptyJSON)
3289         return d
3290
3291     def failUnlessIsEmptyJSON(self, res):
3292         data = simplejson.loads(res)
3293         self.failUnlessEqual(data[0], "dirnode", data)
3294         self.failUnlessReallyEqual(len(data[1]["children"]), 0)
3295
3296     def test_POST_rename_file_slash_fail(self):
3297         d = self.POST(self.public_url + "/foo", t="rename",
3298                       from_name="bar.txt", to_name='kirk/spock.txt')
3299         d.addBoth(self.shouldFail, error.Error,
3300                   "test_POST_rename_file_slash_fail",
3301                   "400 Bad Request",
3302                   "to_name= may not contain a slash",
3303                   )
3304         d.addCallback(lambda res:
3305                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3306         return d
3307
3308     def test_POST_rename_dir(self):
3309         d = self.POST(self.public_url, t="rename",
3310                       from_name="foo", to_name='plunk')
3311         d.addCallback(lambda res:
3312                       self.failIfNodeHasChild(self.public_root, u"foo"))
3313         d.addCallback(lambda res:
3314                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
3315         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
3316         d.addCallback(self.failUnlessIsFooJSON)
3317         return d
3318
3319     def test_POST_move_file(self):
3320         d = self.POST(self.public_url + "/foo", t="move",
3321                       from_name="bar.txt", to_dir="sub")
3322         d.addCallback(lambda res:
3323                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3324         d.addCallback(lambda res:
3325                       self.failUnlessNodeHasChild(self._sub_node, u"bar.txt"))
3326         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3327         d.addCallback(self.failUnlessIsBarDotTxt)
3328         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3329         d.addCallback(self.failUnlessIsBarJSON)
3330         return d
3331
3332     def test_POST_move_file_new_name(self):
3333         d = self.POST(self.public_url + "/foo", t="move",
3334                       from_name="bar.txt", to_name="wibble.txt", to_dir="sub")
3335         d.addCallback(lambda res:
3336                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3337         d.addCallback(lambda res:
3338                       self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3339         d.addCallback(lambda res:
3340                       self.failUnlessNodeHasChild(self._sub_node, u"wibble.txt"))
3341         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt"))
3342         d.addCallback(self.failUnlessIsBarDotTxt)
3343         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt?t=json"))
3344         d.addCallback(self.failUnlessIsBarJSON)
3345         return d
3346
3347     def test_POST_move_file_replace(self):
3348         d = self.POST(self.public_url + "/foo", t="move",
3349                       from_name="bar.txt", to_name="baz.txt", to_dir="sub")
3350         d.addCallback(lambda res:
3351                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3352         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3353         d.addCallback(self.failUnlessIsBarDotTxt)
3354         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt?t=json"))
3355         d.addCallback(self.failUnlessIsBarJSON)
3356         return d
3357
3358     def test_POST_move_file_no_replace(self):
3359         d = self.shouldFail2(error.Error, "POST_move_file_no_replace",
3360                              "409 Conflict",
3361                              "There was already a child by that name, and you asked me to not replace it",
3362                              self.POST, self.public_url + "/foo", t="move",
3363                              replace="false", from_name="bar.txt",
3364                              to_name="baz.txt", to_dir="sub")
3365         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3366         d.addCallback(self.failUnlessIsBarDotTxt)
3367         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3368         d.addCallback(self.failUnlessIsBarJSON)
3369         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3370         d.addCallback(self.failUnlessIsSubBazDotTxt)
3371         return d
3372
3373     def test_POST_move_file_slash_fail(self):
3374         d = self.shouldFail2(error.Error, "test_POST_rename_file_slash_fail",
3375                              "400 Bad Request",
3376                              "to_name= may not contain a slash",
3377                              self.POST, self.public_url + "/foo", t="move",
3378                              from_name="bar.txt",
3379                              to_name="slash/fail.txt", to_dir="sub")
3380         d.addCallback(lambda res:
3381                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3382         d.addCallback(lambda res:
3383                       self.failIfNodeHasChild(self._sub_node, u"slash/fail.txt"))
3384         d.addCallback(lambda ign:
3385                       self.shouldFail2(error.Error,
3386                                        "test_POST_rename_file_slash_fail2",
3387                                        "400 Bad Request",
3388                                        "from_name= may not contain a slash",
3389                                        self.POST, self.public_url + "/foo",
3390                                        t="move",
3391                                        from_name="nope/bar.txt",
3392                                        to_name="fail.txt", to_dir="sub"))
3393         return d
3394
3395     def test_POST_move_file_no_target(self):
3396         d = self.shouldFail2(error.Error, "POST_move_file_no_target",
3397                              "400 Bad Request",
3398                              "move requires from_name and to_dir",
3399                              self.POST, self.public_url + "/foo", t="move",
3400                              from_name="bar.txt", to_name="baz.txt")
3401         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3402         d.addCallback(self.failUnlessIsBarDotTxt)
3403         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3404         d.addCallback(self.failUnlessIsBarJSON)
3405         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3406         d.addCallback(self.failUnlessIsBazDotTxt)
3407         return d
3408
3409     def test_POST_move_file_bad_target_type(self):
3410         d = self.shouldFail2(error.Error, "test_POST_move_file_bad_target_type",
3411                              "400 Bad Request", "invalid target_type parameter",
3412                              self.POST,
3413                              self.public_url + "/foo", t="move",
3414                              target_type="*D", from_name="bar.txt",
3415                              to_dir="sub")
3416         return d
3417
3418     def test_POST_move_file_multi_level(self):
3419         d = self.POST(self.public_url + "/foo/sub/level2?t=mkdir", "")
3420         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t="move",
3421                       from_name="bar.txt", to_dir="sub/level2"))
3422         d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3423         d.addCallback(lambda res: self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3424         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt"))
3425         d.addCallback(self.failUnlessIsBarDotTxt)
3426         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt?t=json"))
3427         d.addCallback(self.failUnlessIsBarJSON)
3428         return d
3429
3430     def test_POST_move_file_to_uri(self):
3431         d = self.POST(self.public_url + "/foo", t="move", target_type="uri",
3432                       from_name="bar.txt", to_dir=self._sub_uri)
3433         d.addCallback(lambda res:
3434                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3435         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3436         d.addCallback(self.failUnlessIsBarDotTxt)
3437         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3438         d.addCallback(self.failUnlessIsBarJSON)
3439         return d
3440
3441     def test_POST_move_file_to_nonexist_dir(self):
3442         d = self.shouldFail2(error.Error, "POST_move_file_to_nonexist_dir",
3443                             "404 Not Found", "No such child: nopechucktesta",
3444                             self.POST, self.public_url + "/foo", t="move",
3445                             from_name="bar.txt", to_dir="nopechucktesta")
3446         return d
3447
3448     def test_POST_move_file_into_file(self):
3449         d = self.shouldFail2(error.Error, "POST_move_file_into_file",
3450                              "400 Bad Request", "to_dir is not a directory",
3451                              self.POST, self.public_url + "/foo", t="move",
3452                              from_name="bar.txt", to_dir="baz.txt")
3453         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3454         d.addCallback(self.failUnlessIsBazDotTxt)
3455         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3456         d.addCallback(self.failUnlessIsBarDotTxt)
3457         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3458         d.addCallback(self.failUnlessIsBarJSON)
3459         return d
3460
3461     def test_POST_move_file_to_bad_uri(self):
3462         d =  self.shouldFail2(error.Error, "POST_move_file_to_bad_uri",
3463                               "400 Bad Request", "to_dir is not a directory",
3464                               self.POST, self.public_url + "/foo", t="move",
3465                               from_name="bar.txt", target_type="uri",
3466                               to_dir="URI:DIR2:mn5jlyjnrjeuydyswlzyui72i:rmneifcj6k6sycjljjhj3f6majsq2zqffydnnul5hfa4j577arma")
3467         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3468         d.addCallback(self.failUnlessIsBarDotTxt)
3469         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3470         d.addCallback(self.failUnlessIsBarJSON)
3471         return d
3472
3473     def test_POST_move_dir(self):
3474         d = self.POST(self.public_url + "/foo", t="move",
3475                       from_name="bar.txt", to_dir="empty")
3476         d.addCallback(lambda res: self.POST(self.public_url + "/foo",
3477                       t="move", from_name="empty", to_dir="sub"))
3478         d.addCallback(lambda res:
3479                       self.failIfNodeHasChild(self._foo_node, u"empty"))
3480         d.addCallback(lambda res:
3481                       self.failUnlessNodeHasChild(self._sub_node, u"empty"))
3482         d.addCallback(lambda res:
3483                       self._sub_node.get_child_at_path(u"empty"))
3484         d.addCallback(lambda node:
3485                       self.failUnlessNodeHasChild(node, u"bar.txt"))
3486         d.addCallback(lambda res:
3487                       self.GET(self.public_url + "/foo/sub/empty/bar.txt"))
3488         d.addCallback(self.failUnlessIsBarDotTxt)
3489         return d
3490
3491     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
3492         """ If target is not None then the redirection has to go to target.  If
3493         statuscode is not None then the redirection has to be accomplished with
3494         that HTTP status code."""
3495         if not isinstance(res, failure.Failure):
3496             to_where = (target is None) and "somewhere" or ("to " + target)
3497             self.fail("%s: we were expecting to get redirected %s, not get an"
3498                       " actual page: %s" % (which, to_where, res))
3499         res.trap(error.PageRedirect)
3500         if statuscode is not None:
3501             self.failUnlessReallyEqual(res.value.status, statuscode,
3502                                        "%s: not a redirect" % which)
3503         if target is not None:
3504             # the PageRedirect does not seem to capture the uri= query arg
3505             # properly, so we can't check for it.
3506             realtarget = self.webish_url + target
3507             self.failUnlessReallyEqual(res.value.location, realtarget,
3508                                        "%s: wrong target" % which)
3509         return res.value.location
3510
3511     def test_GET_URI_form(self):
3512         base = "/uri?uri=%s" % self._bar_txt_uri
3513         # this is supposed to give us a redirect to /uri/$URI, plus arguments
3514         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
3515         d = self.GET(base)
3516         d.addBoth(self.shouldRedirect, targetbase)
3517         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
3518         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
3519         d.addCallback(lambda res: self.GET(base+"&t=json"))
3520         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
3521         d.addCallback(self.log, "about to get file by uri")
3522         d.addCallback(lambda res: self.GET(base, followRedirect=True))
3523         d.addCallback(self.failUnlessIsBarDotTxt)
3524         d.addCallback(self.log, "got file by uri, about to get dir by uri")
3525         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
3526                                            followRedirect=True))
3527         d.addCallback(self.failUnlessIsFooJSON)
3528         d.addCallback(self.log, "got dir by uri")
3529
3530         return d
3531
3532     def test_GET_URI_form_bad(self):
3533         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
3534                              "400 Bad Request", "GET /uri requires uri=",
3535                              self.GET, "/uri")
3536         return d
3537
3538     def test_GET_rename_form(self):
3539         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
3540                      followRedirect=True)
3541         def _check(res):
3542             self.failUnlessIn('name="when_done" value="."', res)
3543             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
3544             self.failUnlessIn(FAVICON_MARKUP, res)
3545         d.addCallback(_check)
3546         return d
3547
3548     def log(self, res, msg):
3549         #print "MSG: %s  RES: %s" % (msg, res)
3550         log.msg(msg)
3551         return res
3552
3553     def test_GET_URI_URL(self):
3554         base = "/uri/%s" % self._bar_txt_uri
3555         d = self.GET(base)
3556         d.addCallback(self.failUnlessIsBarDotTxt)
3557         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
3558         d.addCallback(self.failUnlessIsBarDotTxt)
3559         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
3560         d.addCallback(self.failUnlessIsBarDotTxt)
3561         return d
3562
3563     def test_GET_URI_URL_dir(self):
3564         base = "/uri/%s?t=json" % self._foo_uri
3565         d = self.GET(base)
3566         d.addCallback(self.failUnlessIsFooJSON)
3567         return d
3568
3569     def test_GET_URI_URL_missing(self):
3570         base = "/uri/%s" % self._bad_file_uri
3571         d = self.shouldHTTPError("test_GET_URI_URL_missing",
3572                                  http.GONE, None, "NotEnoughSharesError",
3573                                  self.GET, base)
3574         # TODO: how can we exercise both sides of WebDownloadTarget.fail
3575         # here? we must arrange for a download to fail after target.open()
3576         # has been called, and then inspect the response to see that it is
3577         # shorter than we expected.
3578         return d
3579
3580     def test_PUT_DIRURL_uri(self):
3581         d = self.s.create_dirnode()
3582         def _made_dir(dn):
3583             new_uri = dn.get_uri()
3584             # replace /foo with a new (empty) directory
3585             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
3586             d.addCallback(lambda res:
3587                           self.failUnlessReallyEqual(res.strip(), new_uri))
3588             d.addCallback(lambda res:
3589                           self.failUnlessRWChildURIIs(self.public_root,
3590                                                       u"foo",
3591                                                       new_uri))
3592             return d
3593         d.addCallback(_made_dir)
3594         return d
3595
3596     def test_PUT_DIRURL_uri_noreplace(self):
3597         d = self.s.create_dirnode()
3598         def _made_dir(dn):
3599             new_uri = dn.get_uri()
3600             # replace /foo with a new (empty) directory, but ask that
3601             # replace=false, so it should fail
3602             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
3603                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
3604                                  self.PUT,
3605                                  self.public_url + "/foo?t=uri&replace=false",
3606                                  new_uri)
3607             d.addCallback(lambda res:
3608                           self.failUnlessRWChildURIIs(self.public_root,
3609                                                       u"foo",
3610                                                       self._foo_uri))
3611             return d
3612         d.addCallback(_made_dir)
3613         return d
3614
3615     def test_PUT_DIRURL_bad_t(self):
3616         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
3617                              "400 Bad Request", "PUT to a directory",
3618                              self.PUT, self.public_url + "/foo?t=BOGUS", "")
3619         d.addCallback(lambda res:
3620                       self.failUnlessRWChildURIIs(self.public_root,
3621                                                   u"foo",
3622                                                   self._foo_uri))
3623         return d
3624
3625     def test_PUT_NEWFILEURL_uri(self):
3626         contents, n, new_uri = self.makefile(8)
3627         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
3628         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3629         d.addCallback(lambda res:
3630                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3631                                                       contents))
3632         return d
3633
3634     def test_PUT_NEWFILEURL_mdmf(self):
3635         new_contents = self.NEWFILE_CONTENTS * 300000
3636         d = self.PUT(self.public_url + \
3637                      "/foo/mdmf.txt?format=mdmf",
3638                      new_contents)
3639         d.addCallback(lambda ignored:
3640             self.GET(self.public_url + "/foo/mdmf.txt?t=json"))
3641         def _got_json(json):
3642             data = simplejson.loads(json)
3643             data = data[1]
3644             self.failUnlessIn("format", data)
3645             self.failUnlessEqual(data["format"], "MDMF")
3646             self.failUnless(data['rw_uri'].startswith("URI:MDMF"))
3647             self.failUnless(data['ro_uri'].startswith("URI:MDMF"))
3648         d.addCallback(_got_json)
3649         return d
3650
3651     def test_PUT_NEWFILEURL_sdmf(self):
3652         new_contents = self.NEWFILE_CONTENTS * 300000
3653         d = self.PUT(self.public_url + \
3654                      "/foo/sdmf.txt?format=sdmf",
3655                      new_contents)
3656         d.addCallback(lambda ignored:
3657             self.GET(self.public_url + "/foo/sdmf.txt?t=json"))
3658         def _got_json(json):
3659             data = simplejson.loads(json)
3660             data = data[1]
3661             self.failUnlessIn("format", data)
3662             self.failUnlessEqual(data["format"], "SDMF")
3663         d.addCallback(_got_json)
3664         return d
3665
3666     def test_PUT_NEWFILEURL_bad_format(self):
3667        new_contents = self.NEWFILE_CONTENTS * 300000
3668        return self.shouldHTTPError("PUT_NEWFILEURL_bad_format",
3669                                    400, "Bad Request", "Unknown format: foo",
3670                                    self.PUT, self.public_url + \
3671                                    "/foo/foo.txt?format=foo",
3672                                    new_contents)
3673
3674     def test_PUT_NEWFILEURL_uri_replace(self):
3675         contents, n, new_uri = self.makefile(8)
3676         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
3677         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3678         d.addCallback(lambda res:
3679                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3680                                                       contents))
3681         return d
3682
3683     def test_PUT_NEWFILEURL_uri_no_replace(self):
3684         contents, n, new_uri = self.makefile(8)
3685         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
3686         d.addBoth(self.shouldFail, error.Error,
3687                   "PUT_NEWFILEURL_uri_no_replace",
3688                   "409 Conflict",
3689                   "There was already a child by that name, and you asked me "
3690                   "to not replace it")
3691         return d
3692
3693     def test_PUT_NEWFILEURL_uri_unknown_bad(self):
3694         d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
3695         d.addBoth(self.shouldFail, error.Error,
3696                   "POST_put_uri_unknown_bad",
3697                   "400 Bad Request",
3698                   "unknown cap in a write slot")
3699         return d
3700
3701     def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
3702         d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
3703         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3704                       u"put-future-ro.txt")
3705         return d
3706
3707     def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
3708         d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
3709         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3710                       u"put-future-imm.txt")
3711         return d
3712
3713     def test_PUT_NEWFILE_URI(self):
3714         file_contents = "New file contents here\n"
3715         d = self.PUT("/uri", file_contents)
3716         def _check(uri):
3717             assert isinstance(uri, str), uri
3718             self.failUnlessIn(uri, FakeCHKFileNode.all_contents)
3719             self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
3720                                        file_contents)
3721             return self.GET("/uri/%s" % uri)
3722         d.addCallback(_check)
3723         def _check2(res):
3724             self.failUnlessReallyEqual(res, file_contents)
3725         d.addCallback(_check2)
3726         return d
3727
3728     def test_PUT_NEWFILE_URI_not_mutable(self):
3729         file_contents = "New file contents here\n"
3730         d = self.PUT("/uri?mutable=false", file_contents)
3731         def _check(uri):
3732             assert isinstance(uri, str), uri
3733             self.failUnlessIn(uri, FakeCHKFileNode.all_contents)
3734             self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
3735                                        file_contents)
3736             return self.GET("/uri/%s" % uri)
3737         d.addCallback(_check)
3738         def _check2(res):
3739             self.failUnlessReallyEqual(res, file_contents)
3740         d.addCallback(_check2)
3741         return d
3742
3743     def test_PUT_NEWFILE_URI_only_PUT(self):
3744         d = self.PUT("/uri?t=bogus", "")
3745         d.addBoth(self.shouldFail, error.Error,
3746                   "PUT_NEWFILE_URI_only_PUT",
3747                   "400 Bad Request",
3748                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
3749         return d
3750
3751     def test_PUT_NEWFILE_URI_mutable(self):
3752         file_contents = "New file contents here\n"
3753         d = self.PUT("/uri?mutable=true", file_contents)
3754         def _check1(filecap):
3755             filecap = filecap.strip()
3756             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
3757             self.filecap = filecap
3758             u = uri.WriteableSSKFileURI.init_from_string(filecap)
3759             self.failUnlessIn(u.get_storage_index(), FakeMutableFileNode.all_contents)
3760             n = self.s.create_node_from_uri(filecap)
3761             return n.download_best_version()
3762         d.addCallback(_check1)
3763         def _check2(data):
3764             self.failUnlessReallyEqual(data, file_contents)
3765             return self.GET("/uri/%s" % urllib.quote(self.filecap))
3766         d.addCallback(_check2)
3767         def _check3(res):
3768             self.failUnlessReallyEqual(res, file_contents)
3769         d.addCallback(_check3)
3770         return d
3771
3772     def test_PUT_mkdir(self):
3773         d = self.PUT("/uri?t=mkdir", "")
3774         def _check(uri):
3775             n = self.s.create_node_from_uri(uri.strip())
3776             d2 = self.failUnlessNodeKeysAre(n, [])
3777             d2.addCallback(lambda res:
3778                            self.GET("/uri/%s?t=json" % uri))
3779             return d2
3780         d.addCallback(_check)
3781         d.addCallback(self.failUnlessIsEmptyJSON)
3782         return d
3783
3784     def test_PUT_mkdir_mdmf(self):
3785         d = self.PUT("/uri?t=mkdir&format=mdmf", "")
3786         def _got(res):
3787             u = uri.from_string(res)
3788             # Check that this is an MDMF writecap
3789             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
3790         d.addCallback(_got)
3791         return d
3792
3793     def test_PUT_mkdir_sdmf(self):
3794         d = self.PUT("/uri?t=mkdir&format=sdmf", "")
3795         def _got(res):
3796             u = uri.from_string(res)
3797             self.failUnlessIsInstance(u, uri.DirectoryURI)
3798         d.addCallback(_got)
3799         return d
3800
3801     def test_PUT_mkdir_bad_format(self):
3802         return self.shouldHTTPError("PUT_mkdir_bad_format",
3803                                     400, "Bad Request", "Unknown format: foo",
3804                                     self.PUT, "/uri?t=mkdir&format=foo",
3805                                     "")
3806
3807     def test_POST_check(self):
3808         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
3809         def _done(res):
3810             # this returns a string form of the results, which are probably
3811             # None since we're using fake filenodes.
3812             # TODO: verify that the check actually happened, by changing
3813             # FakeCHKFileNode to count how many times .check() has been
3814             # called.
3815             pass
3816         d.addCallback(_done)
3817         return d
3818
3819
3820     def test_PUT_update_at_offset(self):
3821         file_contents = "test file" * 100000 # about 900 KiB
3822         d = self.PUT("/uri?mutable=true", file_contents)
3823         def _then(filecap):
3824             self.filecap = filecap
3825             new_data = file_contents[:100]
3826             new = "replaced and so on"
3827             new_data += new
3828             new_data += file_contents[len(new_data):]
3829             assert len(new_data) == len(file_contents)
3830             self.new_data = new_data
3831         d.addCallback(_then)
3832         d.addCallback(lambda ignored:
3833             self.PUT("/uri/%s?replace=True&offset=100" % self.filecap,
3834                      "replaced and so on"))
3835         def _get_data(filecap):
3836             n = self.s.create_node_from_uri(filecap)
3837             return n.download_best_version()
3838         d.addCallback(_get_data)
3839         d.addCallback(lambda results:
3840             self.failUnlessEqual(results, self.new_data))
3841         # Now try appending things to the file
3842         d.addCallback(lambda ignored:
3843             self.PUT("/uri/%s?offset=%d" % (self.filecap, len(self.new_data)),
3844                      "puppies" * 100))
3845         d.addCallback(_get_data)
3846         d.addCallback(lambda results:
3847             self.failUnlessEqual(results, self.new_data + ("puppies" * 100)))
3848         # and try replacing the beginning of the file
3849         d.addCallback(lambda ignored:
3850             self.PUT("/uri/%s?offset=0" % self.filecap, "begin"))
3851         d.addCallback(_get_data)
3852         d.addCallback(lambda results:
3853             self.failUnlessEqual(results, "begin"+self.new_data[len("begin"):]+("puppies"*100)))
3854         return d
3855
3856     def test_PUT_update_at_invalid_offset(self):
3857         file_contents = "test file" * 100000 # about 900 KiB
3858         d = self.PUT("/uri?mutable=true", file_contents)
3859         def _then(filecap):
3860             self.filecap = filecap
3861         d.addCallback(_then)
3862         # Negative offsets should cause an error.
3863         d.addCallback(lambda ignored:
3864             self.shouldHTTPError("PUT_update_at_invalid_offset",
3865                                  400, "Bad Request",
3866                                  "Invalid offset",
3867                                  self.PUT,
3868                                  "/uri/%s?offset=-1" % self.filecap,
3869                                  "foo"))
3870         return d
3871
3872     def test_PUT_update_at_offset_immutable(self):
3873         file_contents = "Test file" * 100000
3874         d = self.PUT("/uri", file_contents)
3875         def _then(filecap):
3876             self.filecap = filecap
3877         d.addCallback(_then)
3878         d.addCallback(lambda ignored:
3879             self.shouldHTTPError("PUT_update_at_offset_immutable",
3880                                  400, "Bad Request",
3881                                  "immutable",
3882                                  self.PUT,
3883                                  "/uri/%s?offset=50" % self.filecap,
3884                                  "foo"))
3885         return d
3886
3887
3888     def test_bad_method(self):
3889         url = self.webish_url + self.public_url + "/foo/bar.txt"
3890         d = self.shouldHTTPError("bad_method",
3891                                  501, "Not Implemented",
3892                                  "I don't know how to treat a BOGUS request.",
3893                                  client.getPage, url, method="BOGUS")
3894         return d
3895
3896     def test_short_url(self):
3897         url = self.webish_url + "/uri"
3898         d = self.shouldHTTPError("short_url", 501, "Not Implemented",
3899                                  "I don't know how to treat a DELETE request.",
3900                                  client.getPage, url, method="DELETE")
3901         return d
3902
3903     def test_ophandle_bad(self):
3904         url = self.webish_url + "/operations/bogus?t=status"
3905         d = self.shouldHTTPError("ophandle_bad", 404, "404 Not Found",
3906                                  "unknown/expired handle 'bogus'",
3907                                  client.getPage, url)
3908         return d
3909
3910     def test_ophandle_cancel(self):
3911         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
3912                       followRedirect=True)
3913         d.addCallback(lambda ignored:
3914                       self.GET("/operations/128?t=status&output=JSON"))
3915         def _check1(res):
3916             data = simplejson.loads(res)
3917             self.failUnless("finished" in data, res)
3918             monitor = self.ws.root.child_operations.handles["128"][0]
3919             d = self.POST("/operations/128?t=cancel&output=JSON")
3920             def _check2(res):
3921                 data = simplejson.loads(res)
3922                 self.failUnless("finished" in data, res)
3923                 # t=cancel causes the handle to be forgotten
3924                 self.failUnless(monitor.is_cancelled())
3925             d.addCallback(_check2)
3926             return d
3927         d.addCallback(_check1)
3928         d.addCallback(lambda ignored:
3929                       self.shouldHTTPError("ophandle_cancel",
3930                                            404, "404 Not Found",
3931                                            "unknown/expired handle '128'",
3932                                            self.GET,
3933                                            "/operations/128?t=status&output=JSON"))
3934         return d
3935
3936     def test_ophandle_retainfor(self):
3937         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
3938                       followRedirect=True)
3939         d.addCallback(lambda ignored:
3940                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
3941         def _check1(res):
3942             data = simplejson.loads(res)
3943             self.failUnless("finished" in data, res)
3944         d.addCallback(_check1)
3945         # the retain-for=0 will cause the handle to be expired very soon
3946         d.addCallback(lambda ign:
3947             self.clock.advance(2.0))
3948         d.addCallback(lambda ignored:
3949                       self.shouldHTTPError("ophandle_retainfor",
3950                                            404, "404 Not Found",
3951                                            "unknown/expired handle '129'",
3952                                            self.GET,
3953                                            "/operations/129?t=status&output=JSON"))
3954         return d
3955
3956     def test_ophandle_release_after_complete(self):
3957         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
3958                       followRedirect=True)
3959         d.addCallback(self.wait_for_operation, "130")
3960         d.addCallback(lambda ignored:
3961                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
3962         # the release-after-complete=true will cause the handle to be expired
3963         d.addCallback(lambda ignored:
3964                       self.shouldHTTPError("ophandle_release_after_complete",
3965                                            404, "404 Not Found",
3966                                            "unknown/expired handle '130'",
3967                                            self.GET,
3968                                            "/operations/130?t=status&output=JSON"))
3969         return d
3970
3971     def test_uncollected_ophandle_expiration(self):
3972         # uncollected ophandles should expire after 4 days
3973         def _make_uncollected_ophandle(ophandle):
3974             d = self.POST(self.public_url +
3975                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3976                           followRedirect=False)
3977             # When we start the operation, the webapi server will want
3978             # to redirect us to the page for the ophandle, so we get
3979             # confirmation that the operation has started. If the
3980             # manifest operation has finished by the time we get there,
3981             # following that redirect (by setting followRedirect=True
3982             # above) has the side effect of collecting the ophandle that
3983             # we've just created, which means that we can't use the
3984             # ophandle to test the uncollected timeout anymore. So,
3985             # instead, catch the 302 here and don't follow it.
3986             d.addBoth(self.should302, "uncollected_ophandle_creation")
3987             return d
3988         # Create an ophandle, don't collect it, then advance the clock by
3989         # 4 days - 1 second and make sure that the ophandle is still there.
3990         d = _make_uncollected_ophandle(131)
3991         d.addCallback(lambda ign:
3992             self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
3993         d.addCallback(lambda ign:
3994             self.GET("/operations/131?t=status&output=JSON"))
3995         def _check1(res):
3996             data = simplejson.loads(res)
3997             self.failUnless("finished" in data, res)
3998         d.addCallback(_check1)
3999         # Create an ophandle, don't collect it, then try to collect it
4000         # after 4 days. It should be gone.
4001         d.addCallback(lambda ign:
4002             _make_uncollected_ophandle(132))
4003         d.addCallback(lambda ign:
4004             self.clock.advance(96*60*60))
4005         d.addCallback(lambda ign:
4006             self.shouldHTTPError("uncollected_ophandle_expired_after_100_hours",
4007                                  404, "404 Not Found",
4008                                  "unknown/expired handle '132'",
4009                                  self.GET,
4010                                  "/operations/132?t=status&output=JSON"))
4011         return d
4012
4013     def test_collected_ophandle_expiration(self):
4014         # collected ophandles should expire after 1 day
4015         def _make_collected_ophandle(ophandle):
4016             d = self.POST(self.public_url +
4017                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
4018                           followRedirect=True)
4019             # By following the initial redirect, we collect the ophandle
4020             # we've just created.
4021             return d
4022         # Create a collected ophandle, then collect it after 23 hours
4023         # and 59 seconds to make sure that it is still there.
4024         d = _make_collected_ophandle(133)
4025         d.addCallback(lambda ign:
4026             self.clock.advance((24*60*60) - 1))
4027         d.addCallback(lambda ign:
4028             self.GET("/operations/133?t=status&output=JSON"))
4029         def _check1(res):
4030             data = simplejson.loads(res)
4031             self.failUnless("finished" in data, res)
4032         d.addCallback(_check1)
4033         # Create another uncollected ophandle, then try to collect it
4034         # after 24 hours to make sure that it is gone.
4035         d.addCallback(lambda ign:
4036             _make_collected_ophandle(134))
4037         d.addCallback(lambda ign:
4038             self.clock.advance(24*60*60))
4039         d.addCallback(lambda ign:
4040             self.shouldHTTPError("collected_ophandle_expired_after_1_day",
4041                                  404, "404 Not Found",
4042                                  "unknown/expired handle '134'",
4043                                  self.GET,
4044                                  "/operations/134?t=status&output=JSON"))
4045         return d
4046
4047     def test_incident(self):
4048         d = self.POST("/report_incident", details="eek")
4049         def _done(res):
4050             self.failIfIn("<html>", res)
4051             self.failUnlessIn("Thank you for your report!", res)
4052         d.addCallback(_done)
4053         return d
4054
4055     def test_static(self):
4056         webdir = os.path.join(self.staticdir, "subdir")
4057         fileutil.make_dirs(webdir)
4058         f = open(os.path.join(webdir, "hello.txt"), "wb")
4059         f.write("hello")
4060         f.close()
4061
4062         d = self.GET("/static/subdir/hello.txt")
4063         def _check(res):
4064             self.failUnlessReallyEqual(res, "hello")
4065         d.addCallback(_check)
4066         return d
4067
4068
4069 class IntroducerWeb(unittest.TestCase):
4070     def setUp(self):
4071         self.node = None
4072
4073     def tearDown(self):
4074         d = defer.succeed(None)
4075         if self.node:
4076             d.addCallback(lambda ign: self.node.stopService())
4077         d.addCallback(flushEventualQueue)
4078         return d
4079
4080     def test_welcome(self):
4081         basedir = "web.IntroducerWeb.test_welcome"
4082         os.mkdir(basedir)
4083         fileutil.write(os.path.join(basedir, "tahoe.cfg"), "[node]\nweb.port = tcp:0\n")
4084         self.node = IntroducerNode(basedir)
4085         self.ws = self.node.getServiceNamed("webish")
4086
4087         d = fireEventually(None)
4088         d.addCallback(lambda ign: self.node.startService())
4089         d.addCallback(lambda ign: self.node.when_tub_ready())
4090
4091         d.addCallback(lambda ign: self.GET("/"))
4092         def _check(res):
4093             self.failUnlessIn('Welcome to the Tahoe-LAFS Introducer', res)
4094             self.failUnlessIn(FAVICON_MARKUP, res)
4095         d.addCallback(_check)
4096         return d
4097
4098     def GET(self, urlpath, followRedirect=False, return_response=False,
4099             **kwargs):
4100         # if return_response=True, this fires with (data, statuscode,
4101         # respheaders) instead of just data.
4102         assert not isinstance(urlpath, unicode)
4103         url = self.ws.getURL().rstrip('/') + urlpath
4104         factory = HTTPClientGETFactory(url, method="GET",
4105                                        followRedirect=followRedirect, **kwargs)
4106         reactor.connectTCP("localhost", self.ws.getPortnum(), factory)
4107         d = factory.deferred
4108         def _got_data(data):
4109             return (data, factory.status, factory.response_headers)
4110         if return_response:
4111             d.addCallback(_got_data)
4112         return factory.deferred
4113
4114
4115 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4116     def test_load_file(self):
4117         # This will raise an exception unless a well-formed XML file is found under that name.
4118         common.getxmlfile('directory.xhtml').load()
4119
4120     def test_parse_replace_arg(self):
4121         self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
4122         self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
4123         self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
4124                                    "only-files")
4125         self.shouldFail(AssertionError, "test_parse_replace_arg", "",
4126                         common.parse_replace_arg, "only_fles")
4127
4128     def test_abbreviate_time(self):
4129         self.failUnlessReallyEqual(common.abbreviate_time(None), "")
4130         self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
4131         self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
4132         self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
4133         self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
4134         self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
4135
4136     def test_compute_rate(self):
4137         self.failUnlessReallyEqual(common.compute_rate(None, None), None)
4138         self.failUnlessReallyEqual(common.compute_rate(None, 1), None)
4139         self.failUnlessReallyEqual(common.compute_rate(250000, None), None)
4140         self.failUnlessReallyEqual(common.compute_rate(250000, 0), None)
4141         self.failUnlessReallyEqual(common.compute_rate(250000, 10), 25000.0)
4142         self.failUnlessReallyEqual(common.compute_rate(0, 10), 0.0)
4143         self.shouldFail(AssertionError, "test_compute_rate", "",
4144                         common.compute_rate, -100, 10)
4145         self.shouldFail(AssertionError, "test_compute_rate", "",
4146                         common.compute_rate, 100, -10)
4147
4148         # Sanity check
4149         rate = common.compute_rate(10*1000*1000, 1)
4150         self.failUnlessReallyEqual(common.abbreviate_rate(rate), "10.00MBps")
4151
4152     def test_abbreviate_rate(self):
4153         self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
4154         self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
4155         self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
4156         self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
4157
4158     def test_abbreviate_size(self):
4159         self.failUnlessReallyEqual(common.abbreviate_size(None), "")
4160         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
4161         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
4162         self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
4163         self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
4164
4165     def test_plural(self):
4166         def convert(s):
4167             return "%d second%s" % (s, status.plural(s))
4168         self.failUnlessReallyEqual(convert(0), "0 seconds")
4169         self.failUnlessReallyEqual(convert(1), "1 second")
4170         self.failUnlessReallyEqual(convert(2), "2 seconds")
4171         def convert2(s):
4172             return "has share%s: %s" % (status.plural(s), ",".join(s))
4173         self.failUnlessReallyEqual(convert2([]), "has shares: ")
4174         self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
4175         self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
4176
4177
4178 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4179
4180     def CHECK(self, ign, which, args, clientnum=0):
4181         fileurl = self.fileurls[which]
4182         url = fileurl + "?" + args
4183         return self.GET(url, method="POST", clientnum=clientnum)
4184
4185     def test_filecheck(self):
4186         self.basedir = "web/Grid/filecheck"
4187         self.set_up_grid()
4188         c0 = self.g.clients[0]
4189         self.uris = {}
4190         DATA = "data" * 100
4191         d = c0.upload(upload.Data(DATA, convergence=""))
4192         def _stash_uri(ur, which):
4193             self.uris[which] = ur.uri
4194         d.addCallback(_stash_uri, "good")
4195         d.addCallback(lambda ign:
4196                       c0.upload(upload.Data(DATA+"1", convergence="")))
4197         d.addCallback(_stash_uri, "sick")
4198         d.addCallback(lambda ign:
4199                       c0.upload(upload.Data(DATA+"2", convergence="")))
4200         d.addCallback(_stash_uri, "dead")
4201         def _stash_mutable_uri(n, which):
4202             self.uris[which] = n.get_uri()
4203             assert isinstance(self.uris[which], str)
4204         d.addCallback(lambda ign:
4205             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4206         d.addCallback(_stash_mutable_uri, "corrupt")
4207         d.addCallback(lambda ign:
4208                       c0.upload(upload.Data("literal", convergence="")))
4209         d.addCallback(_stash_uri, "small")
4210         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
4211         d.addCallback(_stash_mutable_uri, "smalldir")
4212
4213         def _compute_fileurls(ignored):
4214             self.fileurls = {}
4215             for which in self.uris:
4216                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4217         d.addCallback(_compute_fileurls)
4218
4219         def _clobber_shares(ignored):
4220             good_shares = self.find_uri_shares(self.uris["good"])
4221             self.failUnlessReallyEqual(len(good_shares), 10)
4222             sick_shares = self.find_uri_shares(self.uris["sick"])
4223             os.unlink(sick_shares[0][2])
4224             dead_shares = self.find_uri_shares(self.uris["dead"])
4225             for i in range(1, 10):
4226                 os.unlink(dead_shares[i][2])
4227             c_shares = self.find_uri_shares(self.uris["corrupt"])
4228             cso = CorruptShareOptions()
4229             cso.stdout = StringIO()
4230             cso.parseOptions([c_shares[0][2]])
4231             corrupt_share(cso)
4232         d.addCallback(_clobber_shares)
4233
4234         d.addCallback(self.CHECK, "good", "t=check")
4235         def _got_html_good(res):
4236             self.failUnlessIn("Healthy", res)
4237             self.failIfIn("Not Healthy", res)
4238             self.failUnlessIn(FAVICON_MARKUP, res)
4239         d.addCallback(_got_html_good)
4240         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
4241         def _got_html_good_return_to(res):
4242             self.failUnlessIn("Healthy", res)
4243             self.failIfIn("Not Healthy", res)
4244             self.failUnlessIn('<a href="somewhere">Return to file', res)
4245         d.addCallback(_got_html_good_return_to)
4246         d.addCallback(self.CHECK, "good", "t=check&output=json")
4247         def _got_json_good(res):
4248             r = simplejson.loads(res)
4249             self.failUnlessEqual(r["summary"], "Healthy")
4250             self.failUnless(r["results"]["healthy"])
4251             self.failIf(r["results"]["needs-rebalancing"])
4252             self.failUnless(r["results"]["recoverable"])
4253         d.addCallback(_got_json_good)
4254
4255         d.addCallback(self.CHECK, "small", "t=check")
4256         def _got_html_small(res):
4257             self.failUnlessIn("Literal files are always healthy", res)
4258             self.failIfIn("Not Healthy", res)
4259         d.addCallback(_got_html_small)
4260         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
4261         def _got_html_small_return_to(res):
4262             self.failUnlessIn("Literal files are always healthy", res)
4263             self.failIfIn("Not Healthy", res)
4264             self.failUnlessIn('<a href="somewhere">Return to file', res)
4265         d.addCallback(_got_html_small_return_to)
4266         d.addCallback(self.CHECK, "small", "t=check&output=json")
4267         def _got_json_small(res):
4268             r = simplejson.loads(res)
4269             self.failUnlessEqual(r["storage-index"], "")
4270             self.failUnless(r["results"]["healthy"])
4271         d.addCallback(_got_json_small)
4272
4273         d.addCallback(self.CHECK, "smalldir", "t=check")
4274         def _got_html_smalldir(res):
4275             self.failUnlessIn("Literal files are always healthy", res)
4276             self.failIfIn("Not Healthy", res)
4277         d.addCallback(_got_html_smalldir)
4278         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
4279         def _got_json_smalldir(res):
4280             r = simplejson.loads(res)
4281             self.failUnlessEqual(r["storage-index"], "")
4282             self.failUnless(r["results"]["healthy"])
4283         d.addCallback(_got_json_smalldir)
4284
4285         d.addCallback(self.CHECK, "sick", "t=check")
4286         def _got_html_sick(res):
4287             self.failUnlessIn("Not Healthy", res)
4288         d.addCallback(_got_html_sick)
4289         d.addCallback(self.CHECK, "sick", "t=check&output=json")
4290         def _got_json_sick(res):
4291             r = simplejson.loads(res)
4292             self.failUnlessEqual(r["summary"],
4293                                  "Not Healthy: 9 shares (enc 3-of-10)")
4294             self.failIf(r["results"]["healthy"])
4295             self.failIf(r["results"]["needs-rebalancing"])
4296             self.failUnless(r["results"]["recoverable"])
4297         d.addCallback(_got_json_sick)
4298
4299         d.addCallback(self.CHECK, "dead", "t=check")
4300         def _got_html_dead(res):
4301             self.failUnlessIn("Not Healthy", res)
4302         d.addCallback(_got_html_dead)
4303         d.addCallback(self.CHECK, "dead", "t=check&output=json")
4304         def _got_json_dead(res):
4305             r = simplejson.loads(res)
4306             self.failUnlessEqual(r["summary"],
4307                                  "Not Healthy: 1 shares (enc 3-of-10)")
4308             self.failIf(r["results"]["healthy"])
4309             self.failIf(r["results"]["needs-rebalancing"])
4310             self.failIf(r["results"]["recoverable"])
4311         d.addCallback(_got_json_dead)
4312
4313         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
4314         def _got_html_corrupt(res):
4315             self.failUnlessIn("Not Healthy! : Unhealthy", res)
4316         d.addCallback(_got_html_corrupt)
4317         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
4318         def _got_json_corrupt(res):
4319             r = simplejson.loads(res)
4320             self.failUnlessIn("Unhealthy: 9 shares (enc 3-of-10)", r["summary"])
4321             self.failIf(r["results"]["healthy"])
4322             self.failUnless(r["results"]["recoverable"])
4323             self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
4324             self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
4325         d.addCallback(_got_json_corrupt)
4326
4327         d.addErrback(self.explain_web_error)
4328         return d
4329
4330     def test_repair_html(self):
4331         self.basedir = "web/Grid/repair_html"
4332         self.set_up_grid()
4333         c0 = self.g.clients[0]
4334         self.uris = {}
4335         DATA = "data" * 100
4336         d = c0.upload(upload.Data(DATA, convergence=""))
4337         def _stash_uri(ur, which):
4338             self.uris[which] = ur.uri
4339         d.addCallback(_stash_uri, "good")
4340         d.addCallback(lambda ign:
4341                       c0.upload(upload.Data(DATA+"1", convergence="")))
4342         d.addCallback(_stash_uri, "sick")
4343         d.addCallback(lambda ign:
4344                       c0.upload(upload.Data(DATA+"2", convergence="")))
4345         d.addCallback(_stash_uri, "dead")
4346         def _stash_mutable_uri(n, which):
4347             self.uris[which] = n.get_uri()
4348             assert isinstance(self.uris[which], str)
4349         d.addCallback(lambda ign:
4350             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4351         d.addCallback(_stash_mutable_uri, "corrupt")
4352
4353         def _compute_fileurls(ignored):
4354             self.fileurls = {}
4355             for which in self.uris:
4356                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4357         d.addCallback(_compute_fileurls)
4358
4359         def _clobber_shares(ignored):
4360             good_shares = self.find_uri_shares(self.uris["good"])
4361             self.failUnlessReallyEqual(len(good_shares), 10)
4362             sick_shares = self.find_uri_shares(self.uris["sick"])
4363             os.unlink(sick_shares[0][2])
4364             dead_shares = self.find_uri_shares(self.uris["dead"])
4365             for i in range(1, 10):
4366                 os.unlink(dead_shares[i][2])
4367             c_shares = self.find_uri_shares(self.uris["corrupt"])
4368             cso = CorruptShareOptions()
4369             cso.stdout = StringIO()
4370             cso.parseOptions([c_shares[0][2]])
4371             corrupt_share(cso)
4372         d.addCallback(_clobber_shares)
4373
4374         d.addCallback(self.CHECK, "good", "t=check&repair=true")
4375         def _got_html_good(res):
4376             self.failUnlessIn("Healthy", res)
4377             self.failIfIn("Not Healthy", res)
4378             self.failUnlessIn("No repair necessary", res)
4379             self.failUnlessIn(FAVICON_MARKUP, res)
4380         d.addCallback(_got_html_good)
4381
4382         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
4383         def _got_html_sick(res):
4384             self.failUnlessIn("Healthy : healthy", res)
4385             self.failIfIn("Not Healthy", res)
4386             self.failUnlessIn("Repair successful", res)
4387         d.addCallback(_got_html_sick)
4388
4389         # repair of a dead file will fail, of course, but it isn't yet
4390         # clear how this should be reported. Right now it shows up as
4391         # a "410 Gone".
4392         #
4393         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
4394         #def _got_html_dead(res):
4395         #    print res
4396         #    self.failUnlessIn("Healthy : healthy", res)
4397         #    self.failIfIn("Not Healthy", res)
4398         #    self.failUnlessIn("No repair necessary", res)
4399         #d.addCallback(_got_html_dead)
4400
4401         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
4402         def _got_html_corrupt(res):
4403             self.failUnlessIn("Healthy : Healthy", res)
4404             self.failIfIn("Not Healthy", res)
4405             self.failUnlessIn("Repair successful", res)
4406         d.addCallback(_got_html_corrupt)
4407
4408         d.addErrback(self.explain_web_error)
4409         return d
4410
4411     def test_repair_json(self):
4412         self.basedir = "web/Grid/repair_json"
4413         self.set_up_grid()
4414         c0 = self.g.clients[0]
4415         self.uris = {}
4416         DATA = "data" * 100
4417         d = c0.upload(upload.Data(DATA+"1", convergence=""))
4418         def _stash_uri(ur, which):
4419             self.uris[which] = ur.uri
4420         d.addCallback(_stash_uri, "sick")
4421
4422         def _compute_fileurls(ignored):
4423             self.fileurls = {}
4424             for which in self.uris:
4425                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4426         d.addCallback(_compute_fileurls)
4427
4428         def _clobber_shares(ignored):
4429             sick_shares = self.find_uri_shares(self.uris["sick"])
4430             os.unlink(sick_shares[0][2])
4431         d.addCallback(_clobber_shares)
4432
4433         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
4434         def _got_json_sick(res):
4435             r = simplejson.loads(res)
4436             self.failUnlessReallyEqual(r["repair-attempted"], True)
4437             self.failUnlessReallyEqual(r["repair-successful"], True)
4438             self.failUnlessEqual(r["pre-repair-results"]["summary"],
4439                                  "Not Healthy: 9 shares (enc 3-of-10)")
4440             self.failIf(r["pre-repair-results"]["results"]["healthy"])
4441             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
4442             self.failUnless(r["post-repair-results"]["results"]["healthy"])
4443         d.addCallback(_got_json_sick)
4444
4445         d.addErrback(self.explain_web_error)
4446         return d
4447
4448     def test_unknown(self, immutable=False):
4449         self.basedir = "web/Grid/unknown"
4450         if immutable:
4451             self.basedir = "web/Grid/unknown-immutable"
4452
4453         self.set_up_grid()
4454         c0 = self.g.clients[0]
4455         self.uris = {}
4456         self.fileurls = {}
4457
4458         # the future cap format may contain slashes, which must be tolerated
4459         expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
4460                                                            safe="")
4461
4462         if immutable:
4463             name = u"future-imm"
4464             future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
4465             d = c0.create_immutable_dirnode({name: (future_node, {})})
4466         else:
4467             name = u"future"
4468             future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4469             d = c0.create_dirnode()
4470
4471         def _stash_root_and_create_file(n):
4472             self.rootnode = n
4473             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
4474             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
4475             if not immutable:
4476                 return self.rootnode.set_node(name, future_node)
4477         d.addCallback(_stash_root_and_create_file)
4478
4479         # make sure directory listing tolerates unknown nodes
4480         d.addCallback(lambda ign: self.GET(self.rooturl))
4481         def _check_directory_html(res, expected_type_suffix):
4482             pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
4483                                   '<td>%s</td>' % (expected_type_suffix, str(name)),
4484                                  re.DOTALL)
4485             self.failUnless(re.search(pattern, res), res)
4486             # find the More Info link for name, should be relative
4487             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4488             info_url = mo.group(1)
4489             self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
4490         if immutable:
4491             d.addCallback(_check_directory_html, "-IMM")
4492         else:
4493             d.addCallback(_check_directory_html, "")
4494
4495         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4496         def _check_directory_json(res, expect_rw_uri):
4497             data = simplejson.loads(res)
4498             self.failUnlessEqual(data[0], "dirnode")
4499             f = data[1]["children"][name]
4500             self.failUnlessEqual(f[0], "unknown")
4501             if expect_rw_uri:
4502                 self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data)
4503             else:
4504                 self.failIfIn("rw_uri", f[1])
4505             if immutable:
4506                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
4507             else:
4508                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
4509             self.failUnlessIn("metadata", f[1])
4510         d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
4511
4512         def _check_info(res, expect_rw_uri, expect_ro_uri):
4513             self.failUnlessIn("Object Type: <span>unknown</span>", res)
4514             if expect_rw_uri:
4515                 self.failUnlessIn(unknown_rwcap, res)
4516             if expect_ro_uri:
4517                 if immutable:
4518                     self.failUnlessIn(unknown_immcap, res)
4519                 else:
4520                     self.failUnlessIn(unknown_rocap, res)
4521             else:
4522                 self.failIfIn(unknown_rocap, res)
4523             self.failIfIn("Raw data as", res)
4524             self.failIfIn("Directory writecap", res)
4525             self.failIfIn("Checker Operations", res)
4526             self.failIfIn("Mutable File Operations", res)
4527             self.failIfIn("Directory Operations", res)
4528
4529         # FIXME: these should have expect_rw_uri=not immutable; I don't know
4530         # why they fail. Possibly related to ticket #922.
4531
4532         d.addCallback(lambda ign: self.GET(expected_info_url))
4533         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
4534         d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
4535         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
4536
4537         def _check_json(res, expect_rw_uri):
4538             data = simplejson.loads(res)
4539             self.failUnlessEqual(data[0], "unknown")
4540             if expect_rw_uri:
4541                 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data)
4542             else:
4543                 self.failIfIn("rw_uri", data[1])
4544
4545             if immutable:
4546                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data)
4547                 self.failUnlessReallyEqual(data[1]["mutable"], False)
4548             elif expect_rw_uri:
4549                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4550                 self.failUnlessReallyEqual(data[1]["mutable"], True)
4551             else:
4552                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4553                 self.failIfIn("mutable", data[1])
4554
4555             # TODO: check metadata contents
4556             self.failUnlessIn("metadata", data[1])
4557
4558         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
4559         d.addCallback(_check_json, expect_rw_uri=not immutable)
4560
4561         # and make sure that a read-only version of the directory can be
4562         # rendered too. This version will not have unknown_rwcap, whether
4563         # or not future_node was immutable.
4564         d.addCallback(lambda ign: self.GET(self.rourl))
4565         if immutable:
4566             d.addCallback(_check_directory_html, "-IMM")
4567         else:
4568             d.addCallback(_check_directory_html, "-RO")
4569
4570         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
4571         d.addCallback(_check_directory_json, expect_rw_uri=False)
4572
4573         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
4574         d.addCallback(_check_json, expect_rw_uri=False)
4575
4576         # TODO: check that getting t=info from the Info link in the ro directory
4577         # works, and does not include the writecap URI.
4578         return d
4579
4580     def test_immutable_unknown(self):
4581         return self.test_unknown(immutable=True)
4582
4583     def test_mutant_dirnodes_are_omitted(self):
4584         self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
4585
4586         self.set_up_grid()
4587         c = self.g.clients[0]
4588         nm = c.nodemaker
4589         self.uris = {}
4590         self.fileurls = {}
4591
4592         lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
4593         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
4594         mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
4595
4596         # This method tests mainly dirnode, but we'd have to duplicate code in order to
4597         # test the dirnode and web layers separately.
4598
4599         # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
4600         # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
4601         # When the directory is read, the mutants should be silently disposed of, leaving
4602         # their lonely sibling.
4603         # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
4604         # because immutable directories don't have a writecap and therefore that field
4605         # isn't (and can't be) decrypted.
4606         # TODO: The field still exists in the netstring. Technically we should check what
4607         # happens if something is put there (_unpack_contents should raise ValueError),
4608         # but that can wait.
4609
4610         lonely_child = nm.create_from_cap(lonely_uri)
4611         mutant_ro_child = nm.create_from_cap(mut_read_uri)
4612         mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
4613
4614         def _by_hook_or_by_crook():
4615             return True
4616         for n in [mutant_ro_child, mutant_write_in_ro_child]:
4617             n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
4618
4619         mutant_write_in_ro_child.get_write_uri    = lambda: None
4620         mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
4621
4622         kids = {u"lonely":      (lonely_child, {}),
4623                 u"ro":          (mutant_ro_child, {}),
4624                 u"write-in-ro": (mutant_write_in_ro_child, {}),
4625                 }
4626         d = c.create_immutable_dirnode(kids)
4627
4628         def _created(dn):
4629             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
4630             self.failIf(dn.is_mutable())
4631             self.failUnless(dn.is_readonly())
4632             # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
4633             self.failIf(hasattr(dn._node, 'get_writekey'))
4634             rep = str(dn)
4635             self.failUnlessIn("RO-IMM", rep)
4636             cap = dn.get_cap()
4637             self.failUnlessIn("CHK", cap.to_string())
4638             self.cap = cap
4639             self.rootnode = dn
4640             self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
4641             return download_to_data(dn._node)
4642         d.addCallback(_created)
4643
4644         def _check_data(data):
4645             # Decode the netstring representation of the directory to check that all children
4646             # are present. This is a bit of an abstraction violation, but there's not really
4647             # any other way to do it given that the real DirectoryNode._unpack_contents would
4648             # strip the mutant children out (which is what we're trying to test, later).
4649             position = 0
4650             numkids = 0
4651             while position < len(data):
4652                 entries, position = split_netstring(data, 1, position)
4653                 entry = entries[0]
4654                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
4655                 name = name_utf8.decode("utf-8")
4656                 self.failUnlessEqual(rwcapdata, "")
4657                 self.failUnlessIn(name, kids)
4658                 (expected_child, ign) = kids[name]
4659                 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
4660                 numkids += 1
4661
4662             self.failUnlessReallyEqual(numkids, 3)
4663             return self.rootnode.list()
4664         d.addCallback(_check_data)
4665
4666         # Now when we use the real directory listing code, the mutants should be absent.
4667         def _check_kids(children):
4668             self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
4669             lonely_node, lonely_metadata = children[u"lonely"]
4670
4671             self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
4672             self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
4673         d.addCallback(_check_kids)
4674
4675         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
4676         d.addCallback(lambda n: n.list())
4677         d.addCallback(_check_kids)  # again with dirnode recreated from cap
4678
4679         # Make sure the lonely child can be listed in HTML...
4680         d.addCallback(lambda ign: self.GET(self.rooturl))
4681         def _check_html(res):
4682             self.failIfIn("URI:SSK", res)
4683             get_lonely = "".join([r'<td>FILE</td>',
4684                                   r'\s+<td>',
4685                                   r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
4686                                   r'</td>',
4687                                   r'\s+<td align="right">%d</td>' % len("one"),
4688                                   ])
4689             self.failUnless(re.search(get_lonely, res), res)
4690
4691             # find the More Info link for name, should be relative
4692             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4693             info_url = mo.group(1)
4694             self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
4695         d.addCallback(_check_html)
4696
4697         # ... and in JSON.
4698         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4699         def _check_json(res):
4700             data = simplejson.loads(res)
4701             self.failUnlessEqual(data[0], "dirnode")
4702             listed_children = data[1]["children"]
4703             self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
4704             ll_type, ll_data = listed_children[u"lonely"]
4705             self.failUnlessEqual(ll_type, "filenode")
4706             self.failIfIn("rw_uri", ll_data)
4707             self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
4708         d.addCallback(_check_json)
4709         return d
4710
4711     def test_deep_check(self):
4712         self.basedir = "web/Grid/deep_check"
4713         self.set_up_grid()
4714         c0 = self.g.clients[0]
4715         self.uris = {}
4716         self.fileurls = {}
4717         DATA = "data" * 100
4718         d = c0.create_dirnode()
4719         def _stash_root_and_create_file(n):
4720             self.rootnode = n
4721             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4722             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4723         d.addCallback(_stash_root_and_create_file)
4724         def _stash_uri(fn, which):
4725             self.uris[which] = fn.get_uri()
4726             return fn
4727         d.addCallback(_stash_uri, "good")
4728         d.addCallback(lambda ign:
4729                       self.rootnode.add_file(u"small",
4730                                              upload.Data("literal",
4731                                                         convergence="")))
4732         d.addCallback(_stash_uri, "small")
4733         d.addCallback(lambda ign:
4734                       self.rootnode.add_file(u"sick",
4735                                              upload.Data(DATA+"1",
4736                                                         convergence="")))
4737         d.addCallback(_stash_uri, "sick")
4738
4739         # this tests that deep-check and stream-manifest will ignore
4740         # UnknownNode instances. Hopefully this will also cover deep-stats.
4741         future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4742         d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
4743
4744         def _clobber_shares(ignored):
4745             self.delete_shares_numbered(self.uris["sick"], [0,1])
4746         d.addCallback(_clobber_shares)
4747
4748         # root
4749         # root/good
4750         # root/small
4751         # root/sick
4752         # root/future
4753
4754         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4755         def _done(res):
4756             try:
4757                 units = [simplejson.loads(line)
4758                          for line in res.splitlines()
4759                          if line]
4760             except ValueError:
4761                 print "response is:", res
4762                 print "undecodeable line was '%s'" % line
4763                 raise
4764             self.failUnlessReallyEqual(len(units), 5+1)
4765             # should be parent-first
4766             u0 = units[0]
4767             self.failUnlessEqual(u0["path"], [])
4768             self.failUnlessEqual(u0["type"], "directory")
4769             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4770             u0cr = u0["check-results"]
4771             self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
4772
4773             ugood = [u for u in units
4774                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4775             self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
4776             ugoodcr = ugood["check-results"]
4777             self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
4778
4779             stats = units[-1]
4780             self.failUnlessEqual(stats["type"], "stats")
4781             s = stats["stats"]
4782             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4783             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4784             self.failUnlessReallyEqual(s["count-directories"], 1)
4785             self.failUnlessReallyEqual(s["count-unknown"], 1)
4786         d.addCallback(_done)
4787
4788         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4789         def _check_manifest(res):
4790             self.failUnless(res.endswith("\n"))
4791             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
4792             self.failUnlessReallyEqual(len(units), 5+1)
4793             self.failUnlessEqual(units[-1]["type"], "stats")
4794             first = units[0]
4795             self.failUnlessEqual(first["path"], [])
4796             self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri())
4797             self.failUnlessEqual(first["type"], "directory")
4798             stats = units[-1]["stats"]
4799             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4800             self.failUnlessReallyEqual(stats["count-literal-files"], 1)
4801             self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
4802             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4803             self.failUnlessReallyEqual(stats["count-unknown"], 1)
4804         d.addCallback(_check_manifest)
4805
4806         # now add root/subdir and root/subdir/grandchild, then make subdir
4807         # unrecoverable, then see what happens
4808
4809         d.addCallback(lambda ign:
4810                       self.rootnode.create_subdirectory(u"subdir"))
4811         d.addCallback(_stash_uri, "subdir")
4812         d.addCallback(lambda subdir_node:
4813                       subdir_node.add_file(u"grandchild",
4814                                            upload.Data(DATA+"2",
4815                                                        convergence="")))
4816         d.addCallback(_stash_uri, "grandchild")
4817
4818         d.addCallback(lambda ign:
4819                       self.delete_shares_numbered(self.uris["subdir"],
4820                                                   range(1, 10)))
4821
4822         # root
4823         # root/good
4824         # root/small
4825         # root/sick
4826         # root/future
4827         # root/subdir [unrecoverable]
4828         # root/subdir/grandchild
4829
4830         # how should a streaming-JSON API indicate fatal error?
4831         # answer: emit ERROR: instead of a JSON string
4832
4833         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4834         def _check_broken_manifest(res):
4835             lines = res.splitlines()
4836             error_lines = [i
4837                            for (i,line) in enumerate(lines)
4838                            if line.startswith("ERROR:")]
4839             if not error_lines:
4840                 self.fail("no ERROR: in output: %s" % (res,))
4841             first_error = error_lines[0]
4842             error_line = lines[first_error]
4843             error_msg = lines[first_error+1:]
4844             error_msg_s = "\n".join(error_msg) + "\n"
4845             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4846                               error_line)
4847             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4848             units = [simplejson.loads(line) for line in lines[:first_error]]
4849             self.failUnlessReallyEqual(len(units), 6) # includes subdir
4850             last_unit = units[-1]
4851             self.failUnlessEqual(last_unit["path"], ["subdir"])
4852         d.addCallback(_check_broken_manifest)
4853
4854         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4855         def _check_broken_deepcheck(res):
4856             lines = res.splitlines()
4857             error_lines = [i
4858                            for (i,line) in enumerate(lines)
4859                            if line.startswith("ERROR:")]
4860             if not error_lines:
4861                 self.fail("no ERROR: in output: %s" % (res,))
4862             first_error = error_lines[0]
4863             error_line = lines[first_error]
4864             error_msg = lines[first_error+1:]
4865             error_msg_s = "\n".join(error_msg) + "\n"
4866             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4867                               error_line)
4868             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4869             units = [simplejson.loads(line) for line in lines[:first_error]]
4870             self.failUnlessReallyEqual(len(units), 6) # includes subdir
4871             last_unit = units[-1]
4872             self.failUnlessEqual(last_unit["path"], ["subdir"])
4873             r = last_unit["check-results"]["results"]
4874             self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
4875             self.failUnlessReallyEqual(r["count-shares-good"], 1)
4876             self.failUnlessReallyEqual(r["recoverable"], False)
4877         d.addCallback(_check_broken_deepcheck)
4878
4879         d.addErrback(self.explain_web_error)
4880         return d
4881
4882     def test_deep_check_and_repair(self):
4883         self.basedir = "web/Grid/deep_check_and_repair"
4884         self.set_up_grid()
4885         c0 = self.g.clients[0]
4886         self.uris = {}
4887         self.fileurls = {}
4888         DATA = "data" * 100
4889         d = c0.create_dirnode()
4890         def _stash_root_and_create_file(n):
4891             self.rootnode = n
4892             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4893             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4894         d.addCallback(_stash_root_and_create_file)
4895         def _stash_uri(fn, which):
4896             self.uris[which] = fn.get_uri()
4897         d.addCallback(_stash_uri, "good")
4898         d.addCallback(lambda ign:
4899                       self.rootnode.add_file(u"small",
4900                                              upload.Data("literal",
4901                                                         convergence="")))
4902         d.addCallback(_stash_uri, "small")
4903         d.addCallback(lambda ign:
4904                       self.rootnode.add_file(u"sick",
4905                                              upload.Data(DATA+"1",
4906                                                         convergence="")))
4907         d.addCallback(_stash_uri, "sick")
4908         #d.addCallback(lambda ign:
4909         #              self.rootnode.add_file(u"dead",
4910         #                                     upload.Data(DATA+"2",
4911         #                                                convergence="")))
4912         #d.addCallback(_stash_uri, "dead")
4913
4914         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
4915         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
4916         #d.addCallback(_stash_uri, "corrupt")
4917
4918         def _clobber_shares(ignored):
4919             good_shares = self.find_uri_shares(self.uris["good"])
4920             self.failUnlessReallyEqual(len(good_shares), 10)
4921             sick_shares = self.find_uri_shares(self.uris["sick"])
4922             os.unlink(sick_shares[0][2])
4923             #dead_shares = self.find_uri_shares(self.uris["dead"])
4924             #for i in range(1, 10):
4925             #    os.unlink(dead_shares[i][2])
4926
4927             #c_shares = self.find_uri_shares(self.uris["corrupt"])
4928             #cso = CorruptShareOptions()
4929             #cso.stdout = StringIO()
4930             #cso.parseOptions([c_shares[0][2]])
4931             #corrupt_share(cso)
4932         d.addCallback(_clobber_shares)
4933
4934         # root
4935         # root/good   CHK, 10 shares
4936         # root/small  LIT
4937         # root/sick   CHK, 9 shares
4938
4939         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
4940         def _done(res):
4941             units = [simplejson.loads(line)
4942                      for line in res.splitlines()
4943                      if line]
4944             self.failUnlessReallyEqual(len(units), 4+1)
4945             # should be parent-first
4946             u0 = units[0]
4947             self.failUnlessEqual(u0["path"], [])
4948             self.failUnlessEqual(u0["type"], "directory")
4949             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4950             u0crr = u0["check-and-repair-results"]
4951             self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
4952             self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
4953
4954             ugood = [u for u in units
4955                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4956             self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
4957             ugoodcrr = ugood["check-and-repair-results"]
4958             self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
4959             self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
4960
4961             usick = [u for u in units
4962                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
4963             self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"])
4964             usickcrr = usick["check-and-repair-results"]
4965             self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
4966             self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
4967             self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
4968             self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
4969
4970             stats = units[-1]
4971             self.failUnlessEqual(stats["type"], "stats")
4972             s = stats["stats"]
4973             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4974             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4975             self.failUnlessReallyEqual(s["count-directories"], 1)
4976         d.addCallback(_done)
4977
4978         d.addErrback(self.explain_web_error)
4979         return d
4980
4981     def _count_leases(self, ignored, which):
4982         u = self.uris[which]
4983         shares = self.find_uri_shares(u)
4984         lease_counts = []
4985         for shnum, serverid, fn in shares:
4986             sf = get_share_file(fn)
4987             num_leases = len(list(sf.get_leases()))
4988             lease_counts.append( (fn, num_leases) )
4989         return lease_counts
4990
4991     def _assert_leasecount(self, lease_counts, expected):
4992         for (fn, num_leases) in lease_counts:
4993             if num_leases != expected:
4994                 self.fail("expected %d leases, have %d, on %s" %
4995                           (expected, num_leases, fn))
4996
4997     def test_add_lease(self):
4998         self.basedir = "web/Grid/add_lease"
4999         self.set_up_grid(num_clients=2)
5000         c0 = self.g.clients[0]
5001         self.uris = {}
5002         DATA = "data" * 100
5003         d = c0.upload(upload.Data(DATA, convergence=""))
5004         def _stash_uri(ur, which):
5005             self.uris[which] = ur.uri
5006         d.addCallback(_stash_uri, "one")
5007         d.addCallback(lambda ign:
5008                       c0.upload(upload.Data(DATA+"1", convergence="")))
5009         d.addCallback(_stash_uri, "two")
5010         def _stash_mutable_uri(n, which):
5011             self.uris[which] = n.get_uri()
5012             assert isinstance(self.uris[which], str)
5013         d.addCallback(lambda ign:
5014             c0.create_mutable_file(publish.MutableData(DATA+"2")))
5015         d.addCallback(_stash_mutable_uri, "mutable")
5016
5017         def _compute_fileurls(ignored):
5018             self.fileurls = {}
5019             for which in self.uris:
5020                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
5021         d.addCallback(_compute_fileurls)
5022
5023         d.addCallback(self._count_leases, "one")
5024         d.addCallback(self._assert_leasecount, 1)
5025         d.addCallback(self._count_leases, "two")
5026         d.addCallback(self._assert_leasecount, 1)
5027         d.addCallback(self._count_leases, "mutable")
5028         d.addCallback(self._assert_leasecount, 1)
5029
5030         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
5031         def _got_html_good(res):
5032             self.failUnlessIn("Healthy", res)
5033             self.failIfIn("Not Healthy", res)
5034         d.addCallback(_got_html_good)
5035
5036         d.addCallback(self._count_leases, "one")
5037         d.addCallback(self._assert_leasecount, 1)
5038         d.addCallback(self._count_leases, "two")
5039         d.addCallback(self._assert_leasecount, 1)
5040         d.addCallback(self._count_leases, "mutable")
5041         d.addCallback(self._assert_leasecount, 1)
5042
5043         # this CHECK uses the original client, which uses the same
5044         # lease-secrets, so it will just renew the original lease
5045         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
5046         d.addCallback(_got_html_good)
5047
5048         d.addCallback(self._count_leases, "one")
5049         d.addCallback(self._assert_leasecount, 1)
5050         d.addCallback(self._count_leases, "two")
5051         d.addCallback(self._assert_leasecount, 1)
5052         d.addCallback(self._count_leases, "mutable")
5053         d.addCallback(self._assert_leasecount, 1)
5054
5055         # this CHECK uses an alternate client, which adds a second lease
5056         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
5057         d.addCallback(_got_html_good)
5058
5059         d.addCallback(self._count_leases, "one")
5060         d.addCallback(self._assert_leasecount, 2)
5061         d.addCallback(self._count_leases, "two")
5062         d.addCallback(self._assert_leasecount, 1)
5063         d.addCallback(self._count_leases, "mutable")
5064         d.addCallback(self._assert_leasecount, 1)
5065
5066         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
5067         d.addCallback(_got_html_good)
5068
5069         d.addCallback(self._count_leases, "one")
5070         d.addCallback(self._assert_leasecount, 2)
5071         d.addCallback(self._count_leases, "two")
5072         d.addCallback(self._assert_leasecount, 1)
5073         d.addCallback(self._count_leases, "mutable")
5074         d.addCallback(self._assert_leasecount, 1)
5075
5076         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
5077                       clientnum=1)
5078         d.addCallback(_got_html_good)
5079
5080         d.addCallback(self._count_leases, "one")
5081         d.addCallback(self._assert_leasecount, 2)
5082         d.addCallback(self._count_leases, "two")
5083         d.addCallback(self._assert_leasecount, 1)
5084         d.addCallback(self._count_leases, "mutable")
5085         d.addCallback(self._assert_leasecount, 2)
5086
5087         d.addErrback(self.explain_web_error)
5088         return d
5089
5090     def test_deep_add_lease(self):
5091         self.basedir = "web/Grid/deep_add_lease"
5092         self.set_up_grid(num_clients=2)
5093         c0 = self.g.clients[0]
5094         self.uris = {}
5095         self.fileurls = {}
5096         DATA = "data" * 100
5097         d = c0.create_dirnode()
5098         def _stash_root_and_create_file(n):
5099             self.rootnode = n
5100             self.uris["root"] = n.get_uri()
5101             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5102             return n.add_file(u"one", upload.Data(DATA, convergence=""))
5103         d.addCallback(_stash_root_and_create_file)
5104         def _stash_uri(fn, which):
5105             self.uris[which] = fn.get_uri()
5106         d.addCallback(_stash_uri, "one")
5107         d.addCallback(lambda ign:
5108                       self.rootnode.add_file(u"small",
5109                                              upload.Data("literal",
5110                                                         convergence="")))
5111         d.addCallback(_stash_uri, "small")
5112
5113         d.addCallback(lambda ign:
5114             c0.create_mutable_file(publish.MutableData("mutable")))
5115         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
5116         d.addCallback(_stash_uri, "mutable")
5117
5118         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
5119         def _done(res):
5120             units = [simplejson.loads(line)
5121                      for line in res.splitlines()
5122                      if line]
5123             # root, one, small, mutable,   stats
5124             self.failUnlessReallyEqual(len(units), 4+1)
5125         d.addCallback(_done)
5126
5127         d.addCallback(self._count_leases, "root")
5128         d.addCallback(self._assert_leasecount, 1)
5129         d.addCallback(self._count_leases, "one")
5130         d.addCallback(self._assert_leasecount, 1)
5131         d.addCallback(self._count_leases, "mutable")
5132         d.addCallback(self._assert_leasecount, 1)
5133
5134         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
5135         d.addCallback(_done)
5136
5137         d.addCallback(self._count_leases, "root")
5138         d.addCallback(self._assert_leasecount, 1)
5139         d.addCallback(self._count_leases, "one")
5140         d.addCallback(self._assert_leasecount, 1)
5141         d.addCallback(self._count_leases, "mutable")
5142         d.addCallback(self._assert_leasecount, 1)
5143
5144         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
5145                       clientnum=1)
5146         d.addCallback(_done)
5147
5148         d.addCallback(self._count_leases, "root")
5149         d.addCallback(self._assert_leasecount, 2)
5150         d.addCallback(self._count_leases, "one")
5151         d.addCallback(self._assert_leasecount, 2)
5152         d.addCallback(self._count_leases, "mutable")
5153         d.addCallback(self._assert_leasecount, 2)
5154
5155         d.addErrback(self.explain_web_error)
5156         return d
5157
5158
5159     def test_exceptions(self):
5160         self.basedir = "web/Grid/exceptions"
5161         self.set_up_grid(num_clients=1, num_servers=2)
5162         c0 = self.g.clients[0]
5163         c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 2
5164         self.fileurls = {}
5165         DATA = "data" * 100
5166         d = c0.create_dirnode()
5167         def _stash_root(n):
5168             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5169             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
5170             return n
5171         d.addCallback(_stash_root)
5172         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
5173         def _stash_bad(ur):
5174             self.fileurls["1share"] = "uri/" + urllib.quote(ur.uri)
5175             self.delete_shares_numbered(ur.uri, range(1,10))
5176
5177             u = uri.from_string(ur.uri)
5178             u.key = testutil.flip_bit(u.key, 0)
5179             baduri = u.to_string()
5180             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
5181         d.addCallback(_stash_bad)
5182         d.addCallback(lambda ign: c0.create_dirnode())
5183         def _mangle_dirnode_1share(n):
5184             u = n.get_uri()
5185             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
5186             self.fileurls["dir-1share-json"] = url + "?t=json"
5187             self.delete_shares_numbered(u, range(1,10))
5188         d.addCallback(_mangle_dirnode_1share)
5189         d.addCallback(lambda ign: c0.create_dirnode())
5190         def _mangle_dirnode_0share(n):
5191             u = n.get_uri()
5192             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
5193             self.fileurls["dir-0share-json"] = url + "?t=json"
5194             self.delete_shares_numbered(u, range(0,10))
5195         d.addCallback(_mangle_dirnode_0share)
5196
5197         # NotEnoughSharesError should be reported sensibly, with a
5198         # text/plain explanation of the problem, and perhaps some
5199         # information on which shares *could* be found.
5200
5201         d.addCallback(lambda ignored:
5202                       self.shouldHTTPError("GET unrecoverable",
5203                                            410, "Gone", "NoSharesError",
5204                                            self.GET, self.fileurls["0shares"]))
5205         def _check_zero_shares(body):
5206             self.failIfIn("<html>", body)
5207             body = " ".join(body.strip().split())
5208             exp = ("NoSharesError: no shares could be found. "
5209                    "Zero shares usually indicates a corrupt URI, or that "
5210                    "no servers were connected, but it might also indicate "
5211                    "severe corruption. You should perform a filecheck on "
5212                    "this object to learn more. The full error message is: "
5213                    "no shares (need 3). Last failure: None")
5214             self.failUnlessReallyEqual(exp, body)
5215         d.addCallback(_check_zero_shares)
5216
5217
5218         d.addCallback(lambda ignored:
5219                       self.shouldHTTPError("GET 1share",
5220                                            410, "Gone", "NotEnoughSharesError",
5221                                            self.GET, self.fileurls["1share"]))
5222         def _check_one_share(body):
5223             self.failIfIn("<html>", body)
5224             body = " ".join(body.strip().split())
5225             msgbase = ("NotEnoughSharesError: This indicates that some "
5226                        "servers were unavailable, or that shares have been "
5227                        "lost to server departure, hard drive failure, or disk "
5228                        "corruption. You should perform a filecheck on "
5229                        "this object to learn more. The full error message is:"
5230                        )
5231             msg1 = msgbase + (" ran out of shares:"
5232                               " complete=sh0"
5233                               " pending="
5234                               " overdue= unused= need 3. Last failure: None")
5235             msg2 = msgbase + (" ran out of shares:"
5236                               " complete="
5237                               " pending=Share(sh0-on-xgru5)"
5238                               " overdue= unused= need 3. Last failure: None")
5239             self.failUnless(body == msg1 or body == msg2, body)
5240         d.addCallback(_check_one_share)
5241
5242         d.addCallback(lambda ignored:
5243                       self.shouldHTTPError("GET imaginary",
5244                                            404, "Not Found", None,
5245                                            self.GET, self.fileurls["imaginary"]))
5246         def _missing_child(body):
5247             self.failUnlessIn("No such child: imaginary", body)
5248         d.addCallback(_missing_child)
5249
5250         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
5251         def _check_0shares_dir_html(body):
5252             self.failUnlessIn("<html>", body)
5253             # we should see the regular page, but without the child table or
5254             # the dirops forms
5255             body = " ".join(body.strip().split())
5256             self.failUnlessIn('href="?t=info">More info on this directory',
5257                               body)
5258             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5259                    "could not be retrieved, because there were insufficient "
5260                    "good shares. This might indicate that no servers were "
5261                    "connected, insufficient servers were connected, the URI "
5262                    "was corrupt, or that shares have been lost due to server "
5263                    "departure, hard drive failure, or disk corruption. You "
5264                    "should perform a filecheck on this object to learn more.")
5265             self.failUnlessIn(exp, body)
5266             self.failUnlessIn("No upload forms: directory is unreadable", body)
5267         d.addCallback(_check_0shares_dir_html)
5268
5269         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
5270         def _check_1shares_dir_html(body):
5271             # at some point, we'll split UnrecoverableFileError into 0-shares
5272             # and some-shares like we did for immutable files (since there
5273             # are different sorts of advice to offer in each case). For now,
5274             # they present the same way.
5275             self.failUnlessIn("<html>", body)
5276             body = " ".join(body.strip().split())
5277             self.failUnlessIn('href="?t=info">More info on this directory',
5278                               body)
5279             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5280                    "could not be retrieved, because there were insufficient "
5281                    "good shares. This might indicate that no servers were "
5282                    "connected, insufficient servers were connected, the URI "
5283                    "was corrupt, or that shares have been lost due to server "
5284                    "departure, hard drive failure, or disk corruption. You "
5285                    "should perform a filecheck on this object to learn more.")
5286             self.failUnlessIn(exp, body)
5287             self.failUnlessIn("No upload forms: directory is unreadable", body)
5288         d.addCallback(_check_1shares_dir_html)
5289
5290         d.addCallback(lambda ignored:
5291                       self.shouldHTTPError("GET dir-0share-json",
5292                                            410, "Gone", "UnrecoverableFileError",
5293                                            self.GET,
5294                                            self.fileurls["dir-0share-json"]))
5295         def _check_unrecoverable_file(body):
5296             self.failIfIn("<html>", body)
5297             body = " ".join(body.strip().split())
5298             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5299                    "could not be retrieved, because there were insufficient "
5300                    "good shares. This might indicate that no servers were "
5301                    "connected, insufficient servers were connected, the URI "
5302                    "was corrupt, or that shares have been lost due to server "
5303                    "departure, hard drive failure, or disk corruption. You "
5304                    "should perform a filecheck on this object to learn more.")
5305             self.failUnlessReallyEqual(exp, body)
5306         d.addCallback(_check_unrecoverable_file)
5307
5308         d.addCallback(lambda ignored:
5309                       self.shouldHTTPError("GET dir-1share-json",
5310                                            410, "Gone", "UnrecoverableFileError",
5311                                            self.GET,
5312                                            self.fileurls["dir-1share-json"]))
5313         d.addCallback(_check_unrecoverable_file)
5314
5315         d.addCallback(lambda ignored:
5316                       self.shouldHTTPError("GET imaginary",
5317                                            404, "Not Found", None,
5318                                            self.GET, self.fileurls["imaginary"]))
5319
5320         # attach a webapi child that throws a random error, to test how it
5321         # gets rendered.
5322         w = c0.getServiceNamed("webish")
5323         w.root.putChild("ERRORBOOM", ErrorBoom())
5324
5325         # "Accept: */*" :        should get a text/html stack trace
5326         # "Accept: text/plain" : should get a text/plain stack trace
5327         # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
5328         # no Accept header:      should get a text/html stack trace
5329
5330         d.addCallback(lambda ignored:
5331                       self.shouldHTTPError("GET errorboom_html",
5332                                            500, "Internal Server Error", None,
5333                                            self.GET, "ERRORBOOM",
5334                                            headers={"accept": "*/*"}))
5335         def _internal_error_html1(body):
5336             self.failUnlessIn("<html>", "expected HTML, not '%s'" % body)
5337         d.addCallback(_internal_error_html1)
5338
5339         d.addCallback(lambda ignored:
5340                       self.shouldHTTPError("GET errorboom_text",
5341                                            500, "Internal Server Error", None,
5342                                            self.GET, "ERRORBOOM",
5343                                            headers={"accept": "text/plain"}))
5344         def _internal_error_text2(body):
5345             self.failIfIn("<html>", body)
5346             self.failUnless(body.startswith("Traceback "), body)
5347         d.addCallback(_internal_error_text2)
5348
5349         CLI_accepts = "text/plain, application/octet-stream"
5350         d.addCallback(lambda ignored:
5351                       self.shouldHTTPError("GET errorboom_text",
5352                                            500, "Internal Server Error", None,
5353                                            self.GET, "ERRORBOOM",
5354                                            headers={"accept": CLI_accepts}))
5355         def _internal_error_text3(body):
5356             self.failIfIn("<html>", body)
5357             self.failUnless(body.startswith("Traceback "), body)
5358         d.addCallback(_internal_error_text3)
5359
5360         d.addCallback(lambda ignored:
5361                       self.shouldHTTPError("GET errorboom_text",
5362                                            500, "Internal Server Error", None,
5363                                            self.GET, "ERRORBOOM"))
5364         def _internal_error_html4(body):
5365             self.failUnlessIn("<html>", body)
5366         d.addCallback(_internal_error_html4)
5367
5368         def _flush_errors(res):
5369             # Trial: please ignore the CompletelyUnhandledError in the logs
5370             self.flushLoggedErrors(CompletelyUnhandledError)
5371             return res
5372         d.addBoth(_flush_errors)
5373
5374         return d
5375
5376     def test_blacklist(self):
5377         # download from a blacklisted URI, get an error
5378         self.basedir = "web/Grid/blacklist"
5379         self.set_up_grid()
5380         c0 = self.g.clients[0]
5381         c0_basedir = c0.basedir
5382         fn = os.path.join(c0_basedir, "access.blacklist")
5383         self.uris = {}
5384         DATA = "off-limits " * 50
5385
5386         d = c0.upload(upload.Data(DATA, convergence=""))
5387         def _stash_uri_and_create_dir(ur):
5388             self.uri = ur.uri
5389             self.url = "uri/"+self.uri
5390             u = uri.from_string_filenode(self.uri)
5391             self.si = u.get_storage_index()
5392             childnode = c0.create_node_from_uri(self.uri, None)
5393             return c0.create_dirnode({u"blacklisted.txt": (childnode,{}) })
5394         d.addCallback(_stash_uri_and_create_dir)
5395         def _stash_dir(node):
5396             self.dir_node = node
5397             self.dir_uri = node.get_uri()
5398             self.dir_url = "uri/"+self.dir_uri
5399         d.addCallback(_stash_dir)
5400         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5401         def _check_dir_html(body):
5402             self.failUnlessIn("<html>", body)
5403             self.failUnlessIn("blacklisted.txt</a>", body)
5404         d.addCallback(_check_dir_html)
5405         d.addCallback(lambda ign: self.GET(self.url))
5406         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5407
5408         def _blacklist(ign):
5409             f = open(fn, "w")
5410             f.write(" # this is a comment\n")
5411             f.write(" \n")
5412             f.write("\n") # also exercise blank lines
5413             f.write("%s %s\n" % (base32.b2a(self.si), "off-limits to you"))
5414             f.close()
5415             # clients should be checking the blacklist each time, so we don't
5416             # need to restart the client
5417         d.addCallback(_blacklist)
5418         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_uri",
5419                                                        403, "Forbidden",
5420                                                        "Access Prohibited: off-limits",
5421                                                        self.GET, self.url))
5422
5423         # We should still be able to list the parent directory, in HTML...
5424         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5425         def _check_dir_html2(body):
5426             self.failUnlessIn("<html>", body)
5427             self.failUnlessIn("blacklisted.txt</strike>", body)
5428         d.addCallback(_check_dir_html2)
5429
5430         # ... and in JSON (used by CLI).
5431         d.addCallback(lambda ign: self.GET(self.dir_url+"?t=json", followRedirect=True))
5432         def _check_dir_json(res):
5433             data = simplejson.loads(res)
5434             self.failUnless(isinstance(data, list), data)
5435             self.failUnlessEqual(data[0], "dirnode")
5436             self.failUnless(isinstance(data[1], dict), data)
5437             self.failUnlessIn("children", data[1])
5438             self.failUnlessIn("blacklisted.txt", data[1]["children"])
5439             childdata = data[1]["children"]["blacklisted.txt"]
5440             self.failUnless(isinstance(childdata, list), data)
5441             self.failUnlessEqual(childdata[0], "filenode")
5442             self.failUnless(isinstance(childdata[1], dict), data)
5443         d.addCallback(_check_dir_json)
5444
5445         def _unblacklist(ign):
5446             open(fn, "w").close()
5447             # the Blacklist object watches mtime to tell when the file has
5448             # changed, but on windows this test will run faster than the
5449             # filesystem's mtime resolution. So we edit Blacklist.last_mtime
5450             # to force a reload.
5451             self.g.clients[0].blacklist.last_mtime -= 2.0
5452         d.addCallback(_unblacklist)
5453
5454         # now a read should work
5455         d.addCallback(lambda ign: self.GET(self.url))
5456         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5457
5458         # read again to exercise the blacklist-is-unchanged logic
5459         d.addCallback(lambda ign: self.GET(self.url))
5460         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5461
5462         # now add a blacklisted directory, and make sure files under it are
5463         # refused too
5464         def _add_dir(ign):
5465             childnode = c0.create_node_from_uri(self.uri, None)
5466             return c0.create_dirnode({u"child": (childnode,{}) })
5467         d.addCallback(_add_dir)
5468         def _get_dircap(dn):
5469             self.dir_si_b32 = base32.b2a(dn.get_storage_index())
5470             self.dir_url_base = "uri/"+dn.get_write_uri()
5471             self.dir_url_json1 = "uri/"+dn.get_write_uri()+"?t=json"
5472             self.dir_url_json2 = "uri/"+dn.get_write_uri()+"/?t=json"
5473             self.dir_url_json_ro = "uri/"+dn.get_readonly_uri()+"/?t=json"
5474             self.child_url = "uri/"+dn.get_readonly_uri()+"/child"
5475         d.addCallback(_get_dircap)
5476         d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True))
5477         d.addCallback(lambda body: self.failUnlessIn("<html>", body))
5478         d.addCallback(lambda ign: self.GET(self.dir_url_json1))
5479         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5480         d.addCallback(lambda ign: self.GET(self.dir_url_json2))
5481         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5482         d.addCallback(lambda ign: self.GET(self.dir_url_json_ro))
5483         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5484         d.addCallback(lambda ign: self.GET(self.child_url))
5485         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5486
5487         def _block_dir(ign):
5488             f = open(fn, "w")
5489             f.write("%s %s\n" % (self.dir_si_b32, "dir-off-limits to you"))
5490             f.close()
5491             self.g.clients[0].blacklist.last_mtime -= 2.0
5492         d.addCallback(_block_dir)
5493         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir base",
5494                                                        403, "Forbidden",
5495                                                        "Access Prohibited: dir-off-limits",
5496                                                        self.GET, self.dir_url_base))
5497         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json1",
5498                                                        403, "Forbidden",
5499                                                        "Access Prohibited: dir-off-limits",
5500                                                        self.GET, self.dir_url_json1))
5501         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json2",
5502                                                        403, "Forbidden",
5503                                                        "Access Prohibited: dir-off-limits",
5504                                                        self.GET, self.dir_url_json2))
5505         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json_ro",
5506                                                        403, "Forbidden",
5507                                                        "Access Prohibited: dir-off-limits",
5508                                                        self.GET, self.dir_url_json_ro))
5509         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir child",
5510                                                        403, "Forbidden",
5511                                                        "Access Prohibited: dir-off-limits",
5512                                                        self.GET, self.child_url))
5513         return d
5514
5515
5516 class CompletelyUnhandledError(Exception):
5517     pass
5518 class ErrorBoom(rend.Page):
5519     def beforeRender(self, ctx):
5520         raise CompletelyUnhandledError("whoops")