src/allmydata/test/test_web.py

   1 import os.path, re, urllib, time
   2 import simplejson
   3 from StringIO import StringIO
   4
   5 from twisted.application import service
   6 from twisted.trial import unittest
   7 from twisted.internet import defer, reactor
   8 from twisted.internet.task import Clock
   9 from twisted.web import client, error, http
  10 from twisted.python import failure, log
  11
  12 from foolscap.api import fireEventually, flushEventualQueue
  13
  14 from nevow import rend
  15
  16 from allmydata import interfaces, uri, webish, dirnode
  17 from allmydata.storage.shares import get_share_file
  18 from allmydata.storage_client import StorageFarmBroker
  19 from allmydata.immutable import upload
  20 from allmydata.immutable.downloader.status import DownloadStatus
  21 from allmydata.dirnode import DirectoryNode
  22 from allmydata.nodemaker import NodeMaker
  23 from allmydata.unknown import UnknownNode
  24 from allmydata.web import status, common
  25 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  26 from allmydata.util import fileutil, base32, hashutil
  27 from allmydata.util.consumer import download_to_data
  28 from allmydata.util.netstring import split_netstring
  29 from allmydata.util.encodingutil import to_str
  30 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  31      create_chk_filenode, WebErrorMixin, ShouldFailMixin, \
  32      make_mutable_file_uri, create_mutable_filenode
  33 from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
  34 from allmydata.mutable import servermap, publish, retrieve
  35 import allmydata.test.common_util as testutil
  36 from allmydata.test.no_network import GridTestMixin
  37 from allmydata.test.common_web import HTTPClientGETFactory, \
  38      HTTPClientHEADFactory
  39 from allmydata.client import Client, SecretHolder
  40 from allmydata.introducer import IntroducerNode
  41
  42 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  43 # create a webserver that works against them
  44
  45 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  46
  47 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
  48 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
  49 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
  50
  51 FAVICON_MARKUP = '<link href="/icon.png" rel="shortcut icon" />'
  52
  53
  54 class FakeStatsProvider:
  55     def get_stats(self):
  56         stats = {'stats': {}, 'counters': {}}
  57         return stats
  58
  59 class FakeNodeMaker(NodeMaker):
  60     encoding_params = {
  61         'k': 3,
  62         'n': 10,
  63         'happy': 7,
  64         'max_segment_size':128*1024 # 1024=KiB
  65     }
  66     def _create_lit(self, cap):
  67         return FakeCHKFileNode(cap)
  68     def _create_immutable(self, cap):
  69         return FakeCHKFileNode(cap)
  70     def _create_mutable(self, cap):
  71         return FakeMutableFileNode(None,
  72                                    None,
  73                                    self.encoding_params, None).init_from_cap(cap)
  74     def create_mutable_file(self, contents="", keysize=None,
  75                             version=SDMF_VERSION):
  76         n = FakeMutableFileNode(None, None, self.encoding_params, None)
  77         return n.create(contents, version=version)
  78
  79 class FakeUploader(service.Service):
  80     name = "uploader"
  81     def upload(self, uploadable):
  82         d = uploadable.get_size()
  83         d.addCallback(lambda size: uploadable.read(size))
  84         def _got_data(datav):
  85             data = "".join(datav)
  86             n = create_chk_filenode(data)
  87             ur = upload.UploadResults(file_size=len(data),
  88                                       ciphertext_fetched=0,
  89                                       preexisting_shares=0,
  90                                       pushed_shares=10,
  91                                       sharemap={},
  92                                       servermap={},
  93                                       timings={},
  94                                       uri_extension_data={},
  95                                       uri_extension_hash="fake",
  96                                       verifycapstr="fakevcap")
  97             ur.set_uri(n.get_uri())
  98             return ur
  99         d.addCallback(_got_data)
 100         return d
 101     def get_helper_info(self):
 102         return (None, False)
 103
 104 class FakeIServer:
 105     def __init__(self, binaryserverid):
 106         self.binaryserverid = binaryserverid
 107     def get_name(self): return "short"
 108     def get_longname(self): return "long"
 109     def get_serverid(self): return self.binaryserverid
 110
 111 def build_one_ds():
 112     ds = DownloadStatus("storage_index", 1234)
 113     now = time.time()
 114
 115     serverA = FakeIServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
 116     serverB = FakeIServer(hashutil.tagged_hash("foo", "serverid_b")[:20])
 117     storage_index = hashutil.storage_index_hash("SI")
 118     e0 = ds.add_segment_request(0, now)
 119     e0.activate(now+0.5)
 120     e0.deliver(now+1, 0, 100, 0.5) # when, start,len, decodetime
 121     e1 = ds.add_segment_request(1, now+2)
 122     e1.error(now+3)
 123     # two outstanding requests
 124     e2 = ds.add_segment_request(2, now+4)
 125     e3 = ds.add_segment_request(3, now+5)
 126     del e2,e3 # hush pyflakes
 127
 128     # simulate a segment which gets delivered faster than a system clock tick (ticket #1166)
 129     e = ds.add_segment_request(4, now)
 130     e.activate(now)
 131     e.deliver(now, 0, 140, 0.5)
 132
 133     e = ds.add_dyhb_request(serverA, now)
 134     e.finished([1,2], now+1)
 135     e = ds.add_dyhb_request(serverB, now+2) # left unfinished
 136
 137     e = ds.add_read_event(0, 120, now)
 138     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 139     e.finished(now+1)
 140     e = ds.add_read_event(120, 30, now+2) # left unfinished
 141
 142     e = ds.add_block_request(serverA, 1, 100, 20, now)
 143     e.finished(20, now+1)
 144     e = ds.add_block_request(serverB, 1, 120, 30, now+1) # left unfinished
 145
 146     # make sure that add_read_event() can come first too
 147     ds1 = DownloadStatus(storage_index, 1234)
 148     e = ds1.add_read_event(0, 120, now)
 149     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 150     e.finished(now+1)
 151
 152     return ds
 153
 154 class FakeHistory:
 155     _all_upload_status = [upload.UploadStatus()]
 156     _all_download_status = [build_one_ds()]
 157     _all_mapupdate_statuses = [servermap.UpdateStatus()]
 158     _all_publish_statuses = [publish.PublishStatus()]
 159     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
 160
 161     def list_all_upload_statuses(self):
 162         return self._all_upload_status
 163     def list_all_download_statuses(self):
 164         return self._all_download_status
 165     def list_all_mapupdate_statuses(self):
 166         return self._all_mapupdate_statuses
 167     def list_all_publish_statuses(self):
 168         return self._all_publish_statuses
 169     def list_all_retrieve_statuses(self):
 170         return self._all_retrieve_statuses
 171     def list_all_helper_statuses(self):
 172         return []
 173
 174 class FakeClient(Client):
 175     def __init__(self):
 176         # don't upcall to Client.__init__, since we only want to initialize a
 177         # minimal subset
 178         service.MultiService.__init__(self)
 179         self.nodeid = "fake_nodeid"
 180         self.nickname = "fake_nickname"
 181         self.introducer_furl = "None"
 182         self.stats_provider = FakeStatsProvider()
 183         self._secret_holder = SecretHolder("lease secret", "convergence secret")
 184         self.helper = None
 185         self.convergence = "some random string"
 186         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 187         self.introducer_client = None
 188         self.history = FakeHistory()
 189         self.uploader = FakeUploader()
 190         self.uploader.setServiceParent(self)
 191         self.blacklist = None
 192         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 193                                        self.uploader, None,
 194                                        None, None, None)
 195         self.mutable_file_default = SDMF_VERSION
 196
 197     def startService(self):
 198         return service.MultiService.startService(self)
 199     def stopService(self):
 200         return service.MultiService.stopService(self)
 201
 202     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 203
 204 class WebMixin(object):
 205     def setUp(self):
 206         self.s = FakeClient()
 207         self.s.startService()
 208         self.staticdir = self.mktemp()
 209         self.clock = Clock()
 210         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
 211                                       clock=self.clock)
 212         self.ws.setServiceParent(self.s)
 213         self.webish_port = self.ws.getPortnum()
 214         self.webish_url = self.ws.getURL()
 215         assert self.webish_url.endswith("/")
 216         self.webish_url = self.webish_url[:-1] # these tests add their own /
 217
 218         l = [ self.s.create_dirnode() for x in range(6) ]
 219         d = defer.DeferredList(l)
 220         def _then(res):
 221             self.public_root = res[0][1]
 222             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 223             self.public_url = "/uri/" + self.public_root.get_uri()
 224             self.private_root = res[1][1]
 225
 226             foo = res[2][1]
 227             self._foo_node = foo
 228             self._foo_uri = foo.get_uri()
 229             self._foo_readonly_uri = foo.get_readonly_uri()
 230             self._foo_verifycap = foo.get_verify_cap().to_string()
 231             # NOTE: we ignore the deferred on all set_uri() calls, because we
 232             # know the fake nodes do these synchronously
 233             self.public_root.set_uri(u"foo", foo.get_uri(),
 234                                      foo.get_readonly_uri())
 235
 236             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 237             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 238             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 239
 240             # sdmf
 241             # XXX: Do we ever use this?
 242             self.BAZ_CONTENTS, n, self._baz_txt_uri, self._baz_txt_readonly_uri = self.makefile_mutable(0)
 243
 244             foo.set_uri(u"baz.txt", self._baz_txt_uri, self._baz_txt_readonly_uri)
 245
 246             # mdmf
 247             self.QUUX_CONTENTS, n, self._quux_txt_uri, self._quux_txt_readonly_uri = self.makefile_mutable(0, mdmf=True)
 248             assert self._quux_txt_uri.startswith("URI:MDMF")
 249             foo.set_uri(u"quux.txt", self._quux_txt_uri, self._quux_txt_readonly_uri)
 250
 251             foo.set_uri(u"empty", res[3][1].get_uri(),
 252                         res[3][1].get_readonly_uri())
 253             sub_uri = res[4][1].get_uri()
 254             self._sub_uri = sub_uri
 255             foo.set_uri(u"sub", sub_uri, sub_uri)
 256             sub = self.s.create_node_from_uri(sub_uri)
 257             self._sub_node = sub
 258
 259             _ign, n, blocking_uri = self.makefile(1)
 260             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 261
 262             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 263             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 264             # still think of it as an umlaut
 265             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 266
 267             self.SUBBAZ_CONTENTS, n, baz_file = self.makefile(2)
 268             self._baz_file_uri = baz_file
 269             sub.set_uri(u"baz.txt", baz_file, baz_file)
 270
 271             _ign, n, self._bad_file_uri = self.makefile(3)
 272             # this uri should not be downloadable
 273             del FakeCHKFileNode.all_contents[self._bad_file_uri]
 274
 275             rodir = res[5][1]
 276             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 277                                      rodir.get_readonly_uri())
 278             rodir.set_uri(u"nor", baz_file, baz_file)
 279
 280             # public/
 281             # public/foo/
 282             # public/foo/bar.txt
 283             # public/foo/baz.txt
 284             # public/foo/quux.txt
 285             # public/foo/blockingfile
 286             # public/foo/empty/
 287             # public/foo/sub/
 288             # public/foo/sub/baz.txt
 289             # public/reedownlee/
 290             # public/reedownlee/nor
 291             self.NEWFILE_CONTENTS = "newfile contents\n"
 292
 293             return foo.get_metadata_for(u"bar.txt")
 294         d.addCallback(_then)
 295         def _got_metadata(metadata):
 296             self._bar_txt_metadata = metadata
 297         d.addCallback(_got_metadata)
 298         return d
 299
 300     def makefile(self, number):
 301         contents = "contents of file %s\n" % number
 302         n = create_chk_filenode(contents)
 303         return contents, n, n.get_uri()
 304
 305     def makefile_mutable(self, number, mdmf=False):
 306         contents = "contents of mutable file %s\n" % number
 307         n = create_mutable_filenode(contents, mdmf)
 308         return contents, n, n.get_uri(), n.get_readonly_uri()
 309
 310     def tearDown(self):
 311         return self.s.stopService()
 312
 313     def failUnlessIsBarDotTxt(self, res):
 314         self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
 315
 316     def failUnlessIsQuuxDotTxt(self, res):
 317         self.failUnlessReallyEqual(res, self.QUUX_CONTENTS, res)
 318
 319     def failUnlessIsBazDotTxt(self, res):
 320         self.failUnlessReallyEqual(res, self.BAZ_CONTENTS, res)
 321
 322     def failUnlessIsSubBazDotTxt(self, res):
 323         self.failUnlessReallyEqual(res, self.SUBBAZ_CONTENTS, res)
 324
 325     def failUnlessIsBarJSON(self, res):
 326         data = simplejson.loads(res)
 327         self.failUnless(isinstance(data, list))
 328         self.failUnlessEqual(data[0], "filenode")
 329         self.failUnless(isinstance(data[1], dict))
 330         self.failIf(data[1]["mutable"])
 331         self.failIfIn("rw_uri", data[1]) # immutable
 332         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
 333         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
 334         self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
 335
 336     def failUnlessIsQuuxJSON(self, res, readonly=False):
 337         data = simplejson.loads(res)
 338         self.failUnless(isinstance(data, list))
 339         self.failUnlessEqual(data[0], "filenode")
 340         self.failUnless(isinstance(data[1], dict))
 341         metadata = data[1]
 342         return self.failUnlessIsQuuxDotTxtMetadata(metadata, readonly)
 343
 344     def failUnlessIsQuuxDotTxtMetadata(self, metadata, readonly):
 345         self.failUnless(metadata['mutable'])
 346         if readonly:
 347             self.failIfIn("rw_uri", metadata)
 348         else:
 349             self.failUnlessIn("rw_uri", metadata)
 350             self.failUnlessEqual(metadata['rw_uri'], self._quux_txt_uri)
 351         self.failUnlessIn("ro_uri", metadata)
 352         self.failUnlessEqual(metadata['ro_uri'], self._quux_txt_readonly_uri)
 353         self.failUnlessReallyEqual(metadata['size'], len(self.QUUX_CONTENTS))
 354
 355     def failUnlessIsFooJSON(self, res):
 356         data = simplejson.loads(res)
 357         self.failUnless(isinstance(data, list))
 358         self.failUnlessEqual(data[0], "dirnode", res)
 359         self.failUnless(isinstance(data[1], dict))
 360         self.failUnless(data[1]["mutable"])
 361         self.failUnlessIn("rw_uri", data[1]) # mutable
 362         self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
 363         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
 364         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
 365
 366         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 367         self.failUnlessEqual(kidnames,
 368                              [u"bar.txt", u"baz.txt", u"blockingfile",
 369                               u"empty", u"n\u00fc.txt", u"quux.txt", u"sub"])
 370         kids = dict( [(unicode(name),value)
 371                       for (name,value)
 372                       in data[1]["children"].iteritems()] )
 373         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 374         self.failUnlessIn("metadata", kids[u"sub"][1])
 375         self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
 376         tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
 377         self.failUnlessIn("linkcrtime", tahoe_md)
 378         self.failUnlessIn("linkmotime", tahoe_md)
 379         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 380         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 381         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri)
 382         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]),
 383                                    self._bar_txt_verifycap)
 384         self.failUnlessIn("metadata", kids[u"bar.txt"][1])
 385         self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
 386         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
 387                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
 388         self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
 389                                    self._bar_txt_uri)
 390         self.failUnlessIn("quux.txt", kids)
 391         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["rw_uri"]),
 392                                    self._quux_txt_uri)
 393         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["ro_uri"]),
 394                                    self._quux_txt_readonly_uri)
 395
 396     def GET(self, urlpath, followRedirect=False, return_response=False,
 397             **kwargs):
 398         # if return_response=True, this fires with (data, statuscode,
 399         # respheaders) instead of just data.
 400         assert not isinstance(urlpath, unicode)
 401         url = self.webish_url + urlpath
 402         factory = HTTPClientGETFactory(url, method="GET",
 403                                        followRedirect=followRedirect, **kwargs)
 404         reactor.connectTCP("localhost", self.webish_port, factory)
 405         d = factory.deferred
 406         def _got_data(data):
 407             return (data, factory.status, factory.response_headers)
 408         if return_response:
 409             d.addCallback(_got_data)
 410         return factory.deferred
 411
 412     def HEAD(self, urlpath, return_response=False, **kwargs):
 413         # this requires some surgery, because twisted.web.client doesn't want
 414         # to give us back the response headers.
 415         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 416         reactor.connectTCP("localhost", self.webish_port, factory)
 417         d = factory.deferred
 418         def _got_data(data):
 419             return (data, factory.status, factory.response_headers)
 420         if return_response:
 421             d.addCallback(_got_data)
 422         return factory.deferred
 423
 424     def PUT(self, urlpath, data, **kwargs):
 425         url = self.webish_url + urlpath
 426         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 427
 428     def DELETE(self, urlpath):
 429         url = self.webish_url + urlpath
 430         return client.getPage(url, method="DELETE")
 431
 432     def POST(self, urlpath, followRedirect=False, **fields):
 433         sepbase = "boogabooga"
 434         sep = "--" + sepbase
 435         form = []
 436         form.append(sep)
 437         form.append('Content-Disposition: form-data; name="_charset"')
 438         form.append('')
 439         form.append('UTF-8')
 440         form.append(sep)
 441         for name, value in fields.iteritems():
 442             if isinstance(value, tuple):
 443                 filename, value = value
 444                 form.append('Content-Disposition: form-data; name="%s"; '
 445                             'filename="%s"' % (name, filename.encode("utf-8")))
 446             else:
 447                 form.append('Content-Disposition: form-data; name="%s"' % name)
 448             form.append('')
 449             if isinstance(value, unicode):
 450                 value = value.encode("utf-8")
 451             else:
 452                 value = str(value)
 453             assert isinstance(value, str)
 454             form.append(value)
 455             form.append(sep)
 456         form[-1] += "--"
 457         body = ""
 458         headers = {}
 459         if fields:
 460             body = "\r\n".join(form) + "\r\n"
 461             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 462         return self.POST2(urlpath, body, headers, followRedirect)
 463
 464     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 465         url = self.webish_url + urlpath
 466         return client.getPage(url, method="POST", postdata=body,
 467                               headers=headers, followRedirect=followRedirect)
 468
 469     def shouldFail(self, res, expected_failure, which,
 470                    substring=None, response_substring=None):
 471         if isinstance(res, failure.Failure):
 472             res.trap(expected_failure)
 473             if substring:
 474                 self.failUnlessIn(substring, str(res), which)
 475             if response_substring:
 476                 self.failUnlessIn(response_substring, res.value.response, which)
 477         else:
 478             self.fail("%s was supposed to raise %s, not get '%s'" %
 479                       (which, expected_failure, res))
 480
 481     def shouldFail2(self, expected_failure, which, substring,
 482                     response_substring,
 483                     callable, *args, **kwargs):
 484         assert substring is None or isinstance(substring, str)
 485         assert response_substring is None or isinstance(response_substring, str)
 486         d = defer.maybeDeferred(callable, *args, **kwargs)
 487         def done(res):
 488             if isinstance(res, failure.Failure):
 489                 res.trap(expected_failure)
 490                 if substring:
 491                     self.failUnlessIn(substring, str(res),
 492                                       "'%s' not in '%s' for test '%s'" % \
 493                                       (substring, str(res), which))
 494                 if response_substring:
 495                     self.failUnlessIn(response_substring, res.value.response,
 496                                       "'%s' not in '%s' for test '%s'" % \
 497                                       (response_substring, res.value.response,
 498                                        which))
 499             else:
 500                 self.fail("%s was supposed to raise %s, not get '%s'" %
 501                           (which, expected_failure, res))
 502         d.addBoth(done)
 503         return d
 504
 505     def should404(self, res, which):
 506         if isinstance(res, failure.Failure):
 507             res.trap(error.Error)
 508             self.failUnlessReallyEqual(res.value.status, "404")
 509         else:
 510             self.fail("%s was supposed to Error(404), not get '%s'" %
 511                       (which, res))
 512
 513     def should302(self, res, which):
 514         if isinstance(res, failure.Failure):
 515             res.trap(error.Error)
 516             self.failUnlessReallyEqual(res.value.status, "302")
 517         else:
 518             self.fail("%s was supposed to Error(302), not get '%s'" %
 519                       (which, res))
 520
 521
 522 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
 523     def test_create(self):
 524         pass
 525
 526     def test_welcome(self):
 527         d = self.GET("/")
 528         def _check(res):
 529             self.failUnlessIn('Welcome to Tahoe-LAFS', res)
 530             self.failUnlessIn(FAVICON_MARKUP, res)
 531             self.failUnlessIn('href="https://tahoe-lafs.org/"', res)
 532
 533             self.s.basedir = 'web/test_welcome'
 534             fileutil.make_dirs("web/test_welcome")
 535             fileutil.make_dirs("web/test_welcome/private")
 536             return self.GET("/")
 537         d.addCallback(_check)
 538         return d
 539
 540     def test_status(self):
 541         h = self.s.get_history()
 542         dl_num = h.list_all_download_statuses()[0].get_counter()
 543         ul_num = h.list_all_upload_statuses()[0].get_counter()
 544         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 545         pub_num = h.list_all_publish_statuses()[0].get_counter()
 546         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 547         d = self.GET("/status", followRedirect=True)
 548         def _check(res):
 549             self.failUnlessIn('Upload and Download Status', res)
 550             self.failUnlessIn('"down-%d"' % dl_num, res)
 551             self.failUnlessIn('"up-%d"' % ul_num, res)
 552             self.failUnlessIn('"mapupdate-%d"' % mu_num, res)
 553             self.failUnlessIn('"publish-%d"' % pub_num, res)
 554             self.failUnlessIn('"retrieve-%d"' % ret_num, res)
 555         d.addCallback(_check)
 556         d.addCallback(lambda res: self.GET("/status/?t=json"))
 557         def _check_json(res):
 558             data = simplejson.loads(res)
 559             self.failUnless(isinstance(data, dict))
 560             #active = data["active"]
 561             # TODO: test more. We need a way to fake an active operation
 562             # here.
 563         d.addCallback(_check_json)
 564
 565         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 566         def _check_dl(res):
 567             self.failUnlessIn("File Download Status", res)
 568         d.addCallback(_check_dl)
 569         d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
 570         def _check_dl_json(res):
 571             data = simplejson.loads(res)
 572             self.failUnless(isinstance(data, dict))
 573             self.failUnlessIn("read", data)
 574             self.failUnlessEqual(data["read"][0]["length"], 120)
 575             self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
 576             self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
 577             self.failUnlessEqual(data["segment"][2]["finish_time"], None)
 578             phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
 579             cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
 580             # serverids[] keys are strings, since that's what JSON does, but
 581             # we'd really like them to be ints
 582             self.failUnlessEqual(data["serverids"]["0"], "phwr")
 583             self.failUnless(data["serverids"].has_key("1"), data["serverids"])
 584             self.failUnlessEqual(data["serverids"]["1"], "cmpu", data["serverids"])
 585             self.failUnlessEqual(data["server_info"][phwr_id]["short"], "phwr")
 586             self.failUnlessEqual(data["server_info"][cmpu_id]["short"], "cmpu")
 587             self.failUnlessIn("dyhb", data)
 588             self.failUnlessIn("misc", data)
 589         d.addCallback(_check_dl_json)
 590         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 591         def _check_ul(res):
 592             self.failUnlessIn("File Upload Status", res)
 593         d.addCallback(_check_ul)
 594         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 595         def _check_mapupdate(res):
 596             self.failUnlessIn("Mutable File Servermap Update Status", res)
 597         d.addCallback(_check_mapupdate)
 598         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 599         def _check_publish(res):
 600             self.failUnlessIn("Mutable File Publish Status", res)
 601         d.addCallback(_check_publish)
 602         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 603         def _check_retrieve(res):
 604             self.failUnlessIn("Mutable File Retrieve Status", res)
 605         d.addCallback(_check_retrieve)
 606
 607         return d
 608
 609     def test_status_numbers(self):
 610         drrm = status.DownloadResultsRendererMixin()
 611         self.failUnlessReallyEqual(drrm.render_time(None, None), "")
 612         self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
 613         self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
 614         self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
 615         self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
 616         self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
 617         self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 618         self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
 619         self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
 620
 621         urrm = status.UploadResultsRendererMixin()
 622         self.failUnlessReallyEqual(urrm.render_time(None, None), "")
 623         self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
 624         self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
 625         self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
 626         self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
 627         self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
 628         self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 629         self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
 630         self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
 631
 632     def test_GET_FILEURL(self):
 633         d = self.GET(self.public_url + "/foo/bar.txt")
 634         d.addCallback(self.failUnlessIsBarDotTxt)
 635         return d
 636
 637     def test_GET_FILEURL_range(self):
 638         headers = {"range": "bytes=1-10"}
 639         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 640                      return_response=True)
 641         def _got((res, status, headers)):
 642             self.failUnlessReallyEqual(int(status), 206)
 643             self.failUnless(headers.has_key("content-range"))
 644             self.failUnlessReallyEqual(headers["content-range"][0],
 645                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 646             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
 647         d.addCallback(_got)
 648         return d
 649
 650     def test_GET_FILEURL_partial_range(self):
 651         headers = {"range": "bytes=5-"}
 652         length  = len(self.BAR_CONTENTS)
 653         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 654                      return_response=True)
 655         def _got((res, status, headers)):
 656             self.failUnlessReallyEqual(int(status), 206)
 657             self.failUnless(headers.has_key("content-range"))
 658             self.failUnlessReallyEqual(headers["content-range"][0],
 659                                        "bytes 5-%d/%d" % (length-1, length))
 660             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
 661         d.addCallback(_got)
 662         return d
 663
 664     def test_GET_FILEURL_partial_end_range(self):
 665         headers = {"range": "bytes=-5"}
 666         length  = len(self.BAR_CONTENTS)
 667         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 668                      return_response=True)
 669         def _got((res, status, headers)):
 670             self.failUnlessReallyEqual(int(status), 206)
 671             self.failUnless(headers.has_key("content-range"))
 672             self.failUnlessReallyEqual(headers["content-range"][0],
 673                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 674             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
 675         d.addCallback(_got)
 676         return d
 677
 678     def test_GET_FILEURL_partial_range_overrun(self):
 679         headers = {"range": "bytes=100-200"}
 680         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
 681                              "416 Requested Range not satisfiable",
 682                              "First beyond end of file",
 683                              self.GET, self.public_url + "/foo/bar.txt",
 684                              headers=headers)
 685         return d
 686
 687     def test_HEAD_FILEURL_range(self):
 688         headers = {"range": "bytes=1-10"}
 689         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 690                      return_response=True)
 691         def _got((res, status, headers)):
 692             self.failUnlessReallyEqual(res, "")
 693             self.failUnlessReallyEqual(int(status), 206)
 694             self.failUnless(headers.has_key("content-range"))
 695             self.failUnlessReallyEqual(headers["content-range"][0],
 696                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 697         d.addCallback(_got)
 698         return d
 699
 700     def test_HEAD_FILEURL_partial_range(self):
 701         headers = {"range": "bytes=5-"}
 702         length  = len(self.BAR_CONTENTS)
 703         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 704                      return_response=True)
 705         def _got((res, status, headers)):
 706             self.failUnlessReallyEqual(int(status), 206)
 707             self.failUnless(headers.has_key("content-range"))
 708             self.failUnlessReallyEqual(headers["content-range"][0],
 709                                        "bytes 5-%d/%d" % (length-1, length))
 710         d.addCallback(_got)
 711         return d
 712
 713     def test_HEAD_FILEURL_partial_end_range(self):
 714         headers = {"range": "bytes=-5"}
 715         length  = len(self.BAR_CONTENTS)
 716         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 717                      return_response=True)
 718         def _got((res, status, headers)):
 719             self.failUnlessReallyEqual(int(status), 206)
 720             self.failUnless(headers.has_key("content-range"))
 721             self.failUnlessReallyEqual(headers["content-range"][0],
 722                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 723         d.addCallback(_got)
 724         return d
 725
 726     def test_HEAD_FILEURL_partial_range_overrun(self):
 727         headers = {"range": "bytes=100-200"}
 728         d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
 729                              "416 Requested Range not satisfiable",
 730                              "",
 731                              self.HEAD, self.public_url + "/foo/bar.txt",
 732                              headers=headers)
 733         return d
 734
 735     def test_GET_FILEURL_range_bad(self):
 736         headers = {"range": "BOGUS=fizbop-quarnak"}
 737         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 738                      return_response=True)
 739         def _got((res, status, headers)):
 740             self.failUnlessReallyEqual(int(status), 200)
 741             self.failUnless(not headers.has_key("content-range"))
 742             self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
 743         d.addCallback(_got)
 744         return d
 745
 746     def test_HEAD_FILEURL(self):
 747         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 748         def _got((res, status, headers)):
 749             self.failUnlessReallyEqual(res, "")
 750             self.failUnlessReallyEqual(headers["content-length"][0],
 751                                        str(len(self.BAR_CONTENTS)))
 752             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
 753         d.addCallback(_got)
 754         return d
 755
 756     def test_GET_FILEURL_named(self):
 757         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 758         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 759         d = self.GET(base + "/@@name=/blah.txt")
 760         d.addCallback(self.failUnlessIsBarDotTxt)
 761         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 762         d.addCallback(self.failUnlessIsBarDotTxt)
 763         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 764         d.addCallback(self.failUnlessIsBarDotTxt)
 765         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 766         d.addCallback(self.failUnlessIsBarDotTxt)
 767         save_url = base + "?save=true&filename=blah.txt"
 768         d.addCallback(lambda res: self.GET(save_url))
 769         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 770         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 771         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 772         u_url = base + "?save=true&filename=" + u_fn_e
 773         d.addCallback(lambda res: self.GET(u_url))
 774         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 775         return d
 776
 777     def test_PUT_FILEURL_named_bad(self):
 778         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 779         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 780                              "400 Bad Request",
 781                              "/file can only be used with GET or HEAD",
 782                              self.PUT, base + "/@@name=/blah.txt", "")
 783         return d
 784
 785
 786     def test_GET_DIRURL_named_bad(self):
 787         base = "/file/%s" % urllib.quote(self._foo_uri)
 788         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 789                              "400 Bad Request",
 790                              "is not a file-cap",
 791                              self.GET, base + "/@@name=/blah.txt")
 792         return d
 793
 794     def test_GET_slash_file_bad(self):
 795         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 796                              "404 Not Found",
 797                              "/file must be followed by a file-cap and a name",
 798                              self.GET, "/file")
 799         return d
 800
 801     def test_GET_unhandled_URI_named(self):
 802         contents, n, newuri = self.makefile(12)
 803         verifier_cap = n.get_verify_cap().to_string()
 804         base = "/file/%s" % urllib.quote(verifier_cap)
 805         # client.create_node_from_uri() can't handle verify-caps
 806         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 807                              "400 Bad Request", "is not a file-cap",
 808                              self.GET, base)
 809         return d
 810
 811     def test_GET_unhandled_URI(self):
 812         contents, n, newuri = self.makefile(12)
 813         verifier_cap = n.get_verify_cap().to_string()
 814         base = "/uri/%s" % urllib.quote(verifier_cap)
 815         # client.create_node_from_uri() can't handle verify-caps
 816         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 817                              "400 Bad Request",
 818                              "GET unknown URI type: can only do t=info",
 819                              self.GET, base)
 820         return d
 821
 822     def test_GET_FILE_URI(self):
 823         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 824         d = self.GET(base)
 825         d.addCallback(self.failUnlessIsBarDotTxt)
 826         return d
 827
 828     def test_GET_FILE_URI_mdmf(self):
 829         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 830         d = self.GET(base)
 831         d.addCallback(self.failUnlessIsQuuxDotTxt)
 832         return d
 833
 834     def test_GET_FILE_URI_mdmf_extensions(self):
 835         base = "/uri/%s" % urllib.quote("%s:RANDOMSTUFF" % self._quux_txt_uri)
 836         d = self.GET(base)
 837         d.addCallback(self.failUnlessIsQuuxDotTxt)
 838         return d
 839
 840     def test_GET_FILE_URI_mdmf_readonly(self):
 841         base = "/uri/%s" % urllib.quote(self._quux_txt_readonly_uri)
 842         d = self.GET(base)
 843         d.addCallback(self.failUnlessIsQuuxDotTxt)
 844         return d
 845
 846     def test_GET_FILE_URI_badchild(self):
 847         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 848         errmsg = "Files have no children, certainly not named 'boguschild'"
 849         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 850                              "400 Bad Request", errmsg,
 851                              self.GET, base)
 852         return d
 853
 854     def test_PUT_FILE_URI_badchild(self):
 855         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 856         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 857         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 858                              "400 Bad Request", errmsg,
 859                              self.PUT, base, "")
 860         return d
 861
 862     def test_PUT_FILE_URI_mdmf(self):
 863         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 864         self._quux_new_contents = "new_contents"
 865         d = self.GET(base)
 866         d.addCallback(lambda res:
 867             self.failUnlessIsQuuxDotTxt(res))
 868         d.addCallback(lambda ignored:
 869             self.PUT(base, self._quux_new_contents))
 870         d.addCallback(lambda ignored:
 871             self.GET(base))
 872         d.addCallback(lambda res:
 873             self.failUnlessReallyEqual(res, self._quux_new_contents))
 874         return d
 875
 876     def test_PUT_FILE_URI_mdmf_extensions(self):
 877         base = "/uri/%s" % urllib.quote("%s:EXTENSIONSTUFF" % self._quux_txt_uri)
 878         self._quux_new_contents = "new_contents"
 879         d = self.GET(base)
 880         d.addCallback(lambda res: self.failUnlessIsQuuxDotTxt(res))
 881         d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
 882         d.addCallback(lambda ignored: self.GET(base))
 883         d.addCallback(lambda res: self.failUnlessEqual(self._quux_new_contents,
 884                                                        res))
 885         return d
 886
 887     def test_PUT_FILE_URI_mdmf_readonly(self):
 888         # We're not allowed to PUT things to a readonly cap.
 889         base = "/uri/%s" % self._quux_txt_readonly_uri
 890         d = self.GET(base)
 891         d.addCallback(lambda res:
 892             self.failUnlessIsQuuxDotTxt(res))
 893         # What should we get here? We get a 500 error now; that's not right.
 894         d.addCallback(lambda ignored:
 895             self.shouldFail2(error.Error, "test_PUT_FILE_URI_mdmf_readonly",
 896                              "400 Bad Request", "read-only cap",
 897                              self.PUT, base, "new data"))
 898         return d
 899
 900     def test_PUT_FILE_URI_sdmf_readonly(self):
 901         # We're not allowed to put things to a readonly cap.
 902         base = "/uri/%s" % self._baz_txt_readonly_uri
 903         d = self.GET(base)
 904         d.addCallback(lambda res:
 905             self.failUnlessIsBazDotTxt(res))
 906         d.addCallback(lambda ignored:
 907             self.shouldFail2(error.Error, "test_PUT_FILE_URI_sdmf_readonly",
 908                              "400 Bad Request", "read-only cap",
 909                              self.PUT, base, "new_data"))
 910         return d
 911
 912     def test_GET_etags(self):
 913
 914         def _check_etags(uri):
 915             d1 = _get_etag(uri)
 916             d2 = _get_etag(uri, 'json')
 917             d = defer.DeferredList([d1, d2], consumeErrors=True)
 918             def _check(results):
 919                 # All deferred must succeed
 920                 self.failUnless(all([r[0] for r in results]))
 921                 # the etag for the t=json form should be just like the etag
 922                 # fo the default t='' form, but with a 'json' suffix
 923                 self.failUnlessEqual(results[0][1] + 'json', results[1][1])
 924             d.addCallback(_check)
 925             return d
 926
 927         def _get_etag(uri, t=''):
 928             targetbase = "/uri/%s?t=%s" % (urllib.quote(uri.strip()), t)
 929             d = self.GET(targetbase, return_response=True, followRedirect=True)
 930             def _just_the_etag(result):
 931                 data, response, headers = result
 932                 etag = headers['etag'][0]
 933                 if uri.startswith('URI:DIR'):
 934                     self.failUnless(etag.startswith('DIR:'), etag)
 935                 return etag
 936             return d.addCallback(_just_the_etag)
 937
 938         # Check that etags work with immutable directories
 939         (newkids, caps) = self._create_immutable_children()
 940         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
 941                       simplejson.dumps(newkids))
 942         def _stash_immdir_uri(uri):
 943             self._immdir_uri = uri
 944             return uri
 945         d.addCallback(_stash_immdir_uri)
 946         d.addCallback(_check_etags)
 947
 948         # Check that etags work with immutable files
 949         d.addCallback(lambda _: _check_etags(self._bar_txt_uri))
 950
 951         # use the ETag on GET
 952         def _check_match(ign):
 953             uri = "/uri/%s" % self._bar_txt_uri
 954             d = self.GET(uri, return_response=True)
 955             # extract the ETag
 956             d.addCallback(lambda (data, code, headers):
 957                           headers['etag'][0])
 958             # do a GET that's supposed to match the ETag
 959             d.addCallback(lambda etag:
 960                           self.GET(uri, return_response=True,
 961                                    headers={"If-None-Match": etag}))
 962             # make sure it short-circuited (304 instead of 200)
 963             d.addCallback(lambda (data, code, headers):
 964                           self.failUnlessEqual(int(code), http.NOT_MODIFIED))
 965             return d
 966         d.addCallback(_check_match)
 967
 968         def _no_etag(uri, t):
 969             target = "/uri/%s?t=%s" % (uri, t)
 970             d = self.GET(target, return_response=True, followRedirect=True)
 971             d.addCallback(lambda (data, code, headers):
 972                           self.failIf("etag" in headers, target))
 973             return d
 974         def _yes_etag(uri, t):
 975             target = "/uri/%s?t=%s" % (uri, t)
 976             d = self.GET(target, return_response=True, followRedirect=True)
 977             d.addCallback(lambda (data, code, headers):
 978                           self.failUnless("etag" in headers, target))
 979             return d
 980
 981         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, ""))
 982         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "json"))
 983         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "uri"))
 984         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "readonly-uri"))
 985         d.addCallback(lambda ign: _no_etag(self._bar_txt_uri, "info"))
 986
 987         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, ""))
 988         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "json"))
 989         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "uri"))
 990         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "readonly-uri"))
 991         d.addCallback(lambda ign: _no_etag(self._immdir_uri, "info"))
 992         d.addCallback(lambda ign: _no_etag(self._immdir_uri, "rename-form"))
 993
 994         return d
 995
 996     # TODO: version of this with a Unicode filename
 997     def test_GET_FILEURL_save(self):
 998         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
 999                      return_response=True)
1000         def _got((res, statuscode, headers)):
1001             content_disposition = headers["content-disposition"][0]
1002             self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
1003             self.failUnlessIsBarDotTxt(res)
1004         d.addCallback(_got)
1005         return d
1006
1007     def test_GET_FILEURL_missing(self):
1008         d = self.GET(self.public_url + "/foo/missing")
1009         d.addBoth(self.should404, "test_GET_FILEURL_missing")
1010         return d
1011
1012     def test_GET_FILEURL_info_mdmf(self):
1013         d = self.GET("/uri/%s?t=info" % self._quux_txt_uri)
1014         def _got(res):
1015             self.failUnlessIn("mutable file (mdmf)", res)
1016             self.failUnlessIn(self._quux_txt_uri, res)
1017             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1018         d.addCallback(_got)
1019         return d
1020
1021     def test_GET_FILEURL_info_mdmf_readonly(self):
1022         d = self.GET("/uri/%s?t=info" % self._quux_txt_readonly_uri)
1023         def _got(res):
1024             self.failUnlessIn("mutable file (mdmf)", res)
1025             self.failIfIn(self._quux_txt_uri, res)
1026             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1027         d.addCallback(_got)
1028         return d
1029
1030     def test_GET_FILEURL_info_sdmf(self):
1031         d = self.GET("/uri/%s?t=info" % self._baz_txt_uri)
1032         def _got(res):
1033             self.failUnlessIn("mutable file (sdmf)", res)
1034             self.failUnlessIn(self._baz_txt_uri, res)
1035         d.addCallback(_got)
1036         return d
1037
1038     def test_GET_FILEURL_info_mdmf_extensions(self):
1039         d = self.GET("/uri/%s:STUFF?t=info" % self._quux_txt_uri)
1040         def _got(res):
1041             self.failUnlessIn("mutable file (mdmf)", res)
1042             self.failUnlessIn(self._quux_txt_uri, res)
1043             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1044         d.addCallback(_got)
1045         return d
1046
1047     def test_PUT_overwrite_only_files(self):
1048         # create a directory, put a file in that directory.
1049         contents, n, filecap = self.makefile(8)
1050         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
1051         d.addCallback(lambda res:
1052             self.PUT(self.public_url + "/foo/dir/file1.txt",
1053                      self.NEWFILE_CONTENTS))
1054         # try to overwrite the file with replace=only-files
1055         # (this should work)
1056         d.addCallback(lambda res:
1057             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
1058                      filecap))
1059         d.addCallback(lambda res:
1060             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
1061                  "There was already a child by that name, and you asked me "
1062                  "to not replace it",
1063                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
1064                  filecap))
1065         return d
1066
1067     def test_PUT_NEWFILEURL(self):
1068         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
1069         # TODO: we lose the response code, so we can't check this
1070         #self.failUnlessReallyEqual(responsecode, 201)
1071         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1072         d.addCallback(lambda res:
1073                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1074                                                       self.NEWFILE_CONTENTS))
1075         return d
1076
1077     def test_PUT_NEWFILEURL_not_mutable(self):
1078         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
1079                      self.NEWFILE_CONTENTS)
1080         # TODO: we lose the response code, so we can't check this
1081         #self.failUnlessReallyEqual(responsecode, 201)
1082         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1083         d.addCallback(lambda res:
1084                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1085                                                       self.NEWFILE_CONTENTS))
1086         return d
1087
1088     def test_PUT_NEWFILEURL_unlinked_mdmf(self):
1089         # this should get us a few segments of an MDMF mutable file,
1090         # which we can then test for.
1091         contents = self.NEWFILE_CONTENTS * 300000
1092         d = self.PUT("/uri?format=mdmf",
1093                      contents)
1094         def _got_filecap(filecap):
1095             self.failUnless(filecap.startswith("URI:MDMF"))
1096             return filecap
1097         d.addCallback(_got_filecap)
1098         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1099         d.addCallback(lambda json: self.failUnlessIn("MDMF", json))
1100         return d
1101
1102     def test_PUT_NEWFILEURL_unlinked_sdmf(self):
1103         contents = self.NEWFILE_CONTENTS * 300000
1104         d = self.PUT("/uri?format=sdmf",
1105                      contents)
1106         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1107         d.addCallback(lambda json: self.failUnlessIn("SDMF", json))
1108         return d
1109
1110     def test_PUT_NEWFILEURL_unlinked_bad_format(self):
1111         contents = self.NEWFILE_CONTENTS * 300000
1112         return self.shouldHTTPError("PUT_NEWFILEURL_unlinked_bad_format",
1113                                     400, "Bad Request", "Unknown format: foo",
1114                                     self.PUT, "/uri?format=foo",
1115                                     contents)
1116
1117     def test_PUT_NEWFILEURL_range_bad(self):
1118         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
1119         target = self.public_url + "/foo/new.txt"
1120         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
1121                              "501 Not Implemented",
1122                              "Content-Range in PUT not yet supported",
1123                              # (and certainly not for immutable files)
1124                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
1125                              headers=headers)
1126         d.addCallback(lambda res:
1127                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
1128         return d
1129
1130     def test_PUT_NEWFILEURL_mutable(self):
1131         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1132                      self.NEWFILE_CONTENTS)
1133         # TODO: we lose the response code, so we can't check this
1134         #self.failUnlessReallyEqual(responsecode, 201)
1135         def _check_uri(res):
1136             u = uri.from_string_mutable_filenode(res)
1137             self.failUnless(u.is_mutable())
1138             self.failIf(u.is_readonly())
1139             return res
1140         d.addCallback(_check_uri)
1141         d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
1142         d.addCallback(lambda res:
1143                       self.failUnlessMutableChildContentsAre(self._foo_node,
1144                                                              u"new.txt",
1145                                                              self.NEWFILE_CONTENTS))
1146         return d
1147
1148     def test_PUT_NEWFILEURL_mutable_toobig(self):
1149         # It is okay to upload large mutable files, so we should be able
1150         # to do that.
1151         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1152                      "b" * (self.s.MUTABLE_SIZELIMIT + 1))
1153         return d
1154
1155     def test_PUT_NEWFILEURL_replace(self):
1156         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
1157         # TODO: we lose the response code, so we can't check this
1158         #self.failUnlessReallyEqual(responsecode, 200)
1159         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
1160         d.addCallback(lambda res:
1161                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1162                                                       self.NEWFILE_CONTENTS))
1163         return d
1164
1165     def test_PUT_NEWFILEURL_bad_t(self):
1166         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
1167                              "PUT to a file: bad t=bogus",
1168                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
1169                              "contents")
1170         return d
1171
1172     def test_PUT_NEWFILEURL_no_replace(self):
1173         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
1174                      self.NEWFILE_CONTENTS)
1175         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
1176                   "409 Conflict",
1177                   "There was already a child by that name, and you asked me "
1178                   "to not replace it")
1179         return d
1180
1181     def test_PUT_NEWFILEURL_mkdirs(self):
1182         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
1183         fn = self._foo_node
1184         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
1185         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
1186         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
1187         d.addCallback(lambda res:
1188                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
1189                                                       self.NEWFILE_CONTENTS))
1190         return d
1191
1192     def test_PUT_NEWFILEURL_blocked(self):
1193         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
1194                      self.NEWFILE_CONTENTS)
1195         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
1196                   "409 Conflict",
1197                   "Unable to create directory 'blockingfile': a file was in the way")
1198         return d
1199
1200     def test_PUT_NEWFILEURL_emptyname(self):
1201         # an empty pathname component (i.e. a double-slash) is disallowed
1202         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
1203                              "400 Bad Request",
1204                              "The webapi does not allow empty pathname components",
1205                              self.PUT, self.public_url + "/foo//new.txt", "")
1206         return d
1207
1208     def test_DELETE_FILEURL(self):
1209         d = self.DELETE(self.public_url + "/foo/bar.txt")
1210         d.addCallback(lambda res:
1211                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1212         return d
1213
1214     def test_DELETE_FILEURL_missing(self):
1215         d = self.DELETE(self.public_url + "/foo/missing")
1216         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
1217         return d
1218
1219     def test_DELETE_FILEURL_missing2(self):
1220         d = self.DELETE(self.public_url + "/missing/missing")
1221         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
1222         return d
1223
1224     def failUnlessHasBarDotTxtMetadata(self, res):
1225         data = simplejson.loads(res)
1226         self.failUnless(isinstance(data, list))
1227         self.failUnlessIn("metadata", data[1])
1228         self.failUnlessIn("tahoe", data[1]["metadata"])
1229         self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
1230         self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
1231         self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
1232                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
1233
1234     def test_GET_FILEURL_json(self):
1235         # twisted.web.http.parse_qs ignores any query args without an '=', so
1236         # I can't do "GET /path?json", I have to do "GET /path/t=json"
1237         # instead. This may make it tricky to emulate the S3 interface
1238         # completely.
1239         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
1240         def _check1(data):
1241             self.failUnlessIsBarJSON(data)
1242             self.failUnlessHasBarDotTxtMetadata(data)
1243             return
1244         d.addCallback(_check1)
1245         return d
1246
1247     def test_GET_FILEURL_json_mutable_type(self):
1248         # The JSON should include format, which says whether the
1249         # file is SDMF or MDMF
1250         d = self.PUT("/uri?format=mdmf",
1251                      self.NEWFILE_CONTENTS * 300000)
1252         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1253         def _got_json(json, version):
1254             data = simplejson.loads(json)
1255             assert "filenode" == data[0]
1256             data = data[1]
1257             assert isinstance(data, dict)
1258
1259             self.failUnlessIn("format", data)
1260             self.failUnlessEqual(data["format"], version)
1261
1262         d.addCallback(_got_json, "MDMF")
1263         # Now make an SDMF file and check that it is reported correctly.
1264         d.addCallback(lambda ignored:
1265             self.PUT("/uri?format=sdmf",
1266                       self.NEWFILE_CONTENTS * 300000))
1267         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1268         d.addCallback(_got_json, "SDMF")
1269         return d
1270
1271     def test_GET_FILEURL_json_mdmf(self):
1272         d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
1273         d.addCallback(self.failUnlessIsQuuxJSON)
1274         return d
1275
1276     def test_GET_FILEURL_json_missing(self):
1277         d = self.GET(self.public_url + "/foo/missing?json")
1278         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
1279         return d
1280
1281     def test_GET_FILEURL_uri(self):
1282         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
1283         def _check(res):
1284             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1285         d.addCallback(_check)
1286         d.addCallback(lambda res:
1287                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
1288         def _check2(res):
1289             # for now, for files, uris and readonly-uris are the same
1290             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1291         d.addCallback(_check2)
1292         return d
1293
1294     def test_GET_FILEURL_badtype(self):
1295         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
1296                                  "bad t=bogus",
1297                                  self.GET,
1298                                  self.public_url + "/foo/bar.txt?t=bogus")
1299         return d
1300
1301     def test_CSS_FILE(self):
1302         d = self.GET("/tahoe.css", followRedirect=True)
1303         def _check(res):
1304             CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
1305             self.failUnless(CSS_STYLE.search(res), res)
1306         d.addCallback(_check)
1307         return d
1308
1309     def test_GET_FILEURL_uri_missing(self):
1310         d = self.GET(self.public_url + "/foo/missing?t=uri")
1311         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
1312         return d
1313
1314     def _check_upload_and_mkdir_forms(self, html):
1315         # We should have a form to create a file, with radio buttons that allow
1316         # the user to toggle whether it is a CHK/LIT (default), SDMF, or MDMF file.
1317         self.failUnlessIn('name="t" value="upload"', html)
1318         self.failUnlessIn('input checked="checked" type="radio" id="upload-chk" value="chk" name="format"', html)
1319         self.failUnlessIn('input type="radio" id="upload-sdmf" value="sdmf" name="format"', html)
1320         self.failUnlessIn('input type="radio" id="upload-mdmf" value="mdmf" name="format"', html)
1321
1322         # We should also have the ability to create a mutable directory, with
1323         # radio buttons that allow the user to toggle whether it is an SDMF (default)
1324         # or MDMF directory.
1325         self.failUnlessIn('name="t" value="mkdir"', html)
1326         self.failUnlessIn('input checked="checked" type="radio" id="mkdir-sdmf" value="sdmf" name="format"', html)
1327         self.failUnlessIn('input type="radio" id="mkdir-mdmf" value="mdmf" name="format"', html)
1328
1329         self.failUnlessIn(FAVICON_MARKUP, html)
1330
1331     def test_GET_DIRECTORY_html(self):
1332         d = self.GET(self.public_url + "/foo", followRedirect=True)
1333         def _check(html):
1334             self.failUnlessIn('<div class="toolbar-item"><a href="../../..">Return to Welcome page</a></div>', html)
1335             self._check_upload_and_mkdir_forms(html)
1336             self.failUnlessIn("quux", html)
1337         d.addCallback(_check)
1338         return d
1339
1340     def test_GET_root_html(self):
1341         d = self.GET("/")
1342         d.addCallback(self._check_upload_and_mkdir_forms)
1343         return d
1344
1345     def test_GET_DIRURL(self):
1346         # the addSlash means we get a redirect here
1347         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1348         ROOT = "../../.."
1349         d = self.GET(self.public_url + "/foo", followRedirect=True)
1350         def _check(res):
1351             self.failUnlessIn('<a href="%s">Return to Welcome page' % ROOT, res)
1352
1353             # the FILE reference points to a URI, but it should end in bar.txt
1354             bar_url = ("%s/file/%s/@@named=/bar.txt" %
1355                        (ROOT, urllib.quote(self._bar_txt_uri)))
1356             get_bar = "".join([r'<td>FILE</td>',
1357                                r'\s+<td>',
1358                                r'<a href="%s">bar.txt</a>' % bar_url,
1359                                r'</td>',
1360                                r'\s+<td align="right">%d</td>' % len(self.BAR_CONTENTS),
1361                                ])
1362             self.failUnless(re.search(get_bar, res), res)
1363             for label in ['unlink', 'rename/move']:
1364                 for line in res.split("\n"):
1365                     # find the line that contains the relevant button for bar.txt
1366                     if ("form action" in line and
1367                         ('value="%s"' % (label,)) in line and
1368                         'value="bar.txt"' in line):
1369                         # the form target should use a relative URL
1370                         foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1371                         self.failUnlessIn('action="%s"' % foo_url, line)
1372                         # and the when_done= should too
1373                         #done_url = urllib.quote(???)
1374                         #self.failUnlessIn('name="when_done" value="%s"' % done_url, line)
1375
1376                         # 'unlink' needs to use POST because it directly has a side effect
1377                         if label == 'unlink':
1378                             self.failUnlessIn('method="post"', line)
1379                         break
1380                 else:
1381                     self.fail("unable to find '%s bar.txt' line" % (label,))
1382
1383             # the DIR reference just points to a URI
1384             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1385             get_sub = ((r'<td>DIR</td>')
1386                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1387             self.failUnless(re.search(get_sub, res), res)
1388         d.addCallback(_check)
1389
1390         # look at a readonly directory
1391         d.addCallback(lambda res:
1392                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
1393         def _check2(res):
1394             self.failUnlessIn("(read-only)", res)
1395             self.failIfIn("Upload a file", res)
1396         d.addCallback(_check2)
1397
1398         # and at a directory that contains a readonly directory
1399         d.addCallback(lambda res:
1400                       self.GET(self.public_url, followRedirect=True))
1401         def _check3(res):
1402             self.failUnless(re.search('<td>DIR-RO</td>'
1403                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1404         d.addCallback(_check3)
1405
1406         # and an empty directory
1407         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1408         def _check4(res):
1409             self.failUnlessIn("directory is empty", res)
1410             MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
1411             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1412         d.addCallback(_check4)
1413
1414         # and at a literal directory
1415         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1416         d.addCallback(lambda res:
1417                       self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1418         def _check5(res):
1419             self.failUnlessIn('(immutable)', res)
1420             self.failUnless(re.search('<td>FILE</td>'
1421                                       r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1422         d.addCallback(_check5)
1423         return d
1424
1425     def test_GET_DIRURL_badtype(self):
1426         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1427                                  400, "Bad Request",
1428                                  "bad t=bogus",
1429                                  self.GET,
1430                                  self.public_url + "/foo?t=bogus")
1431         return d
1432
1433     def test_GET_DIRURL_json(self):
1434         d = self.GET(self.public_url + "/foo?t=json")
1435         d.addCallback(self.failUnlessIsFooJSON)
1436         return d
1437
1438     def test_GET_DIRURL_json_format(self):
1439         d = self.PUT(self.public_url + \
1440                      "/foo/sdmf.txt?format=sdmf",
1441                      self.NEWFILE_CONTENTS * 300000)
1442         d.addCallback(lambda ignored:
1443             self.PUT(self.public_url + \
1444                      "/foo/mdmf.txt?format=mdmf",
1445                      self.NEWFILE_CONTENTS * 300000))
1446         # Now we have an MDMF and SDMF file in the directory. If we GET
1447         # its JSON, we should see their encodings.
1448         d.addCallback(lambda ignored:
1449             self.GET(self.public_url + "/foo?t=json"))
1450         def _got_json(json):
1451             data = simplejson.loads(json)
1452             assert data[0] == "dirnode"
1453
1454             data = data[1]
1455             kids = data['children']
1456
1457             mdmf_data = kids['mdmf.txt'][1]
1458             self.failUnlessIn("format", mdmf_data)
1459             self.failUnlessEqual(mdmf_data["format"], "MDMF")
1460
1461             sdmf_data = kids['sdmf.txt'][1]
1462             self.failUnlessIn("format", sdmf_data)
1463             self.failUnlessEqual(sdmf_data["format"], "SDMF")
1464         d.addCallback(_got_json)
1465         return d
1466
1467
1468     def test_POST_DIRURL_manifest_no_ophandle(self):
1469         d = self.shouldFail2(error.Error,
1470                              "test_POST_DIRURL_manifest_no_ophandle",
1471                              "400 Bad Request",
1472                              "slow operation requires ophandle=",
1473                              self.POST, self.public_url, t="start-manifest")
1474         return d
1475
1476     def test_POST_DIRURL_manifest(self):
1477         d = defer.succeed(None)
1478         def getman(ignored, output):
1479             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1480                           followRedirect=True)
1481             d.addCallback(self.wait_for_operation, "125")
1482             d.addCallback(self.get_operation_results, "125", output)
1483             return d
1484         d.addCallback(getman, None)
1485         def _got_html(manifest):
1486             self.failUnlessIn("Manifest of SI=", manifest)
1487             self.failUnlessIn("<td>sub</td>", manifest)
1488             self.failUnlessIn(self._sub_uri, manifest)
1489             self.failUnlessIn("<td>sub/baz.txt</td>", manifest)
1490             self.failUnlessIn(FAVICON_MARKUP, manifest)
1491         d.addCallback(_got_html)
1492
1493         # both t=status and unadorned GET should be identical
1494         d.addCallback(lambda res: self.GET("/operations/125"))
1495         d.addCallback(_got_html)
1496
1497         d.addCallback(getman, "html")
1498         d.addCallback(_got_html)
1499         d.addCallback(getman, "text")
1500         def _got_text(manifest):
1501             self.failUnlessIn("\nsub " + self._sub_uri + "\n", manifest)
1502             self.failUnlessIn("\nsub/baz.txt URI:CHK:", manifest)
1503         d.addCallback(_got_text)
1504         d.addCallback(getman, "JSON")
1505         def _got_json(res):
1506             data = res["manifest"]
1507             got = {}
1508             for (path_list, cap) in data:
1509                 got[tuple(path_list)] = cap
1510             self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
1511             self.failUnlessIn((u"sub", u"baz.txt"), got)
1512             self.failUnlessIn("finished", res)
1513             self.failUnlessIn("origin", res)
1514             self.failUnlessIn("storage-index", res)
1515             self.failUnlessIn("verifycaps", res)
1516             self.failUnlessIn("stats", res)
1517         d.addCallback(_got_json)
1518         return d
1519
1520     def test_POST_DIRURL_deepsize_no_ophandle(self):
1521         d = self.shouldFail2(error.Error,
1522                              "test_POST_DIRURL_deepsize_no_ophandle",
1523                              "400 Bad Request",
1524                              "slow operation requires ophandle=",
1525                              self.POST, self.public_url, t="start-deep-size")
1526         return d
1527
1528     def test_POST_DIRURL_deepsize(self):
1529         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1530                       followRedirect=True)
1531         d.addCallback(self.wait_for_operation, "126")
1532         d.addCallback(self.get_operation_results, "126", "json")
1533         def _got_json(data):
1534             self.failUnlessReallyEqual(data["finished"], True)
1535             size = data["size"]
1536             self.failUnless(size > 1000)
1537         d.addCallback(_got_json)
1538         d.addCallback(self.get_operation_results, "126", "text")
1539         def _got_text(res):
1540             mo = re.search(r'^size: (\d+)$', res, re.M)
1541             self.failUnless(mo, res)
1542             size = int(mo.group(1))
1543             # with directories, the size varies.
1544             self.failUnless(size > 1000)
1545         d.addCallback(_got_text)
1546         return d
1547
1548     def test_POST_DIRURL_deepstats_no_ophandle(self):
1549         d = self.shouldFail2(error.Error,
1550                              "test_POST_DIRURL_deepstats_no_ophandle",
1551                              "400 Bad Request",
1552                              "slow operation requires ophandle=",
1553                              self.POST, self.public_url, t="start-deep-stats")
1554         return d
1555
1556     def test_POST_DIRURL_deepstats(self):
1557         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1558                       followRedirect=True)
1559         d.addCallback(self.wait_for_operation, "127")
1560         d.addCallback(self.get_operation_results, "127", "json")
1561         def _got_json(stats):
1562             expected = {"count-immutable-files": 3,
1563                         "count-mutable-files": 2,
1564                         "count-literal-files": 0,
1565                         "count-files": 5,
1566                         "count-directories": 3,
1567                         "size-immutable-files": 57,
1568                         "size-literal-files": 0,
1569                         #"size-directories": 1912, # varies
1570                         #"largest-directory": 1590,
1571                         "largest-directory-children": 7,
1572                         "largest-immutable-file": 19,
1573                         }
1574             for k,v in expected.iteritems():
1575                 self.failUnlessReallyEqual(stats[k], v,
1576                                            "stats[%s] was %s, not %s" %
1577                                            (k, stats[k], v))
1578             self.failUnlessReallyEqual(stats["size-files-histogram"],
1579                                        [ [11, 31, 3] ])
1580         d.addCallback(_got_json)
1581         return d
1582
1583     def test_POST_DIRURL_stream_manifest(self):
1584         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1585         def _check(res):
1586             self.failUnless(res.endswith("\n"))
1587             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1588             self.failUnlessReallyEqual(len(units), 9)
1589             self.failUnlessEqual(units[-1]["type"], "stats")
1590             first = units[0]
1591             self.failUnlessEqual(first["path"], [])
1592             self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri)
1593             self.failUnlessEqual(first["type"], "directory")
1594             baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0]
1595             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1596             self.failIfEqual(baz["storage-index"], None)
1597             self.failIfEqual(baz["verifycap"], None)
1598             self.failIfEqual(baz["repaircap"], None)
1599             # XXX: Add quux and baz to this test.
1600             return
1601         d.addCallback(_check)
1602         return d
1603
1604     def test_GET_DIRURL_uri(self):
1605         d = self.GET(self.public_url + "/foo?t=uri")
1606         def _check(res):
1607             self.failUnlessReallyEqual(to_str(res), self._foo_uri)
1608         d.addCallback(_check)
1609         return d
1610
1611     def test_GET_DIRURL_readonly_uri(self):
1612         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1613         def _check(res):
1614             self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri)
1615         d.addCallback(_check)
1616         return d
1617
1618     def test_PUT_NEWDIRURL(self):
1619         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1620         d.addCallback(lambda res:
1621                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1622         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1623         d.addCallback(self.failUnlessNodeKeysAre, [])
1624         return d
1625
1626     def test_PUT_NEWDIRURL_mdmf(self):
1627         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1628         d.addCallback(lambda res:
1629                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1630         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1631         d.addCallback(lambda node:
1632             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1633         return d
1634
1635     def test_PUT_NEWDIRURL_sdmf(self):
1636         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=sdmf",
1637                      "")
1638         d.addCallback(lambda res:
1639                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1640         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1641         d.addCallback(lambda node:
1642             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1643         return d
1644
1645     def test_PUT_NEWDIRURL_bad_format(self):
1646         return self.shouldHTTPError("PUT_NEWDIRURL_bad_format",
1647                                     400, "Bad Request", "Unknown format: foo",
1648                                     self.PUT, self.public_url +
1649                                     "/foo/newdir=?t=mkdir&format=foo", "")
1650
1651     def test_POST_NEWDIRURL(self):
1652         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1653         d.addCallback(lambda res:
1654                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1655         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1656         d.addCallback(self.failUnlessNodeKeysAre, [])
1657         return d
1658
1659     def test_POST_NEWDIRURL_mdmf(self):
1660         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1661         d.addCallback(lambda res:
1662                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1663         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1664         d.addCallback(lambda node:
1665             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1666         return d
1667
1668     def test_POST_NEWDIRURL_sdmf(self):
1669         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=sdmf", "")
1670         d.addCallback(lambda res:
1671             self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1672         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1673         d.addCallback(lambda node:
1674             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1675         return d
1676
1677     def test_POST_NEWDIRURL_bad_format(self):
1678         return self.shouldHTTPError("POST_NEWDIRURL_bad_format",
1679                                     400, "Bad Request", "Unknown format: foo",
1680                                     self.POST2, self.public_url + \
1681                                     "/foo/newdir?t=mkdir&format=foo", "")
1682
1683     def test_POST_NEWDIRURL_emptyname(self):
1684         # an empty pathname component (i.e. a double-slash) is disallowed
1685         d = self.shouldFail2(error.Error, "POST_NEWDIRURL_emptyname",
1686                              "400 Bad Request",
1687                              "The webapi does not allow empty pathname components, i.e. a double slash",
1688                              self.POST, self.public_url + "//?t=mkdir")
1689         return d
1690
1691     def _do_POST_NEWDIRURL_initial_children_test(self, version=None):
1692         (newkids, caps) = self._create_initial_children()
1693         query = "/foo/newdir?t=mkdir-with-children"
1694         if version == MDMF_VERSION:
1695             query += "&format=mdmf"
1696         elif version == SDMF_VERSION:
1697             query += "&format=sdmf"
1698         else:
1699             version = SDMF_VERSION # for later
1700         d = self.POST2(self.public_url + query,
1701                        simplejson.dumps(newkids))
1702         def _check(uri):
1703             n = self.s.create_node_from_uri(uri.strip())
1704             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1705             self.failUnlessEqual(n._node.get_version(), version)
1706             d2.addCallback(lambda ign:
1707                            self.failUnlessROChildURIIs(n, u"child-imm",
1708                                                        caps['filecap1']))
1709             d2.addCallback(lambda ign:
1710                            self.failUnlessRWChildURIIs(n, u"child-mutable",
1711                                                        caps['filecap2']))
1712             d2.addCallback(lambda ign:
1713                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1714                                                        caps['filecap3']))
1715             d2.addCallback(lambda ign:
1716                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1717                                                        caps['unknown_rocap']))
1718             d2.addCallback(lambda ign:
1719                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1720                                                        caps['unknown_rwcap']))
1721             d2.addCallback(lambda ign:
1722                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1723                                                        caps['unknown_immcap']))
1724             d2.addCallback(lambda ign:
1725                            self.failUnlessRWChildURIIs(n, u"dirchild",
1726                                                        caps['dircap']))
1727             d2.addCallback(lambda ign:
1728                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1729                                                        caps['litdircap']))
1730             d2.addCallback(lambda ign:
1731                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1732                                                        caps['emptydircap']))
1733             return d2
1734         d.addCallback(_check)
1735         d.addCallback(lambda res:
1736                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1737         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1738         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1739         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1740         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1741         return d
1742
1743     def test_POST_NEWDIRURL_initial_children(self):
1744         return self._do_POST_NEWDIRURL_initial_children_test()
1745
1746     def test_POST_NEWDIRURL_initial_children_mdmf(self):
1747         return self._do_POST_NEWDIRURL_initial_children_test(MDMF_VERSION)
1748
1749     def test_POST_NEWDIRURL_initial_children_sdmf(self):
1750         return self._do_POST_NEWDIRURL_initial_children_test(SDMF_VERSION)
1751
1752     def test_POST_NEWDIRURL_initial_children_bad_format(self):
1753         (newkids, caps) = self._create_initial_children()
1754         return self.shouldHTTPError("POST_NEWDIRURL_initial_children_bad_format",
1755                                     400, "Bad Request", "Unknown format: foo",
1756                                     self.POST2, self.public_url + \
1757                                     "/foo/newdir?t=mkdir-with-children&format=foo",
1758                                     simplejson.dumps(newkids))
1759
1760     def test_POST_NEWDIRURL_immutable(self):
1761         (newkids, caps) = self._create_immutable_children()
1762         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1763                        simplejson.dumps(newkids))
1764         def _check(uri):
1765             n = self.s.create_node_from_uri(uri.strip())
1766             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1767             d2.addCallback(lambda ign:
1768                            self.failUnlessROChildURIIs(n, u"child-imm",
1769                                                        caps['filecap1']))
1770             d2.addCallback(lambda ign:
1771                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1772                                                        caps['unknown_immcap']))
1773             d2.addCallback(lambda ign:
1774                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
1775                                                        caps['immdircap']))
1776             d2.addCallback(lambda ign:
1777                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1778                                                        caps['litdircap']))
1779             d2.addCallback(lambda ign:
1780                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1781                                                        caps['emptydircap']))
1782             return d2
1783         d.addCallback(_check)
1784         d.addCallback(lambda res:
1785                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1786         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1787         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1788         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1789         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1790         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1791         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1792         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1793         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
1794         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1795         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
1796         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1797         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
1798         d.addErrback(self.explain_web_error)
1799         return d
1800
1801     def test_POST_NEWDIRURL_immutable_bad(self):
1802         (newkids, caps) = self._create_initial_children()
1803         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
1804                              "400 Bad Request",
1805                              "needed to be immutable but was not",
1806                              self.POST2,
1807                              self.public_url + "/foo/newdir?t=mkdir-immutable",
1808                              simplejson.dumps(newkids))
1809         return d
1810
1811     def test_PUT_NEWDIRURL_exists(self):
1812         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1813         d.addCallback(lambda res:
1814                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1815         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1816         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1817         return d
1818
1819     def test_PUT_NEWDIRURL_blocked(self):
1820         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1821                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1822                              self.PUT,
1823                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1824         d.addCallback(lambda res:
1825                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1826         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1827         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1828         return d
1829
1830     def test_PUT_NEWDIRURL_mkdirs(self):
1831         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1832         d.addCallback(lambda res:
1833                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1834         d.addCallback(lambda res:
1835                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1836         d.addCallback(lambda res:
1837                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1838         d.addCallback(self.failUnlessNodeKeysAre, [])
1839         return d
1840
1841     def test_PUT_NEWDIRURL_mkdirs_mdmf(self):
1842         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=mdmf", "")
1843         d.addCallback(lambda ignored:
1844             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1845         d.addCallback(lambda ignored:
1846             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1847         d.addCallback(lambda ignored:
1848             self._foo_node.get_child_at_path(u"subdir"))
1849         def _got_subdir(subdir):
1850             # XXX: What we want?
1851             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1852             self.failUnlessNodeHasChild(subdir, u"newdir")
1853             return subdir.get_child_at_path(u"newdir")
1854         d.addCallback(_got_subdir)
1855         d.addCallback(lambda newdir:
1856             self.failUnlessEqual(newdir._node.get_version(), MDMF_VERSION))
1857         return d
1858
1859     def test_PUT_NEWDIRURL_mkdirs_sdmf(self):
1860         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=sdmf", "")
1861         d.addCallback(lambda ignored:
1862             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1863         d.addCallback(lambda ignored:
1864             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1865         d.addCallback(lambda ignored:
1866             self._foo_node.get_child_at_path(u"subdir"))
1867         def _got_subdir(subdir):
1868             # XXX: What we want?
1869             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1870             self.failUnlessNodeHasChild(subdir, u"newdir")
1871             return subdir.get_child_at_path(u"newdir")
1872         d.addCallback(_got_subdir)
1873         d.addCallback(lambda newdir:
1874             self.failUnlessEqual(newdir._node.get_version(), SDMF_VERSION))
1875         return d
1876
1877     def test_PUT_NEWDIRURL_mkdirs_bad_format(self):
1878         return self.shouldHTTPError("PUT_NEWDIRURL_mkdirs_bad_format",
1879                                     400, "Bad Request", "Unknown format: foo",
1880                                     self.PUT, self.public_url + \
1881                                     "/foo/subdir/newdir?t=mkdir&format=foo",
1882                                     "")
1883
1884     def test_DELETE_DIRURL(self):
1885         d = self.DELETE(self.public_url + "/foo")
1886         d.addCallback(lambda res:
1887                       self.failIfNodeHasChild(self.public_root, u"foo"))
1888         return d
1889
1890     def test_DELETE_DIRURL_missing(self):
1891         d = self.DELETE(self.public_url + "/foo/missing")
1892         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1893         d.addCallback(lambda res:
1894                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1895         return d
1896
1897     def test_DELETE_DIRURL_missing2(self):
1898         d = self.DELETE(self.public_url + "/missing")
1899         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1900         return d
1901
1902     def dump_root(self):
1903         print "NODEWALK"
1904         w = webish.DirnodeWalkerMixin()
1905         def visitor(childpath, childnode, metadata):
1906             print childpath
1907         d = w.walk(self.public_root, visitor)
1908         return d
1909
1910     def failUnlessNodeKeysAre(self, node, expected_keys):
1911         for k in expected_keys:
1912             assert isinstance(k, unicode)
1913         d = node.list()
1914         def _check(children):
1915             self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
1916         d.addCallback(_check)
1917         return d
1918     def failUnlessNodeHasChild(self, node, name):
1919         assert isinstance(name, unicode)
1920         d = node.list()
1921         def _check(children):
1922             self.failUnlessIn(name, children)
1923         d.addCallback(_check)
1924         return d
1925     def failIfNodeHasChild(self, node, name):
1926         assert isinstance(name, unicode)
1927         d = node.list()
1928         def _check(children):
1929             self.failIfIn(name, children)
1930         d.addCallback(_check)
1931         return d
1932
1933     def failUnlessChildContentsAre(self, node, name, expected_contents):
1934         assert isinstance(name, unicode)
1935         d = node.get_child_at_path(name)
1936         d.addCallback(lambda node: download_to_data(node))
1937         def _check(contents):
1938             self.failUnlessReallyEqual(contents, expected_contents)
1939         d.addCallback(_check)
1940         return d
1941
1942     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1943         assert isinstance(name, unicode)
1944         d = node.get_child_at_path(name)
1945         d.addCallback(lambda node: node.download_best_version())
1946         def _check(contents):
1947             self.failUnlessReallyEqual(contents, expected_contents)
1948         d.addCallback(_check)
1949         return d
1950
1951     def failUnlessRWChildURIIs(self, node, name, expected_uri):
1952         assert isinstance(name, unicode)
1953         d = node.get_child_at_path(name)
1954         def _check(child):
1955             self.failUnless(child.is_unknown() or not child.is_readonly())
1956             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1957             self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
1958             expected_ro_uri = self._make_readonly(expected_uri)
1959             if expected_ro_uri:
1960                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1961         d.addCallback(_check)
1962         return d
1963
1964     def failUnlessROChildURIIs(self, node, name, expected_uri):
1965         assert isinstance(name, unicode)
1966         d = node.get_child_at_path(name)
1967         def _check(child):
1968             self.failUnless(child.is_unknown() or child.is_readonly())
1969             self.failUnlessReallyEqual(child.get_write_uri(), None)
1970             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1971             self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
1972         d.addCallback(_check)
1973         return d
1974
1975     def failUnlessURIMatchesRWChild(self, got_uri, node, name):
1976         assert isinstance(name, unicode)
1977         d = node.get_child_at_path(name)
1978         def _check(child):
1979             self.failUnless(child.is_unknown() or not child.is_readonly())
1980             self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
1981             self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
1982             expected_ro_uri = self._make_readonly(got_uri)
1983             if expected_ro_uri:
1984                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1985         d.addCallback(_check)
1986         return d
1987
1988     def failUnlessURIMatchesROChild(self, got_uri, node, name):
1989         assert isinstance(name, unicode)
1990         d = node.get_child_at_path(name)
1991         def _check(child):
1992             self.failUnless(child.is_unknown() or child.is_readonly())
1993             self.failUnlessReallyEqual(child.get_write_uri(), None)
1994             self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
1995             self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
1996         d.addCallback(_check)
1997         return d
1998
1999     def failUnlessCHKURIHasContents(self, got_uri, contents):
2000         self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
2001
2002     def test_POST_upload(self):
2003         d = self.POST(self.public_url + "/foo", t="upload",
2004                       file=("new.txt", self.NEWFILE_CONTENTS))
2005         fn = self._foo_node
2006         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2007         d.addCallback(lambda res:
2008                       self.failUnlessChildContentsAre(fn, u"new.txt",
2009                                                       self.NEWFILE_CONTENTS))
2010         return d
2011
2012     def test_POST_upload_unicode(self):
2013         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2014         d = self.POST(self.public_url + "/foo", t="upload",
2015                       file=(filename, self.NEWFILE_CONTENTS))
2016         fn = self._foo_node
2017         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2018         d.addCallback(lambda res:
2019                       self.failUnlessChildContentsAre(fn, filename,
2020                                                       self.NEWFILE_CONTENTS))
2021         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2022         d.addCallback(lambda res: self.GET(target_url))
2023         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2024                                                                   self.NEWFILE_CONTENTS,
2025                                                                   contents))
2026         return d
2027
2028     def test_POST_upload_unicode_named(self):
2029         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2030         d = self.POST(self.public_url + "/foo", t="upload",
2031                       name=filename,
2032                       file=("overridden", self.NEWFILE_CONTENTS))
2033         fn = self._foo_node
2034         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2035         d.addCallback(lambda res:
2036                       self.failUnlessChildContentsAre(fn, filename,
2037                                                       self.NEWFILE_CONTENTS))
2038         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2039         d.addCallback(lambda res: self.GET(target_url))
2040         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2041                                                                   self.NEWFILE_CONTENTS,
2042                                                                   contents))
2043         return d
2044
2045     def test_POST_upload_no_link(self):
2046         d = self.POST("/uri", t="upload",
2047                       file=("new.txt", self.NEWFILE_CONTENTS))
2048         def _check_upload_results(page):
2049             # this should be a page which describes the results of the upload
2050             # that just finished.
2051             self.failUnlessIn("Upload Results:", page)
2052             self.failUnlessIn("URI:", page)
2053             uri_re = re.compile("URI: <tt><span>(.*)</span>")
2054             mo = uri_re.search(page)
2055             self.failUnless(mo, page)
2056             new_uri = mo.group(1)
2057             return new_uri
2058         d.addCallback(_check_upload_results)
2059         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
2060         return d
2061
2062     def test_POST_upload_no_link_whendone(self):
2063         d = self.POST("/uri", t="upload", when_done="/",
2064                       file=("new.txt", self.NEWFILE_CONTENTS))
2065         d.addBoth(self.shouldRedirect, "/")
2066         return d
2067
2068     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
2069         d = defer.maybeDeferred(callable, *args, **kwargs)
2070         def done(res):
2071             if isinstance(res, failure.Failure):
2072                 res.trap(error.PageRedirect)
2073                 statuscode = res.value.status
2074                 target = res.value.location
2075                 return checker(statuscode, target)
2076             self.fail("%s: callable was supposed to redirect, not return '%s'"
2077                       % (which, res))
2078         d.addBoth(done)
2079         return d
2080
2081     def test_POST_upload_no_link_whendone_results(self):
2082         def check(statuscode, target):
2083             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2084             self.failUnless(target.startswith(self.webish_url), target)
2085             return client.getPage(target, method="GET")
2086         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
2087                                  check,
2088                                  self.POST, "/uri", t="upload",
2089                                  when_done="/uri/%(uri)s",
2090                                  file=("new.txt", self.NEWFILE_CONTENTS))
2091         d.addCallback(lambda res:
2092                       self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
2093         return d
2094
2095     def test_POST_upload_no_link_mutable(self):
2096         d = self.POST("/uri", t="upload", mutable="true",
2097                       file=("new.txt", self.NEWFILE_CONTENTS))
2098         def _check(filecap):
2099             filecap = filecap.strip()
2100             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2101             self.filecap = filecap
2102             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2103             self.failUnlessIn(u.get_storage_index(), FakeMutableFileNode.all_contents)
2104             n = self.s.create_node_from_uri(filecap)
2105             return n.download_best_version()
2106         d.addCallback(_check)
2107         def _check2(data):
2108             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2109             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2110         d.addCallback(_check2)
2111         def _check3(data):
2112             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2113             return self.GET("/file/%s" % urllib.quote(self.filecap))
2114         d.addCallback(_check3)
2115         def _check4(data):
2116             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2117         d.addCallback(_check4)
2118         return d
2119
2120     def test_POST_upload_no_link_mutable_toobig(self):
2121         # The SDMF size limit is no longer in place, so we should be
2122         # able to upload mutable files that are as large as we want them
2123         # to be.
2124         d = self.POST("/uri", t="upload", mutable="true",
2125                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2126         return d
2127
2128
2129     def test_POST_upload_format_unlinked(self):
2130         def _check_upload_unlinked(ign, format, uri_prefix):
2131             filename = format + ".txt"
2132             d = self.POST("/uri?t=upload&format=" + format,
2133                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2134             def _got_results(results):
2135                 if format.upper() in ("SDMF", "MDMF"):
2136                     # webapi.rst says this returns a filecap
2137                     filecap = results
2138                 else:
2139                     # for immutable, it returns an "upload results page", and
2140                     # the filecap is buried inside
2141                     line = [l for l in results.split("\n") if "URI: " in l][0]
2142                     mo = re.search(r'<span>([^<]+)</span>', line)
2143                     filecap = mo.group(1)
2144                 self.failUnless(filecap.startswith(uri_prefix),
2145                                 (uri_prefix, filecap))
2146                 return self.GET("/uri/%s?t=json" % filecap)
2147             d.addCallback(_got_results)
2148             def _got_json(json):
2149                 data = simplejson.loads(json)
2150                 data = data[1]
2151                 self.failUnlessIn("format", data)
2152                 self.failUnlessEqual(data["format"], format.upper())
2153             d.addCallback(_got_json)
2154             return d
2155         d = defer.succeed(None)
2156         d.addCallback(_check_upload_unlinked, "chk", "URI:CHK")
2157         d.addCallback(_check_upload_unlinked, "CHK", "URI:CHK")
2158         d.addCallback(_check_upload_unlinked, "sdmf", "URI:SSK")
2159         d.addCallback(_check_upload_unlinked, "mdmf", "URI:MDMF")
2160         return d
2161
2162     def test_POST_upload_bad_format_unlinked(self):
2163         return self.shouldHTTPError("POST_upload_bad_format_unlinked",
2164                                     400, "Bad Request", "Unknown format: foo",
2165                                     self.POST,
2166                                     "/uri?t=upload&format=foo",
2167                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2168
2169     def test_POST_upload_format(self):
2170         def _check_upload(ign, format, uri_prefix, fn=None):
2171             filename = format + ".txt"
2172             d = self.POST(self.public_url +
2173                           "/foo?t=upload&format=" + format,
2174                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2175             def _got_filecap(filecap):
2176                 if fn is not None:
2177                     filenameu = unicode(filename)
2178                     self.failUnlessURIMatchesRWChild(filecap, fn, filenameu)
2179                 self.failUnless(filecap.startswith(uri_prefix))
2180                 return self.GET(self.public_url + "/foo/%s?t=json" % filename)
2181             d.addCallback(_got_filecap)
2182             def _got_json(json):
2183                 data = simplejson.loads(json)
2184                 data = data[1]
2185                 self.failUnlessIn("format", data)
2186                 self.failUnlessEqual(data["format"], format.upper())
2187             d.addCallback(_got_json)
2188             return d
2189
2190         d = defer.succeed(None)
2191         d.addCallback(_check_upload, "chk", "URI:CHK")
2192         d.addCallback(_check_upload, "sdmf", "URI:SSK", self._foo_node)
2193         d.addCallback(_check_upload, "mdmf", "URI:MDMF")
2194         d.addCallback(_check_upload, "MDMF", "URI:MDMF")
2195         return d
2196
2197     def test_POST_upload_bad_format(self):
2198         return self.shouldHTTPError("POST_upload_bad_format",
2199                                     400, "Bad Request", "Unknown format: foo",
2200                                     self.POST, self.public_url + \
2201                                     "/foo?t=upload&format=foo",
2202                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2203
2204     def test_POST_upload_mutable(self):
2205         # this creates a mutable file
2206         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
2207                       file=("new.txt", self.NEWFILE_CONTENTS))
2208         fn = self._foo_node
2209         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2210         d.addCallback(lambda res:
2211                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2212                                                              self.NEWFILE_CONTENTS))
2213         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2214         def _got(newnode):
2215             self.failUnless(IMutableFileNode.providedBy(newnode))
2216             self.failUnless(newnode.is_mutable())
2217             self.failIf(newnode.is_readonly())
2218             self._mutable_node = newnode
2219             self._mutable_uri = newnode.get_uri()
2220         d.addCallback(_got)
2221
2222         # now upload it again and make sure that the URI doesn't change
2223         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
2224         d.addCallback(lambda res:
2225                       self.POST(self.public_url + "/foo", t="upload",
2226                                 mutable="true",
2227                                 file=("new.txt", NEWER_CONTENTS)))
2228         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2229         d.addCallback(lambda res:
2230                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2231                                                              NEWER_CONTENTS))
2232         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2233         def _got2(newnode):
2234             self.failUnless(IMutableFileNode.providedBy(newnode))
2235             self.failUnless(newnode.is_mutable())
2236             self.failIf(newnode.is_readonly())
2237             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2238         d.addCallback(_got2)
2239
2240         # upload a second time, using PUT instead of POST
2241         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
2242         d.addCallback(lambda res:
2243                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
2244         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2245         d.addCallback(lambda res:
2246                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2247                                                              NEW2_CONTENTS))
2248
2249         # finally list the directory, since mutable files are displayed
2250         # slightly differently
2251
2252         d.addCallback(lambda res:
2253                       self.GET(self.public_url + "/foo/",
2254                                followRedirect=True))
2255         def _check_page(res):
2256             # TODO: assert more about the contents
2257             self.failUnlessIn("SSK", res)
2258             return res
2259         d.addCallback(_check_page)
2260
2261         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2262         def _got3(newnode):
2263             self.failUnless(IMutableFileNode.providedBy(newnode))
2264             self.failUnless(newnode.is_mutable())
2265             self.failIf(newnode.is_readonly())
2266             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2267         d.addCallback(_got3)
2268
2269         # look at the JSON form of the enclosing directory
2270         d.addCallback(lambda res:
2271                       self.GET(self.public_url + "/foo/?t=json",
2272                                followRedirect=True))
2273         def _check_page_json(res):
2274             parsed = simplejson.loads(res)
2275             self.failUnlessEqual(parsed[0], "dirnode")
2276             children = dict( [(unicode(name),value)
2277                               for (name,value)
2278                               in parsed[1]["children"].iteritems()] )
2279             self.failUnlessIn(u"new.txt", children)
2280             new_json = children[u"new.txt"]
2281             self.failUnlessEqual(new_json[0], "filenode")
2282             self.failUnless(new_json[1]["mutable"])
2283             self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri)
2284             ro_uri = self._mutable_node.get_readonly().to_string()
2285             self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri)
2286         d.addCallback(_check_page_json)
2287
2288         # and the JSON form of the file
2289         d.addCallback(lambda res:
2290                       self.GET(self.public_url + "/foo/new.txt?t=json"))
2291         def _check_file_json(res):
2292             parsed = simplejson.loads(res)
2293             self.failUnlessEqual(parsed[0], "filenode")
2294             self.failUnless(parsed[1]["mutable"])
2295             self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri)
2296             ro_uri = self._mutable_node.get_readonly().to_string()
2297             self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri)
2298         d.addCallback(_check_file_json)
2299
2300         # and look at t=uri and t=readonly-uri
2301         d.addCallback(lambda res:
2302                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
2303         d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
2304         d.addCallback(lambda res:
2305                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
2306         def _check_ro_uri(res):
2307             ro_uri = self._mutable_node.get_readonly().to_string()
2308             self.failUnlessReallyEqual(res, ro_uri)
2309         d.addCallback(_check_ro_uri)
2310
2311         # make sure we can get to it from /uri/URI
2312         d.addCallback(lambda res:
2313                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
2314         d.addCallback(lambda res:
2315                       self.failUnlessReallyEqual(res, NEW2_CONTENTS))
2316
2317         # and that HEAD computes the size correctly
2318         d.addCallback(lambda res:
2319                       self.HEAD(self.public_url + "/foo/new.txt",
2320                                 return_response=True))
2321         def _got_headers((res, status, headers)):
2322             self.failUnlessReallyEqual(res, "")
2323             self.failUnlessReallyEqual(headers["content-length"][0],
2324                                        str(len(NEW2_CONTENTS)))
2325             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
2326         d.addCallback(_got_headers)
2327
2328         # make sure that outdated size limits aren't enforced anymore.
2329         d.addCallback(lambda ignored:
2330             self.POST(self.public_url + "/foo", t="upload",
2331                       mutable="true",
2332                       file=("new.txt",
2333                             "b" * (self.s.MUTABLE_SIZELIMIT+1))))
2334         d.addErrback(self.dump_error)
2335         return d
2336
2337     def test_POST_upload_mutable_toobig(self):
2338         # SDMF had a size limti that was removed a while ago. MDMF has
2339         # never had a size limit. Test to make sure that we do not
2340         # encounter errors when trying to upload large mutable files,
2341         # since there should be no coded prohibitions regarding large
2342         # mutable files.
2343         d = self.POST(self.public_url + "/foo",
2344                       t="upload", mutable="true",
2345                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2346         return d
2347
2348     def dump_error(self, f):
2349         # if the web server returns an error code (like 400 Bad Request),
2350         # web.client.getPage puts the HTTP response body into the .response
2351         # attribute of the exception object that it gives back. It does not
2352         # appear in the Failure's repr(), so the ERROR that trial displays
2353         # will be rather terse and unhelpful. addErrback this method to the
2354         # end of your chain to get more information out of these errors.
2355         if f.check(error.Error):
2356             print "web.error.Error:"
2357             print f
2358             print f.value.response
2359         return f
2360
2361     def test_POST_upload_replace(self):
2362         d = self.POST(self.public_url + "/foo", t="upload",
2363                       file=("bar.txt", self.NEWFILE_CONTENTS))
2364         fn = self._foo_node
2365         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
2366         d.addCallback(lambda res:
2367                       self.failUnlessChildContentsAre(fn, u"bar.txt",
2368                                                       self.NEWFILE_CONTENTS))
2369         return d
2370
2371     def test_POST_upload_no_replace_ok(self):
2372         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2373                       file=("new.txt", self.NEWFILE_CONTENTS))
2374         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
2375         d.addCallback(lambda res: self.failUnlessReallyEqual(res,
2376                                                              self.NEWFILE_CONTENTS))
2377         return d
2378
2379     def test_POST_upload_no_replace_queryarg(self):
2380         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2381                       file=("bar.txt", self.NEWFILE_CONTENTS))
2382         d.addBoth(self.shouldFail, error.Error,
2383                   "POST_upload_no_replace_queryarg",
2384                   "409 Conflict",
2385                   "There was already a child by that name, and you asked me "
2386                   "to not replace it")
2387         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2388         d.addCallback(self.failUnlessIsBarDotTxt)
2389         return d
2390
2391     def test_POST_upload_no_replace_field(self):
2392         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
2393                       file=("bar.txt", self.NEWFILE_CONTENTS))
2394         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
2395                   "409 Conflict",
2396                   "There was already a child by that name, and you asked me "
2397                   "to not replace it")
2398         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2399         d.addCallback(self.failUnlessIsBarDotTxt)
2400         return d
2401
2402     def test_POST_upload_whendone(self):
2403         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
2404                       file=("new.txt", self.NEWFILE_CONTENTS))
2405         d.addBoth(self.shouldRedirect, "/THERE")
2406         fn = self._foo_node
2407         d.addCallback(lambda res:
2408                       self.failUnlessChildContentsAre(fn, u"new.txt",
2409                                                       self.NEWFILE_CONTENTS))
2410         return d
2411
2412     def test_POST_upload_named(self):
2413         fn = self._foo_node
2414         d = self.POST(self.public_url + "/foo", t="upload",
2415                       name="new.txt", file=self.NEWFILE_CONTENTS)
2416         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2417         d.addCallback(lambda res:
2418                       self.failUnlessChildContentsAre(fn, u"new.txt",
2419                                                       self.NEWFILE_CONTENTS))
2420         return d
2421
2422     def test_POST_upload_named_badfilename(self):
2423         d = self.POST(self.public_url + "/foo", t="upload",
2424                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
2425         d.addBoth(self.shouldFail, error.Error,
2426                   "test_POST_upload_named_badfilename",
2427                   "400 Bad Request",
2428                   "name= may not contain a slash",
2429                   )
2430         # make sure that nothing was added
2431         d.addCallback(lambda res:
2432                       self.failUnlessNodeKeysAre(self._foo_node,
2433                                                  [u"bar.txt", u"baz.txt", u"blockingfile",
2434                                                   u"empty", u"n\u00fc.txt", u"quux.txt",
2435                                                   u"sub"]))
2436         return d
2437
2438     def test_POST_FILEURL_check(self):
2439         bar_url = self.public_url + "/foo/bar.txt"
2440         d = self.POST(bar_url, t="check")
2441         def _check(res):
2442             self.failUnlessIn("Healthy :", res)
2443         d.addCallback(_check)
2444         redir_url = "http://allmydata.org/TARGET"
2445         def _check2(statuscode, target):
2446             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2447             self.failUnlessReallyEqual(target, redir_url)
2448         d.addCallback(lambda res:
2449                       self.shouldRedirect2("test_POST_FILEURL_check",
2450                                            _check2,
2451                                            self.POST, bar_url,
2452                                            t="check",
2453                                            when_done=redir_url))
2454         d.addCallback(lambda res:
2455                       self.POST(bar_url, t="check", return_to=redir_url))
2456         def _check3(res):
2457             self.failUnlessIn("Healthy :", res)
2458             self.failUnlessIn("Return to file", res)
2459             self.failUnlessIn(redir_url, res)
2460         d.addCallback(_check3)
2461
2462         d.addCallback(lambda res:
2463                       self.POST(bar_url, t="check", output="JSON"))
2464         def _check_json(res):
2465             data = simplejson.loads(res)
2466             self.failUnlessIn("storage-index", data)
2467             self.failUnless(data["results"]["healthy"])
2468         d.addCallback(_check_json)
2469
2470         return d
2471
2472     def test_POST_FILEURL_check_and_repair(self):
2473         bar_url = self.public_url + "/foo/bar.txt"
2474         d = self.POST(bar_url, t="check", repair="true")
2475         def _check(res):
2476             self.failUnlessIn("Healthy :", res)
2477         d.addCallback(_check)
2478         redir_url = "http://allmydata.org/TARGET"
2479         def _check2(statuscode, target):
2480             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2481             self.failUnlessReallyEqual(target, redir_url)
2482         d.addCallback(lambda res:
2483                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
2484                                            _check2,
2485                                            self.POST, bar_url,
2486                                            t="check", repair="true",
2487                                            when_done=redir_url))
2488         d.addCallback(lambda res:
2489                       self.POST(bar_url, t="check", return_to=redir_url))
2490         def _check3(res):
2491             self.failUnlessIn("Healthy :", res)
2492             self.failUnlessIn("Return to file", res)
2493             self.failUnlessIn(redir_url, res)
2494         d.addCallback(_check3)
2495         return d
2496
2497     def test_POST_DIRURL_check(self):
2498         foo_url = self.public_url + "/foo/"
2499         d = self.POST(foo_url, t="check")
2500         def _check(res):
2501             self.failUnlessIn("Healthy :", res)
2502         d.addCallback(_check)
2503         redir_url = "http://allmydata.org/TARGET"
2504         def _check2(statuscode, target):
2505             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2506             self.failUnlessReallyEqual(target, redir_url)
2507         d.addCallback(lambda res:
2508                       self.shouldRedirect2("test_POST_DIRURL_check",
2509                                            _check2,
2510                                            self.POST, foo_url,
2511                                            t="check",
2512                                            when_done=redir_url))
2513         d.addCallback(lambda res:
2514                       self.POST(foo_url, t="check", return_to=redir_url))
2515         def _check3(res):
2516             self.failUnlessIn("Healthy :", res)
2517             self.failUnlessIn("Return to file/directory", res)
2518             self.failUnlessIn(redir_url, res)
2519         d.addCallback(_check3)
2520
2521         d.addCallback(lambda res:
2522                       self.POST(foo_url, t="check", output="JSON"))
2523         def _check_json(res):
2524             data = simplejson.loads(res)
2525             self.failUnlessIn("storage-index", data)
2526             self.failUnless(data["results"]["healthy"])
2527         d.addCallback(_check_json)
2528
2529         return d
2530
2531     def test_POST_DIRURL_check_and_repair(self):
2532         foo_url = self.public_url + "/foo/"
2533         d = self.POST(foo_url, t="check", repair="true")
2534         def _check(res):
2535             self.failUnlessIn("Healthy :", res)
2536         d.addCallback(_check)
2537         redir_url = "http://allmydata.org/TARGET"
2538         def _check2(statuscode, target):
2539             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2540             self.failUnlessReallyEqual(target, redir_url)
2541         d.addCallback(lambda res:
2542                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2543                                            _check2,
2544                                            self.POST, foo_url,
2545                                            t="check", repair="true",
2546                                            when_done=redir_url))
2547         d.addCallback(lambda res:
2548                       self.POST(foo_url, t="check", return_to=redir_url))
2549         def _check3(res):
2550             self.failUnlessIn("Healthy :", res)
2551             self.failUnlessIn("Return to file/directory", res)
2552             self.failUnlessIn(redir_url, res)
2553         d.addCallback(_check3)
2554         return d
2555
2556     def test_POST_FILEURL_mdmf_check(self):
2557         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2558         d = self.POST(quux_url, t="check")
2559         def _check(res):
2560             self.failUnlessIn("Healthy", res)
2561         d.addCallback(_check)
2562         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2563         d.addCallback(lambda ignored:
2564                       self.POST(quux_extension_url, t="check"))
2565         d.addCallback(_check)
2566         return d
2567
2568     def test_POST_FILEURL_mdmf_check_and_repair(self):
2569         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2570         d = self.POST(quux_url, t="check", repair="true")
2571         def _check(res):
2572             self.failUnlessIn("Healthy", res)
2573         d.addCallback(_check)
2574         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2575         d.addCallback(lambda ignored:
2576                       self.POST(quux_extension_url, t="check", repair="true"))
2577         d.addCallback(_check)
2578         return d
2579
2580     def wait_for_operation(self, ignored, ophandle):
2581         url = "/operations/" + ophandle
2582         url += "?t=status&output=JSON"
2583         d = self.GET(url)
2584         def _got(res):
2585             data = simplejson.loads(res)
2586             if not data["finished"]:
2587                 d = self.stall(delay=1.0)
2588                 d.addCallback(self.wait_for_operation, ophandle)
2589                 return d
2590             return data
2591         d.addCallback(_got)
2592         return d
2593
2594     def get_operation_results(self, ignored, ophandle, output=None):
2595         url = "/operations/" + ophandle
2596         url += "?t=status"
2597         if output:
2598             url += "&output=" + output
2599         d = self.GET(url)
2600         def _got(res):
2601             if output and output.lower() == "json":
2602                 return simplejson.loads(res)
2603             return res
2604         d.addCallback(_got)
2605         return d
2606
2607     def test_POST_DIRURL_deepcheck_no_ophandle(self):
2608         d = self.shouldFail2(error.Error,
2609                              "test_POST_DIRURL_deepcheck_no_ophandle",
2610                              "400 Bad Request",
2611                              "slow operation requires ophandle=",
2612                              self.POST, self.public_url, t="start-deep-check")
2613         return d
2614
2615     def test_POST_DIRURL_deepcheck(self):
2616         def _check_redirect(statuscode, target):
2617             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2618             self.failUnless(target.endswith("/operations/123"))
2619         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2620                                  self.POST, self.public_url,
2621                                  t="start-deep-check", ophandle="123")
2622         d.addCallback(self.wait_for_operation, "123")
2623         def _check_json(data):
2624             self.failUnlessReallyEqual(data["finished"], True)
2625             self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2626             self.failUnlessReallyEqual(data["count-objects-healthy"], 10)
2627         d.addCallback(_check_json)
2628         d.addCallback(self.get_operation_results, "123", "html")
2629         def _check_html(res):
2630             self.failUnlessIn("Objects Checked: <span>10</span>", res)
2631             self.failUnlessIn("Objects Healthy: <span>10</span>", res)
2632             self.failUnlessIn(FAVICON_MARKUP, res)
2633         d.addCallback(_check_html)
2634
2635         d.addCallback(lambda res:
2636                       self.GET("/operations/123/"))
2637         d.addCallback(_check_html) # should be the same as without the slash
2638
2639         d.addCallback(lambda res:
2640                       self.shouldFail2(error.Error, "one", "404 Not Found",
2641                                        "No detailed results for SI bogus",
2642                                        self.GET, "/operations/123/bogus"))
2643
2644         foo_si = self._foo_node.get_storage_index()
2645         foo_si_s = base32.b2a(foo_si)
2646         d.addCallback(lambda res:
2647                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2648         def _check_foo_json(res):
2649             data = simplejson.loads(res)
2650             self.failUnlessEqual(data["storage-index"], foo_si_s)
2651             self.failUnless(data["results"]["healthy"])
2652         d.addCallback(_check_foo_json)
2653         return d
2654
2655     def test_POST_DIRURL_deepcheck_and_repair(self):
2656         d = self.POST(self.public_url, t="start-deep-check", repair="true",
2657                       ophandle="124", output="json", followRedirect=True)
2658         d.addCallback(self.wait_for_operation, "124")
2659         def _check_json(data):
2660             self.failUnlessReallyEqual(data["finished"], True)
2661             self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2662             self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 10)
2663             self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2664             self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2665             self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2666             self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2667             self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2668             self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 10)
2669             self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2670             self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2671         d.addCallback(_check_json)
2672         d.addCallback(self.get_operation_results, "124", "html")
2673         def _check_html(res):
2674             self.failUnlessIn("Objects Checked: <span>10</span>", res)
2675
2676             self.failUnlessIn("Objects Healthy (before repair): <span>10</span>", res)
2677             self.failUnlessIn("Objects Unhealthy (before repair): <span>0</span>", res)
2678             self.failUnlessIn("Corrupt Shares (before repair): <span>0</span>", res)
2679
2680             self.failUnlessIn("Repairs Attempted: <span>0</span>", res)
2681             self.failUnlessIn("Repairs Successful: <span>0</span>", res)
2682             self.failUnlessIn("Repairs Unsuccessful: <span>0</span>", res)
2683
2684             self.failUnlessIn("Objects Healthy (after repair): <span>10</span>", res)
2685             self.failUnlessIn("Objects Unhealthy (after repair): <span>0</span>", res)
2686             self.failUnlessIn("Corrupt Shares (after repair): <span>0</span>", res)
2687
2688             self.failUnlessIn(FAVICON_MARKUP, res)
2689         d.addCallback(_check_html)
2690         return d
2691
2692     def test_POST_FILEURL_bad_t(self):
2693         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2694                              "POST to file: bad t=bogus",
2695                              self.POST, self.public_url + "/foo/bar.txt",
2696                              t="bogus")
2697         return d
2698
2699     def test_POST_mkdir(self): # return value?
2700         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2701         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2702         d.addCallback(self.failUnlessNodeKeysAre, [])
2703         return d
2704
2705     def test_POST_mkdir_mdmf(self):
2706         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=mdmf")
2707         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2708         d.addCallback(lambda node:
2709             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2710         return d
2711
2712     def test_POST_mkdir_sdmf(self):
2713         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=sdmf")
2714         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2715         d.addCallback(lambda node:
2716             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2717         return d
2718
2719     def test_POST_mkdir_bad_format(self):
2720         return self.shouldHTTPError("POST_mkdir_bad_format",
2721                                     400, "Bad Request", "Unknown format: foo",
2722                                     self.POST, self.public_url +
2723                                     "/foo?t=mkdir&name=newdir&format=foo")
2724
2725     def test_POST_mkdir_initial_children(self):
2726         (newkids, caps) = self._create_initial_children()
2727         d = self.POST2(self.public_url +
2728                        "/foo?t=mkdir-with-children&name=newdir",
2729                        simplejson.dumps(newkids))
2730         d.addCallback(lambda res:
2731                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2732         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2733         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2734         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2735         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2736         return d
2737
2738     def test_POST_mkdir_initial_children_mdmf(self):
2739         (newkids, caps) = self._create_initial_children()
2740         d = self.POST2(self.public_url +
2741                        "/foo?t=mkdir-with-children&name=newdir&format=mdmf",
2742                        simplejson.dumps(newkids))
2743         d.addCallback(lambda res:
2744                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2745         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2746         d.addCallback(lambda node:
2747             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2748         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2749         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2750                        caps['filecap1'])
2751         return d
2752
2753     # XXX: Duplication.
2754     def test_POST_mkdir_initial_children_sdmf(self):
2755         (newkids, caps) = self._create_initial_children()
2756         d = self.POST2(self.public_url +
2757                        "/foo?t=mkdir-with-children&name=newdir&format=sdmf",
2758                        simplejson.dumps(newkids))
2759         d.addCallback(lambda res:
2760                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2761         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2762         d.addCallback(lambda node:
2763             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2764         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2765         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2766                        caps['filecap1'])
2767         return d
2768
2769     def test_POST_mkdir_initial_children_bad_format(self):
2770         (newkids, caps) = self._create_initial_children()
2771         return self.shouldHTTPError("POST_mkdir_initial_children_bad_format",
2772                                     400, "Bad Request", "Unknown format: foo",
2773                                     self.POST, self.public_url + \
2774                                     "/foo?t=mkdir-with-children&name=newdir&format=foo",
2775                                     simplejson.dumps(newkids))
2776
2777     def test_POST_mkdir_immutable(self):
2778         (newkids, caps) = self._create_immutable_children()
2779         d = self.POST2(self.public_url +
2780                        "/foo?t=mkdir-immutable&name=newdir",
2781                        simplejson.dumps(newkids))
2782         d.addCallback(lambda res:
2783                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2784         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2785         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2786         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2787         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2788         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2789         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2790         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2791         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2792         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2793         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2794         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2795         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2796         return d
2797
2798     def test_POST_mkdir_immutable_bad(self):
2799         (newkids, caps) = self._create_initial_children()
2800         d = self.shouldFail2(error.Error, "POST_mkdir_immutable_bad",
2801                              "400 Bad Request",
2802                              "needed to be immutable but was not",
2803                              self.POST2,
2804                              self.public_url +
2805                              "/foo?t=mkdir-immutable&name=newdir",
2806                              simplejson.dumps(newkids))
2807         return d
2808
2809     def test_POST_mkdir_2(self):
2810         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
2811         d.addCallback(lambda res:
2812                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2813         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2814         d.addCallback(self.failUnlessNodeKeysAre, [])
2815         return d
2816
2817     def test_POST_mkdirs_2(self):
2818         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
2819         d.addCallback(lambda res:
2820                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
2821         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
2822         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
2823         d.addCallback(self.failUnlessNodeKeysAre, [])
2824         return d
2825
2826     def test_POST_mkdir_no_parentdir_noredirect(self):
2827         d = self.POST("/uri?t=mkdir")
2828         def _after_mkdir(res):
2829             uri.DirectoryURI.init_from_string(res)
2830         d.addCallback(_after_mkdir)
2831         return d
2832
2833     def test_POST_mkdir_no_parentdir_noredirect_mdmf(self):
2834         d = self.POST("/uri?t=mkdir&format=mdmf")
2835         def _after_mkdir(res):
2836             u = uri.from_string(res)
2837             # Check that this is an MDMF writecap
2838             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
2839         d.addCallback(_after_mkdir)
2840         return d
2841
2842     def test_POST_mkdir_no_parentdir_noredirect_sdmf(self):
2843         d = self.POST("/uri?t=mkdir&format=sdmf")
2844         def _after_mkdir(res):
2845             u = uri.from_string(res)
2846             self.failUnlessIsInstance(u, uri.DirectoryURI)
2847         d.addCallback(_after_mkdir)
2848         return d
2849
2850     def test_POST_mkdir_no_parentdir_noredirect_bad_format(self):
2851         return self.shouldHTTPError("POST_mkdir_no_parentdir_noredirect_bad_format",
2852                                     400, "Bad Request", "Unknown format: foo",
2853                                     self.POST, self.public_url +
2854                                     "/uri?t=mkdir&format=foo")
2855
2856     def test_POST_mkdir_no_parentdir_noredirect2(self):
2857         # make sure form-based arguments (as on the welcome page) still work
2858         d = self.POST("/uri", t="mkdir")
2859         def _after_mkdir(res):
2860             uri.DirectoryURI.init_from_string(res)
2861         d.addCallback(_after_mkdir)
2862         d.addErrback(self.explain_web_error)
2863         return d
2864
2865     def test_POST_mkdir_no_parentdir_redirect(self):
2866         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
2867         d.addBoth(self.shouldRedirect, None, statuscode='303')
2868         def _check_target(target):
2869             target = urllib.unquote(target)
2870             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2871         d.addCallback(_check_target)
2872         return d
2873
2874     def test_POST_mkdir_no_parentdir_redirect2(self):
2875         d = self.POST("/uri", t="mkdir", redirect_to_result="true")
2876         d.addBoth(self.shouldRedirect, None, statuscode='303')
2877         def _check_target(target):
2878             target = urllib.unquote(target)
2879             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2880         d.addCallback(_check_target)
2881         d.addErrback(self.explain_web_error)
2882         return d
2883
2884     def _make_readonly(self, u):
2885         ro_uri = uri.from_string(u).get_readonly()
2886         if ro_uri is None:
2887             return None
2888         return ro_uri.to_string()
2889
2890     def _create_initial_children(self):
2891         contents, n, filecap1 = self.makefile(12)
2892         md1 = {"metakey1": "metavalue1"}
2893         filecap2 = make_mutable_file_uri()
2894         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
2895         filecap3 = node3.get_readonly_uri()
2896         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
2897         dircap = DirectoryNode(node4, None, None).get_uri()
2898         mdmfcap = make_mutable_file_uri(mdmf=True)
2899         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2900         emptydircap = "URI:DIR2-LIT:"
2901         newkids = {u"child-imm":        ["filenode", {"rw_uri": filecap1,
2902                                                       "ro_uri": self._make_readonly(filecap1),
2903                                                       "metadata": md1, }],
2904                    u"child-mutable":    ["filenode", {"rw_uri": filecap2,
2905                                                       "ro_uri": self._make_readonly(filecap2)}],
2906                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
2907                    u"unknownchild-rw":  ["unknown",  {"rw_uri": unknown_rwcap,
2908                                                       "ro_uri": unknown_rocap}],
2909                    u"unknownchild-ro":  ["unknown",  {"ro_uri": unknown_rocap}],
2910                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2911                    u"dirchild":         ["dirnode",  {"rw_uri": dircap,
2912                                                       "ro_uri": self._make_readonly(dircap)}],
2913                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2914                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2915                    u"child-mutable-mdmf": ["filenode", {"rw_uri": mdmfcap,
2916                                                         "ro_uri": self._make_readonly(mdmfcap)}],
2917                    }
2918         return newkids, {'filecap1': filecap1,
2919                          'filecap2': filecap2,
2920                          'filecap3': filecap3,
2921                          'unknown_rwcap': unknown_rwcap,
2922                          'unknown_rocap': unknown_rocap,
2923                          'unknown_immcap': unknown_immcap,
2924                          'dircap': dircap,
2925                          'litdircap': litdircap,
2926                          'emptydircap': emptydircap,
2927                          'mdmfcap': mdmfcap}
2928
2929     def _create_immutable_children(self):
2930         contents, n, filecap1 = self.makefile(12)
2931         md1 = {"metakey1": "metavalue1"}
2932         tnode = create_chk_filenode("immutable directory contents\n"*10)
2933         dnode = DirectoryNode(tnode, None, None)
2934         assert not dnode.is_mutable()
2935         immdircap = dnode.get_uri()
2936         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2937         emptydircap = "URI:DIR2-LIT:"
2938         newkids = {u"child-imm":        ["filenode", {"ro_uri": filecap1,
2939                                                       "metadata": md1, }],
2940                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2941                    u"dirchild-imm":     ["dirnode",  {"ro_uri": immdircap}],
2942                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2943                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2944                    }
2945         return newkids, {'filecap1': filecap1,
2946                          'unknown_immcap': unknown_immcap,
2947                          'immdircap': immdircap,
2948                          'litdircap': litdircap,
2949                          'emptydircap': emptydircap}
2950
2951     def test_POST_mkdir_no_parentdir_initial_children(self):
2952         (newkids, caps) = self._create_initial_children()
2953         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
2954         def _after_mkdir(res):
2955             self.failUnless(res.startswith("URI:DIR"), res)
2956             n = self.s.create_node_from_uri(res)
2957             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2958             d2.addCallback(lambda ign:
2959                            self.failUnlessROChildURIIs(n, u"child-imm",
2960                                                        caps['filecap1']))
2961             d2.addCallback(lambda ign:
2962                            self.failUnlessRWChildURIIs(n, u"child-mutable",
2963                                                        caps['filecap2']))
2964             d2.addCallback(lambda ign:
2965                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
2966                                                        caps['filecap3']))
2967             d2.addCallback(lambda ign:
2968                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
2969                                                        caps['unknown_rwcap']))
2970             d2.addCallback(lambda ign:
2971                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
2972                                                        caps['unknown_rocap']))
2973             d2.addCallback(lambda ign:
2974                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2975                                                        caps['unknown_immcap']))
2976             d2.addCallback(lambda ign:
2977                            self.failUnlessRWChildURIIs(n, u"dirchild",
2978                                                        caps['dircap']))
2979             return d2
2980         d.addCallback(_after_mkdir)
2981         return d
2982
2983     def test_POST_mkdir_no_parentdir_unexpected_children(self):
2984         # the regular /uri?t=mkdir operation is specified to ignore its body.
2985         # Only t=mkdir-with-children pays attention to it.
2986         (newkids, caps) = self._create_initial_children()
2987         d = self.shouldHTTPError("POST_mkdir_no_parentdir_unexpected_children",
2988                                  400, "Bad Request",
2989                                  "t=mkdir does not accept children=, "
2990                                  "try t=mkdir-with-children instead",
2991                                  self.POST2, "/uri?t=mkdir", # without children
2992                                  simplejson.dumps(newkids))
2993         return d
2994
2995     def test_POST_noparent_bad(self):
2996         d = self.shouldHTTPError("POST_noparent_bad",
2997                                  400, "Bad Request",
2998                                  "/uri accepts only PUT, PUT?t=mkdir, "
2999                                  "POST?t=upload, and POST?t=mkdir",
3000                                  self.POST, "/uri?t=bogus")
3001         return d
3002
3003     def test_POST_mkdir_no_parentdir_immutable(self):
3004         (newkids, caps) = self._create_immutable_children()
3005         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
3006         def _after_mkdir(res):
3007             self.failUnless(res.startswith("URI:DIR"), res)
3008             n = self.s.create_node_from_uri(res)
3009             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3010             d2.addCallback(lambda ign:
3011                            self.failUnlessROChildURIIs(n, u"child-imm",
3012                                                           caps['filecap1']))
3013             d2.addCallback(lambda ign:
3014                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3015                                                           caps['unknown_immcap']))
3016             d2.addCallback(lambda ign:
3017                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
3018                                                           caps['immdircap']))
3019             d2.addCallback(lambda ign:
3020                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
3021                                                           caps['litdircap']))
3022             d2.addCallback(lambda ign:
3023                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
3024                                                           caps['emptydircap']))
3025             return d2
3026         d.addCallback(_after_mkdir)
3027         return d
3028
3029     def test_POST_mkdir_no_parentdir_immutable_bad(self):
3030         (newkids, caps) = self._create_initial_children()
3031         d = self.shouldFail2(error.Error,
3032                              "test_POST_mkdir_no_parentdir_immutable_bad",
3033                              "400 Bad Request",
3034                              "needed to be immutable but was not",
3035                              self.POST2,
3036                              "/uri?t=mkdir-immutable",
3037                              simplejson.dumps(newkids))
3038         return d
3039
3040     def test_welcome_page_mkdir_button(self):
3041         # Fetch the welcome page.
3042         d = self.GET("/")
3043         def _after_get_welcome_page(res):
3044             MKDIR_BUTTON_RE = re.compile(
3045                 '<form action="([^"]*)" method="post".*?'
3046                 '<input type="hidden" name="t" value="([^"]*)" />'
3047                 '<input type="hidden" name="([^"]*)" value="([^"]*)" />'
3048                 '<input type="submit" value="Create a directory" />',
3049                 re.I)
3050             mo = MKDIR_BUTTON_RE.search(res)
3051             formaction = mo.group(1)
3052             formt = mo.group(2)
3053             formaname = mo.group(3)
3054             formavalue = mo.group(4)
3055             return (formaction, formt, formaname, formavalue)
3056         d.addCallback(_after_get_welcome_page)
3057         def _after_parse_form(res):
3058             (formaction, formt, formaname, formavalue) = res
3059             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
3060         d.addCallback(_after_parse_form)
3061         d.addBoth(self.shouldRedirect, None, statuscode='303')
3062         return d
3063
3064     def test_POST_mkdir_replace(self): # return value?
3065         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
3066         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3067         d.addCallback(self.failUnlessNodeKeysAre, [])
3068         return d
3069
3070     def test_POST_mkdir_no_replace_queryarg(self): # return value?
3071         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
3072         d.addBoth(self.shouldFail, error.Error,
3073                   "POST_mkdir_no_replace_queryarg",
3074                   "409 Conflict",
3075                   "There was already a child by that name, and you asked me "
3076                   "to not replace it")
3077         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3078         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3079         return d
3080
3081     def test_POST_mkdir_no_replace_field(self): # return value?
3082         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
3083                       replace="false")
3084         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
3085                   "409 Conflict",
3086                   "There was already a child by that name, and you asked me "
3087                   "to not replace it")
3088         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3089         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3090         return d
3091
3092     def test_POST_mkdir_whendone_field(self):
3093         d = self.POST(self.public_url + "/foo",
3094                       t="mkdir", name="newdir", when_done="/THERE")
3095         d.addBoth(self.shouldRedirect, "/THERE")
3096         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3097         d.addCallback(self.failUnlessNodeKeysAre, [])
3098         return d
3099
3100     def test_POST_mkdir_whendone_queryarg(self):
3101         d = self.POST(self.public_url + "/foo?when_done=/THERE",
3102                       t="mkdir", name="newdir")
3103         d.addBoth(self.shouldRedirect, "/THERE")
3104         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3105         d.addCallback(self.failUnlessNodeKeysAre, [])
3106         return d
3107
3108     def test_POST_bad_t(self):
3109         d = self.shouldFail2(error.Error, "POST_bad_t",
3110                              "400 Bad Request",
3111                              "POST to a directory with bad t=BOGUS",
3112                              self.POST, self.public_url + "/foo", t="BOGUS")
3113         return d
3114
3115     def test_POST_set_children(self, command_name="set_children"):
3116         contents9, n9, newuri9 = self.makefile(9)
3117         contents10, n10, newuri10 = self.makefile(10)
3118         contents11, n11, newuri11 = self.makefile(11)
3119
3120         reqbody = """{
3121                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
3122                                                 "size": 0,
3123                                                 "metadata": {
3124                                                   "ctime": 1002777696.7564139,
3125                                                   "mtime": 1002777696.7564139
3126                                                  }
3127                                                } ],
3128                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
3129                                                 "size": 1,
3130                                                 "metadata": {
3131                                                   "ctime": 1002777696.7564139,
3132                                                   "mtime": 1002777696.7564139
3133                                                  }
3134                                                } ],
3135                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
3136                                                 "size": 2,
3137                                                 "metadata": {
3138                                                   "ctime": 1002777696.7564139,
3139                                                   "mtime": 1002777696.7564139
3140                                                  }
3141                                                } ]
3142                     }""" % (newuri9, newuri10, newuri11)
3143
3144         url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
3145
3146         d = client.getPage(url, method="POST", postdata=reqbody)
3147         def _then(res):
3148             self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
3149             self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
3150             self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
3151
3152         d.addCallback(_then)
3153         d.addErrback(self.dump_error)
3154         return d
3155
3156     def test_POST_set_children_with_hyphen(self):
3157         return self.test_POST_set_children(command_name="set-children")
3158
3159     def test_POST_link_uri(self):
3160         contents, n, newuri = self.makefile(8)
3161         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
3162         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
3163         d.addCallback(lambda res:
3164                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3165                                                       contents))
3166         return d
3167
3168     def test_POST_link_uri_replace(self):
3169         contents, n, newuri = self.makefile(8)
3170         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
3171         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
3172         d.addCallback(lambda res:
3173                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3174                                                       contents))
3175         return d
3176
3177     def test_POST_link_uri_unknown_bad(self):
3178         d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
3179         d.addBoth(self.shouldFail, error.Error,
3180                   "POST_link_uri_unknown_bad",
3181                   "400 Bad Request",
3182                   "unknown cap in a write slot")
3183         return d
3184
3185     def test_POST_link_uri_unknown_ro_good(self):
3186         d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
3187         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
3188         return d
3189
3190     def test_POST_link_uri_unknown_imm_good(self):
3191         d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
3192         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
3193         return d
3194
3195     def test_POST_link_uri_no_replace_queryarg(self):
3196         contents, n, newuri = self.makefile(8)
3197         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
3198                       name="bar.txt", uri=newuri)
3199         d.addBoth(self.shouldFail, error.Error,
3200                   "POST_link_uri_no_replace_queryarg",
3201                   "409 Conflict",
3202                   "There was already a child by that name, and you asked me "
3203                   "to not replace it")
3204         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3205         d.addCallback(self.failUnlessIsBarDotTxt)
3206         return d
3207
3208     def test_POST_link_uri_no_replace_field(self):
3209         contents, n, newuri = self.makefile(8)
3210         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
3211                       name="bar.txt", uri=newuri)
3212         d.addBoth(self.shouldFail, error.Error,
3213                   "POST_link_uri_no_replace_field",
3214                   "409 Conflict",
3215                   "There was already a child by that name, and you asked me "
3216                   "to not replace it")
3217         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3218         d.addCallback(self.failUnlessIsBarDotTxt)
3219         return d
3220
3221     def test_POST_delete(self, command_name='delete'):
3222         d = self._foo_node.list()
3223         def _check_before(children):
3224             self.failUnlessIn(u"bar.txt", children)
3225         d.addCallback(_check_before)
3226         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t=command_name, name="bar.txt"))
3227         d.addCallback(lambda res: self._foo_node.list())
3228         def _check_after(children):
3229             self.failIfIn(u"bar.txt", children)
3230         d.addCallback(_check_after)
3231         return d
3232
3233     def test_POST_unlink(self):
3234         return self.test_POST_delete(command_name='unlink')
3235
3236     def test_POST_rename_file(self):
3237         d = self.POST(self.public_url + "/foo", t="rename",
3238                       from_name="bar.txt", to_name='wibble.txt')
3239         d.addCallback(lambda res:
3240                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3241         d.addCallback(lambda res:
3242                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
3243         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
3244         d.addCallback(self.failUnlessIsBarDotTxt)
3245         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
3246         d.addCallback(self.failUnlessIsBarJSON)
3247         return d
3248
3249     def test_POST_rename_file_redundant(self):
3250         d = self.POST(self.public_url + "/foo", t="rename",
3251                       from_name="bar.txt", to_name='bar.txt')
3252         d.addCallback(lambda res:
3253                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3254         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3255         d.addCallback(self.failUnlessIsBarDotTxt)
3256         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3257         d.addCallback(self.failUnlessIsBarJSON)
3258         return d
3259
3260     def test_POST_rename_file_replace(self):
3261         # rename a file and replace a directory with it
3262         d = self.POST(self.public_url + "/foo", t="rename",
3263                       from_name="bar.txt", to_name='empty')
3264         d.addCallback(lambda res:
3265                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3266         d.addCallback(lambda res:
3267                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
3268         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
3269         d.addCallback(self.failUnlessIsBarDotTxt)
3270         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3271         d.addCallback(self.failUnlessIsBarJSON)
3272         return d
3273
3274     def test_POST_rename_file_no_replace_queryarg(self):
3275         # rename a file and replace a directory with it
3276         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
3277                       from_name="bar.txt", to_name='empty')
3278         d.addBoth(self.shouldFail, error.Error,
3279                   "POST_rename_file_no_replace_queryarg",
3280                   "409 Conflict",
3281                   "There was already a child by that name, and you asked me "
3282                   "to not replace it")
3283         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3284         d.addCallback(self.failUnlessIsEmptyJSON)
3285         return d
3286
3287     def test_POST_rename_file_no_replace_field(self):
3288         # rename a file and replace a directory with it
3289         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
3290                       from_name="bar.txt", to_name='empty')
3291         d.addBoth(self.shouldFail, error.Error,
3292                   "POST_rename_file_no_replace_field",
3293                   "409 Conflict",
3294                   "There was already a child by that name, and you asked me "
3295                   "to not replace it")
3296         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3297         d.addCallback(self.failUnlessIsEmptyJSON)
3298         return d
3299
3300     def failUnlessIsEmptyJSON(self, res):
3301         data = simplejson.loads(res)
3302         self.failUnlessEqual(data[0], "dirnode", data)
3303         self.failUnlessReallyEqual(len(data[1]["children"]), 0)
3304
3305     def test_POST_rename_file_slash_fail(self):
3306         d = self.POST(self.public_url + "/foo", t="rename",
3307                       from_name="bar.txt", to_name='kirk/spock.txt')
3308         d.addBoth(self.shouldFail, error.Error,
3309                   "test_POST_rename_file_slash_fail",
3310                   "400 Bad Request",
3311                   "to_name= may not contain a slash",
3312                   )
3313         d.addCallback(lambda res:
3314                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3315         return d
3316
3317     def test_POST_rename_dir(self):
3318         d = self.POST(self.public_url, t="rename",
3319                       from_name="foo", to_name='plunk')
3320         d.addCallback(lambda res:
3321                       self.failIfNodeHasChild(self.public_root, u"foo"))
3322         d.addCallback(lambda res:
3323                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
3324         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
3325         d.addCallback(self.failUnlessIsFooJSON)
3326         return d
3327
3328     def test_POST_move_file(self):
3329         d = self.POST(self.public_url + "/foo", t="move",
3330                       from_name="bar.txt", to_dir="sub")
3331         d.addCallback(lambda res:
3332                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3333         d.addCallback(lambda res:
3334                       self.failUnlessNodeHasChild(self._sub_node, u"bar.txt"))
3335         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3336         d.addCallback(self.failUnlessIsBarDotTxt)
3337         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3338         d.addCallback(self.failUnlessIsBarJSON)
3339         return d
3340
3341     def test_POST_move_file_new_name(self):
3342         d = self.POST(self.public_url + "/foo", t="move",
3343                       from_name="bar.txt", to_name="wibble.txt", to_dir="sub")
3344         d.addCallback(lambda res:
3345                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3346         d.addCallback(lambda res:
3347                       self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3348         d.addCallback(lambda res:
3349                       self.failUnlessNodeHasChild(self._sub_node, u"wibble.txt"))
3350         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt"))
3351         d.addCallback(self.failUnlessIsBarDotTxt)
3352         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt?t=json"))
3353         d.addCallback(self.failUnlessIsBarJSON)
3354         return d
3355
3356     def test_POST_move_file_replace(self):
3357         d = self.POST(self.public_url + "/foo", t="move",
3358                       from_name="bar.txt", to_name="baz.txt", to_dir="sub")
3359         d.addCallback(lambda res:
3360                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3361         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3362         d.addCallback(self.failUnlessIsBarDotTxt)
3363         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt?t=json"))
3364         d.addCallback(self.failUnlessIsBarJSON)
3365         return d
3366
3367     def test_POST_move_file_no_replace(self):
3368         d = self.shouldFail2(error.Error, "POST_move_file_no_replace",
3369                              "409 Conflict",
3370                              "There was already a child by that name, and you asked me to not replace it",
3371                              self.POST, self.public_url + "/foo", t="move",
3372                              replace="false", from_name="bar.txt",
3373                              to_name="baz.txt", to_dir="sub")
3374         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3375         d.addCallback(self.failUnlessIsBarDotTxt)
3376         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3377         d.addCallback(self.failUnlessIsBarJSON)
3378         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3379         d.addCallback(self.failUnlessIsSubBazDotTxt)
3380         return d
3381
3382     def test_POST_move_file_slash_fail(self):
3383         d = self.shouldFail2(error.Error, "test_POST_rename_file_slash_fail",
3384                              "400 Bad Request",
3385                              "to_name= may not contain a slash",
3386                              self.POST, self.public_url + "/foo", t="move",
3387                              from_name="bar.txt",
3388                              to_name="slash/fail.txt", to_dir="sub")
3389         d.addCallback(lambda res:
3390                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3391         d.addCallback(lambda res:
3392                       self.failIfNodeHasChild(self._sub_node, u"slash/fail.txt"))
3393         d.addCallback(lambda ign:
3394                       self.shouldFail2(error.Error,
3395                                        "test_POST_rename_file_slash_fail2",
3396                                        "400 Bad Request",
3397                                        "from_name= may not contain a slash",
3398                                        self.POST, self.public_url + "/foo",
3399                                        t="move",
3400                                        from_name="nope/bar.txt",
3401                                        to_name="fail.txt", to_dir="sub"))
3402         return d
3403
3404     def test_POST_move_file_no_target(self):
3405         d = self.shouldFail2(error.Error, "POST_move_file_no_target",
3406                              "400 Bad Request",
3407                              "move requires from_name and to_dir",
3408                              self.POST, self.public_url + "/foo", t="move",
3409                              from_name="bar.txt", to_name="baz.txt")
3410         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3411         d.addCallback(self.failUnlessIsBarDotTxt)
3412         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3413         d.addCallback(self.failUnlessIsBarJSON)
3414         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3415         d.addCallback(self.failUnlessIsBazDotTxt)
3416         return d
3417
3418     def test_POST_move_file_bad_target_type(self):
3419         d = self.shouldFail2(error.Error, "test_POST_move_file_bad_target_type",
3420                              "400 Bad Request", "invalid target_type parameter",
3421                              self.POST,
3422                              self.public_url + "/foo", t="move",
3423                              target_type="*D", from_name="bar.txt",
3424                              to_dir="sub")
3425         return d
3426
3427     def test_POST_move_file_multi_level(self):
3428         d = self.POST(self.public_url + "/foo/sub/level2?t=mkdir", "")
3429         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t="move",
3430                       from_name="bar.txt", to_dir="sub/level2"))
3431         d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3432         d.addCallback(lambda res: self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3433         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt"))
3434         d.addCallback(self.failUnlessIsBarDotTxt)
3435         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt?t=json"))
3436         d.addCallback(self.failUnlessIsBarJSON)
3437         return d
3438
3439     def test_POST_move_file_to_uri(self):
3440         d = self.POST(self.public_url + "/foo", t="move", target_type="uri",
3441                       from_name="bar.txt", to_dir=self._sub_uri)
3442         d.addCallback(lambda res:
3443                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3444         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3445         d.addCallback(self.failUnlessIsBarDotTxt)
3446         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3447         d.addCallback(self.failUnlessIsBarJSON)
3448         return d
3449
3450     def test_POST_move_file_to_nonexist_dir(self):
3451         d = self.shouldFail2(error.Error, "POST_move_file_to_nonexist_dir",
3452                             "404 Not Found", "No such child: nopechucktesta",
3453                             self.POST, self.public_url + "/foo", t="move",
3454                             from_name="bar.txt", to_dir="nopechucktesta")
3455         return d
3456
3457     def test_POST_move_file_into_file(self):
3458         d = self.shouldFail2(error.Error, "POST_move_file_into_file",
3459                              "400 Bad Request", "to_dir is not a directory",
3460                              self.POST, self.public_url + "/foo", t="move",
3461                              from_name="bar.txt", to_dir="baz.txt")
3462         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3463         d.addCallback(self.failUnlessIsBazDotTxt)
3464         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3465         d.addCallback(self.failUnlessIsBarDotTxt)
3466         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3467         d.addCallback(self.failUnlessIsBarJSON)
3468         return d
3469
3470     def test_POST_move_file_to_bad_uri(self):
3471         d =  self.shouldFail2(error.Error, "POST_move_file_to_bad_uri",
3472                               "400 Bad Request", "to_dir is not a directory",
3473                               self.POST, self.public_url + "/foo", t="move",
3474                               from_name="bar.txt", target_type="uri",
3475                               to_dir="URI:DIR2:mn5jlyjnrjeuydyswlzyui72i:rmneifcj6k6sycjljjhj3f6majsq2zqffydnnul5hfa4j577arma")
3476         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3477         d.addCallback(self.failUnlessIsBarDotTxt)
3478         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3479         d.addCallback(self.failUnlessIsBarJSON)
3480         return d
3481
3482     def test_POST_move_dir(self):
3483         d = self.POST(self.public_url + "/foo", t="move",
3484                       from_name="bar.txt", to_dir="empty")
3485         d.addCallback(lambda res: self.POST(self.public_url + "/foo",
3486                       t="move", from_name="empty", to_dir="sub"))
3487         d.addCallback(lambda res:
3488                       self.failIfNodeHasChild(self._foo_node, u"empty"))
3489         d.addCallback(lambda res:
3490                       self.failUnlessNodeHasChild(self._sub_node, u"empty"))
3491         d.addCallback(lambda res:
3492                       self._sub_node.get_child_at_path(u"empty"))
3493         d.addCallback(lambda node:
3494                       self.failUnlessNodeHasChild(node, u"bar.txt"))
3495         d.addCallback(lambda res:
3496                       self.GET(self.public_url + "/foo/sub/empty/bar.txt"))
3497         d.addCallback(self.failUnlessIsBarDotTxt)
3498         return d
3499
3500     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
3501         """ If target is not None then the redirection has to go to target.  If
3502         statuscode is not None then the redirection has to be accomplished with
3503         that HTTP status code."""
3504         if not isinstance(res, failure.Failure):
3505             to_where = (target is None) and "somewhere" or ("to " + target)
3506             self.fail("%s: we were expecting to get redirected %s, not get an"
3507                       " actual page: %s" % (which, to_where, res))
3508         res.trap(error.PageRedirect)
3509         if statuscode is not None:
3510             self.failUnlessReallyEqual(res.value.status, statuscode,
3511                                        "%s: not a redirect" % which)
3512         if target is not None:
3513             # the PageRedirect does not seem to capture the uri= query arg
3514             # properly, so we can't check for it.
3515             realtarget = self.webish_url + target
3516             self.failUnlessReallyEqual(res.value.location, realtarget,
3517                                        "%s: wrong target" % which)
3518         return res.value.location
3519
3520     def test_GET_URI_form(self):
3521         base = "/uri?uri=%s" % self._bar_txt_uri
3522         # this is supposed to give us a redirect to /uri/$URI, plus arguments
3523         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
3524         d = self.GET(base)
3525         d.addBoth(self.shouldRedirect, targetbase)
3526         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
3527         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
3528         d.addCallback(lambda res: self.GET(base+"&t=json"))
3529         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
3530         d.addCallback(self.log, "about to get file by uri")
3531         d.addCallback(lambda res: self.GET(base, followRedirect=True))
3532         d.addCallback(self.failUnlessIsBarDotTxt)
3533         d.addCallback(self.log, "got file by uri, about to get dir by uri")
3534         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
3535                                            followRedirect=True))
3536         d.addCallback(self.failUnlessIsFooJSON)
3537         d.addCallback(self.log, "got dir by uri")
3538
3539         return d
3540
3541     def test_GET_URI_form_bad(self):
3542         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
3543                              "400 Bad Request", "GET /uri requires uri=",
3544                              self.GET, "/uri")
3545         return d
3546
3547     def test_GET_rename_form(self):
3548         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
3549                      followRedirect=True)
3550         def _check(res):
3551             self.failUnlessIn('name="when_done" value="."', res)
3552             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
3553             self.failUnlessIn(FAVICON_MARKUP, res)
3554         d.addCallback(_check)
3555         return d
3556
3557     def log(self, res, msg):
3558         #print "MSG: %s  RES: %s" % (msg, res)
3559         log.msg(msg)
3560         return res
3561
3562     def test_GET_URI_URL(self):
3563         base = "/uri/%s" % self._bar_txt_uri
3564         d = self.GET(base)
3565         d.addCallback(self.failUnlessIsBarDotTxt)
3566         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
3567         d.addCallback(self.failUnlessIsBarDotTxt)
3568         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
3569         d.addCallback(self.failUnlessIsBarDotTxt)
3570         return d
3571
3572     def test_GET_URI_URL_dir(self):
3573         base = "/uri/%s?t=json" % self._foo_uri
3574         d = self.GET(base)
3575         d.addCallback(self.failUnlessIsFooJSON)
3576         return d
3577
3578     def test_GET_URI_URL_missing(self):
3579         base = "/uri/%s" % self._bad_file_uri
3580         d = self.shouldHTTPError("test_GET_URI_URL_missing",
3581                                  http.GONE, None, "NotEnoughSharesError",
3582                                  self.GET, base)
3583         # TODO: how can we exercise both sides of WebDownloadTarget.fail
3584         # here? we must arrange for a download to fail after target.open()
3585         # has been called, and then inspect the response to see that it is
3586         # shorter than we expected.
3587         return d
3588
3589     def test_PUT_DIRURL_uri(self):
3590         d = self.s.create_dirnode()
3591         def _made_dir(dn):
3592             new_uri = dn.get_uri()
3593             # replace /foo with a new (empty) directory
3594             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
3595             d.addCallback(lambda res:
3596                           self.failUnlessReallyEqual(res.strip(), new_uri))
3597             d.addCallback(lambda res:
3598                           self.failUnlessRWChildURIIs(self.public_root,
3599                                                       u"foo",
3600                                                       new_uri))
3601             return d
3602         d.addCallback(_made_dir)
3603         return d
3604
3605     def test_PUT_DIRURL_uri_noreplace(self):
3606         d = self.s.create_dirnode()
3607         def _made_dir(dn):
3608             new_uri = dn.get_uri()
3609             # replace /foo with a new (empty) directory, but ask that
3610             # replace=false, so it should fail
3611             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
3612                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
3613                                  self.PUT,
3614                                  self.public_url + "/foo?t=uri&replace=false",
3615                                  new_uri)
3616             d.addCallback(lambda res:
3617                           self.failUnlessRWChildURIIs(self.public_root,
3618                                                       u"foo",
3619                                                       self._foo_uri))
3620             return d
3621         d.addCallback(_made_dir)
3622         return d
3623
3624     def test_PUT_DIRURL_bad_t(self):
3625         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
3626                              "400 Bad Request", "PUT to a directory",
3627                              self.PUT, self.public_url + "/foo?t=BOGUS", "")
3628         d.addCallback(lambda res:
3629                       self.failUnlessRWChildURIIs(self.public_root,
3630                                                   u"foo",
3631                                                   self._foo_uri))
3632         return d
3633
3634     def test_PUT_NEWFILEURL_uri(self):
3635         contents, n, new_uri = self.makefile(8)
3636         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
3637         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3638         d.addCallback(lambda res:
3639                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3640                                                       contents))
3641         return d
3642
3643     def test_PUT_NEWFILEURL_mdmf(self):
3644         new_contents = self.NEWFILE_CONTENTS * 300000
3645         d = self.PUT(self.public_url + \
3646                      "/foo/mdmf.txt?format=mdmf",
3647                      new_contents)
3648         d.addCallback(lambda ignored:
3649             self.GET(self.public_url + "/foo/mdmf.txt?t=json"))
3650         def _got_json(json):
3651             data = simplejson.loads(json)
3652             data = data[1]
3653             self.failUnlessIn("format", data)
3654             self.failUnlessEqual(data["format"], "MDMF")
3655             self.failUnless(data['rw_uri'].startswith("URI:MDMF"))
3656             self.failUnless(data['ro_uri'].startswith("URI:MDMF"))
3657         d.addCallback(_got_json)
3658         return d
3659
3660     def test_PUT_NEWFILEURL_sdmf(self):
3661         new_contents = self.NEWFILE_CONTENTS * 300000
3662         d = self.PUT(self.public_url + \
3663                      "/foo/sdmf.txt?format=sdmf",
3664                      new_contents)
3665         d.addCallback(lambda ignored:
3666             self.GET(self.public_url + "/foo/sdmf.txt?t=json"))
3667         def _got_json(json):
3668             data = simplejson.loads(json)
3669             data = data[1]
3670             self.failUnlessIn("format", data)
3671             self.failUnlessEqual(data["format"], "SDMF")
3672         d.addCallback(_got_json)
3673         return d
3674
3675     def test_PUT_NEWFILEURL_bad_format(self):
3676        new_contents = self.NEWFILE_CONTENTS * 300000
3677        return self.shouldHTTPError("PUT_NEWFILEURL_bad_format",
3678                                    400, "Bad Request", "Unknown format: foo",
3679                                    self.PUT, self.public_url + \
3680                                    "/foo/foo.txt?format=foo",
3681                                    new_contents)
3682
3683     def test_PUT_NEWFILEURL_uri_replace(self):
3684         contents, n, new_uri = self.makefile(8)
3685         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
3686         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3687         d.addCallback(lambda res:
3688                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3689                                                       contents))
3690         return d
3691
3692     def test_PUT_NEWFILEURL_uri_no_replace(self):
3693         contents, n, new_uri = self.makefile(8)
3694         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
3695         d.addBoth(self.shouldFail, error.Error,
3696                   "PUT_NEWFILEURL_uri_no_replace",
3697                   "409 Conflict",
3698                   "There was already a child by that name, and you asked me "
3699                   "to not replace it")
3700         return d
3701
3702     def test_PUT_NEWFILEURL_uri_unknown_bad(self):
3703         d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
3704         d.addBoth(self.shouldFail, error.Error,
3705                   "POST_put_uri_unknown_bad",
3706                   "400 Bad Request",
3707                   "unknown cap in a write slot")
3708         return d
3709
3710     def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
3711         d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
3712         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3713                       u"put-future-ro.txt")
3714         return d
3715
3716     def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
3717         d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
3718         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3719                       u"put-future-imm.txt")
3720         return d
3721
3722     def test_PUT_NEWFILE_URI(self):
3723         file_contents = "New file contents here\n"
3724         d = self.PUT("/uri", file_contents)
3725         def _check(uri):
3726             assert isinstance(uri, str), uri
3727             self.failUnlessIn(uri, FakeCHKFileNode.all_contents)
3728             self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
3729                                        file_contents)
3730             return self.GET("/uri/%s" % uri)
3731         d.addCallback(_check)
3732         def _check2(res):
3733             self.failUnlessReallyEqual(res, file_contents)
3734         d.addCallback(_check2)
3735         return d
3736
3737     def test_PUT_NEWFILE_URI_not_mutable(self):
3738         file_contents = "New file contents here\n"
3739         d = self.PUT("/uri?mutable=false", file_contents)
3740         def _check(uri):
3741             assert isinstance(uri, str), uri
3742             self.failUnlessIn(uri, FakeCHKFileNode.all_contents)
3743             self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
3744                                        file_contents)
3745             return self.GET("/uri/%s" % uri)
3746         d.addCallback(_check)
3747         def _check2(res):
3748             self.failUnlessReallyEqual(res, file_contents)
3749         d.addCallback(_check2)
3750         return d
3751
3752     def test_PUT_NEWFILE_URI_only_PUT(self):
3753         d = self.PUT("/uri?t=bogus", "")
3754         d.addBoth(self.shouldFail, error.Error,
3755                   "PUT_NEWFILE_URI_only_PUT",
3756                   "400 Bad Request",
3757                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
3758         return d
3759
3760     def test_PUT_NEWFILE_URI_mutable(self):
3761         file_contents = "New file contents here\n"
3762         d = self.PUT("/uri?mutable=true", file_contents)
3763         def _check1(filecap):
3764             filecap = filecap.strip()
3765             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
3766             self.filecap = filecap
3767             u = uri.WriteableSSKFileURI.init_from_string(filecap)
3768             self.failUnlessIn(u.get_storage_index(), FakeMutableFileNode.all_contents)
3769             n = self.s.create_node_from_uri(filecap)
3770             return n.download_best_version()
3771         d.addCallback(_check1)
3772         def _check2(data):
3773             self.failUnlessReallyEqual(data, file_contents)
3774             return self.GET("/uri/%s" % urllib.quote(self.filecap))
3775         d.addCallback(_check2)
3776         def _check3(res):
3777             self.failUnlessReallyEqual(res, file_contents)
3778         d.addCallback(_check3)
3779         return d
3780
3781     def test_PUT_mkdir(self):
3782         d = self.PUT("/uri?t=mkdir", "")
3783         def _check(uri):
3784             n = self.s.create_node_from_uri(uri.strip())
3785             d2 = self.failUnlessNodeKeysAre(n, [])
3786             d2.addCallback(lambda res:
3787                            self.GET("/uri/%s?t=json" % uri))
3788             return d2
3789         d.addCallback(_check)
3790         d.addCallback(self.failUnlessIsEmptyJSON)
3791         return d
3792
3793     def test_PUT_mkdir_mdmf(self):
3794         d = self.PUT("/uri?t=mkdir&format=mdmf", "")
3795         def _got(res):
3796             u = uri.from_string(res)
3797             # Check that this is an MDMF writecap
3798             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
3799         d.addCallback(_got)
3800         return d
3801
3802     def test_PUT_mkdir_sdmf(self):
3803         d = self.PUT("/uri?t=mkdir&format=sdmf", "")
3804         def _got(res):
3805             u = uri.from_string(res)
3806             self.failUnlessIsInstance(u, uri.DirectoryURI)
3807         d.addCallback(_got)
3808         return d
3809
3810     def test_PUT_mkdir_bad_format(self):
3811         return self.shouldHTTPError("PUT_mkdir_bad_format",
3812                                     400, "Bad Request", "Unknown format: foo",
3813                                     self.PUT, "/uri?t=mkdir&format=foo",
3814                                     "")
3815
3816     def test_POST_check(self):
3817         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
3818         def _done(res):
3819             # this returns a string form of the results, which are probably
3820             # None since we're using fake filenodes.
3821             # TODO: verify that the check actually happened, by changing
3822             # FakeCHKFileNode to count how many times .check() has been
3823             # called.
3824             pass
3825         d.addCallback(_done)
3826         return d
3827
3828
3829     def test_PUT_update_at_offset(self):
3830         file_contents = "test file" * 100000 # about 900 KiB
3831         d = self.PUT("/uri?mutable=true", file_contents)
3832         def _then(filecap):
3833             self.filecap = filecap
3834             new_data = file_contents[:100]
3835             new = "replaced and so on"
3836             new_data += new
3837             new_data += file_contents[len(new_data):]
3838             assert len(new_data) == len(file_contents)
3839             self.new_data = new_data
3840         d.addCallback(_then)
3841         d.addCallback(lambda ignored:
3842             self.PUT("/uri/%s?replace=True&offset=100" % self.filecap,
3843                      "replaced and so on"))
3844         def _get_data(filecap):
3845             n = self.s.create_node_from_uri(filecap)
3846             return n.download_best_version()
3847         d.addCallback(_get_data)
3848         d.addCallback(lambda results:
3849             self.failUnlessEqual(results, self.new_data))
3850         # Now try appending things to the file
3851         d.addCallback(lambda ignored:
3852             self.PUT("/uri/%s?offset=%d" % (self.filecap, len(self.new_data)),
3853                      "puppies" * 100))
3854         d.addCallback(_get_data)
3855         d.addCallback(lambda results:
3856             self.failUnlessEqual(results, self.new_data + ("puppies" * 100)))
3857         # and try replacing the beginning of the file
3858         d.addCallback(lambda ignored:
3859             self.PUT("/uri/%s?offset=0" % self.filecap, "begin"))
3860         d.addCallback(_get_data)
3861         d.addCallback(lambda results:
3862             self.failUnlessEqual(results, "begin"+self.new_data[len("begin"):]+("puppies"*100)))
3863         return d
3864
3865     def test_PUT_update_at_invalid_offset(self):
3866         file_contents = "test file" * 100000 # about 900 KiB
3867         d = self.PUT("/uri?mutable=true", file_contents)
3868         def _then(filecap):
3869             self.filecap = filecap
3870         d.addCallback(_then)
3871         # Negative offsets should cause an error.
3872         d.addCallback(lambda ignored:
3873             self.shouldHTTPError("PUT_update_at_invalid_offset",
3874                                  400, "Bad Request",
3875                                  "Invalid offset",
3876                                  self.PUT,
3877                                  "/uri/%s?offset=-1" % self.filecap,
3878                                  "foo"))
3879         return d
3880
3881     def test_PUT_update_at_offset_immutable(self):
3882         file_contents = "Test file" * 100000
3883         d = self.PUT("/uri", file_contents)
3884         def _then(filecap):
3885             self.filecap = filecap
3886         d.addCallback(_then)
3887         d.addCallback(lambda ignored:
3888             self.shouldHTTPError("PUT_update_at_offset_immutable",
3889                                  400, "Bad Request",
3890                                  "immutable",
3891                                  self.PUT,
3892                                  "/uri/%s?offset=50" % self.filecap,
3893                                  "foo"))
3894         return d
3895
3896
3897     def test_bad_method(self):
3898         url = self.webish_url + self.public_url + "/foo/bar.txt"
3899         d = self.shouldHTTPError("bad_method",
3900                                  501, "Not Implemented",
3901                                  "I don't know how to treat a BOGUS request.",
3902                                  client.getPage, url, method="BOGUS")
3903         return d
3904
3905     def test_short_url(self):
3906         url = self.webish_url + "/uri"
3907         d = self.shouldHTTPError("short_url", 501, "Not Implemented",
3908                                  "I don't know how to treat a DELETE request.",
3909                                  client.getPage, url, method="DELETE")
3910         return d
3911
3912     def test_ophandle_bad(self):
3913         url = self.webish_url + "/operations/bogus?t=status"
3914         d = self.shouldHTTPError("ophandle_bad", 404, "404 Not Found",
3915                                  "unknown/expired handle 'bogus'",
3916                                  client.getPage, url)
3917         return d
3918
3919     def test_ophandle_cancel(self):
3920         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
3921                       followRedirect=True)
3922         d.addCallback(lambda ignored:
3923                       self.GET("/operations/128?t=status&output=JSON"))
3924         def _check1(res):
3925             data = simplejson.loads(res)
3926             self.failUnless("finished" in data, res)
3927             monitor = self.ws.root.child_operations.handles["128"][0]
3928             d = self.POST("/operations/128?t=cancel&output=JSON")
3929             def _check2(res):
3930                 data = simplejson.loads(res)
3931                 self.failUnless("finished" in data, res)
3932                 # t=cancel causes the handle to be forgotten
3933                 self.failUnless(monitor.is_cancelled())
3934             d.addCallback(_check2)
3935             return d
3936         d.addCallback(_check1)
3937         d.addCallback(lambda ignored:
3938                       self.shouldHTTPError("ophandle_cancel",
3939                                            404, "404 Not Found",
3940                                            "unknown/expired handle '128'",
3941                                            self.GET,
3942                                            "/operations/128?t=status&output=JSON"))
3943         return d
3944
3945     def test_ophandle_retainfor(self):
3946         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
3947                       followRedirect=True)
3948         d.addCallback(lambda ignored:
3949                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
3950         def _check1(res):
3951             data = simplejson.loads(res)
3952             self.failUnless("finished" in data, res)
3953         d.addCallback(_check1)
3954         # the retain-for=0 will cause the handle to be expired very soon
3955         d.addCallback(lambda ign:
3956             self.clock.advance(2.0))
3957         d.addCallback(lambda ignored:
3958                       self.shouldHTTPError("ophandle_retainfor",
3959                                            404, "404 Not Found",
3960                                            "unknown/expired handle '129'",
3961                                            self.GET,
3962                                            "/operations/129?t=status&output=JSON"))
3963         return d
3964
3965     def test_ophandle_release_after_complete(self):
3966         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
3967                       followRedirect=True)
3968         d.addCallback(self.wait_for_operation, "130")
3969         d.addCallback(lambda ignored:
3970                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
3971         # the release-after-complete=true will cause the handle to be expired
3972         d.addCallback(lambda ignored:
3973                       self.shouldHTTPError("ophandle_release_after_complete",
3974                                            404, "404 Not Found",
3975                                            "unknown/expired handle '130'",
3976                                            self.GET,
3977                                            "/operations/130?t=status&output=JSON"))
3978         return d
3979
3980     def test_uncollected_ophandle_expiration(self):
3981         # uncollected ophandles should expire after 4 days
3982         def _make_uncollected_ophandle(ophandle):
3983             d = self.POST(self.public_url +
3984                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3985                           followRedirect=False)
3986             # When we start the operation, the webapi server will want
3987             # to redirect us to the page for the ophandle, so we get
3988             # confirmation that the operation has started. If the
3989             # manifest operation has finished by the time we get there,
3990             # following that redirect (by setting followRedirect=True
3991             # above) has the side effect of collecting the ophandle that
3992             # we've just created, which means that we can't use the
3993             # ophandle to test the uncollected timeout anymore. So,
3994             # instead, catch the 302 here and don't follow it.
3995             d.addBoth(self.should302, "uncollected_ophandle_creation")
3996             return d
3997         # Create an ophandle, don't collect it, then advance the clock by
3998         # 4 days - 1 second and make sure that the ophandle is still there.
3999         d = _make_uncollected_ophandle(131)
4000         d.addCallback(lambda ign:
4001             self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
4002         d.addCallback(lambda ign:
4003             self.GET("/operations/131?t=status&output=JSON"))
4004         def _check1(res):
4005             data = simplejson.loads(res)
4006             self.failUnless("finished" in data, res)
4007         d.addCallback(_check1)
4008         # Create an ophandle, don't collect it, then try to collect it
4009         # after 4 days. It should be gone.
4010         d.addCallback(lambda ign:
4011             _make_uncollected_ophandle(132))
4012         d.addCallback(lambda ign:
4013             self.clock.advance(96*60*60))
4014         d.addCallback(lambda ign:
4015             self.shouldHTTPError("uncollected_ophandle_expired_after_100_hours",
4016                                  404, "404 Not Found",
4017                                  "unknown/expired handle '132'",
4018                                  self.GET,
4019                                  "/operations/132?t=status&output=JSON"))
4020         return d
4021
4022     def test_collected_ophandle_expiration(self):
4023         # collected ophandles should expire after 1 day
4024         def _make_collected_ophandle(ophandle):
4025             d = self.POST(self.public_url +
4026                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
4027                           followRedirect=True)
4028             # By following the initial redirect, we collect the ophandle
4029             # we've just created.
4030             return d
4031         # Create a collected ophandle, then collect it after 23 hours
4032         # and 59 seconds to make sure that it is still there.
4033         d = _make_collected_ophandle(133)
4034         d.addCallback(lambda ign:
4035             self.clock.advance((24*60*60) - 1))
4036         d.addCallback(lambda ign:
4037             self.GET("/operations/133?t=status&output=JSON"))
4038         def _check1(res):
4039             data = simplejson.loads(res)
4040             self.failUnless("finished" in data, res)
4041         d.addCallback(_check1)
4042         # Create another uncollected ophandle, then try to collect it
4043         # after 24 hours to make sure that it is gone.
4044         d.addCallback(lambda ign:
4045             _make_collected_ophandle(134))
4046         d.addCallback(lambda ign:
4047             self.clock.advance(24*60*60))
4048         d.addCallback(lambda ign:
4049             self.shouldHTTPError("collected_ophandle_expired_after_1_day",
4050                                  404, "404 Not Found",
4051                                  "unknown/expired handle '134'",
4052                                  self.GET,
4053                                  "/operations/134?t=status&output=JSON"))
4054         return d
4055
4056     def test_incident(self):
4057         d = self.POST("/report_incident", details="eek")
4058         def _done(res):
4059             self.failIfIn("<html>", res)
4060             self.failUnlessIn("Thank you for your report!", res)
4061         d.addCallback(_done)
4062         return d
4063
4064     def test_static(self):
4065         webdir = os.path.join(self.staticdir, "subdir")
4066         fileutil.make_dirs(webdir)
4067         f = open(os.path.join(webdir, "hello.txt"), "wb")
4068         f.write("hello")
4069         f.close()
4070
4071         d = self.GET("/static/subdir/hello.txt")
4072         def _check(res):
4073             self.failUnlessReallyEqual(res, "hello")
4074         d.addCallback(_check)
4075         return d
4076
4077
4078 class IntroducerWeb(unittest.TestCase):
4079     def setUp(self):
4080         self.node = None
4081
4082     def tearDown(self):
4083         d = defer.succeed(None)
4084         if self.node:
4085             d.addCallback(lambda ign: self.node.stopService())
4086         d.addCallback(flushEventualQueue)
4087         return d
4088
4089     def test_welcome(self):
4090         basedir = "web.IntroducerWeb.test_welcome"
4091         os.mkdir(basedir)
4092         fileutil.write(os.path.join(basedir, "tahoe.cfg"), "[node]\nweb.port = tcp:0\n")
4093         self.node = IntroducerNode(basedir)
4094         self.ws = self.node.getServiceNamed("webish")
4095
4096         d = fireEventually(None)
4097         d.addCallback(lambda ign: self.node.startService())
4098         d.addCallback(lambda ign: self.node.when_tub_ready())
4099
4100         d.addCallback(lambda ign: self.GET("/"))
4101         def _check(res):
4102             self.failUnlessIn('Welcome to the Tahoe-LAFS Introducer', res)
4103             self.failUnlessIn(FAVICON_MARKUP, res)
4104         d.addCallback(_check)
4105         return d
4106
4107     def GET(self, urlpath, followRedirect=False, return_response=False,
4108             **kwargs):
4109         # if return_response=True, this fires with (data, statuscode,
4110         # respheaders) instead of just data.
4111         assert not isinstance(urlpath, unicode)
4112         url = self.ws.getURL().rstrip('/') + urlpath
4113         factory = HTTPClientGETFactory(url, method="GET",
4114                                        followRedirect=followRedirect, **kwargs)
4115         reactor.connectTCP("localhost", self.ws.getPortnum(), factory)
4116         d = factory.deferred
4117         def _got_data(data):
4118             return (data, factory.status, factory.response_headers)
4119         if return_response:
4120             d.addCallback(_got_data)
4121         return factory.deferred
4122
4123
4124 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4125     def test_load_file(self):
4126         # This will raise an exception unless a well-formed XML file is found under that name.
4127         common.getxmlfile('directory.xhtml').load()
4128
4129     def test_parse_replace_arg(self):
4130         self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
4131         self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
4132         self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
4133                                    "only-files")
4134         self.shouldFail(AssertionError, "test_parse_replace_arg", "",
4135                         common.parse_replace_arg, "only_fles")
4136
4137     def test_abbreviate_time(self):
4138         self.failUnlessReallyEqual(common.abbreviate_time(None), "")
4139         self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
4140         self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
4141         self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
4142         self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
4143         self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
4144
4145     def test_compute_rate(self):
4146         self.failUnlessReallyEqual(common.compute_rate(None, None), None)
4147         self.failUnlessReallyEqual(common.compute_rate(None, 1), None)
4148         self.failUnlessReallyEqual(common.compute_rate(250000, None), None)
4149         self.failUnlessReallyEqual(common.compute_rate(250000, 0), None)
4150         self.failUnlessReallyEqual(common.compute_rate(250000, 10), 25000.0)
4151         self.failUnlessReallyEqual(common.compute_rate(0, 10), 0.0)
4152         self.shouldFail(AssertionError, "test_compute_rate", "",
4153                         common.compute_rate, -100, 10)
4154         self.shouldFail(AssertionError, "test_compute_rate", "",
4155                         common.compute_rate, 100, -10)
4156
4157         # Sanity check
4158         rate = common.compute_rate(10*1000*1000, 1)
4159         self.failUnlessReallyEqual(common.abbreviate_rate(rate), "10.00MBps")
4160
4161     def test_abbreviate_rate(self):
4162         self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
4163         self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
4164         self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
4165         self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
4166
4167     def test_abbreviate_size(self):
4168         self.failUnlessReallyEqual(common.abbreviate_size(None), "")
4169         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
4170         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
4171         self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
4172         self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
4173
4174     def test_plural(self):
4175         def convert(s):
4176             return "%d second%s" % (s, status.plural(s))
4177         self.failUnlessReallyEqual(convert(0), "0 seconds")
4178         self.failUnlessReallyEqual(convert(1), "1 second")
4179         self.failUnlessReallyEqual(convert(2), "2 seconds")
4180         def convert2(s):
4181             return "has share%s: %s" % (status.plural(s), ",".join(s))
4182         self.failUnlessReallyEqual(convert2([]), "has shares: ")
4183         self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
4184         self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
4185
4186
4187 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4188
4189     def CHECK(self, ign, which, args, clientnum=0):
4190         fileurl = self.fileurls[which]
4191         url = fileurl + "?" + args
4192         return self.GET(url, method="POST", clientnum=clientnum)
4193
4194     def test_filecheck(self):
4195         self.basedir = "web/Grid/filecheck"
4196         self.set_up_grid()
4197         c0 = self.g.clients[0]
4198         self.uris = {}
4199         DATA = "data" * 100
4200         d = c0.upload(upload.Data(DATA, convergence=""))
4201         def _stash_uri(ur, which):
4202             self.uris[which] = ur.get_uri()
4203         d.addCallback(_stash_uri, "good")
4204         d.addCallback(lambda ign:
4205                       c0.upload(upload.Data(DATA+"1", convergence="")))
4206         d.addCallback(_stash_uri, "sick")
4207         d.addCallback(lambda ign:
4208                       c0.upload(upload.Data(DATA+"2", convergence="")))
4209         d.addCallback(_stash_uri, "dead")
4210         def _stash_mutable_uri(n, which):
4211             self.uris[which] = n.get_uri()
4212             assert isinstance(self.uris[which], str)
4213         d.addCallback(lambda ign:
4214             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4215         d.addCallback(_stash_mutable_uri, "corrupt")
4216         d.addCallback(lambda ign:
4217                       c0.upload(upload.Data("literal", convergence="")))
4218         d.addCallback(_stash_uri, "small")
4219         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
4220         d.addCallback(_stash_mutable_uri, "smalldir")
4221
4222         def _compute_fileurls(ignored):
4223             self.fileurls = {}
4224             for which in self.uris:
4225                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4226         d.addCallback(_compute_fileurls)
4227
4228         def _clobber_shares(ignored):
4229             good_shares = self.find_uri_shares(self.uris["good"])
4230             self.failUnlessReallyEqual(len(good_shares), 10)
4231             sick_shares = self.find_uri_shares(self.uris["sick"])
4232             os.unlink(sick_shares[0][2])
4233             dead_shares = self.find_uri_shares(self.uris["dead"])
4234             for i in range(1, 10):
4235                 os.unlink(dead_shares[i][2])
4236             c_shares = self.find_uri_shares(self.uris["corrupt"])
4237             cso = CorruptShareOptions()
4238             cso.stdout = StringIO()
4239             cso.parseOptions([c_shares[0][2]])
4240             corrupt_share(cso)
4241         d.addCallback(_clobber_shares)
4242
4243         d.addCallback(self.CHECK, "good", "t=check")
4244         def _got_html_good(res):
4245             self.failUnlessIn("Healthy", res)
4246             self.failIfIn("Not Healthy", res)
4247             self.failUnlessIn(FAVICON_MARKUP, res)
4248         d.addCallback(_got_html_good)
4249         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
4250         def _got_html_good_return_to(res):
4251             self.failUnlessIn("Healthy", res)
4252             self.failIfIn("Not Healthy", res)
4253             self.failUnlessIn('<a href="somewhere">Return to file', res)
4254         d.addCallback(_got_html_good_return_to)
4255         d.addCallback(self.CHECK, "good", "t=check&output=json")
4256         def _got_json_good(res):
4257             r = simplejson.loads(res)
4258             self.failUnlessEqual(r["summary"], "Healthy")
4259             self.failUnless(r["results"]["healthy"])
4260             self.failIf(r["results"]["needs-rebalancing"])
4261             self.failUnless(r["results"]["recoverable"])
4262         d.addCallback(_got_json_good)
4263
4264         d.addCallback(self.CHECK, "small", "t=check")
4265         def _got_html_small(res):
4266             self.failUnlessIn("Literal files are always healthy", res)
4267             self.failIfIn("Not Healthy", res)
4268         d.addCallback(_got_html_small)
4269         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
4270         def _got_html_small_return_to(res):
4271             self.failUnlessIn("Literal files are always healthy", res)
4272             self.failIfIn("Not Healthy", res)
4273             self.failUnlessIn('<a href="somewhere">Return to file', res)
4274         d.addCallback(_got_html_small_return_to)
4275         d.addCallback(self.CHECK, "small", "t=check&output=json")
4276         def _got_json_small(res):
4277             r = simplejson.loads(res)
4278             self.failUnlessEqual(r["storage-index"], "")
4279             self.failUnless(r["results"]["healthy"])
4280         d.addCallback(_got_json_small)
4281
4282         d.addCallback(self.CHECK, "smalldir", "t=check")
4283         def _got_html_smalldir(res):
4284             self.failUnlessIn("Literal files are always healthy", res)
4285             self.failIfIn("Not Healthy", res)
4286         d.addCallback(_got_html_smalldir)
4287         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
4288         def _got_json_smalldir(res):
4289             r = simplejson.loads(res)
4290             self.failUnlessEqual(r["storage-index"], "")
4291             self.failUnless(r["results"]["healthy"])
4292         d.addCallback(_got_json_smalldir)
4293
4294         d.addCallback(self.CHECK, "sick", "t=check")
4295         def _got_html_sick(res):
4296             self.failUnlessIn("Not Healthy", res)
4297         d.addCallback(_got_html_sick)
4298         d.addCallback(self.CHECK, "sick", "t=check&output=json")
4299         def _got_json_sick(res):
4300             r = simplejson.loads(res)
4301             self.failUnlessEqual(r["summary"],
4302                                  "Not Healthy: 9 shares (enc 3-of-10)")
4303             self.failIf(r["results"]["healthy"])
4304             self.failIf(r["results"]["needs-rebalancing"])
4305             self.failUnless(r["results"]["recoverable"])
4306         d.addCallback(_got_json_sick)
4307
4308         d.addCallback(self.CHECK, "dead", "t=check")
4309         def _got_html_dead(res):
4310             self.failUnlessIn("Not Healthy", res)
4311         d.addCallback(_got_html_dead)
4312         d.addCallback(self.CHECK, "dead", "t=check&output=json")
4313         def _got_json_dead(res):
4314             r = simplejson.loads(res)
4315             self.failUnlessEqual(r["summary"],
4316                                  "Not Healthy: 1 shares (enc 3-of-10)")
4317             self.failIf(r["results"]["healthy"])
4318             self.failIf(r["results"]["needs-rebalancing"])
4319             self.failIf(r["results"]["recoverable"])
4320         d.addCallback(_got_json_dead)
4321
4322         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
4323         def _got_html_corrupt(res):
4324             self.failUnlessIn("Not Healthy! : Unhealthy", res)
4325         d.addCallback(_got_html_corrupt)
4326         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
4327         def _got_json_corrupt(res):
4328             r = simplejson.loads(res)
4329             self.failUnlessIn("Unhealthy: 9 shares (enc 3-of-10)", r["summary"])
4330             self.failIf(r["results"]["healthy"])
4331             self.failUnless(r["results"]["recoverable"])
4332             self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
4333             self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
4334         d.addCallback(_got_json_corrupt)
4335
4336         d.addErrback(self.explain_web_error)
4337         return d
4338
4339     def test_repair_html(self):
4340         self.basedir = "web/Grid/repair_html"
4341         self.set_up_grid()
4342         c0 = self.g.clients[0]
4343         self.uris = {}
4344         DATA = "data" * 100
4345         d = c0.upload(upload.Data(DATA, convergence=""))
4346         def _stash_uri(ur, which):
4347             self.uris[which] = ur.get_uri()
4348         d.addCallback(_stash_uri, "good")
4349         d.addCallback(lambda ign:
4350                       c0.upload(upload.Data(DATA+"1", convergence="")))
4351         d.addCallback(_stash_uri, "sick")
4352         d.addCallback(lambda ign:
4353                       c0.upload(upload.Data(DATA+"2", convergence="")))
4354         d.addCallback(_stash_uri, "dead")
4355         def _stash_mutable_uri(n, which):
4356             self.uris[which] = n.get_uri()
4357             assert isinstance(self.uris[which], str)
4358         d.addCallback(lambda ign:
4359             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4360         d.addCallback(_stash_mutable_uri, "corrupt")
4361
4362         def _compute_fileurls(ignored):
4363             self.fileurls = {}
4364             for which in self.uris:
4365                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4366         d.addCallback(_compute_fileurls)
4367
4368         def _clobber_shares(ignored):
4369             good_shares = self.find_uri_shares(self.uris["good"])
4370             self.failUnlessReallyEqual(len(good_shares), 10)
4371             sick_shares = self.find_uri_shares(self.uris["sick"])
4372             os.unlink(sick_shares[0][2])
4373             dead_shares = self.find_uri_shares(self.uris["dead"])
4374             for i in range(1, 10):
4375                 os.unlink(dead_shares[i][2])
4376             c_shares = self.find_uri_shares(self.uris["corrupt"])
4377             cso = CorruptShareOptions()
4378             cso.stdout = StringIO()
4379             cso.parseOptions([c_shares[0][2]])
4380             corrupt_share(cso)
4381         d.addCallback(_clobber_shares)
4382
4383         d.addCallback(self.CHECK, "good", "t=check&repair=true")
4384         def _got_html_good(res):
4385             self.failUnlessIn("Healthy", res)
4386             self.failIfIn("Not Healthy", res)
4387             self.failUnlessIn("No repair necessary", res)
4388             self.failUnlessIn(FAVICON_MARKUP, res)
4389         d.addCallback(_got_html_good)
4390
4391         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
4392         def _got_html_sick(res):
4393             self.failUnlessIn("Healthy : healthy", res)
4394             self.failIfIn("Not Healthy", res)
4395             self.failUnlessIn("Repair successful", res)
4396         d.addCallback(_got_html_sick)
4397
4398         # repair of a dead file will fail, of course, but it isn't yet
4399         # clear how this should be reported. Right now it shows up as
4400         # a "410 Gone".
4401         #
4402         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
4403         #def _got_html_dead(res):
4404         #    print res
4405         #    self.failUnlessIn("Healthy : healthy", res)
4406         #    self.failIfIn("Not Healthy", res)
4407         #    self.failUnlessIn("No repair necessary", res)
4408         #d.addCallback(_got_html_dead)
4409
4410         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
4411         def _got_html_corrupt(res):
4412             self.failUnlessIn("Healthy : Healthy", res)
4413             self.failIfIn("Not Healthy", res)
4414             self.failUnlessIn("Repair successful", res)
4415         d.addCallback(_got_html_corrupt)
4416
4417         d.addErrback(self.explain_web_error)
4418         return d
4419
4420     def test_repair_json(self):
4421         self.basedir = "web/Grid/repair_json"
4422         self.set_up_grid()
4423         c0 = self.g.clients[0]
4424         self.uris = {}
4425         DATA = "data" * 100
4426         d = c0.upload(upload.Data(DATA+"1", convergence=""))
4427         def _stash_uri(ur, which):
4428             self.uris[which] = ur.get_uri()
4429         d.addCallback(_stash_uri, "sick")
4430
4431         def _compute_fileurls(ignored):
4432             self.fileurls = {}
4433             for which in self.uris:
4434                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4435         d.addCallback(_compute_fileurls)
4436
4437         def _clobber_shares(ignored):
4438             sick_shares = self.find_uri_shares(self.uris["sick"])
4439             os.unlink(sick_shares[0][2])
4440         d.addCallback(_clobber_shares)
4441
4442         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
4443         def _got_json_sick(res):
4444             r = simplejson.loads(res)
4445             self.failUnlessReallyEqual(r["repair-attempted"], True)
4446             self.failUnlessReallyEqual(r["repair-successful"], True)
4447             self.failUnlessEqual(r["pre-repair-results"]["summary"],
4448                                  "Not Healthy: 9 shares (enc 3-of-10)")
4449             self.failIf(r["pre-repair-results"]["results"]["healthy"])
4450             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
4451             self.failUnless(r["post-repair-results"]["results"]["healthy"])
4452         d.addCallback(_got_json_sick)
4453
4454         d.addErrback(self.explain_web_error)
4455         return d
4456
4457     def test_unknown(self, immutable=False):
4458         self.basedir = "web/Grid/unknown"
4459         if immutable:
4460             self.basedir = "web/Grid/unknown-immutable"
4461
4462         self.set_up_grid()
4463         c0 = self.g.clients[0]
4464         self.uris = {}
4465         self.fileurls = {}
4466
4467         # the future cap format may contain slashes, which must be tolerated
4468         expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
4469                                                            safe="")
4470
4471         if immutable:
4472             name = u"future-imm"
4473             future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
4474             d = c0.create_immutable_dirnode({name: (future_node, {})})
4475         else:
4476             name = u"future"
4477             future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4478             d = c0.create_dirnode()
4479
4480         def _stash_root_and_create_file(n):
4481             self.rootnode = n
4482             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
4483             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
4484             if not immutable:
4485                 return self.rootnode.set_node(name, future_node)
4486         d.addCallback(_stash_root_and_create_file)
4487
4488         # make sure directory listing tolerates unknown nodes
4489         d.addCallback(lambda ign: self.GET(self.rooturl))
4490         def _check_directory_html(res, expected_type_suffix):
4491             pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
4492                                   '<td>%s</td>' % (expected_type_suffix, str(name)),
4493                                  re.DOTALL)
4494             self.failUnless(re.search(pattern, res), res)
4495             # find the More Info link for name, should be relative
4496             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4497             info_url = mo.group(1)
4498             self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
4499         if immutable:
4500             d.addCallback(_check_directory_html, "-IMM")
4501         else:
4502             d.addCallback(_check_directory_html, "")
4503
4504         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4505         def _check_directory_json(res, expect_rw_uri):
4506             data = simplejson.loads(res)
4507             self.failUnlessEqual(data[0], "dirnode")
4508             f = data[1]["children"][name]
4509             self.failUnlessEqual(f[0], "unknown")
4510             if expect_rw_uri:
4511                 self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data)
4512             else:
4513                 self.failIfIn("rw_uri", f[1])
4514             if immutable:
4515                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
4516             else:
4517                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
4518             self.failUnlessIn("metadata", f[1])
4519         d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
4520
4521         def _check_info(res, expect_rw_uri, expect_ro_uri):
4522             self.failUnlessIn("Object Type: <span>unknown</span>", res)
4523             if expect_rw_uri:
4524                 self.failUnlessIn(unknown_rwcap, res)
4525             if expect_ro_uri:
4526                 if immutable:
4527                     self.failUnlessIn(unknown_immcap, res)
4528                 else:
4529                     self.failUnlessIn(unknown_rocap, res)
4530             else:
4531                 self.failIfIn(unknown_rocap, res)
4532             self.failIfIn("Raw data as", res)
4533             self.failIfIn("Directory writecap", res)
4534             self.failIfIn("Checker Operations", res)
4535             self.failIfIn("Mutable File Operations", res)
4536             self.failIfIn("Directory Operations", res)
4537
4538         # FIXME: these should have expect_rw_uri=not immutable; I don't know
4539         # why they fail. Possibly related to ticket #922.
4540
4541         d.addCallback(lambda ign: self.GET(expected_info_url))
4542         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
4543         d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
4544         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
4545
4546         def _check_json(res, expect_rw_uri):
4547             data = simplejson.loads(res)
4548             self.failUnlessEqual(data[0], "unknown")
4549             if expect_rw_uri:
4550                 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data)
4551             else:
4552                 self.failIfIn("rw_uri", data[1])
4553
4554             if immutable:
4555                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data)
4556                 self.failUnlessReallyEqual(data[1]["mutable"], False)
4557             elif expect_rw_uri:
4558                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4559                 self.failUnlessReallyEqual(data[1]["mutable"], True)
4560             else:
4561                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4562                 self.failIfIn("mutable", data[1])
4563
4564             # TODO: check metadata contents
4565             self.failUnlessIn("metadata", data[1])
4566
4567         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
4568         d.addCallback(_check_json, expect_rw_uri=not immutable)
4569
4570         # and make sure that a read-only version of the directory can be
4571         # rendered too. This version will not have unknown_rwcap, whether
4572         # or not future_node was immutable.
4573         d.addCallback(lambda ign: self.GET(self.rourl))
4574         if immutable:
4575             d.addCallback(_check_directory_html, "-IMM")
4576         else:
4577             d.addCallback(_check_directory_html, "-RO")
4578
4579         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
4580         d.addCallback(_check_directory_json, expect_rw_uri=False)
4581
4582         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
4583         d.addCallback(_check_json, expect_rw_uri=False)
4584
4585         # TODO: check that getting t=info from the Info link in the ro directory
4586         # works, and does not include the writecap URI.
4587         return d
4588
4589     def test_immutable_unknown(self):
4590         return self.test_unknown(immutable=True)
4591
4592     def test_mutant_dirnodes_are_omitted(self):
4593         self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
4594
4595         self.set_up_grid()
4596         c = self.g.clients[0]
4597         nm = c.nodemaker
4598         self.uris = {}
4599         self.fileurls = {}
4600
4601         lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
4602         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
4603         mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
4604
4605         # This method tests mainly dirnode, but we'd have to duplicate code in order to
4606         # test the dirnode and web layers separately.
4607
4608         # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
4609         # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
4610         # When the directory is read, the mutants should be silently disposed of, leaving
4611         # their lonely sibling.
4612         # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
4613         # because immutable directories don't have a writecap and therefore that field
4614         # isn't (and can't be) decrypted.
4615         # TODO: The field still exists in the netstring. Technically we should check what
4616         # happens if something is put there (_unpack_contents should raise ValueError),
4617         # but that can wait.
4618
4619         lonely_child = nm.create_from_cap(lonely_uri)
4620         mutant_ro_child = nm.create_from_cap(mut_read_uri)
4621         mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
4622
4623         def _by_hook_or_by_crook():
4624             return True
4625         for n in [mutant_ro_child, mutant_write_in_ro_child]:
4626             n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
4627
4628         mutant_write_in_ro_child.get_write_uri    = lambda: None
4629         mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
4630
4631         kids = {u"lonely":      (lonely_child, {}),
4632                 u"ro":          (mutant_ro_child, {}),
4633                 u"write-in-ro": (mutant_write_in_ro_child, {}),
4634                 }
4635         d = c.create_immutable_dirnode(kids)
4636
4637         def _created(dn):
4638             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
4639             self.failIf(dn.is_mutable())
4640             self.failUnless(dn.is_readonly())
4641             # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
4642             self.failIf(hasattr(dn._node, 'get_writekey'))
4643             rep = str(dn)
4644             self.failUnlessIn("RO-IMM", rep)
4645             cap = dn.get_cap()
4646             self.failUnlessIn("CHK", cap.to_string())
4647             self.cap = cap
4648             self.rootnode = dn
4649             self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
4650             return download_to_data(dn._node)
4651         d.addCallback(_created)
4652
4653         def _check_data(data):
4654             # Decode the netstring representation of the directory to check that all children
4655             # are present. This is a bit of an abstraction violation, but there's not really
4656             # any other way to do it given that the real DirectoryNode._unpack_contents would
4657             # strip the mutant children out (which is what we're trying to test, later).
4658             position = 0
4659             numkids = 0
4660             while position < len(data):
4661                 entries, position = split_netstring(data, 1, position)
4662                 entry = entries[0]
4663                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
4664                 name = name_utf8.decode("utf-8")
4665                 self.failUnlessEqual(rwcapdata, "")
4666                 self.failUnlessIn(name, kids)
4667                 (expected_child, ign) = kids[name]
4668                 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
4669                 numkids += 1
4670
4671             self.failUnlessReallyEqual(numkids, 3)
4672             return self.rootnode.list()
4673         d.addCallback(_check_data)
4674
4675         # Now when we use the real directory listing code, the mutants should be absent.
4676         def _check_kids(children):
4677             self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
4678             lonely_node, lonely_metadata = children[u"lonely"]
4679
4680             self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
4681             self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
4682         d.addCallback(_check_kids)
4683
4684         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
4685         d.addCallback(lambda n: n.list())
4686         d.addCallback(_check_kids)  # again with dirnode recreated from cap
4687
4688         # Make sure the lonely child can be listed in HTML...
4689         d.addCallback(lambda ign: self.GET(self.rooturl))
4690         def _check_html(res):
4691             self.failIfIn("URI:SSK", res)
4692             get_lonely = "".join([r'<td>FILE</td>',
4693                                   r'\s+<td>',
4694                                   r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
4695                                   r'</td>',
4696                                   r'\s+<td align="right">%d</td>' % len("one"),
4697                                   ])
4698             self.failUnless(re.search(get_lonely, res), res)
4699
4700             # find the More Info link for name, should be relative
4701             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4702             info_url = mo.group(1)
4703             self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
4704         d.addCallback(_check_html)
4705
4706         # ... and in JSON.
4707         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4708         def _check_json(res):
4709             data = simplejson.loads(res)
4710             self.failUnlessEqual(data[0], "dirnode")
4711             listed_children = data[1]["children"]
4712             self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
4713             ll_type, ll_data = listed_children[u"lonely"]
4714             self.failUnlessEqual(ll_type, "filenode")
4715             self.failIfIn("rw_uri", ll_data)
4716             self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
4717         d.addCallback(_check_json)
4718         return d
4719
4720     def test_deep_check(self):
4721         self.basedir = "web/Grid/deep_check"
4722         self.set_up_grid()
4723         c0 = self.g.clients[0]
4724         self.uris = {}
4725         self.fileurls = {}
4726         DATA = "data" * 100
4727         d = c0.create_dirnode()
4728         def _stash_root_and_create_file(n):
4729             self.rootnode = n
4730             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4731             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4732         d.addCallback(_stash_root_and_create_file)
4733         def _stash_uri(fn, which):
4734             self.uris[which] = fn.get_uri()
4735             return fn
4736         d.addCallback(_stash_uri, "good")
4737         d.addCallback(lambda ign:
4738                       self.rootnode.add_file(u"small",
4739                                              upload.Data("literal",
4740                                                         convergence="")))
4741         d.addCallback(_stash_uri, "small")
4742         d.addCallback(lambda ign:
4743                       self.rootnode.add_file(u"sick",
4744                                              upload.Data(DATA+"1",
4745                                                         convergence="")))
4746         d.addCallback(_stash_uri, "sick")
4747
4748         # this tests that deep-check and stream-manifest will ignore
4749         # UnknownNode instances. Hopefully this will also cover deep-stats.
4750         future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4751         d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
4752
4753         def _clobber_shares(ignored):
4754             self.delete_shares_numbered(self.uris["sick"], [0,1])
4755         d.addCallback(_clobber_shares)
4756
4757         # root
4758         # root/good
4759         # root/small
4760         # root/sick
4761         # root/future
4762
4763         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4764         def _done(res):
4765             try:
4766                 units = [simplejson.loads(line)
4767                          for line in res.splitlines()
4768                          if line]
4769             except ValueError:
4770                 print "response is:", res
4771                 print "undecodeable line was '%s'" % line
4772                 raise
4773             self.failUnlessReallyEqual(len(units), 5+1)
4774             # should be parent-first
4775             u0 = units[0]
4776             self.failUnlessEqual(u0["path"], [])
4777             self.failUnlessEqual(u0["type"], "directory")
4778             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4779             u0cr = u0["check-results"]
4780             self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
4781
4782             ugood = [u for u in units
4783                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4784             self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
4785             ugoodcr = ugood["check-results"]
4786             self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
4787
4788             stats = units[-1]
4789             self.failUnlessEqual(stats["type"], "stats")
4790             s = stats["stats"]
4791             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4792             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4793             self.failUnlessReallyEqual(s["count-directories"], 1)
4794             self.failUnlessReallyEqual(s["count-unknown"], 1)
4795         d.addCallback(_done)
4796
4797         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4798         def _check_manifest(res):
4799             self.failUnless(res.endswith("\n"))
4800             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
4801             self.failUnlessReallyEqual(len(units), 5+1)
4802             self.failUnlessEqual(units[-1]["type"], "stats")
4803             first = units[0]
4804             self.failUnlessEqual(first["path"], [])
4805             self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri())
4806             self.failUnlessEqual(first["type"], "directory")
4807             stats = units[-1]["stats"]
4808             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4809             self.failUnlessReallyEqual(stats["count-literal-files"], 1)
4810             self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
4811             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4812             self.failUnlessReallyEqual(stats["count-unknown"], 1)
4813         d.addCallback(_check_manifest)
4814
4815         # now add root/subdir and root/subdir/grandchild, then make subdir
4816         # unrecoverable, then see what happens
4817
4818         d.addCallback(lambda ign:
4819                       self.rootnode.create_subdirectory(u"subdir"))
4820         d.addCallback(_stash_uri, "subdir")
4821         d.addCallback(lambda subdir_node:
4822                       subdir_node.add_file(u"grandchild",
4823                                            upload.Data(DATA+"2",
4824                                                        convergence="")))
4825         d.addCallback(_stash_uri, "grandchild")
4826
4827         d.addCallback(lambda ign:
4828                       self.delete_shares_numbered(self.uris["subdir"],
4829                                                   range(1, 10)))
4830
4831         # root
4832         # root/good
4833         # root/small
4834         # root/sick
4835         # root/future
4836         # root/subdir [unrecoverable]
4837         # root/subdir/grandchild
4838
4839         # how should a streaming-JSON API indicate fatal error?
4840         # answer: emit ERROR: instead of a JSON string
4841
4842         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4843         def _check_broken_manifest(res):
4844             lines = res.splitlines()
4845             error_lines = [i
4846                            for (i,line) in enumerate(lines)
4847                            if line.startswith("ERROR:")]
4848             if not error_lines:
4849                 self.fail("no ERROR: in output: %s" % (res,))
4850             first_error = error_lines[0]
4851             error_line = lines[first_error]
4852             error_msg = lines[first_error+1:]
4853             error_msg_s = "\n".join(error_msg) + "\n"
4854             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4855                               error_line)
4856             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4857             units = [simplejson.loads(line) for line in lines[:first_error]]
4858             self.failUnlessReallyEqual(len(units), 6) # includes subdir
4859             last_unit = units[-1]
4860             self.failUnlessEqual(last_unit["path"], ["subdir"])
4861         d.addCallback(_check_broken_manifest)
4862
4863         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4864         def _check_broken_deepcheck(res):
4865             lines = res.splitlines()
4866             error_lines = [i
4867                            for (i,line) in enumerate(lines)
4868                            if line.startswith("ERROR:")]
4869             if not error_lines:
4870                 self.fail("no ERROR: in output: %s" % (res,))
4871             first_error = error_lines[0]
4872             error_line = lines[first_error]
4873             error_msg = lines[first_error+1:]
4874             error_msg_s = "\n".join(error_msg) + "\n"
4875             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4876                               error_line)
4877             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4878             units = [simplejson.loads(line) for line in lines[:first_error]]
4879             self.failUnlessReallyEqual(len(units), 6) # includes subdir
4880             last_unit = units[-1]
4881             self.failUnlessEqual(last_unit["path"], ["subdir"])
4882             r = last_unit["check-results"]["results"]
4883             self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
4884             self.failUnlessReallyEqual(r["count-shares-good"], 1)
4885             self.failUnlessReallyEqual(r["recoverable"], False)
4886         d.addCallback(_check_broken_deepcheck)
4887
4888         d.addErrback(self.explain_web_error)
4889         return d
4890
4891     def test_deep_check_and_repair(self):
4892         self.basedir = "web/Grid/deep_check_and_repair"
4893         self.set_up_grid()
4894         c0 = self.g.clients[0]
4895         self.uris = {}
4896         self.fileurls = {}
4897         DATA = "data" * 100
4898         d = c0.create_dirnode()
4899         def _stash_root_and_create_file(n):
4900             self.rootnode = n
4901             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4902             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4903         d.addCallback(_stash_root_and_create_file)
4904         def _stash_uri(fn, which):
4905             self.uris[which] = fn.get_uri()
4906         d.addCallback(_stash_uri, "good")
4907         d.addCallback(lambda ign:
4908                       self.rootnode.add_file(u"small",
4909                                              upload.Data("literal",
4910                                                         convergence="")))
4911         d.addCallback(_stash_uri, "small")
4912         d.addCallback(lambda ign:
4913                       self.rootnode.add_file(u"sick",
4914                                              upload.Data(DATA+"1",
4915                                                         convergence="")))
4916         d.addCallback(_stash_uri, "sick")
4917         #d.addCallback(lambda ign:
4918         #              self.rootnode.add_file(u"dead",
4919         #                                     upload.Data(DATA+"2",
4920         #                                                convergence="")))
4921         #d.addCallback(_stash_uri, "dead")
4922
4923         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
4924         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
4925         #d.addCallback(_stash_uri, "corrupt")
4926
4927         def _clobber_shares(ignored):
4928             good_shares = self.find_uri_shares(self.uris["good"])
4929             self.failUnlessReallyEqual(len(good_shares), 10)
4930             sick_shares = self.find_uri_shares(self.uris["sick"])
4931             os.unlink(sick_shares[0][2])
4932             #dead_shares = self.find_uri_shares(self.uris["dead"])
4933             #for i in range(1, 10):
4934             #    os.unlink(dead_shares[i][2])
4935
4936             #c_shares = self.find_uri_shares(self.uris["corrupt"])
4937             #cso = CorruptShareOptions()
4938             #cso.stdout = StringIO()
4939             #cso.parseOptions([c_shares[0][2]])
4940             #corrupt_share(cso)
4941         d.addCallback(_clobber_shares)
4942
4943         # root
4944         # root/good   CHK, 10 shares
4945         # root/small  LIT
4946         # root/sick   CHK, 9 shares
4947
4948         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
4949         def _done(res):
4950             units = [simplejson.loads(line)
4951                      for line in res.splitlines()
4952                      if line]
4953             self.failUnlessReallyEqual(len(units), 4+1)
4954             # should be parent-first
4955             u0 = units[0]
4956             self.failUnlessEqual(u0["path"], [])
4957             self.failUnlessEqual(u0["type"], "directory")
4958             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4959             u0crr = u0["check-and-repair-results"]
4960             self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
4961             self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
4962
4963             ugood = [u for u in units
4964                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4965             self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
4966             ugoodcrr = ugood["check-and-repair-results"]
4967             self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
4968             self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
4969
4970             usick = [u for u in units
4971                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
4972             self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"])
4973             usickcrr = usick["check-and-repair-results"]
4974             self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
4975             self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
4976             self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
4977             self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
4978
4979             stats = units[-1]
4980             self.failUnlessEqual(stats["type"], "stats")
4981             s = stats["stats"]
4982             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4983             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4984             self.failUnlessReallyEqual(s["count-directories"], 1)
4985         d.addCallback(_done)
4986
4987         d.addErrback(self.explain_web_error)
4988         return d
4989
4990     def _count_leases(self, ignored, which):
4991         u = self.uris[which]
4992         shares = self.find_uri_shares(u)
4993         lease_counts = []
4994         for shnum, serverid, fn in shares:
4995             sf = get_share_file(fn)
4996             num_leases = len(list(sf.get_leases()))
4997             lease_counts.append( (fn, num_leases) )
4998         return lease_counts
4999
5000     def _assert_leasecount(self, lease_counts, expected):
5001         for (fn, num_leases) in lease_counts:
5002             if num_leases != expected:
5003                 self.fail("expected %d leases, have %d, on %s" %
5004                           (expected, num_leases, fn))
5005
5006     def test_add_lease(self):
5007         self.basedir = "web/Grid/add_lease"
5008         self.set_up_grid(num_clients=2)
5009         c0 = self.g.clients[0]
5010         self.uris = {}
5011         DATA = "data" * 100
5012         d = c0.upload(upload.Data(DATA, convergence=""))
5013         def _stash_uri(ur, which):
5014             self.uris[which] = ur.get_uri()
5015         d.addCallback(_stash_uri, "one")
5016         d.addCallback(lambda ign:
5017                       c0.upload(upload.Data(DATA+"1", convergence="")))
5018         d.addCallback(_stash_uri, "two")
5019         def _stash_mutable_uri(n, which):
5020             self.uris[which] = n.get_uri()
5021             assert isinstance(self.uris[which], str)
5022         d.addCallback(lambda ign:
5023             c0.create_mutable_file(publish.MutableData(DATA+"2")))
5024         d.addCallback(_stash_mutable_uri, "mutable")
5025
5026         def _compute_fileurls(ignored):
5027             self.fileurls = {}
5028             for which in self.uris:
5029                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
5030         d.addCallback(_compute_fileurls)
5031
5032         d.addCallback(self._count_leases, "one")
5033         d.addCallback(self._assert_leasecount, 1)
5034         d.addCallback(self._count_leases, "two")
5035         d.addCallback(self._assert_leasecount, 1)
5036         d.addCallback(self._count_leases, "mutable")
5037         d.addCallback(self._assert_leasecount, 1)
5038
5039         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
5040         def _got_html_good(res):
5041             self.failUnlessIn("Healthy", res)
5042             self.failIfIn("Not Healthy", res)
5043         d.addCallback(_got_html_good)
5044
5045         d.addCallback(self._count_leases, "one")
5046         d.addCallback(self._assert_leasecount, 1)
5047         d.addCallback(self._count_leases, "two")
5048         d.addCallback(self._assert_leasecount, 1)
5049         d.addCallback(self._count_leases, "mutable")
5050         d.addCallback(self._assert_leasecount, 1)
5051
5052         # this CHECK uses the original client, which uses the same
5053         # lease-secrets, so it will just renew the original lease
5054         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
5055         d.addCallback(_got_html_good)
5056
5057         d.addCallback(self._count_leases, "one")
5058         d.addCallback(self._assert_leasecount, 1)
5059         d.addCallback(self._count_leases, "two")
5060         d.addCallback(self._assert_leasecount, 1)
5061         d.addCallback(self._count_leases, "mutable")
5062         d.addCallback(self._assert_leasecount, 1)
5063
5064         # this CHECK uses an alternate client, which adds a second lease
5065         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
5066         d.addCallback(_got_html_good)
5067
5068         d.addCallback(self._count_leases, "one")
5069         d.addCallback(self._assert_leasecount, 2)
5070         d.addCallback(self._count_leases, "two")
5071         d.addCallback(self._assert_leasecount, 1)
5072         d.addCallback(self._count_leases, "mutable")
5073         d.addCallback(self._assert_leasecount, 1)
5074
5075         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
5076         d.addCallback(_got_html_good)
5077
5078         d.addCallback(self._count_leases, "one")
5079         d.addCallback(self._assert_leasecount, 2)
5080         d.addCallback(self._count_leases, "two")
5081         d.addCallback(self._assert_leasecount, 1)
5082         d.addCallback(self._count_leases, "mutable")
5083         d.addCallback(self._assert_leasecount, 1)
5084
5085         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
5086                       clientnum=1)
5087         d.addCallback(_got_html_good)
5088
5089         d.addCallback(self._count_leases, "one")
5090         d.addCallback(self._assert_leasecount, 2)
5091         d.addCallback(self._count_leases, "two")
5092         d.addCallback(self._assert_leasecount, 1)
5093         d.addCallback(self._count_leases, "mutable")
5094         d.addCallback(self._assert_leasecount, 2)
5095
5096         d.addErrback(self.explain_web_error)
5097         return d
5098
5099     def test_deep_add_lease(self):
5100         self.basedir = "web/Grid/deep_add_lease"
5101         self.set_up_grid(num_clients=2)
5102         c0 = self.g.clients[0]
5103         self.uris = {}
5104         self.fileurls = {}
5105         DATA = "data" * 100
5106         d = c0.create_dirnode()
5107         def _stash_root_and_create_file(n):
5108             self.rootnode = n
5109             self.uris["root"] = n.get_uri()
5110             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5111             return n.add_file(u"one", upload.Data(DATA, convergence=""))
5112         d.addCallback(_stash_root_and_create_file)
5113         def _stash_uri(fn, which):
5114             self.uris[which] = fn.get_uri()
5115         d.addCallback(_stash_uri, "one")
5116         d.addCallback(lambda ign:
5117                       self.rootnode.add_file(u"small",
5118                                              upload.Data("literal",
5119                                                         convergence="")))
5120         d.addCallback(_stash_uri, "small")
5121
5122         d.addCallback(lambda ign:
5123             c0.create_mutable_file(publish.MutableData("mutable")))
5124         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
5125         d.addCallback(_stash_uri, "mutable")
5126
5127         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
5128         def _done(res):
5129             units = [simplejson.loads(line)
5130                      for line in res.splitlines()
5131                      if line]
5132             # root, one, small, mutable,   stats
5133             self.failUnlessReallyEqual(len(units), 4+1)
5134         d.addCallback(_done)
5135
5136         d.addCallback(self._count_leases, "root")
5137         d.addCallback(self._assert_leasecount, 1)
5138         d.addCallback(self._count_leases, "one")
5139         d.addCallback(self._assert_leasecount, 1)
5140         d.addCallback(self._count_leases, "mutable")
5141         d.addCallback(self._assert_leasecount, 1)
5142
5143         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
5144         d.addCallback(_done)
5145
5146         d.addCallback(self._count_leases, "root")
5147         d.addCallback(self._assert_leasecount, 1)
5148         d.addCallback(self._count_leases, "one")
5149         d.addCallback(self._assert_leasecount, 1)
5150         d.addCallback(self._count_leases, "mutable")
5151         d.addCallback(self._assert_leasecount, 1)
5152
5153         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
5154                       clientnum=1)
5155         d.addCallback(_done)
5156
5157         d.addCallback(self._count_leases, "root")
5158         d.addCallback(self._assert_leasecount, 2)
5159         d.addCallback(self._count_leases, "one")
5160         d.addCallback(self._assert_leasecount, 2)
5161         d.addCallback(self._count_leases, "mutable")
5162         d.addCallback(self._assert_leasecount, 2)
5163
5164         d.addErrback(self.explain_web_error)
5165         return d
5166
5167
5168     def test_exceptions(self):
5169         self.basedir = "web/Grid/exceptions"
5170         self.set_up_grid(num_clients=1, num_servers=2)
5171         c0 = self.g.clients[0]
5172         c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 2
5173         self.fileurls = {}
5174         DATA = "data" * 100
5175         d = c0.create_dirnode()
5176         def _stash_root(n):
5177             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5178             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
5179             return n
5180         d.addCallback(_stash_root)
5181         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
5182         def _stash_bad(ur):
5183             self.fileurls["1share"] = "uri/" + urllib.quote(ur.get_uri())
5184             self.delete_shares_numbered(ur.get_uri(), range(1,10))
5185
5186             u = uri.from_string(ur.get_uri())
5187             u.key = testutil.flip_bit(u.key, 0)
5188             baduri = u.to_string()
5189             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
5190         d.addCallback(_stash_bad)
5191         d.addCallback(lambda ign: c0.create_dirnode())
5192         def _mangle_dirnode_1share(n):
5193             u = n.get_uri()
5194             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
5195             self.fileurls["dir-1share-json"] = url + "?t=json"
5196             self.delete_shares_numbered(u, range(1,10))
5197         d.addCallback(_mangle_dirnode_1share)
5198         d.addCallback(lambda ign: c0.create_dirnode())
5199         def _mangle_dirnode_0share(n):
5200             u = n.get_uri()
5201             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
5202             self.fileurls["dir-0share-json"] = url + "?t=json"
5203             self.delete_shares_numbered(u, range(0,10))
5204         d.addCallback(_mangle_dirnode_0share)
5205
5206         # NotEnoughSharesError should be reported sensibly, with a
5207         # text/plain explanation of the problem, and perhaps some
5208         # information on which shares *could* be found.
5209
5210         d.addCallback(lambda ignored:
5211                       self.shouldHTTPError("GET unrecoverable",
5212                                            410, "Gone", "NoSharesError",
5213                                            self.GET, self.fileurls["0shares"]))
5214         def _check_zero_shares(body):
5215             self.failIfIn("<html>", body)
5216             body = " ".join(body.strip().split())
5217             exp = ("NoSharesError: no shares could be found. "
5218                    "Zero shares usually indicates a corrupt URI, or that "
5219                    "no servers were connected, but it might also indicate "
5220                    "severe corruption. You should perform a filecheck on "
5221                    "this object to learn more. The full error message is: "
5222                    "no shares (need 3). Last failure: None")
5223             self.failUnlessReallyEqual(exp, body)
5224         d.addCallback(_check_zero_shares)
5225
5226
5227         d.addCallback(lambda ignored:
5228                       self.shouldHTTPError("GET 1share",
5229                                            410, "Gone", "NotEnoughSharesError",
5230                                            self.GET, self.fileurls["1share"]))
5231         def _check_one_share(body):
5232             self.failIfIn("<html>", body)
5233             body = " ".join(body.strip().split())
5234             msgbase = ("NotEnoughSharesError: This indicates that some "
5235                        "servers were unavailable, or that shares have been "
5236                        "lost to server departure, hard drive failure, or disk "
5237                        "corruption. You should perform a filecheck on "
5238                        "this object to learn more. The full error message is:"
5239                        )
5240             msg1 = msgbase + (" ran out of shares:"
5241                               " complete=sh0"
5242                               " pending="
5243                               " overdue= unused= need 3. Last failure: None")
5244             msg2 = msgbase + (" ran out of shares:"
5245                               " complete="
5246                               " pending=Share(sh0-on-xgru5)"
5247                               " overdue= unused= need 3. Last failure: None")
5248             self.failUnless(body == msg1 or body == msg2, body)
5249         d.addCallback(_check_one_share)
5250
5251         d.addCallback(lambda ignored:
5252                       self.shouldHTTPError("GET imaginary",
5253                                            404, "Not Found", None,
5254                                            self.GET, self.fileurls["imaginary"]))
5255         def _missing_child(body):
5256             self.failUnlessIn("No such child: imaginary", body)
5257         d.addCallback(_missing_child)
5258
5259         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
5260         def _check_0shares_dir_html(body):
5261             self.failUnlessIn("<html>", body)
5262             # we should see the regular page, but without the child table or
5263             # the dirops forms
5264             body = " ".join(body.strip().split())
5265             self.failUnlessIn('href="?t=info">More info on this directory',
5266                               body)
5267             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5268                    "could not be retrieved, because there were insufficient "
5269                    "good shares. This might indicate that no servers were "
5270                    "connected, insufficient servers were connected, the URI "
5271                    "was corrupt, or that shares have been lost due to server "
5272                    "departure, hard drive failure, or disk corruption. You "
5273                    "should perform a filecheck on this object to learn more.")
5274             self.failUnlessIn(exp, body)
5275             self.failUnlessIn("No upload forms: directory is unreadable", body)
5276         d.addCallback(_check_0shares_dir_html)
5277
5278         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
5279         def _check_1shares_dir_html(body):
5280             # at some point, we'll split UnrecoverableFileError into 0-shares
5281             # and some-shares like we did for immutable files (since there
5282             # are different sorts of advice to offer in each case). For now,
5283             # they present the same way.
5284             self.failUnlessIn("<html>", body)
5285             body = " ".join(body.strip().split())
5286             self.failUnlessIn('href="?t=info">More info on this directory',
5287                               body)
5288             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5289                    "could not be retrieved, because there were insufficient "
5290                    "good shares. This might indicate that no servers were "
5291                    "connected, insufficient servers were connected, the URI "
5292                    "was corrupt, or that shares have been lost due to server "
5293                    "departure, hard drive failure, or disk corruption. You "
5294                    "should perform a filecheck on this object to learn more.")
5295             self.failUnlessIn(exp, body)
5296             self.failUnlessIn("No upload forms: directory is unreadable", body)
5297         d.addCallback(_check_1shares_dir_html)
5298
5299         d.addCallback(lambda ignored:
5300                       self.shouldHTTPError("GET dir-0share-json",
5301                                            410, "Gone", "UnrecoverableFileError",
5302                                            self.GET,
5303                                            self.fileurls["dir-0share-json"]))
5304         def _check_unrecoverable_file(body):
5305             self.failIfIn("<html>", body)
5306             body = " ".join(body.strip().split())
5307             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5308                    "could not be retrieved, because there were insufficient "
5309                    "good shares. This might indicate that no servers were "
5310                    "connected, insufficient servers were connected, the URI "
5311                    "was corrupt, or that shares have been lost due to server "
5312                    "departure, hard drive failure, or disk corruption. You "
5313                    "should perform a filecheck on this object to learn more.")
5314             self.failUnlessReallyEqual(exp, body)
5315         d.addCallback(_check_unrecoverable_file)
5316
5317         d.addCallback(lambda ignored:
5318                       self.shouldHTTPError("GET dir-1share-json",
5319                                            410, "Gone", "UnrecoverableFileError",
5320                                            self.GET,
5321                                            self.fileurls["dir-1share-json"]))
5322         d.addCallback(_check_unrecoverable_file)
5323
5324         d.addCallback(lambda ignored:
5325                       self.shouldHTTPError("GET imaginary",
5326                                            404, "Not Found", None,
5327                                            self.GET, self.fileurls["imaginary"]))
5328
5329         # attach a webapi child that throws a random error, to test how it
5330         # gets rendered.
5331         w = c0.getServiceNamed("webish")
5332         w.root.putChild("ERRORBOOM", ErrorBoom())
5333
5334         # "Accept: */*" :        should get a text/html stack trace
5335         # "Accept: text/plain" : should get a text/plain stack trace
5336         # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
5337         # no Accept header:      should get a text/html stack trace
5338
5339         d.addCallback(lambda ignored:
5340                       self.shouldHTTPError("GET errorboom_html",
5341                                            500, "Internal Server Error", None,
5342                                            self.GET, "ERRORBOOM",
5343                                            headers={"accept": "*/*"}))
5344         def _internal_error_html1(body):
5345             self.failUnlessIn("<html>", "expected HTML, not '%s'" % body)
5346         d.addCallback(_internal_error_html1)
5347
5348         d.addCallback(lambda ignored:
5349                       self.shouldHTTPError("GET errorboom_text",
5350                                            500, "Internal Server Error", None,
5351                                            self.GET, "ERRORBOOM",
5352                                            headers={"accept": "text/plain"}))
5353         def _internal_error_text2(body):
5354             self.failIfIn("<html>", body)
5355             self.failUnless(body.startswith("Traceback "), body)
5356         d.addCallback(_internal_error_text2)
5357
5358         CLI_accepts = "text/plain, application/octet-stream"
5359         d.addCallback(lambda ignored:
5360                       self.shouldHTTPError("GET errorboom_text",
5361                                            500, "Internal Server Error", None,
5362                                            self.GET, "ERRORBOOM",
5363                                            headers={"accept": CLI_accepts}))
5364         def _internal_error_text3(body):
5365             self.failIfIn("<html>", body)
5366             self.failUnless(body.startswith("Traceback "), body)
5367         d.addCallback(_internal_error_text3)
5368
5369         d.addCallback(lambda ignored:
5370                       self.shouldHTTPError("GET errorboom_text",
5371                                            500, "Internal Server Error", None,
5372                                            self.GET, "ERRORBOOM"))
5373         def _internal_error_html4(body):
5374             self.failUnlessIn("<html>", body)
5375         d.addCallback(_internal_error_html4)
5376
5377         def _flush_errors(res):
5378             # Trial: please ignore the CompletelyUnhandledError in the logs
5379             self.flushLoggedErrors(CompletelyUnhandledError)
5380             return res
5381         d.addBoth(_flush_errors)
5382
5383         return d
5384
5385     def test_blacklist(self):
5386         # download from a blacklisted URI, get an error
5387         self.basedir = "web/Grid/blacklist"
5388         self.set_up_grid()
5389         c0 = self.g.clients[0]
5390         c0_basedir = c0.basedir
5391         fn = os.path.join(c0_basedir, "access.blacklist")
5392         self.uris = {}
5393         DATA = "off-limits " * 50
5394
5395         d = c0.upload(upload.Data(DATA, convergence=""))
5396         def _stash_uri_and_create_dir(ur):
5397             self.uri = ur.get_uri()
5398             self.url = "uri/"+self.uri
5399             u = uri.from_string_filenode(self.uri)
5400             self.si = u.get_storage_index()
5401             childnode = c0.create_node_from_uri(self.uri, None)
5402             return c0.create_dirnode({u"blacklisted.txt": (childnode,{}) })
5403         d.addCallback(_stash_uri_and_create_dir)
5404         def _stash_dir(node):
5405             self.dir_node = node
5406             self.dir_uri = node.get_uri()
5407             self.dir_url = "uri/"+self.dir_uri
5408         d.addCallback(_stash_dir)
5409         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5410         def _check_dir_html(body):
5411             self.failUnlessIn("<html>", body)
5412             self.failUnlessIn("blacklisted.txt</a>", body)
5413         d.addCallback(_check_dir_html)
5414         d.addCallback(lambda ign: self.GET(self.url))
5415         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5416
5417         def _blacklist(ign):
5418             f = open(fn, "w")
5419             f.write(" # this is a comment\n")
5420             f.write(" \n")
5421             f.write("\n") # also exercise blank lines
5422             f.write("%s %s\n" % (base32.b2a(self.si), "off-limits to you"))
5423             f.close()
5424             # clients should be checking the blacklist each time, so we don't
5425             # need to restart the client
5426         d.addCallback(_blacklist)
5427         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_uri",
5428                                                        403, "Forbidden",
5429                                                        "Access Prohibited: off-limits",
5430                                                        self.GET, self.url))
5431
5432         # We should still be able to list the parent directory, in HTML...
5433         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5434         def _check_dir_html2(body):
5435             self.failUnlessIn("<html>", body)
5436             self.failUnlessIn("blacklisted.txt</strike>", body)
5437         d.addCallback(_check_dir_html2)
5438
5439         # ... and in JSON (used by CLI).
5440         d.addCallback(lambda ign: self.GET(self.dir_url+"?t=json", followRedirect=True))
5441         def _check_dir_json(res):
5442             data = simplejson.loads(res)
5443             self.failUnless(isinstance(data, list), data)
5444             self.failUnlessEqual(data[0], "dirnode")
5445             self.failUnless(isinstance(data[1], dict), data)
5446             self.failUnlessIn("children", data[1])
5447             self.failUnlessIn("blacklisted.txt", data[1]["children"])
5448             childdata = data[1]["children"]["blacklisted.txt"]
5449             self.failUnless(isinstance(childdata, list), data)
5450             self.failUnlessEqual(childdata[0], "filenode")
5451             self.failUnless(isinstance(childdata[1], dict), data)
5452         d.addCallback(_check_dir_json)
5453
5454         def _unblacklist(ign):
5455             open(fn, "w").close()
5456             # the Blacklist object watches mtime to tell when the file has
5457             # changed, but on windows this test will run faster than the
5458             # filesystem's mtime resolution. So we edit Blacklist.last_mtime
5459             # to force a reload.
5460             self.g.clients[0].blacklist.last_mtime -= 2.0
5461         d.addCallback(_unblacklist)
5462
5463         # now a read should work
5464         d.addCallback(lambda ign: self.GET(self.url))
5465         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5466
5467         # read again to exercise the blacklist-is-unchanged logic
5468         d.addCallback(lambda ign: self.GET(self.url))
5469         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5470
5471         # now add a blacklisted directory, and make sure files under it are
5472         # refused too
5473         def _add_dir(ign):
5474             childnode = c0.create_node_from_uri(self.uri, None)
5475             return c0.create_dirnode({u"child": (childnode,{}) })
5476         d.addCallback(_add_dir)
5477         def _get_dircap(dn):
5478             self.dir_si_b32 = base32.b2a(dn.get_storage_index())
5479             self.dir_url_base = "uri/"+dn.get_write_uri()
5480             self.dir_url_json1 = "uri/"+dn.get_write_uri()+"?t=json"
5481             self.dir_url_json2 = "uri/"+dn.get_write_uri()+"/?t=json"
5482             self.dir_url_json_ro = "uri/"+dn.get_readonly_uri()+"/?t=json"
5483             self.child_url = "uri/"+dn.get_readonly_uri()+"/child"
5484         d.addCallback(_get_dircap)
5485         d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True))
5486         d.addCallback(lambda body: self.failUnlessIn("<html>", body))
5487         d.addCallback(lambda ign: self.GET(self.dir_url_json1))
5488         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5489         d.addCallback(lambda ign: self.GET(self.dir_url_json2))
5490         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5491         d.addCallback(lambda ign: self.GET(self.dir_url_json_ro))
5492         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5493         d.addCallback(lambda ign: self.GET(self.child_url))
5494         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5495
5496         def _block_dir(ign):
5497             f = open(fn, "w")
5498             f.write("%s %s\n" % (self.dir_si_b32, "dir-off-limits to you"))
5499             f.close()
5500             self.g.clients[0].blacklist.last_mtime -= 2.0
5501         d.addCallback(_block_dir)
5502         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir base",
5503                                                        403, "Forbidden",
5504                                                        "Access Prohibited: dir-off-limits",
5505                                                        self.GET, self.dir_url_base))
5506         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json1",
5507                                                        403, "Forbidden",
5508                                                        "Access Prohibited: dir-off-limits",
5509                                                        self.GET, self.dir_url_json1))
5510         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json2",
5511                                                        403, "Forbidden",
5512                                                        "Access Prohibited: dir-off-limits",
5513                                                        self.GET, self.dir_url_json2))
5514         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json_ro",
5515                                                        403, "Forbidden",
5516                                                        "Access Prohibited: dir-off-limits",
5517                                                        self.GET, self.dir_url_json_ro))
5518         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir child",
5519                                                        403, "Forbidden",
5520                                                        "Access Prohibited: dir-off-limits",
5521                                                        self.GET, self.child_url))
5522         return d
5523
5524
5525 class CompletelyUnhandledError(Exception):
5526     pass
5527 class ErrorBoom(rend.Page):
5528     def beforeRender(self, ctx):
5529         raise CompletelyUnhandledError("whoops")