src/allmydata/test/test_web.py

   1 import os.path, re, urllib, time
   2 import simplejson
   3 from StringIO import StringIO
   4
   5 from twisted.application import service
   6 from twisted.trial import unittest
   7 from twisted.internet import defer, reactor
   8 from twisted.internet.task import Clock
   9 from twisted.web import client, error, http
  10 from twisted.python import failure, log
  11
  12 from foolscap.api import fireEventually, flushEventualQueue
  13
  14 from nevow import rend
  15
  16 from allmydata import interfaces, uri, webish, dirnode
  17 from allmydata.storage.shares import get_share_file
  18 from allmydata.storage_client import StorageFarmBroker
  19 from allmydata.immutable import upload
  20 from allmydata.immutable.downloader.status import DownloadStatus
  21 from allmydata.dirnode import DirectoryNode
  22 from allmydata.nodemaker import NodeMaker
  23 from allmydata.unknown import UnknownNode
  24 from allmydata.web import status, common
  25 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  26 from allmydata.util import fileutil, base32, hashutil
  27 from allmydata.util.consumer import download_to_data
  28 from allmydata.util.netstring import split_netstring
  29 from allmydata.util.encodingutil import to_str
  30 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  31      create_chk_filenode, WebErrorMixin, ShouldFailMixin, \
  32      make_mutable_file_uri, create_mutable_filenode
  33 from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
  34 from allmydata.mutable import servermap, publish, retrieve
  35 import allmydata.test.common_util as testutil
  36 from allmydata.test.no_network import GridTestMixin
  37 from allmydata.test.common_web import HTTPClientGETFactory, \
  38      HTTPClientHEADFactory
  39 from allmydata.client import Client, SecretHolder
  40 from allmydata.introducer import IntroducerNode
  41
  42 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  43 # create a webserver that works against them
  44
  45 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  46
  47 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
  48 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
  49 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
  50
  51 FAVICON_MARKUP = '<link href="/icon.png" rel="shortcut icon" />'
  52
  53
  54 class FakeStatsProvider:
  55     def get_stats(self):
  56         stats = {'stats': {}, 'counters': {}}
  57         return stats
  58
  59 class FakeNodeMaker(NodeMaker):
  60     encoding_params = {
  61         'k': 3,
  62         'n': 10,
  63         'happy': 7,
  64         'max_segment_size':128*1024 # 1024=KiB
  65     }
  66     def _create_lit(self, cap):
  67         return FakeCHKFileNode(cap, self.all_contents)
  68     def _create_immutable(self, cap):
  69         return FakeCHKFileNode(cap, self.all_contents)
  70     def _create_mutable(self, cap):
  71         return FakeMutableFileNode(None, None,
  72                                    self.encoding_params, None,
  73                                    self.all_contents).init_from_cap(cap)
  74     def create_mutable_file(self, contents="", keysize=None,
  75                             version=SDMF_VERSION):
  76         n = FakeMutableFileNode(None, None, self.encoding_params, None,
  77                                 self.all_contents)
  78         return n.create(contents, version=version)
  79
  80 class FakeUploader(service.Service):
  81     name = "uploader"
  82     def upload(self, uploadable):
  83         d = uploadable.get_size()
  84         d.addCallback(lambda size: uploadable.read(size))
  85         def _got_data(datav):
  86             data = "".join(datav)
  87             n = create_chk_filenode(data, self.all_contents)
  88             ur = upload.UploadResults(file_size=len(data),
  89                                       ciphertext_fetched=0,
  90                                       preexisting_shares=0,
  91                                       pushed_shares=10,
  92                                       sharemap={},
  93                                       servermap={},
  94                                       timings={},
  95                                       uri_extension_data={},
  96                                       uri_extension_hash="fake",
  97                                       verifycapstr="fakevcap")
  98             ur.set_uri(n.get_uri())
  99             return ur
 100         d.addCallback(_got_data)
 101         return d
 102     def get_helper_info(self):
 103         return (None, False)
 104
 105 class FakeIServer:
 106     def __init__(self, binaryserverid):
 107         self.binaryserverid = binaryserverid
 108     def get_name(self): return "short"
 109     def get_longname(self): return "long"
 110     def get_serverid(self): return self.binaryserverid
 111
 112 def build_one_ds():
 113     ds = DownloadStatus("storage_index", 1234)
 114     now = time.time()
 115
 116     serverA = FakeIServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
 117     serverB = FakeIServer(hashutil.tagged_hash("foo", "serverid_b")[:20])
 118     storage_index = hashutil.storage_index_hash("SI")
 119     e0 = ds.add_segment_request(0, now)
 120     e0.activate(now+0.5)
 121     e0.deliver(now+1, 0, 100, 0.5) # when, start,len, decodetime
 122     e1 = ds.add_segment_request(1, now+2)
 123     e1.error(now+3)
 124     # two outstanding requests
 125     e2 = ds.add_segment_request(2, now+4)
 126     e3 = ds.add_segment_request(3, now+5)
 127     del e2,e3 # hush pyflakes
 128
 129     # simulate a segment which gets delivered faster than a system clock tick (ticket #1166)
 130     e = ds.add_segment_request(4, now)
 131     e.activate(now)
 132     e.deliver(now, 0, 140, 0.5)
 133
 134     e = ds.add_dyhb_request(serverA, now)
 135     e.finished([1,2], now+1)
 136     e = ds.add_dyhb_request(serverB, now+2) # left unfinished
 137
 138     e = ds.add_read_event(0, 120, now)
 139     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 140     e.finished(now+1)
 141     e = ds.add_read_event(120, 30, now+2) # left unfinished
 142
 143     e = ds.add_block_request(serverA, 1, 100, 20, now)
 144     e.finished(20, now+1)
 145     e = ds.add_block_request(serverB, 1, 120, 30, now+1) # left unfinished
 146
 147     # make sure that add_read_event() can come first too
 148     ds1 = DownloadStatus(storage_index, 1234)
 149     e = ds1.add_read_event(0, 120, now)
 150     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 151     e.finished(now+1)
 152
 153     return ds
 154
 155 class FakeHistory:
 156     _all_upload_status = [upload.UploadStatus()]
 157     _all_download_status = [build_one_ds()]
 158     _all_mapupdate_statuses = [servermap.UpdateStatus()]
 159     _all_publish_statuses = [publish.PublishStatus()]
 160     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
 161
 162     def list_all_upload_statuses(self):
 163         return self._all_upload_status
 164     def list_all_download_statuses(self):
 165         return self._all_download_status
 166     def list_all_mapupdate_statuses(self):
 167         return self._all_mapupdate_statuses
 168     def list_all_publish_statuses(self):
 169         return self._all_publish_statuses
 170     def list_all_retrieve_statuses(self):
 171         return self._all_retrieve_statuses
 172     def list_all_helper_statuses(self):
 173         return []
 174
 175 class FakeClient(Client):
 176     def __init__(self):
 177         # don't upcall to Client.__init__, since we only want to initialize a
 178         # minimal subset
 179         service.MultiService.__init__(self)
 180         self.all_contents = {}
 181         self.nodeid = "fake_nodeid"
 182         self.nickname = "fake_nickname"
 183         self.introducer_furl = "None"
 184         self.stats_provider = FakeStatsProvider()
 185         self._secret_holder = SecretHolder("lease secret", "convergence secret")
 186         self.helper = None
 187         self.convergence = "some random string"
 188         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 189         self.introducer_client = None
 190         self.history = FakeHistory()
 191         self.uploader = FakeUploader()
 192         self.uploader.all_contents = self.all_contents
 193         self.uploader.setServiceParent(self)
 194         self.blacklist = None
 195         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 196                                        self.uploader, None,
 197                                        None, None, None)
 198         self.nodemaker.all_contents = self.all_contents
 199         self.mutable_file_default = SDMF_VERSION
 200
 201     def startService(self):
 202         return service.MultiService.startService(self)
 203     def stopService(self):
 204         return service.MultiService.stopService(self)
 205
 206     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 207
 208 class WebMixin(object):
 209     def setUp(self):
 210         self.s = FakeClient()
 211         self.s.startService()
 212         self.staticdir = self.mktemp()
 213         self.clock = Clock()
 214         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
 215                                       clock=self.clock)
 216         self.ws.setServiceParent(self.s)
 217         self.webish_port = self.ws.getPortnum()
 218         self.webish_url = self.ws.getURL()
 219         assert self.webish_url.endswith("/")
 220         self.webish_url = self.webish_url[:-1] # these tests add their own /
 221
 222         l = [ self.s.create_dirnode() for x in range(6) ]
 223         d = defer.DeferredList(l)
 224         def _then(res):
 225             self.public_root = res[0][1]
 226             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 227             self.public_url = "/uri/" + self.public_root.get_uri()
 228             self.private_root = res[1][1]
 229
 230             foo = res[2][1]
 231             self._foo_node = foo
 232             self._foo_uri = foo.get_uri()
 233             self._foo_readonly_uri = foo.get_readonly_uri()
 234             self._foo_verifycap = foo.get_verify_cap().to_string()
 235             # NOTE: we ignore the deferred on all set_uri() calls, because we
 236             # know the fake nodes do these synchronously
 237             self.public_root.set_uri(u"foo", foo.get_uri(),
 238                                      foo.get_readonly_uri())
 239
 240             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 241             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 242             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 243
 244             # sdmf
 245             # XXX: Do we ever use this?
 246             self.BAZ_CONTENTS, n, self._baz_txt_uri, self._baz_txt_readonly_uri = self.makefile_mutable(0)
 247
 248             foo.set_uri(u"baz.txt", self._baz_txt_uri, self._baz_txt_readonly_uri)
 249
 250             # mdmf
 251             self.QUUX_CONTENTS, n, self._quux_txt_uri, self._quux_txt_readonly_uri = self.makefile_mutable(0, mdmf=True)
 252             assert self._quux_txt_uri.startswith("URI:MDMF")
 253             foo.set_uri(u"quux.txt", self._quux_txt_uri, self._quux_txt_readonly_uri)
 254
 255             foo.set_uri(u"empty", res[3][1].get_uri(),
 256                         res[3][1].get_readonly_uri())
 257             sub_uri = res[4][1].get_uri()
 258             self._sub_uri = sub_uri
 259             foo.set_uri(u"sub", sub_uri, sub_uri)
 260             sub = self.s.create_node_from_uri(sub_uri)
 261             self._sub_node = sub
 262
 263             _ign, n, blocking_uri = self.makefile(1)
 264             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 265
 266             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 267             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 268             # still think of it as an umlaut
 269             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 270
 271             self.SUBBAZ_CONTENTS, n, baz_file = self.makefile(2)
 272             self._baz_file_uri = baz_file
 273             sub.set_uri(u"baz.txt", baz_file, baz_file)
 274
 275             _ign, n, self._bad_file_uri = self.makefile(3)
 276             # this uri should not be downloadable
 277             del self.s.all_contents[self._bad_file_uri]
 278
 279             rodir = res[5][1]
 280             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 281                                      rodir.get_readonly_uri())
 282             rodir.set_uri(u"nor", baz_file, baz_file)
 283
 284             # public/
 285             # public/foo/
 286             # public/foo/bar.txt
 287             # public/foo/baz.txt
 288             # public/foo/quux.txt
 289             # public/foo/blockingfile
 290             # public/foo/empty/
 291             # public/foo/sub/
 292             # public/foo/sub/baz.txt
 293             # public/reedownlee/
 294             # public/reedownlee/nor
 295             self.NEWFILE_CONTENTS = "newfile contents\n"
 296
 297             return foo.get_metadata_for(u"bar.txt")
 298         d.addCallback(_then)
 299         def _got_metadata(metadata):
 300             self._bar_txt_metadata = metadata
 301         d.addCallback(_got_metadata)
 302         return d
 303
 304     def get_all_contents(self):
 305         return self.s.all_contents
 306
 307     def makefile(self, number):
 308         contents = "contents of file %s\n" % number
 309         n = create_chk_filenode(contents, self.get_all_contents())
 310         return contents, n, n.get_uri()
 311
 312     def makefile_mutable(self, number, mdmf=False):
 313         contents = "contents of mutable file %s\n" % number
 314         n = create_mutable_filenode(contents, mdmf, self.s.all_contents)
 315         return contents, n, n.get_uri(), n.get_readonly_uri()
 316
 317     def tearDown(self):
 318         return self.s.stopService()
 319
 320     def failUnlessIsBarDotTxt(self, res):
 321         self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
 322
 323     def failUnlessIsQuuxDotTxt(self, res):
 324         self.failUnlessReallyEqual(res, self.QUUX_CONTENTS, res)
 325
 326     def failUnlessIsBazDotTxt(self, res):
 327         self.failUnlessReallyEqual(res, self.BAZ_CONTENTS, res)
 328
 329     def failUnlessIsSubBazDotTxt(self, res):
 330         self.failUnlessReallyEqual(res, self.SUBBAZ_CONTENTS, res)
 331
 332     def failUnlessIsBarJSON(self, res):
 333         data = simplejson.loads(res)
 334         self.failUnless(isinstance(data, list))
 335         self.failUnlessEqual(data[0], "filenode")
 336         self.failUnless(isinstance(data[1], dict))
 337         self.failIf(data[1]["mutable"])
 338         self.failIfIn("rw_uri", data[1]) # immutable
 339         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
 340         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
 341         self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
 342
 343     def failUnlessIsQuuxJSON(self, res, readonly=False):
 344         data = simplejson.loads(res)
 345         self.failUnless(isinstance(data, list))
 346         self.failUnlessEqual(data[0], "filenode")
 347         self.failUnless(isinstance(data[1], dict))
 348         metadata = data[1]
 349         return self.failUnlessIsQuuxDotTxtMetadata(metadata, readonly)
 350
 351     def failUnlessIsQuuxDotTxtMetadata(self, metadata, readonly):
 352         self.failUnless(metadata['mutable'])
 353         if readonly:
 354             self.failIfIn("rw_uri", metadata)
 355         else:
 356             self.failUnlessIn("rw_uri", metadata)
 357             self.failUnlessEqual(metadata['rw_uri'], self._quux_txt_uri)
 358         self.failUnlessIn("ro_uri", metadata)
 359         self.failUnlessEqual(metadata['ro_uri'], self._quux_txt_readonly_uri)
 360         self.failUnlessReallyEqual(metadata['size'], len(self.QUUX_CONTENTS))
 361
 362     def failUnlessIsFooJSON(self, res):
 363         data = simplejson.loads(res)
 364         self.failUnless(isinstance(data, list))
 365         self.failUnlessEqual(data[0], "dirnode", res)
 366         self.failUnless(isinstance(data[1], dict))
 367         self.failUnless(data[1]["mutable"])
 368         self.failUnlessIn("rw_uri", data[1]) # mutable
 369         self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
 370         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
 371         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
 372
 373         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 374         self.failUnlessEqual(kidnames,
 375                              [u"bar.txt", u"baz.txt", u"blockingfile",
 376                               u"empty", u"n\u00fc.txt", u"quux.txt", u"sub"])
 377         kids = dict( [(unicode(name),value)
 378                       for (name,value)
 379                       in data[1]["children"].iteritems()] )
 380         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 381         self.failUnlessIn("metadata", kids[u"sub"][1])
 382         self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
 383         tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
 384         self.failUnlessIn("linkcrtime", tahoe_md)
 385         self.failUnlessIn("linkmotime", tahoe_md)
 386         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 387         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 388         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri)
 389         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]),
 390                                    self._bar_txt_verifycap)
 391         self.failUnlessIn("metadata", kids[u"bar.txt"][1])
 392         self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
 393         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
 394                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
 395         self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
 396                                    self._bar_txt_uri)
 397         self.failUnlessIn("quux.txt", kids)
 398         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["rw_uri"]),
 399                                    self._quux_txt_uri)
 400         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["ro_uri"]),
 401                                    self._quux_txt_readonly_uri)
 402
 403     def GET(self, urlpath, followRedirect=False, return_response=False,
 404             **kwargs):
 405         # if return_response=True, this fires with (data, statuscode,
 406         # respheaders) instead of just data.
 407         assert not isinstance(urlpath, unicode)
 408         url = self.webish_url + urlpath
 409         factory = HTTPClientGETFactory(url, method="GET",
 410                                        followRedirect=followRedirect, **kwargs)
 411         reactor.connectTCP("localhost", self.webish_port, factory)
 412         d = factory.deferred
 413         def _got_data(data):
 414             return (data, factory.status, factory.response_headers)
 415         if return_response:
 416             d.addCallback(_got_data)
 417         return factory.deferred
 418
 419     def HEAD(self, urlpath, return_response=False, **kwargs):
 420         # this requires some surgery, because twisted.web.client doesn't want
 421         # to give us back the response headers.
 422         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 423         reactor.connectTCP("localhost", self.webish_port, factory)
 424         d = factory.deferred
 425         def _got_data(data):
 426             return (data, factory.status, factory.response_headers)
 427         if return_response:
 428             d.addCallback(_got_data)
 429         return factory.deferred
 430
 431     def PUT(self, urlpath, data, **kwargs):
 432         url = self.webish_url + urlpath
 433         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 434
 435     def DELETE(self, urlpath):
 436         url = self.webish_url + urlpath
 437         return client.getPage(url, method="DELETE")
 438
 439     def POST(self, urlpath, followRedirect=False, **fields):
 440         sepbase = "boogabooga"
 441         sep = "--" + sepbase
 442         form = []
 443         form.append(sep)
 444         form.append('Content-Disposition: form-data; name="_charset"')
 445         form.append('')
 446         form.append('UTF-8')
 447         form.append(sep)
 448         for name, value in fields.iteritems():
 449             if isinstance(value, tuple):
 450                 filename, value = value
 451                 form.append('Content-Disposition: form-data; name="%s"; '
 452                             'filename="%s"' % (name, filename.encode("utf-8")))
 453             else:
 454                 form.append('Content-Disposition: form-data; name="%s"' % name)
 455             form.append('')
 456             if isinstance(value, unicode):
 457                 value = value.encode("utf-8")
 458             else:
 459                 value = str(value)
 460             assert isinstance(value, str)
 461             form.append(value)
 462             form.append(sep)
 463         form[-1] += "--"
 464         body = ""
 465         headers = {}
 466         if fields:
 467             body = "\r\n".join(form) + "\r\n"
 468             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 469         return self.POST2(urlpath, body, headers, followRedirect)
 470
 471     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 472         url = self.webish_url + urlpath
 473         return client.getPage(url, method="POST", postdata=body,
 474                               headers=headers, followRedirect=followRedirect)
 475
 476     def shouldFail(self, res, expected_failure, which,
 477                    substring=None, response_substring=None):
 478         if isinstance(res, failure.Failure):
 479             res.trap(expected_failure)
 480             if substring:
 481                 self.failUnlessIn(substring, str(res), which)
 482             if response_substring:
 483                 self.failUnlessIn(response_substring, res.value.response, which)
 484         else:
 485             self.fail("%s was supposed to raise %s, not get '%s'" %
 486                       (which, expected_failure, res))
 487
 488     def shouldFail2(self, expected_failure, which, substring,
 489                     response_substring,
 490                     callable, *args, **kwargs):
 491         assert substring is None or isinstance(substring, str)
 492         assert response_substring is None or isinstance(response_substring, str)
 493         d = defer.maybeDeferred(callable, *args, **kwargs)
 494         def done(res):
 495             if isinstance(res, failure.Failure):
 496                 res.trap(expected_failure)
 497                 if substring:
 498                     self.failUnlessIn(substring, str(res),
 499                                       "'%s' not in '%s' for test '%s'" % \
 500                                       (substring, str(res), which))
 501                 if response_substring:
 502                     self.failUnlessIn(response_substring, res.value.response,
 503                                       "'%s' not in '%s' for test '%s'" % \
 504                                       (response_substring, res.value.response,
 505                                        which))
 506             else:
 507                 self.fail("%s was supposed to raise %s, not get '%s'" %
 508                           (which, expected_failure, res))
 509         d.addBoth(done)
 510         return d
 511
 512     def should404(self, res, which):
 513         if isinstance(res, failure.Failure):
 514             res.trap(error.Error)
 515             self.failUnlessReallyEqual(res.value.status, "404")
 516         else:
 517             self.fail("%s was supposed to Error(404), not get '%s'" %
 518                       (which, res))
 519
 520     def should302(self, res, which):
 521         if isinstance(res, failure.Failure):
 522             res.trap(error.Error)
 523             self.failUnlessReallyEqual(res.value.status, "302")
 524         else:
 525             self.fail("%s was supposed to Error(302), not get '%s'" %
 526                       (which, res))
 527
 528
 529 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
 530     def test_create(self):
 531         pass
 532
 533     def test_welcome(self):
 534         d = self.GET("/")
 535         def _check(res):
 536             self.failUnlessIn('Welcome to Tahoe-LAFS', res)
 537             self.failUnlessIn(FAVICON_MARKUP, res)
 538             self.failUnlessIn('href="https://tahoe-lafs.org/"', res)
 539
 540             self.s.basedir = 'web/test_welcome'
 541             fileutil.make_dirs("web/test_welcome")
 542             fileutil.make_dirs("web/test_welcome/private")
 543             return self.GET("/")
 544         d.addCallback(_check)
 545         return d
 546
 547     def test_status(self):
 548         h = self.s.get_history()
 549         dl_num = h.list_all_download_statuses()[0].get_counter()
 550         ul_num = h.list_all_upload_statuses()[0].get_counter()
 551         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 552         pub_num = h.list_all_publish_statuses()[0].get_counter()
 553         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 554         d = self.GET("/status", followRedirect=True)
 555         def _check(res):
 556             self.failUnlessIn('Upload and Download Status', res)
 557             self.failUnlessIn('"down-%d"' % dl_num, res)
 558             self.failUnlessIn('"up-%d"' % ul_num, res)
 559             self.failUnlessIn('"mapupdate-%d"' % mu_num, res)
 560             self.failUnlessIn('"publish-%d"' % pub_num, res)
 561             self.failUnlessIn('"retrieve-%d"' % ret_num, res)
 562         d.addCallback(_check)
 563         d.addCallback(lambda res: self.GET("/status/?t=json"))
 564         def _check_json(res):
 565             data = simplejson.loads(res)
 566             self.failUnless(isinstance(data, dict))
 567             #active = data["active"]
 568             # TODO: test more. We need a way to fake an active operation
 569             # here.
 570         d.addCallback(_check_json)
 571
 572         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 573         def _check_dl(res):
 574             self.failUnlessIn("File Download Status", res)
 575         d.addCallback(_check_dl)
 576         d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
 577         def _check_dl_json(res):
 578             data = simplejson.loads(res)
 579             self.failUnless(isinstance(data, dict))
 580             self.failUnlessIn("read", data)
 581             self.failUnlessEqual(data["read"][0]["length"], 120)
 582             self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
 583             self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
 584             self.failUnlessEqual(data["segment"][2]["finish_time"], None)
 585             phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
 586             cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
 587             # serverids[] keys are strings, since that's what JSON does, but
 588             # we'd really like them to be ints
 589             self.failUnlessEqual(data["serverids"]["0"], "phwr")
 590             self.failUnless(data["serverids"].has_key("1"), data["serverids"])
 591             self.failUnlessEqual(data["serverids"]["1"], "cmpu", data["serverids"])
 592             self.failUnlessEqual(data["server_info"][phwr_id]["short"], "phwr")
 593             self.failUnlessEqual(data["server_info"][cmpu_id]["short"], "cmpu")
 594             self.failUnlessIn("dyhb", data)
 595             self.failUnlessIn("misc", data)
 596         d.addCallback(_check_dl_json)
 597         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 598         def _check_ul(res):
 599             self.failUnlessIn("File Upload Status", res)
 600         d.addCallback(_check_ul)
 601         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 602         def _check_mapupdate(res):
 603             self.failUnlessIn("Mutable File Servermap Update Status", res)
 604         d.addCallback(_check_mapupdate)
 605         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 606         def _check_publish(res):
 607             self.failUnlessIn("Mutable File Publish Status", res)
 608         d.addCallback(_check_publish)
 609         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 610         def _check_retrieve(res):
 611             self.failUnlessIn("Mutable File Retrieve Status", res)
 612         d.addCallback(_check_retrieve)
 613
 614         return d
 615
 616     def test_status_numbers(self):
 617         drrm = status.DownloadResultsRendererMixin()
 618         self.failUnlessReallyEqual(drrm.render_time(None, None), "")
 619         self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
 620         self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
 621         self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
 622         self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
 623         self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
 624         self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 625         self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
 626         self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
 627
 628         urrm = status.UploadResultsRendererMixin()
 629         self.failUnlessReallyEqual(urrm.render_time(None, None), "")
 630         self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
 631         self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
 632         self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
 633         self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
 634         self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
 635         self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 636         self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
 637         self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
 638
 639     def test_GET_FILEURL(self):
 640         d = self.GET(self.public_url + "/foo/bar.txt")
 641         d.addCallback(self.failUnlessIsBarDotTxt)
 642         return d
 643
 644     def test_GET_FILEURL_range(self):
 645         headers = {"range": "bytes=1-10"}
 646         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 647                      return_response=True)
 648         def _got((res, status, headers)):
 649             self.failUnlessReallyEqual(int(status), 206)
 650             self.failUnless(headers.has_key("content-range"))
 651             self.failUnlessReallyEqual(headers["content-range"][0],
 652                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 653             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
 654         d.addCallback(_got)
 655         return d
 656
 657     def test_GET_FILEURL_partial_range(self):
 658         headers = {"range": "bytes=5-"}
 659         length  = len(self.BAR_CONTENTS)
 660         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 661                      return_response=True)
 662         def _got((res, status, headers)):
 663             self.failUnlessReallyEqual(int(status), 206)
 664             self.failUnless(headers.has_key("content-range"))
 665             self.failUnlessReallyEqual(headers["content-range"][0],
 666                                        "bytes 5-%d/%d" % (length-1, length))
 667             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
 668         d.addCallback(_got)
 669         return d
 670
 671     def test_GET_FILEURL_partial_end_range(self):
 672         headers = {"range": "bytes=-5"}
 673         length  = len(self.BAR_CONTENTS)
 674         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 675                      return_response=True)
 676         def _got((res, status, headers)):
 677             self.failUnlessReallyEqual(int(status), 206)
 678             self.failUnless(headers.has_key("content-range"))
 679             self.failUnlessReallyEqual(headers["content-range"][0],
 680                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 681             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
 682         d.addCallback(_got)
 683         return d
 684
 685     def test_GET_FILEURL_partial_range_overrun(self):
 686         headers = {"range": "bytes=100-200"}
 687         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
 688                              "416 Requested Range not satisfiable",
 689                              "First beyond end of file",
 690                              self.GET, self.public_url + "/foo/bar.txt",
 691                              headers=headers)
 692         return d
 693
 694     def test_HEAD_FILEURL_range(self):
 695         headers = {"range": "bytes=1-10"}
 696         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 697                      return_response=True)
 698         def _got((res, status, headers)):
 699             self.failUnlessReallyEqual(res, "")
 700             self.failUnlessReallyEqual(int(status), 206)
 701             self.failUnless(headers.has_key("content-range"))
 702             self.failUnlessReallyEqual(headers["content-range"][0],
 703                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 704         d.addCallback(_got)
 705         return d
 706
 707     def test_HEAD_FILEURL_partial_range(self):
 708         headers = {"range": "bytes=5-"}
 709         length  = len(self.BAR_CONTENTS)
 710         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 711                      return_response=True)
 712         def _got((res, status, headers)):
 713             self.failUnlessReallyEqual(int(status), 206)
 714             self.failUnless(headers.has_key("content-range"))
 715             self.failUnlessReallyEqual(headers["content-range"][0],
 716                                        "bytes 5-%d/%d" % (length-1, length))
 717         d.addCallback(_got)
 718         return d
 719
 720     def test_HEAD_FILEURL_partial_end_range(self):
 721         headers = {"range": "bytes=-5"}
 722         length  = len(self.BAR_CONTENTS)
 723         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 724                      return_response=True)
 725         def _got((res, status, headers)):
 726             self.failUnlessReallyEqual(int(status), 206)
 727             self.failUnless(headers.has_key("content-range"))
 728             self.failUnlessReallyEqual(headers["content-range"][0],
 729                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 730         d.addCallback(_got)
 731         return d
 732
 733     def test_HEAD_FILEURL_partial_range_overrun(self):
 734         headers = {"range": "bytes=100-200"}
 735         d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
 736                              "416 Requested Range not satisfiable",
 737                              "",
 738                              self.HEAD, self.public_url + "/foo/bar.txt",
 739                              headers=headers)
 740         return d
 741
 742     def test_GET_FILEURL_range_bad(self):
 743         headers = {"range": "BOGUS=fizbop-quarnak"}
 744         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 745                      return_response=True)
 746         def _got((res, status, headers)):
 747             self.failUnlessReallyEqual(int(status), 200)
 748             self.failUnless(not headers.has_key("content-range"))
 749             self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
 750         d.addCallback(_got)
 751         return d
 752
 753     def test_HEAD_FILEURL(self):
 754         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 755         def _got((res, status, headers)):
 756             self.failUnlessReallyEqual(res, "")
 757             self.failUnlessReallyEqual(headers["content-length"][0],
 758                                        str(len(self.BAR_CONTENTS)))
 759             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
 760         d.addCallback(_got)
 761         return d
 762
 763     def test_GET_FILEURL_named(self):
 764         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 765         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 766         d = self.GET(base + "/@@name=/blah.txt")
 767         d.addCallback(self.failUnlessIsBarDotTxt)
 768         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 769         d.addCallback(self.failUnlessIsBarDotTxt)
 770         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 771         d.addCallback(self.failUnlessIsBarDotTxt)
 772         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 773         d.addCallback(self.failUnlessIsBarDotTxt)
 774         save_url = base + "?save=true&filename=blah.txt"
 775         d.addCallback(lambda res: self.GET(save_url))
 776         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 777         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 778         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 779         u_url = base + "?save=true&filename=" + u_fn_e
 780         d.addCallback(lambda res: self.GET(u_url))
 781         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 782         return d
 783
 784     def test_PUT_FILEURL_named_bad(self):
 785         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 786         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 787                              "400 Bad Request",
 788                              "/file can only be used with GET or HEAD",
 789                              self.PUT, base + "/@@name=/blah.txt", "")
 790         return d
 791
 792
 793     def test_GET_DIRURL_named_bad(self):
 794         base = "/file/%s" % urllib.quote(self._foo_uri)
 795         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 796                              "400 Bad Request",
 797                              "is not a file-cap",
 798                              self.GET, base + "/@@name=/blah.txt")
 799         return d
 800
 801     def test_GET_slash_file_bad(self):
 802         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 803                              "404 Not Found",
 804                              "/file must be followed by a file-cap and a name",
 805                              self.GET, "/file")
 806         return d
 807
 808     def test_GET_unhandled_URI_named(self):
 809         contents, n, newuri = self.makefile(12)
 810         verifier_cap = n.get_verify_cap().to_string()
 811         base = "/file/%s" % urllib.quote(verifier_cap)
 812         # client.create_node_from_uri() can't handle verify-caps
 813         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 814                              "400 Bad Request", "is not a file-cap",
 815                              self.GET, base)
 816         return d
 817
 818     def test_GET_unhandled_URI(self):
 819         contents, n, newuri = self.makefile(12)
 820         verifier_cap = n.get_verify_cap().to_string()
 821         base = "/uri/%s" % urllib.quote(verifier_cap)
 822         # client.create_node_from_uri() can't handle verify-caps
 823         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 824                              "400 Bad Request",
 825                              "GET unknown URI type: can only do t=info",
 826                              self.GET, base)
 827         return d
 828
 829     def test_GET_FILE_URI(self):
 830         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 831         d = self.GET(base)
 832         d.addCallback(self.failUnlessIsBarDotTxt)
 833         return d
 834
 835     def test_GET_FILE_URI_mdmf(self):
 836         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 837         d = self.GET(base)
 838         d.addCallback(self.failUnlessIsQuuxDotTxt)
 839         return d
 840
 841     def test_GET_FILE_URI_mdmf_extensions(self):
 842         base = "/uri/%s" % urllib.quote("%s:RANDOMSTUFF" % self._quux_txt_uri)
 843         d = self.GET(base)
 844         d.addCallback(self.failUnlessIsQuuxDotTxt)
 845         return d
 846
 847     def test_GET_FILE_URI_mdmf_readonly(self):
 848         base = "/uri/%s" % urllib.quote(self._quux_txt_readonly_uri)
 849         d = self.GET(base)
 850         d.addCallback(self.failUnlessIsQuuxDotTxt)
 851         return d
 852
 853     def test_GET_FILE_URI_badchild(self):
 854         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 855         errmsg = "Files have no children, certainly not named 'boguschild'"
 856         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 857                              "400 Bad Request", errmsg,
 858                              self.GET, base)
 859         return d
 860
 861     def test_PUT_FILE_URI_badchild(self):
 862         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 863         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 864         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 865                              "400 Bad Request", errmsg,
 866                              self.PUT, base, "")
 867         return d
 868
 869     def test_PUT_FILE_URI_mdmf(self):
 870         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 871         self._quux_new_contents = "new_contents"
 872         d = self.GET(base)
 873         d.addCallback(lambda res:
 874             self.failUnlessIsQuuxDotTxt(res))
 875         d.addCallback(lambda ignored:
 876             self.PUT(base, self._quux_new_contents))
 877         d.addCallback(lambda ignored:
 878             self.GET(base))
 879         d.addCallback(lambda res:
 880             self.failUnlessReallyEqual(res, self._quux_new_contents))
 881         return d
 882
 883     def test_PUT_FILE_URI_mdmf_extensions(self):
 884         base = "/uri/%s" % urllib.quote("%s:EXTENSIONSTUFF" % self._quux_txt_uri)
 885         self._quux_new_contents = "new_contents"
 886         d = self.GET(base)
 887         d.addCallback(lambda res: self.failUnlessIsQuuxDotTxt(res))
 888         d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
 889         d.addCallback(lambda ignored: self.GET(base))
 890         d.addCallback(lambda res: self.failUnlessEqual(self._quux_new_contents,
 891                                                        res))
 892         return d
 893
 894     def test_PUT_FILE_URI_mdmf_readonly(self):
 895         # We're not allowed to PUT things to a readonly cap.
 896         base = "/uri/%s" % self._quux_txt_readonly_uri
 897         d = self.GET(base)
 898         d.addCallback(lambda res:
 899             self.failUnlessIsQuuxDotTxt(res))
 900         # What should we get here? We get a 500 error now; that's not right.
 901         d.addCallback(lambda ignored:
 902             self.shouldFail2(error.Error, "test_PUT_FILE_URI_mdmf_readonly",
 903                              "400 Bad Request", "read-only cap",
 904                              self.PUT, base, "new data"))
 905         return d
 906
 907     def test_PUT_FILE_URI_sdmf_readonly(self):
 908         # We're not allowed to put things to a readonly cap.
 909         base = "/uri/%s" % self._baz_txt_readonly_uri
 910         d = self.GET(base)
 911         d.addCallback(lambda res:
 912             self.failUnlessIsBazDotTxt(res))
 913         d.addCallback(lambda ignored:
 914             self.shouldFail2(error.Error, "test_PUT_FILE_URI_sdmf_readonly",
 915                              "400 Bad Request", "read-only cap",
 916                              self.PUT, base, "new_data"))
 917         return d
 918
 919     def test_GET_etags(self):
 920
 921         def _check_etags(uri):
 922             d1 = _get_etag(uri)
 923             d2 = _get_etag(uri, 'json')
 924             d = defer.DeferredList([d1, d2], consumeErrors=True)
 925             def _check(results):
 926                 # All deferred must succeed
 927                 self.failUnless(all([r[0] for r in results]))
 928                 # the etag for the t=json form should be just like the etag
 929                 # fo the default t='' form, but with a 'json' suffix
 930                 self.failUnlessEqual(results[0][1] + 'json', results[1][1])
 931             d.addCallback(_check)
 932             return d
 933
 934         def _get_etag(uri, t=''):
 935             targetbase = "/uri/%s?t=%s" % (urllib.quote(uri.strip()), t)
 936             d = self.GET(targetbase, return_response=True, followRedirect=True)
 937             def _just_the_etag(result):
 938                 data, response, headers = result
 939                 etag = headers['etag'][0]
 940                 if uri.startswith('URI:DIR'):
 941                     self.failUnless(etag.startswith('DIR:'), etag)
 942                 return etag
 943             return d.addCallback(_just_the_etag)
 944
 945         # Check that etags work with immutable directories
 946         (newkids, caps) = self._create_immutable_children()
 947         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
 948                       simplejson.dumps(newkids))
 949         def _stash_immdir_uri(uri):
 950             self._immdir_uri = uri
 951             return uri
 952         d.addCallback(_stash_immdir_uri)
 953         d.addCallback(_check_etags)
 954
 955         # Check that etags work with immutable files
 956         d.addCallback(lambda _: _check_etags(self._bar_txt_uri))
 957
 958         # use the ETag on GET
 959         def _check_match(ign):
 960             uri = "/uri/%s" % self._bar_txt_uri
 961             d = self.GET(uri, return_response=True)
 962             # extract the ETag
 963             d.addCallback(lambda (data, code, headers):
 964                           headers['etag'][0])
 965             # do a GET that's supposed to match the ETag
 966             d.addCallback(lambda etag:
 967                           self.GET(uri, return_response=True,
 968                                    headers={"If-None-Match": etag}))
 969             # make sure it short-circuited (304 instead of 200)
 970             d.addCallback(lambda (data, code, headers):
 971                           self.failUnlessEqual(int(code), http.NOT_MODIFIED))
 972             return d
 973         d.addCallback(_check_match)
 974
 975         def _no_etag(uri, t):
 976             target = "/uri/%s?t=%s" % (uri, t)
 977             d = self.GET(target, return_response=True, followRedirect=True)
 978             d.addCallback(lambda (data, code, headers):
 979                           self.failIf("etag" in headers, target))
 980             return d
 981         def _yes_etag(uri, t):
 982             target = "/uri/%s?t=%s" % (uri, t)
 983             d = self.GET(target, return_response=True, followRedirect=True)
 984             d.addCallback(lambda (data, code, headers):
 985                           self.failUnless("etag" in headers, target))
 986             return d
 987
 988         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, ""))
 989         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "json"))
 990         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "uri"))
 991         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "readonly-uri"))
 992         d.addCallback(lambda ign: _no_etag(self._bar_txt_uri, "info"))
 993
 994         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, ""))
 995         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "json"))
 996         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "uri"))
 997         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "readonly-uri"))
 998         d.addCallback(lambda ign: _no_etag(self._immdir_uri, "info"))
 999         d.addCallback(lambda ign: _no_etag(self._immdir_uri, "rename-form"))
1000
1001         return d
1002
1003     # TODO: version of this with a Unicode filename
1004     def test_GET_FILEURL_save(self):
1005         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
1006                      return_response=True)
1007         def _got((res, statuscode, headers)):
1008             content_disposition = headers["content-disposition"][0]
1009             self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
1010             self.failUnlessIsBarDotTxt(res)
1011         d.addCallback(_got)
1012         return d
1013
1014     def test_GET_FILEURL_missing(self):
1015         d = self.GET(self.public_url + "/foo/missing")
1016         d.addBoth(self.should404, "test_GET_FILEURL_missing")
1017         return d
1018
1019     def test_GET_FILEURL_info_mdmf(self):
1020         d = self.GET("/uri/%s?t=info" % self._quux_txt_uri)
1021         def _got(res):
1022             self.failUnlessIn("mutable file (mdmf)", res)
1023             self.failUnlessIn(self._quux_txt_uri, res)
1024             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1025         d.addCallback(_got)
1026         return d
1027
1028     def test_GET_FILEURL_info_mdmf_readonly(self):
1029         d = self.GET("/uri/%s?t=info" % self._quux_txt_readonly_uri)
1030         def _got(res):
1031             self.failUnlessIn("mutable file (mdmf)", res)
1032             self.failIfIn(self._quux_txt_uri, res)
1033             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1034         d.addCallback(_got)
1035         return d
1036
1037     def test_GET_FILEURL_info_sdmf(self):
1038         d = self.GET("/uri/%s?t=info" % self._baz_txt_uri)
1039         def _got(res):
1040             self.failUnlessIn("mutable file (sdmf)", res)
1041             self.failUnlessIn(self._baz_txt_uri, res)
1042         d.addCallback(_got)
1043         return d
1044
1045     def test_GET_FILEURL_info_mdmf_extensions(self):
1046         d = self.GET("/uri/%s:STUFF?t=info" % self._quux_txt_uri)
1047         def _got(res):
1048             self.failUnlessIn("mutable file (mdmf)", res)
1049             self.failUnlessIn(self._quux_txt_uri, res)
1050             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1051         d.addCallback(_got)
1052         return d
1053
1054     def test_PUT_overwrite_only_files(self):
1055         # create a directory, put a file in that directory.
1056         contents, n, filecap = self.makefile(8)
1057         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
1058         d.addCallback(lambda res:
1059             self.PUT(self.public_url + "/foo/dir/file1.txt",
1060                      self.NEWFILE_CONTENTS))
1061         # try to overwrite the file with replace=only-files
1062         # (this should work)
1063         d.addCallback(lambda res:
1064             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
1065                      filecap))
1066         d.addCallback(lambda res:
1067             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
1068                  "There was already a child by that name, and you asked me "
1069                  "to not replace it",
1070                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
1071                  filecap))
1072         return d
1073
1074     def test_PUT_NEWFILEURL(self):
1075         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
1076         # TODO: we lose the response code, so we can't check this
1077         #self.failUnlessReallyEqual(responsecode, 201)
1078         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1079         d.addCallback(lambda res:
1080                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1081                                                       self.NEWFILE_CONTENTS))
1082         return d
1083
1084     def test_PUT_NEWFILEURL_not_mutable(self):
1085         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
1086                      self.NEWFILE_CONTENTS)
1087         # TODO: we lose the response code, so we can't check this
1088         #self.failUnlessReallyEqual(responsecode, 201)
1089         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1090         d.addCallback(lambda res:
1091                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1092                                                       self.NEWFILE_CONTENTS))
1093         return d
1094
1095     def test_PUT_NEWFILEURL_unlinked_mdmf(self):
1096         # this should get us a few segments of an MDMF mutable file,
1097         # which we can then test for.
1098         contents = self.NEWFILE_CONTENTS * 300000
1099         d = self.PUT("/uri?format=mdmf",
1100                      contents)
1101         def _got_filecap(filecap):
1102             self.failUnless(filecap.startswith("URI:MDMF"))
1103             return filecap
1104         d.addCallback(_got_filecap)
1105         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1106         d.addCallback(lambda json: self.failUnlessIn("MDMF", json))
1107         return d
1108
1109     def test_PUT_NEWFILEURL_unlinked_sdmf(self):
1110         contents = self.NEWFILE_CONTENTS * 300000
1111         d = self.PUT("/uri?format=sdmf",
1112                      contents)
1113         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1114         d.addCallback(lambda json: self.failUnlessIn("SDMF", json))
1115         return d
1116
1117     def test_PUT_NEWFILEURL_unlinked_bad_format(self):
1118         contents = self.NEWFILE_CONTENTS * 300000
1119         return self.shouldHTTPError("PUT_NEWFILEURL_unlinked_bad_format",
1120                                     400, "Bad Request", "Unknown format: foo",
1121                                     self.PUT, "/uri?format=foo",
1122                                     contents)
1123
1124     def test_PUT_NEWFILEURL_range_bad(self):
1125         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
1126         target = self.public_url + "/foo/new.txt"
1127         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
1128                              "501 Not Implemented",
1129                              "Content-Range in PUT not yet supported",
1130                              # (and certainly not for immutable files)
1131                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
1132                              headers=headers)
1133         d.addCallback(lambda res:
1134                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
1135         return d
1136
1137     def test_PUT_NEWFILEURL_mutable(self):
1138         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1139                      self.NEWFILE_CONTENTS)
1140         # TODO: we lose the response code, so we can't check this
1141         #self.failUnlessReallyEqual(responsecode, 201)
1142         def _check_uri(res):
1143             u = uri.from_string_mutable_filenode(res)
1144             self.failUnless(u.is_mutable())
1145             self.failIf(u.is_readonly())
1146             return res
1147         d.addCallback(_check_uri)
1148         d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
1149         d.addCallback(lambda res:
1150                       self.failUnlessMutableChildContentsAre(self._foo_node,
1151                                                              u"new.txt",
1152                                                              self.NEWFILE_CONTENTS))
1153         return d
1154
1155     def test_PUT_NEWFILEURL_mutable_toobig(self):
1156         # It is okay to upload large mutable files, so we should be able
1157         # to do that.
1158         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1159                      "b" * (self.s.MUTABLE_SIZELIMIT + 1))
1160         return d
1161
1162     def test_PUT_NEWFILEURL_replace(self):
1163         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
1164         # TODO: we lose the response code, so we can't check this
1165         #self.failUnlessReallyEqual(responsecode, 200)
1166         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
1167         d.addCallback(lambda res:
1168                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1169                                                       self.NEWFILE_CONTENTS))
1170         return d
1171
1172     def test_PUT_NEWFILEURL_bad_t(self):
1173         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
1174                              "PUT to a file: bad t=bogus",
1175                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
1176                              "contents")
1177         return d
1178
1179     def test_PUT_NEWFILEURL_no_replace(self):
1180         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
1181                      self.NEWFILE_CONTENTS)
1182         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
1183                   "409 Conflict",
1184                   "There was already a child by that name, and you asked me "
1185                   "to not replace it")
1186         return d
1187
1188     def test_PUT_NEWFILEURL_mkdirs(self):
1189         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
1190         fn = self._foo_node
1191         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
1192         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
1193         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
1194         d.addCallback(lambda res:
1195                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
1196                                                       self.NEWFILE_CONTENTS))
1197         return d
1198
1199     def test_PUT_NEWFILEURL_blocked(self):
1200         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
1201                      self.NEWFILE_CONTENTS)
1202         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
1203                   "409 Conflict",
1204                   "Unable to create directory 'blockingfile': a file was in the way")
1205         return d
1206
1207     def test_PUT_NEWFILEURL_emptyname(self):
1208         # an empty pathname component (i.e. a double-slash) is disallowed
1209         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
1210                              "400 Bad Request",
1211                              "The webapi does not allow empty pathname components",
1212                              self.PUT, self.public_url + "/foo//new.txt", "")
1213         return d
1214
1215     def test_DELETE_FILEURL(self):
1216         d = self.DELETE(self.public_url + "/foo/bar.txt")
1217         d.addCallback(lambda res:
1218                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1219         return d
1220
1221     def test_DELETE_FILEURL_missing(self):
1222         d = self.DELETE(self.public_url + "/foo/missing")
1223         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
1224         return d
1225
1226     def test_DELETE_FILEURL_missing2(self):
1227         d = self.DELETE(self.public_url + "/missing/missing")
1228         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
1229         return d
1230
1231     def failUnlessHasBarDotTxtMetadata(self, res):
1232         data = simplejson.loads(res)
1233         self.failUnless(isinstance(data, list))
1234         self.failUnlessIn("metadata", data[1])
1235         self.failUnlessIn("tahoe", data[1]["metadata"])
1236         self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
1237         self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
1238         self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
1239                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
1240
1241     def test_GET_FILEURL_json(self):
1242         # twisted.web.http.parse_qs ignores any query args without an '=', so
1243         # I can't do "GET /path?json", I have to do "GET /path/t=json"
1244         # instead. This may make it tricky to emulate the S3 interface
1245         # completely.
1246         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
1247         def _check1(data):
1248             self.failUnlessIsBarJSON(data)
1249             self.failUnlessHasBarDotTxtMetadata(data)
1250             return
1251         d.addCallback(_check1)
1252         return d
1253
1254     def test_GET_FILEURL_json_mutable_type(self):
1255         # The JSON should include format, which says whether the
1256         # file is SDMF or MDMF
1257         d = self.PUT("/uri?format=mdmf",
1258                      self.NEWFILE_CONTENTS * 300000)
1259         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1260         def _got_json(json, version):
1261             data = simplejson.loads(json)
1262             assert "filenode" == data[0]
1263             data = data[1]
1264             assert isinstance(data, dict)
1265
1266             self.failUnlessIn("format", data)
1267             self.failUnlessEqual(data["format"], version)
1268
1269         d.addCallback(_got_json, "MDMF")
1270         # Now make an SDMF file and check that it is reported correctly.
1271         d.addCallback(lambda ignored:
1272             self.PUT("/uri?format=sdmf",
1273                       self.NEWFILE_CONTENTS * 300000))
1274         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1275         d.addCallback(_got_json, "SDMF")
1276         return d
1277
1278     def test_GET_FILEURL_json_mdmf(self):
1279         d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
1280         d.addCallback(self.failUnlessIsQuuxJSON)
1281         return d
1282
1283     def test_GET_FILEURL_json_missing(self):
1284         d = self.GET(self.public_url + "/foo/missing?json")
1285         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
1286         return d
1287
1288     def test_GET_FILEURL_uri(self):
1289         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
1290         def _check(res):
1291             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1292         d.addCallback(_check)
1293         d.addCallback(lambda res:
1294                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
1295         def _check2(res):
1296             # for now, for files, uris and readonly-uris are the same
1297             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1298         d.addCallback(_check2)
1299         return d
1300
1301     def test_GET_FILEURL_badtype(self):
1302         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
1303                                  "bad t=bogus",
1304                                  self.GET,
1305                                  self.public_url + "/foo/bar.txt?t=bogus")
1306         return d
1307
1308     def test_CSS_FILE(self):
1309         d = self.GET("/tahoe.css", followRedirect=True)
1310         def _check(res):
1311             CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
1312             self.failUnless(CSS_STYLE.search(res), res)
1313         d.addCallback(_check)
1314         return d
1315
1316     def test_GET_FILEURL_uri_missing(self):
1317         d = self.GET(self.public_url + "/foo/missing?t=uri")
1318         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
1319         return d
1320
1321     def _check_upload_and_mkdir_forms(self, html):
1322         # We should have a form to create a file, with radio buttons that allow
1323         # the user to toggle whether it is a CHK/LIT (default), SDMF, or MDMF file.
1324         self.failUnlessIn('name="t" value="upload"', html)
1325         self.failUnlessIn('input checked="checked" type="radio" id="upload-chk" value="chk" name="format"', html)
1326         self.failUnlessIn('input type="radio" id="upload-sdmf" value="sdmf" name="format"', html)
1327         self.failUnlessIn('input type="radio" id="upload-mdmf" value="mdmf" name="format"', html)
1328
1329         # We should also have the ability to create a mutable directory, with
1330         # radio buttons that allow the user to toggle whether it is an SDMF (default)
1331         # or MDMF directory.
1332         self.failUnlessIn('name="t" value="mkdir"', html)
1333         self.failUnlessIn('input checked="checked" type="radio" id="mkdir-sdmf" value="sdmf" name="format"', html)
1334         self.failUnlessIn('input type="radio" id="mkdir-mdmf" value="mdmf" name="format"', html)
1335
1336         self.failUnlessIn(FAVICON_MARKUP, html)
1337
1338     def test_GET_DIRECTORY_html(self):
1339         d = self.GET(self.public_url + "/foo", followRedirect=True)
1340         def _check(html):
1341             self.failUnlessIn('<div class="toolbar-item"><a href="../../..">Return to Welcome page</a></div>', html)
1342             self._check_upload_and_mkdir_forms(html)
1343             self.failUnlessIn("quux", html)
1344         d.addCallback(_check)
1345         return d
1346
1347     def test_GET_root_html(self):
1348         d = self.GET("/")
1349         d.addCallback(self._check_upload_and_mkdir_forms)
1350         return d
1351
1352     def test_GET_DIRURL(self):
1353         # the addSlash means we get a redirect here
1354         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1355         ROOT = "../../.."
1356         d = self.GET(self.public_url + "/foo", followRedirect=True)
1357         def _check(res):
1358             self.failUnlessIn('<a href="%s">Return to Welcome page' % ROOT, res)
1359
1360             # the FILE reference points to a URI, but it should end in bar.txt
1361             bar_url = ("%s/file/%s/@@named=/bar.txt" %
1362                        (ROOT, urllib.quote(self._bar_txt_uri)))
1363             get_bar = "".join([r'<td>FILE</td>',
1364                                r'\s+<td>',
1365                                r'<a href="%s">bar.txt</a>' % bar_url,
1366                                r'</td>',
1367                                r'\s+<td align="right">%d</td>' % len(self.BAR_CONTENTS),
1368                                ])
1369             self.failUnless(re.search(get_bar, res), res)
1370             for label in ['unlink', 'rename/move']:
1371                 for line in res.split("\n"):
1372                     # find the line that contains the relevant button for bar.txt
1373                     if ("form action" in line and
1374                         ('value="%s"' % (label,)) in line and
1375                         'value="bar.txt"' in line):
1376                         # the form target should use a relative URL
1377                         foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1378                         self.failUnlessIn('action="%s"' % foo_url, line)
1379                         # and the when_done= should too
1380                         #done_url = urllib.quote(???)
1381                         #self.failUnlessIn('name="when_done" value="%s"' % done_url, line)
1382
1383                         # 'unlink' needs to use POST because it directly has a side effect
1384                         if label == 'unlink':
1385                             self.failUnlessIn('method="post"', line)
1386                         break
1387                 else:
1388                     self.fail("unable to find '%s bar.txt' line" % (label,))
1389
1390             # the DIR reference just points to a URI
1391             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1392             get_sub = ((r'<td>DIR</td>')
1393                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1394             self.failUnless(re.search(get_sub, res), res)
1395         d.addCallback(_check)
1396
1397         # look at a readonly directory
1398         d.addCallback(lambda res:
1399                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
1400         def _check2(res):
1401             self.failUnlessIn("(read-only)", res)
1402             self.failIfIn("Upload a file", res)
1403         d.addCallback(_check2)
1404
1405         # and at a directory that contains a readonly directory
1406         d.addCallback(lambda res:
1407                       self.GET(self.public_url, followRedirect=True))
1408         def _check3(res):
1409             self.failUnless(re.search('<td>DIR-RO</td>'
1410                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1411         d.addCallback(_check3)
1412
1413         # and an empty directory
1414         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1415         def _check4(res):
1416             self.failUnlessIn("directory is empty", res)
1417             MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
1418             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1419         d.addCallback(_check4)
1420
1421         # and at a literal directory
1422         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1423         d.addCallback(lambda res:
1424                       self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1425         def _check5(res):
1426             self.failUnlessIn('(immutable)', res)
1427             self.failUnless(re.search('<td>FILE</td>'
1428                                       r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1429         d.addCallback(_check5)
1430         return d
1431
1432     def test_GET_DIRURL_badtype(self):
1433         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1434                                  400, "Bad Request",
1435                                  "bad t=bogus",
1436                                  self.GET,
1437                                  self.public_url + "/foo?t=bogus")
1438         return d
1439
1440     def test_GET_DIRURL_json(self):
1441         d = self.GET(self.public_url + "/foo?t=json")
1442         d.addCallback(self.failUnlessIsFooJSON)
1443         return d
1444
1445     def test_GET_DIRURL_json_format(self):
1446         d = self.PUT(self.public_url + \
1447                      "/foo/sdmf.txt?format=sdmf",
1448                      self.NEWFILE_CONTENTS * 300000)
1449         d.addCallback(lambda ignored:
1450             self.PUT(self.public_url + \
1451                      "/foo/mdmf.txt?format=mdmf",
1452                      self.NEWFILE_CONTENTS * 300000))
1453         # Now we have an MDMF and SDMF file in the directory. If we GET
1454         # its JSON, we should see their encodings.
1455         d.addCallback(lambda ignored:
1456             self.GET(self.public_url + "/foo?t=json"))
1457         def _got_json(json):
1458             data = simplejson.loads(json)
1459             assert data[0] == "dirnode"
1460
1461             data = data[1]
1462             kids = data['children']
1463
1464             mdmf_data = kids['mdmf.txt'][1]
1465             self.failUnlessIn("format", mdmf_data)
1466             self.failUnlessEqual(mdmf_data["format"], "MDMF")
1467
1468             sdmf_data = kids['sdmf.txt'][1]
1469             self.failUnlessIn("format", sdmf_data)
1470             self.failUnlessEqual(sdmf_data["format"], "SDMF")
1471         d.addCallback(_got_json)
1472         return d
1473
1474
1475     def test_POST_DIRURL_manifest_no_ophandle(self):
1476         d = self.shouldFail2(error.Error,
1477                              "test_POST_DIRURL_manifest_no_ophandle",
1478                              "400 Bad Request",
1479                              "slow operation requires ophandle=",
1480                              self.POST, self.public_url, t="start-manifest")
1481         return d
1482
1483     def test_POST_DIRURL_manifest(self):
1484         d = defer.succeed(None)
1485         def getman(ignored, output):
1486             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1487                           followRedirect=True)
1488             d.addCallback(self.wait_for_operation, "125")
1489             d.addCallback(self.get_operation_results, "125", output)
1490             return d
1491         d.addCallback(getman, None)
1492         def _got_html(manifest):
1493             self.failUnlessIn("Manifest of SI=", manifest)
1494             self.failUnlessIn("<td>sub</td>", manifest)
1495             self.failUnlessIn(self._sub_uri, manifest)
1496             self.failUnlessIn("<td>sub/baz.txt</td>", manifest)
1497             self.failUnlessIn(FAVICON_MARKUP, manifest)
1498         d.addCallback(_got_html)
1499
1500         # both t=status and unadorned GET should be identical
1501         d.addCallback(lambda res: self.GET("/operations/125"))
1502         d.addCallback(_got_html)
1503
1504         d.addCallback(getman, "html")
1505         d.addCallback(_got_html)
1506         d.addCallback(getman, "text")
1507         def _got_text(manifest):
1508             self.failUnlessIn("\nsub " + self._sub_uri + "\n", manifest)
1509             self.failUnlessIn("\nsub/baz.txt URI:CHK:", manifest)
1510         d.addCallback(_got_text)
1511         d.addCallback(getman, "JSON")
1512         def _got_json(res):
1513             data = res["manifest"]
1514             got = {}
1515             for (path_list, cap) in data:
1516                 got[tuple(path_list)] = cap
1517             self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
1518             self.failUnlessIn((u"sub", u"baz.txt"), got)
1519             self.failUnlessIn("finished", res)
1520             self.failUnlessIn("origin", res)
1521             self.failUnlessIn("storage-index", res)
1522             self.failUnlessIn("verifycaps", res)
1523             self.failUnlessIn("stats", res)
1524         d.addCallback(_got_json)
1525         return d
1526
1527     def test_POST_DIRURL_deepsize_no_ophandle(self):
1528         d = self.shouldFail2(error.Error,
1529                              "test_POST_DIRURL_deepsize_no_ophandle",
1530                              "400 Bad Request",
1531                              "slow operation requires ophandle=",
1532                              self.POST, self.public_url, t="start-deep-size")
1533         return d
1534
1535     def test_POST_DIRURL_deepsize(self):
1536         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1537                       followRedirect=True)
1538         d.addCallback(self.wait_for_operation, "126")
1539         d.addCallback(self.get_operation_results, "126", "json")
1540         def _got_json(data):
1541             self.failUnlessReallyEqual(data["finished"], True)
1542             size = data["size"]
1543             self.failUnless(size > 1000)
1544         d.addCallback(_got_json)
1545         d.addCallback(self.get_operation_results, "126", "text")
1546         def _got_text(res):
1547             mo = re.search(r'^size: (\d+)$', res, re.M)
1548             self.failUnless(mo, res)
1549             size = int(mo.group(1))
1550             # with directories, the size varies.
1551             self.failUnless(size > 1000)
1552         d.addCallback(_got_text)
1553         return d
1554
1555     def test_POST_DIRURL_deepstats_no_ophandle(self):
1556         d = self.shouldFail2(error.Error,
1557                              "test_POST_DIRURL_deepstats_no_ophandle",
1558                              "400 Bad Request",
1559                              "slow operation requires ophandle=",
1560                              self.POST, self.public_url, t="start-deep-stats")
1561         return d
1562
1563     def test_POST_DIRURL_deepstats(self):
1564         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1565                       followRedirect=True)
1566         d.addCallback(self.wait_for_operation, "127")
1567         d.addCallback(self.get_operation_results, "127", "json")
1568         def _got_json(stats):
1569             expected = {"count-immutable-files": 3,
1570                         "count-mutable-files": 2,
1571                         "count-literal-files": 0,
1572                         "count-files": 5,
1573                         "count-directories": 3,
1574                         "size-immutable-files": 57,
1575                         "size-literal-files": 0,
1576                         #"size-directories": 1912, # varies
1577                         #"largest-directory": 1590,
1578                         "largest-directory-children": 7,
1579                         "largest-immutable-file": 19,
1580                         }
1581             for k,v in expected.iteritems():
1582                 self.failUnlessReallyEqual(stats[k], v,
1583                                            "stats[%s] was %s, not %s" %
1584                                            (k, stats[k], v))
1585             self.failUnlessReallyEqual(stats["size-files-histogram"],
1586                                        [ [11, 31, 3] ])
1587         d.addCallback(_got_json)
1588         return d
1589
1590     def test_POST_DIRURL_stream_manifest(self):
1591         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1592         def _check(res):
1593             self.failUnless(res.endswith("\n"))
1594             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1595             self.failUnlessReallyEqual(len(units), 9)
1596             self.failUnlessEqual(units[-1]["type"], "stats")
1597             first = units[0]
1598             self.failUnlessEqual(first["path"], [])
1599             self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri)
1600             self.failUnlessEqual(first["type"], "directory")
1601             baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0]
1602             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1603             self.failIfEqual(baz["storage-index"], None)
1604             self.failIfEqual(baz["verifycap"], None)
1605             self.failIfEqual(baz["repaircap"], None)
1606             # XXX: Add quux and baz to this test.
1607             return
1608         d.addCallback(_check)
1609         return d
1610
1611     def test_GET_DIRURL_uri(self):
1612         d = self.GET(self.public_url + "/foo?t=uri")
1613         def _check(res):
1614             self.failUnlessReallyEqual(to_str(res), self._foo_uri)
1615         d.addCallback(_check)
1616         return d
1617
1618     def test_GET_DIRURL_readonly_uri(self):
1619         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1620         def _check(res):
1621             self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri)
1622         d.addCallback(_check)
1623         return d
1624
1625     def test_PUT_NEWDIRURL(self):
1626         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1627         d.addCallback(lambda res:
1628                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1629         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1630         d.addCallback(self.failUnlessNodeKeysAre, [])
1631         return d
1632
1633     def test_PUT_NEWDIRURL_mdmf(self):
1634         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1635         d.addCallback(lambda res:
1636                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1637         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1638         d.addCallback(lambda node:
1639             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1640         return d
1641
1642     def test_PUT_NEWDIRURL_sdmf(self):
1643         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=sdmf",
1644                      "")
1645         d.addCallback(lambda res:
1646                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1647         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1648         d.addCallback(lambda node:
1649             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1650         return d
1651
1652     def test_PUT_NEWDIRURL_bad_format(self):
1653         return self.shouldHTTPError("PUT_NEWDIRURL_bad_format",
1654                                     400, "Bad Request", "Unknown format: foo",
1655                                     self.PUT, self.public_url +
1656                                     "/foo/newdir=?t=mkdir&format=foo", "")
1657
1658     def test_POST_NEWDIRURL(self):
1659         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1660         d.addCallback(lambda res:
1661                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1662         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1663         d.addCallback(self.failUnlessNodeKeysAre, [])
1664         return d
1665
1666     def test_POST_NEWDIRURL_mdmf(self):
1667         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1668         d.addCallback(lambda res:
1669                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1670         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1671         d.addCallback(lambda node:
1672             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1673         return d
1674
1675     def test_POST_NEWDIRURL_sdmf(self):
1676         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=sdmf", "")
1677         d.addCallback(lambda res:
1678             self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1679         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1680         d.addCallback(lambda node:
1681             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1682         return d
1683
1684     def test_POST_NEWDIRURL_bad_format(self):
1685         return self.shouldHTTPError("POST_NEWDIRURL_bad_format",
1686                                     400, "Bad Request", "Unknown format: foo",
1687                                     self.POST2, self.public_url + \
1688                                     "/foo/newdir?t=mkdir&format=foo", "")
1689
1690     def test_POST_NEWDIRURL_emptyname(self):
1691         # an empty pathname component (i.e. a double-slash) is disallowed
1692         d = self.shouldFail2(error.Error, "POST_NEWDIRURL_emptyname",
1693                              "400 Bad Request",
1694                              "The webapi does not allow empty pathname components, i.e. a double slash",
1695                              self.POST, self.public_url + "//?t=mkdir")
1696         return d
1697
1698     def _do_POST_NEWDIRURL_initial_children_test(self, version=None):
1699         (newkids, caps) = self._create_initial_children()
1700         query = "/foo/newdir?t=mkdir-with-children"
1701         if version == MDMF_VERSION:
1702             query += "&format=mdmf"
1703         elif version == SDMF_VERSION:
1704             query += "&format=sdmf"
1705         else:
1706             version = SDMF_VERSION # for later
1707         d = self.POST2(self.public_url + query,
1708                        simplejson.dumps(newkids))
1709         def _check(uri):
1710             n = self.s.create_node_from_uri(uri.strip())
1711             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1712             self.failUnlessEqual(n._node.get_version(), version)
1713             d2.addCallback(lambda ign:
1714                            self.failUnlessROChildURIIs(n, u"child-imm",
1715                                                        caps['filecap1']))
1716             d2.addCallback(lambda ign:
1717                            self.failUnlessRWChildURIIs(n, u"child-mutable",
1718                                                        caps['filecap2']))
1719             d2.addCallback(lambda ign:
1720                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1721                                                        caps['filecap3']))
1722             d2.addCallback(lambda ign:
1723                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1724                                                        caps['unknown_rocap']))
1725             d2.addCallback(lambda ign:
1726                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1727                                                        caps['unknown_rwcap']))
1728             d2.addCallback(lambda ign:
1729                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1730                                                        caps['unknown_immcap']))
1731             d2.addCallback(lambda ign:
1732                            self.failUnlessRWChildURIIs(n, u"dirchild",
1733                                                        caps['dircap']))
1734             d2.addCallback(lambda ign:
1735                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1736                                                        caps['litdircap']))
1737             d2.addCallback(lambda ign:
1738                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1739                                                        caps['emptydircap']))
1740             return d2
1741         d.addCallback(_check)
1742         d.addCallback(lambda res:
1743                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1744         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1745         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1746         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1747         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1748         return d
1749
1750     def test_POST_NEWDIRURL_initial_children(self):
1751         return self._do_POST_NEWDIRURL_initial_children_test()
1752
1753     def test_POST_NEWDIRURL_initial_children_mdmf(self):
1754         return self._do_POST_NEWDIRURL_initial_children_test(MDMF_VERSION)
1755
1756     def test_POST_NEWDIRURL_initial_children_sdmf(self):
1757         return self._do_POST_NEWDIRURL_initial_children_test(SDMF_VERSION)
1758
1759     def test_POST_NEWDIRURL_initial_children_bad_format(self):
1760         (newkids, caps) = self._create_initial_children()
1761         return self.shouldHTTPError("POST_NEWDIRURL_initial_children_bad_format",
1762                                     400, "Bad Request", "Unknown format: foo",
1763                                     self.POST2, self.public_url + \
1764                                     "/foo/newdir?t=mkdir-with-children&format=foo",
1765                                     simplejson.dumps(newkids))
1766
1767     def test_POST_NEWDIRURL_immutable(self):
1768         (newkids, caps) = self._create_immutable_children()
1769         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1770                        simplejson.dumps(newkids))
1771         def _check(uri):
1772             n = self.s.create_node_from_uri(uri.strip())
1773             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1774             d2.addCallback(lambda ign:
1775                            self.failUnlessROChildURIIs(n, u"child-imm",
1776                                                        caps['filecap1']))
1777             d2.addCallback(lambda ign:
1778                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1779                                                        caps['unknown_immcap']))
1780             d2.addCallback(lambda ign:
1781                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
1782                                                        caps['immdircap']))
1783             d2.addCallback(lambda ign:
1784                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1785                                                        caps['litdircap']))
1786             d2.addCallback(lambda ign:
1787                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1788                                                        caps['emptydircap']))
1789             return d2
1790         d.addCallback(_check)
1791         d.addCallback(lambda res:
1792                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1793         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1794         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1795         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1796         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1797         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1798         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1799         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1800         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
1801         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1802         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
1803         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1804         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
1805         d.addErrback(self.explain_web_error)
1806         return d
1807
1808     def test_POST_NEWDIRURL_immutable_bad(self):
1809         (newkids, caps) = self._create_initial_children()
1810         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
1811                              "400 Bad Request",
1812                              "needed to be immutable but was not",
1813                              self.POST2,
1814                              self.public_url + "/foo/newdir?t=mkdir-immutable",
1815                              simplejson.dumps(newkids))
1816         return d
1817
1818     def test_PUT_NEWDIRURL_exists(self):
1819         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1820         d.addCallback(lambda res:
1821                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1822         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1823         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1824         return d
1825
1826     def test_PUT_NEWDIRURL_blocked(self):
1827         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1828                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1829                              self.PUT,
1830                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1831         d.addCallback(lambda res:
1832                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1833         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1834         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1835         return d
1836
1837     def test_PUT_NEWDIRURL_mkdirs(self):
1838         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1839         d.addCallback(lambda res:
1840                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1841         d.addCallback(lambda res:
1842                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1843         d.addCallback(lambda res:
1844                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1845         d.addCallback(self.failUnlessNodeKeysAre, [])
1846         return d
1847
1848     def test_PUT_NEWDIRURL_mkdirs_mdmf(self):
1849         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=mdmf", "")
1850         d.addCallback(lambda ignored:
1851             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1852         d.addCallback(lambda ignored:
1853             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1854         d.addCallback(lambda ignored:
1855             self._foo_node.get_child_at_path(u"subdir"))
1856         def _got_subdir(subdir):
1857             # XXX: What we want?
1858             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1859             self.failUnlessNodeHasChild(subdir, u"newdir")
1860             return subdir.get_child_at_path(u"newdir")
1861         d.addCallback(_got_subdir)
1862         d.addCallback(lambda newdir:
1863             self.failUnlessEqual(newdir._node.get_version(), MDMF_VERSION))
1864         return d
1865
1866     def test_PUT_NEWDIRURL_mkdirs_sdmf(self):
1867         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=sdmf", "")
1868         d.addCallback(lambda ignored:
1869             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1870         d.addCallback(lambda ignored:
1871             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1872         d.addCallback(lambda ignored:
1873             self._foo_node.get_child_at_path(u"subdir"))
1874         def _got_subdir(subdir):
1875             # XXX: What we want?
1876             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1877             self.failUnlessNodeHasChild(subdir, u"newdir")
1878             return subdir.get_child_at_path(u"newdir")
1879         d.addCallback(_got_subdir)
1880         d.addCallback(lambda newdir:
1881             self.failUnlessEqual(newdir._node.get_version(), SDMF_VERSION))
1882         return d
1883
1884     def test_PUT_NEWDIRURL_mkdirs_bad_format(self):
1885         return self.shouldHTTPError("PUT_NEWDIRURL_mkdirs_bad_format",
1886                                     400, "Bad Request", "Unknown format: foo",
1887                                     self.PUT, self.public_url + \
1888                                     "/foo/subdir/newdir?t=mkdir&format=foo",
1889                                     "")
1890
1891     def test_DELETE_DIRURL(self):
1892         d = self.DELETE(self.public_url + "/foo")
1893         d.addCallback(lambda res:
1894                       self.failIfNodeHasChild(self.public_root, u"foo"))
1895         return d
1896
1897     def test_DELETE_DIRURL_missing(self):
1898         d = self.DELETE(self.public_url + "/foo/missing")
1899         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1900         d.addCallback(lambda res:
1901                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1902         return d
1903
1904     def test_DELETE_DIRURL_missing2(self):
1905         d = self.DELETE(self.public_url + "/missing")
1906         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1907         return d
1908
1909     def dump_root(self):
1910         print "NODEWALK"
1911         w = webish.DirnodeWalkerMixin()
1912         def visitor(childpath, childnode, metadata):
1913             print childpath
1914         d = w.walk(self.public_root, visitor)
1915         return d
1916
1917     def failUnlessNodeKeysAre(self, node, expected_keys):
1918         for k in expected_keys:
1919             assert isinstance(k, unicode)
1920         d = node.list()
1921         def _check(children):
1922             self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
1923         d.addCallback(_check)
1924         return d
1925     def failUnlessNodeHasChild(self, node, name):
1926         assert isinstance(name, unicode)
1927         d = node.list()
1928         def _check(children):
1929             self.failUnlessIn(name, children)
1930         d.addCallback(_check)
1931         return d
1932     def failIfNodeHasChild(self, node, name):
1933         assert isinstance(name, unicode)
1934         d = node.list()
1935         def _check(children):
1936             self.failIfIn(name, children)
1937         d.addCallback(_check)
1938         return d
1939
1940     def failUnlessChildContentsAre(self, node, name, expected_contents):
1941         assert isinstance(name, unicode)
1942         d = node.get_child_at_path(name)
1943         d.addCallback(lambda node: download_to_data(node))
1944         def _check(contents):
1945             self.failUnlessReallyEqual(contents, expected_contents)
1946         d.addCallback(_check)
1947         return d
1948
1949     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1950         assert isinstance(name, unicode)
1951         d = node.get_child_at_path(name)
1952         d.addCallback(lambda node: node.download_best_version())
1953         def _check(contents):
1954             self.failUnlessReallyEqual(contents, expected_contents)
1955         d.addCallback(_check)
1956         return d
1957
1958     def failUnlessRWChildURIIs(self, node, name, expected_uri):
1959         assert isinstance(name, unicode)
1960         d = node.get_child_at_path(name)
1961         def _check(child):
1962             self.failUnless(child.is_unknown() or not child.is_readonly())
1963             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1964             self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
1965             expected_ro_uri = self._make_readonly(expected_uri)
1966             if expected_ro_uri:
1967                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1968         d.addCallback(_check)
1969         return d
1970
1971     def failUnlessROChildURIIs(self, node, name, expected_uri):
1972         assert isinstance(name, unicode)
1973         d = node.get_child_at_path(name)
1974         def _check(child):
1975             self.failUnless(child.is_unknown() or child.is_readonly())
1976             self.failUnlessReallyEqual(child.get_write_uri(), None)
1977             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1978             self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
1979         d.addCallback(_check)
1980         return d
1981
1982     def failUnlessURIMatchesRWChild(self, got_uri, node, name):
1983         assert isinstance(name, unicode)
1984         d = node.get_child_at_path(name)
1985         def _check(child):
1986             self.failUnless(child.is_unknown() or not child.is_readonly())
1987             self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
1988             self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
1989             expected_ro_uri = self._make_readonly(got_uri)
1990             if expected_ro_uri:
1991                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1992         d.addCallback(_check)
1993         return d
1994
1995     def failUnlessURIMatchesROChild(self, got_uri, node, name):
1996         assert isinstance(name, unicode)
1997         d = node.get_child_at_path(name)
1998         def _check(child):
1999             self.failUnless(child.is_unknown() or child.is_readonly())
2000             self.failUnlessReallyEqual(child.get_write_uri(), None)
2001             self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
2002             self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
2003         d.addCallback(_check)
2004         return d
2005
2006     def failUnlessCHKURIHasContents(self, got_uri, contents):
2007         self.failUnless(self.get_all_contents()[got_uri] == contents)
2008
2009     def test_POST_upload(self):
2010         d = self.POST(self.public_url + "/foo", t="upload",
2011                       file=("new.txt", self.NEWFILE_CONTENTS))
2012         fn = self._foo_node
2013         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2014         d.addCallback(lambda res:
2015                       self.failUnlessChildContentsAre(fn, u"new.txt",
2016                                                       self.NEWFILE_CONTENTS))
2017         return d
2018
2019     def test_POST_upload_unicode(self):
2020         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2021         d = self.POST(self.public_url + "/foo", t="upload",
2022                       file=(filename, self.NEWFILE_CONTENTS))
2023         fn = self._foo_node
2024         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2025         d.addCallback(lambda res:
2026                       self.failUnlessChildContentsAre(fn, filename,
2027                                                       self.NEWFILE_CONTENTS))
2028         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2029         d.addCallback(lambda res: self.GET(target_url))
2030         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2031                                                                   self.NEWFILE_CONTENTS,
2032                                                                   contents))
2033         return d
2034
2035     def test_POST_upload_unicode_named(self):
2036         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2037         d = self.POST(self.public_url + "/foo", t="upload",
2038                       name=filename,
2039                       file=("overridden", self.NEWFILE_CONTENTS))
2040         fn = self._foo_node
2041         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2042         d.addCallback(lambda res:
2043                       self.failUnlessChildContentsAre(fn, filename,
2044                                                       self.NEWFILE_CONTENTS))
2045         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2046         d.addCallback(lambda res: self.GET(target_url))
2047         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2048                                                                   self.NEWFILE_CONTENTS,
2049                                                                   contents))
2050         return d
2051
2052     def test_POST_upload_no_link(self):
2053         d = self.POST("/uri", t="upload",
2054                       file=("new.txt", self.NEWFILE_CONTENTS))
2055         def _check_upload_results(page):
2056             # this should be a page which describes the results of the upload
2057             # that just finished.
2058             self.failUnlessIn("Upload Results:", page)
2059             self.failUnlessIn("URI:", page)
2060             uri_re = re.compile("URI: <tt><span>(.*)</span>")
2061             mo = uri_re.search(page)
2062             self.failUnless(mo, page)
2063             new_uri = mo.group(1)
2064             return new_uri
2065         d.addCallback(_check_upload_results)
2066         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
2067         return d
2068
2069     def test_POST_upload_no_link_whendone(self):
2070         d = self.POST("/uri", t="upload", when_done="/",
2071                       file=("new.txt", self.NEWFILE_CONTENTS))
2072         d.addBoth(self.shouldRedirect, "/")
2073         return d
2074
2075     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
2076         d = defer.maybeDeferred(callable, *args, **kwargs)
2077         def done(res):
2078             if isinstance(res, failure.Failure):
2079                 res.trap(error.PageRedirect)
2080                 statuscode = res.value.status
2081                 target = res.value.location
2082                 return checker(statuscode, target)
2083             self.fail("%s: callable was supposed to redirect, not return '%s'"
2084                       % (which, res))
2085         d.addBoth(done)
2086         return d
2087
2088     def test_POST_upload_no_link_whendone_results(self):
2089         def check(statuscode, target):
2090             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2091             self.failUnless(target.startswith(self.webish_url), target)
2092             return client.getPage(target, method="GET")
2093         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
2094                                  check,
2095                                  self.POST, "/uri", t="upload",
2096                                  when_done="/uri/%(uri)s",
2097                                  file=("new.txt", self.NEWFILE_CONTENTS))
2098         d.addCallback(lambda res:
2099                       self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
2100         return d
2101
2102     def test_POST_upload_no_link_mutable(self):
2103         d = self.POST("/uri", t="upload", mutable="true",
2104                       file=("new.txt", self.NEWFILE_CONTENTS))
2105         def _check(filecap):
2106             filecap = filecap.strip()
2107             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2108             self.filecap = filecap
2109             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2110             self.failUnlessIn(u.get_storage_index(), self.get_all_contents())
2111             n = self.s.create_node_from_uri(filecap)
2112             return n.download_best_version()
2113         d.addCallback(_check)
2114         def _check2(data):
2115             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2116             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2117         d.addCallback(_check2)
2118         def _check3(data):
2119             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2120             return self.GET("/file/%s" % urllib.quote(self.filecap))
2121         d.addCallback(_check3)
2122         def _check4(data):
2123             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2124         d.addCallback(_check4)
2125         return d
2126
2127     def test_POST_upload_no_link_mutable_toobig(self):
2128         # The SDMF size limit is no longer in place, so we should be
2129         # able to upload mutable files that are as large as we want them
2130         # to be.
2131         d = self.POST("/uri", t="upload", mutable="true",
2132                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2133         return d
2134
2135
2136     def test_POST_upload_format_unlinked(self):
2137         def _check_upload_unlinked(ign, format, uri_prefix):
2138             filename = format + ".txt"
2139             d = self.POST("/uri?t=upload&format=" + format,
2140                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2141             def _got_results(results):
2142                 if format.upper() in ("SDMF", "MDMF"):
2143                     # webapi.rst says this returns a filecap
2144                     filecap = results
2145                 else:
2146                     # for immutable, it returns an "upload results page", and
2147                     # the filecap is buried inside
2148                     line = [l for l in results.split("\n") if "URI: " in l][0]
2149                     mo = re.search(r'<span>([^<]+)</span>', line)
2150                     filecap = mo.group(1)
2151                 self.failUnless(filecap.startswith(uri_prefix),
2152                                 (uri_prefix, filecap))
2153                 return self.GET("/uri/%s?t=json" % filecap)
2154             d.addCallback(_got_results)
2155             def _got_json(json):
2156                 data = simplejson.loads(json)
2157                 data = data[1]
2158                 self.failUnlessIn("format", data)
2159                 self.failUnlessEqual(data["format"], format.upper())
2160             d.addCallback(_got_json)
2161             return d
2162         d = defer.succeed(None)
2163         d.addCallback(_check_upload_unlinked, "chk", "URI:CHK")
2164         d.addCallback(_check_upload_unlinked, "CHK", "URI:CHK")
2165         d.addCallback(_check_upload_unlinked, "sdmf", "URI:SSK")
2166         d.addCallback(_check_upload_unlinked, "mdmf", "URI:MDMF")
2167         return d
2168
2169     def test_POST_upload_bad_format_unlinked(self):
2170         return self.shouldHTTPError("POST_upload_bad_format_unlinked",
2171                                     400, "Bad Request", "Unknown format: foo",
2172                                     self.POST,
2173                                     "/uri?t=upload&format=foo",
2174                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2175
2176     def test_POST_upload_format(self):
2177         def _check_upload(ign, format, uri_prefix, fn=None):
2178             filename = format + ".txt"
2179             d = self.POST(self.public_url +
2180                           "/foo?t=upload&format=" + format,
2181                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2182             def _got_filecap(filecap):
2183                 if fn is not None:
2184                     filenameu = unicode(filename)
2185                     self.failUnlessURIMatchesRWChild(filecap, fn, filenameu)
2186                 self.failUnless(filecap.startswith(uri_prefix))
2187                 return self.GET(self.public_url + "/foo/%s?t=json" % filename)
2188             d.addCallback(_got_filecap)
2189             def _got_json(json):
2190                 data = simplejson.loads(json)
2191                 data = data[1]
2192                 self.failUnlessIn("format", data)
2193                 self.failUnlessEqual(data["format"], format.upper())
2194             d.addCallback(_got_json)
2195             return d
2196
2197         d = defer.succeed(None)
2198         d.addCallback(_check_upload, "chk", "URI:CHK")
2199         d.addCallback(_check_upload, "sdmf", "URI:SSK", self._foo_node)
2200         d.addCallback(_check_upload, "mdmf", "URI:MDMF")
2201         d.addCallback(_check_upload, "MDMF", "URI:MDMF")
2202         return d
2203
2204     def test_POST_upload_bad_format(self):
2205         return self.shouldHTTPError("POST_upload_bad_format",
2206                                     400, "Bad Request", "Unknown format: foo",
2207                                     self.POST, self.public_url + \
2208                                     "/foo?t=upload&format=foo",
2209                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2210
2211     def test_POST_upload_mutable(self):
2212         # this creates a mutable file
2213         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
2214                       file=("new.txt", self.NEWFILE_CONTENTS))
2215         fn = self._foo_node
2216         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2217         d.addCallback(lambda res:
2218                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2219                                                              self.NEWFILE_CONTENTS))
2220         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2221         def _got(newnode):
2222             self.failUnless(IMutableFileNode.providedBy(newnode))
2223             self.failUnless(newnode.is_mutable())
2224             self.failIf(newnode.is_readonly())
2225             self._mutable_node = newnode
2226             self._mutable_uri = newnode.get_uri()
2227         d.addCallback(_got)
2228
2229         # now upload it again and make sure that the URI doesn't change
2230         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
2231         d.addCallback(lambda res:
2232                       self.POST(self.public_url + "/foo", t="upload",
2233                                 mutable="true",
2234                                 file=("new.txt", NEWER_CONTENTS)))
2235         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2236         d.addCallback(lambda res:
2237                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2238                                                              NEWER_CONTENTS))
2239         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2240         def _got2(newnode):
2241             self.failUnless(IMutableFileNode.providedBy(newnode))
2242             self.failUnless(newnode.is_mutable())
2243             self.failIf(newnode.is_readonly())
2244             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2245         d.addCallback(_got2)
2246
2247         # upload a second time, using PUT instead of POST
2248         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
2249         d.addCallback(lambda res:
2250                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
2251         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2252         d.addCallback(lambda res:
2253                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2254                                                              NEW2_CONTENTS))
2255
2256         # finally list the directory, since mutable files are displayed
2257         # slightly differently
2258
2259         d.addCallback(lambda res:
2260                       self.GET(self.public_url + "/foo/",
2261                                followRedirect=True))
2262         def _check_page(res):
2263             # TODO: assert more about the contents
2264             self.failUnlessIn("SSK", res)
2265             return res
2266         d.addCallback(_check_page)
2267
2268         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2269         def _got3(newnode):
2270             self.failUnless(IMutableFileNode.providedBy(newnode))
2271             self.failUnless(newnode.is_mutable())
2272             self.failIf(newnode.is_readonly())
2273             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2274         d.addCallback(_got3)
2275
2276         # look at the JSON form of the enclosing directory
2277         d.addCallback(lambda res:
2278                       self.GET(self.public_url + "/foo/?t=json",
2279                                followRedirect=True))
2280         def _check_page_json(res):
2281             parsed = simplejson.loads(res)
2282             self.failUnlessEqual(parsed[0], "dirnode")
2283             children = dict( [(unicode(name),value)
2284                               for (name,value)
2285                               in parsed[1]["children"].iteritems()] )
2286             self.failUnlessIn(u"new.txt", children)
2287             new_json = children[u"new.txt"]
2288             self.failUnlessEqual(new_json[0], "filenode")
2289             self.failUnless(new_json[1]["mutable"])
2290             self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri)
2291             ro_uri = self._mutable_node.get_readonly().to_string()
2292             self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri)
2293         d.addCallback(_check_page_json)
2294
2295         # and the JSON form of the file
2296         d.addCallback(lambda res:
2297                       self.GET(self.public_url + "/foo/new.txt?t=json"))
2298         def _check_file_json(res):
2299             parsed = simplejson.loads(res)
2300             self.failUnlessEqual(parsed[0], "filenode")
2301             self.failUnless(parsed[1]["mutable"])
2302             self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri)
2303             ro_uri = self._mutable_node.get_readonly().to_string()
2304             self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri)
2305         d.addCallback(_check_file_json)
2306
2307         # and look at t=uri and t=readonly-uri
2308         d.addCallback(lambda res:
2309                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
2310         d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
2311         d.addCallback(lambda res:
2312                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
2313         def _check_ro_uri(res):
2314             ro_uri = self._mutable_node.get_readonly().to_string()
2315             self.failUnlessReallyEqual(res, ro_uri)
2316         d.addCallback(_check_ro_uri)
2317
2318         # make sure we can get to it from /uri/URI
2319         d.addCallback(lambda res:
2320                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
2321         d.addCallback(lambda res:
2322                       self.failUnlessReallyEqual(res, NEW2_CONTENTS))
2323
2324         # and that HEAD computes the size correctly
2325         d.addCallback(lambda res:
2326                       self.HEAD(self.public_url + "/foo/new.txt",
2327                                 return_response=True))
2328         def _got_headers((res, status, headers)):
2329             self.failUnlessReallyEqual(res, "")
2330             self.failUnlessReallyEqual(headers["content-length"][0],
2331                                        str(len(NEW2_CONTENTS)))
2332             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
2333         d.addCallback(_got_headers)
2334
2335         # make sure that outdated size limits aren't enforced anymore.
2336         d.addCallback(lambda ignored:
2337             self.POST(self.public_url + "/foo", t="upload",
2338                       mutable="true",
2339                       file=("new.txt",
2340                             "b" * (self.s.MUTABLE_SIZELIMIT+1))))
2341         d.addErrback(self.dump_error)
2342         return d
2343
2344     def test_POST_upload_mutable_toobig(self):
2345         # SDMF had a size limti that was removed a while ago. MDMF has
2346         # never had a size limit. Test to make sure that we do not
2347         # encounter errors when trying to upload large mutable files,
2348         # since there should be no coded prohibitions regarding large
2349         # mutable files.
2350         d = self.POST(self.public_url + "/foo",
2351                       t="upload", mutable="true",
2352                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2353         return d
2354
2355     def dump_error(self, f):
2356         # if the web server returns an error code (like 400 Bad Request),
2357         # web.client.getPage puts the HTTP response body into the .response
2358         # attribute of the exception object that it gives back. It does not
2359         # appear in the Failure's repr(), so the ERROR that trial displays
2360         # will be rather terse and unhelpful. addErrback this method to the
2361         # end of your chain to get more information out of these errors.
2362         if f.check(error.Error):
2363             print "web.error.Error:"
2364             print f
2365             print f.value.response
2366         return f
2367
2368     def test_POST_upload_replace(self):
2369         d = self.POST(self.public_url + "/foo", t="upload",
2370                       file=("bar.txt", self.NEWFILE_CONTENTS))
2371         fn = self._foo_node
2372         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
2373         d.addCallback(lambda res:
2374                       self.failUnlessChildContentsAre(fn, u"bar.txt",
2375                                                       self.NEWFILE_CONTENTS))
2376         return d
2377
2378     def test_POST_upload_no_replace_ok(self):
2379         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2380                       file=("new.txt", self.NEWFILE_CONTENTS))
2381         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
2382         d.addCallback(lambda res: self.failUnlessReallyEqual(res,
2383                                                              self.NEWFILE_CONTENTS))
2384         return d
2385
2386     def test_POST_upload_no_replace_queryarg(self):
2387         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2388                       file=("bar.txt", self.NEWFILE_CONTENTS))
2389         d.addBoth(self.shouldFail, error.Error,
2390                   "POST_upload_no_replace_queryarg",
2391                   "409 Conflict",
2392                   "There was already a child by that name, and you asked me "
2393                   "to not replace it")
2394         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2395         d.addCallback(self.failUnlessIsBarDotTxt)
2396         return d
2397
2398     def test_POST_upload_no_replace_field(self):
2399         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
2400                       file=("bar.txt", self.NEWFILE_CONTENTS))
2401         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
2402                   "409 Conflict",
2403                   "There was already a child by that name, and you asked me "
2404                   "to not replace it")
2405         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2406         d.addCallback(self.failUnlessIsBarDotTxt)
2407         return d
2408
2409     def test_POST_upload_whendone(self):
2410         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
2411                       file=("new.txt", self.NEWFILE_CONTENTS))
2412         d.addBoth(self.shouldRedirect, "/THERE")
2413         fn = self._foo_node
2414         d.addCallback(lambda res:
2415                       self.failUnlessChildContentsAre(fn, u"new.txt",
2416                                                       self.NEWFILE_CONTENTS))
2417         return d
2418
2419     def test_POST_upload_named(self):
2420         fn = self._foo_node
2421         d = self.POST(self.public_url + "/foo", t="upload",
2422                       name="new.txt", file=self.NEWFILE_CONTENTS)
2423         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2424         d.addCallback(lambda res:
2425                       self.failUnlessChildContentsAre(fn, u"new.txt",
2426                                                       self.NEWFILE_CONTENTS))
2427         return d
2428
2429     def test_POST_upload_named_badfilename(self):
2430         d = self.POST(self.public_url + "/foo", t="upload",
2431                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
2432         d.addBoth(self.shouldFail, error.Error,
2433                   "test_POST_upload_named_badfilename",
2434                   "400 Bad Request",
2435                   "name= may not contain a slash",
2436                   )
2437         # make sure that nothing was added
2438         d.addCallback(lambda res:
2439                       self.failUnlessNodeKeysAre(self._foo_node,
2440                                                  [u"bar.txt", u"baz.txt", u"blockingfile",
2441                                                   u"empty", u"n\u00fc.txt", u"quux.txt",
2442                                                   u"sub"]))
2443         return d
2444
2445     def test_POST_FILEURL_check(self):
2446         bar_url = self.public_url + "/foo/bar.txt"
2447         d = self.POST(bar_url, t="check")
2448         def _check(res):
2449             self.failUnlessIn("Healthy :", res)
2450         d.addCallback(_check)
2451         redir_url = "http://allmydata.org/TARGET"
2452         def _check2(statuscode, target):
2453             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2454             self.failUnlessReallyEqual(target, redir_url)
2455         d.addCallback(lambda res:
2456                       self.shouldRedirect2("test_POST_FILEURL_check",
2457                                            _check2,
2458                                            self.POST, bar_url,
2459                                            t="check",
2460                                            when_done=redir_url))
2461         d.addCallback(lambda res:
2462                       self.POST(bar_url, t="check", return_to=redir_url))
2463         def _check3(res):
2464             self.failUnlessIn("Healthy :", res)
2465             self.failUnlessIn("Return to file", res)
2466             self.failUnlessIn(redir_url, res)
2467         d.addCallback(_check3)
2468
2469         d.addCallback(lambda res:
2470                       self.POST(bar_url, t="check", output="JSON"))
2471         def _check_json(res):
2472             data = simplejson.loads(res)
2473             self.failUnlessIn("storage-index", data)
2474             self.failUnless(data["results"]["healthy"])
2475         d.addCallback(_check_json)
2476
2477         return d
2478
2479     def test_POST_FILEURL_check_and_repair(self):
2480         bar_url = self.public_url + "/foo/bar.txt"
2481         d = self.POST(bar_url, t="check", repair="true")
2482         def _check(res):
2483             self.failUnlessIn("Healthy :", res)
2484         d.addCallback(_check)
2485         redir_url = "http://allmydata.org/TARGET"
2486         def _check2(statuscode, target):
2487             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2488             self.failUnlessReallyEqual(target, redir_url)
2489         d.addCallback(lambda res:
2490                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
2491                                            _check2,
2492                                            self.POST, bar_url,
2493                                            t="check", repair="true",
2494                                            when_done=redir_url))
2495         d.addCallback(lambda res:
2496                       self.POST(bar_url, t="check", return_to=redir_url))
2497         def _check3(res):
2498             self.failUnlessIn("Healthy :", res)
2499             self.failUnlessIn("Return to file", res)
2500             self.failUnlessIn(redir_url, res)
2501         d.addCallback(_check3)
2502         return d
2503
2504     def test_POST_DIRURL_check(self):
2505         foo_url = self.public_url + "/foo/"
2506         d = self.POST(foo_url, t="check")
2507         def _check(res):
2508             self.failUnlessIn("Healthy :", res)
2509         d.addCallback(_check)
2510         redir_url = "http://allmydata.org/TARGET"
2511         def _check2(statuscode, target):
2512             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2513             self.failUnlessReallyEqual(target, redir_url)
2514         d.addCallback(lambda res:
2515                       self.shouldRedirect2("test_POST_DIRURL_check",
2516                                            _check2,
2517                                            self.POST, foo_url,
2518                                            t="check",
2519                                            when_done=redir_url))
2520         d.addCallback(lambda res:
2521                       self.POST(foo_url, t="check", return_to=redir_url))
2522         def _check3(res):
2523             self.failUnlessIn("Healthy :", res)
2524             self.failUnlessIn("Return to file/directory", res)
2525             self.failUnlessIn(redir_url, res)
2526         d.addCallback(_check3)
2527
2528         d.addCallback(lambda res:
2529                       self.POST(foo_url, t="check", output="JSON"))
2530         def _check_json(res):
2531             data = simplejson.loads(res)
2532             self.failUnlessIn("storage-index", data)
2533             self.failUnless(data["results"]["healthy"])
2534         d.addCallback(_check_json)
2535
2536         return d
2537
2538     def test_POST_DIRURL_check_and_repair(self):
2539         foo_url = self.public_url + "/foo/"
2540         d = self.POST(foo_url, t="check", repair="true")
2541         def _check(res):
2542             self.failUnlessIn("Healthy :", res)
2543         d.addCallback(_check)
2544         redir_url = "http://allmydata.org/TARGET"
2545         def _check2(statuscode, target):
2546             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2547             self.failUnlessReallyEqual(target, redir_url)
2548         d.addCallback(lambda res:
2549                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2550                                            _check2,
2551                                            self.POST, foo_url,
2552                                            t="check", repair="true",
2553                                            when_done=redir_url))
2554         d.addCallback(lambda res:
2555                       self.POST(foo_url, t="check", return_to=redir_url))
2556         def _check3(res):
2557             self.failUnlessIn("Healthy :", res)
2558             self.failUnlessIn("Return to file/directory", res)
2559             self.failUnlessIn(redir_url, res)
2560         d.addCallback(_check3)
2561         return d
2562
2563     def test_POST_FILEURL_mdmf_check(self):
2564         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2565         d = self.POST(quux_url, t="check")
2566         def _check(res):
2567             self.failUnlessIn("Healthy", res)
2568         d.addCallback(_check)
2569         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2570         d.addCallback(lambda ignored:
2571                       self.POST(quux_extension_url, t="check"))
2572         d.addCallback(_check)
2573         return d
2574
2575     def test_POST_FILEURL_mdmf_check_and_repair(self):
2576         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2577         d = self.POST(quux_url, t="check", repair="true")
2578         def _check(res):
2579             self.failUnlessIn("Healthy", res)
2580         d.addCallback(_check)
2581         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2582         d.addCallback(lambda ignored:
2583                       self.POST(quux_extension_url, t="check", repair="true"))
2584         d.addCallback(_check)
2585         return d
2586
2587     def wait_for_operation(self, ignored, ophandle):
2588         url = "/operations/" + ophandle
2589         url += "?t=status&output=JSON"
2590         d = self.GET(url)
2591         def _got(res):
2592             data = simplejson.loads(res)
2593             if not data["finished"]:
2594                 d = self.stall(delay=1.0)
2595                 d.addCallback(self.wait_for_operation, ophandle)
2596                 return d
2597             return data
2598         d.addCallback(_got)
2599         return d
2600
2601     def get_operation_results(self, ignored, ophandle, output=None):
2602         url = "/operations/" + ophandle
2603         url += "?t=status"
2604         if output:
2605             url += "&output=" + output
2606         d = self.GET(url)
2607         def _got(res):
2608             if output and output.lower() == "json":
2609                 return simplejson.loads(res)
2610             return res
2611         d.addCallback(_got)
2612         return d
2613
2614     def test_POST_DIRURL_deepcheck_no_ophandle(self):
2615         d = self.shouldFail2(error.Error,
2616                              "test_POST_DIRURL_deepcheck_no_ophandle",
2617                              "400 Bad Request",
2618                              "slow operation requires ophandle=",
2619                              self.POST, self.public_url, t="start-deep-check")
2620         return d
2621
2622     def test_POST_DIRURL_deepcheck(self):
2623         def _check_redirect(statuscode, target):
2624             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2625             self.failUnless(target.endswith("/operations/123"))
2626         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2627                                  self.POST, self.public_url,
2628                                  t="start-deep-check", ophandle="123")
2629         d.addCallback(self.wait_for_operation, "123")
2630         def _check_json(data):
2631             self.failUnlessReallyEqual(data["finished"], True)
2632             self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2633             self.failUnlessReallyEqual(data["count-objects-healthy"], 10)
2634         d.addCallback(_check_json)
2635         d.addCallback(self.get_operation_results, "123", "html")
2636         def _check_html(res):
2637             self.failUnlessIn("Objects Checked: <span>10</span>", res)
2638             self.failUnlessIn("Objects Healthy: <span>10</span>", res)
2639             self.failUnlessIn(FAVICON_MARKUP, res)
2640         d.addCallback(_check_html)
2641
2642         d.addCallback(lambda res:
2643                       self.GET("/operations/123/"))
2644         d.addCallback(_check_html) # should be the same as without the slash
2645
2646         d.addCallback(lambda res:
2647                       self.shouldFail2(error.Error, "one", "404 Not Found",
2648                                        "No detailed results for SI bogus",
2649                                        self.GET, "/operations/123/bogus"))
2650
2651         foo_si = self._foo_node.get_storage_index()
2652         foo_si_s = base32.b2a(foo_si)
2653         d.addCallback(lambda res:
2654                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2655         def _check_foo_json(res):
2656             data = simplejson.loads(res)
2657             self.failUnlessEqual(data["storage-index"], foo_si_s)
2658             self.failUnless(data["results"]["healthy"])
2659         d.addCallback(_check_foo_json)
2660         return d
2661
2662     def test_POST_DIRURL_deepcheck_and_repair(self):
2663         d = self.POST(self.public_url, t="start-deep-check", repair="true",
2664                       ophandle="124", output="json", followRedirect=True)
2665         d.addCallback(self.wait_for_operation, "124")
2666         def _check_json(data):
2667             self.failUnlessReallyEqual(data["finished"], True)
2668             self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2669             self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 10)
2670             self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2671             self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2672             self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2673             self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2674             self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2675             self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 10)
2676             self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2677             self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2678         d.addCallback(_check_json)
2679         d.addCallback(self.get_operation_results, "124", "html")
2680         def _check_html(res):
2681             self.failUnlessIn("Objects Checked: <span>10</span>", res)
2682
2683             self.failUnlessIn("Objects Healthy (before repair): <span>10</span>", res)
2684             self.failUnlessIn("Objects Unhealthy (before repair): <span>0</span>", res)
2685             self.failUnlessIn("Corrupt Shares (before repair): <span>0</span>", res)
2686
2687             self.failUnlessIn("Repairs Attempted: <span>0</span>", res)
2688             self.failUnlessIn("Repairs Successful: <span>0</span>", res)
2689             self.failUnlessIn("Repairs Unsuccessful: <span>0</span>", res)
2690
2691             self.failUnlessIn("Objects Healthy (after repair): <span>10</span>", res)
2692             self.failUnlessIn("Objects Unhealthy (after repair): <span>0</span>", res)
2693             self.failUnlessIn("Corrupt Shares (after repair): <span>0</span>", res)
2694
2695             self.failUnlessIn(FAVICON_MARKUP, res)
2696         d.addCallback(_check_html)
2697         return d
2698
2699     def test_POST_FILEURL_bad_t(self):
2700         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2701                              "POST to file: bad t=bogus",
2702                              self.POST, self.public_url + "/foo/bar.txt",
2703                              t="bogus")
2704         return d
2705
2706     def test_POST_mkdir(self): # return value?
2707         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2708         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2709         d.addCallback(self.failUnlessNodeKeysAre, [])
2710         return d
2711
2712     def test_POST_mkdir_mdmf(self):
2713         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=mdmf")
2714         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2715         d.addCallback(lambda node:
2716             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2717         return d
2718
2719     def test_POST_mkdir_sdmf(self):
2720         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=sdmf")
2721         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2722         d.addCallback(lambda node:
2723             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2724         return d
2725
2726     def test_POST_mkdir_bad_format(self):
2727         return self.shouldHTTPError("POST_mkdir_bad_format",
2728                                     400, "Bad Request", "Unknown format: foo",
2729                                     self.POST, self.public_url +
2730                                     "/foo?t=mkdir&name=newdir&format=foo")
2731
2732     def test_POST_mkdir_initial_children(self):
2733         (newkids, caps) = self._create_initial_children()
2734         d = self.POST2(self.public_url +
2735                        "/foo?t=mkdir-with-children&name=newdir",
2736                        simplejson.dumps(newkids))
2737         d.addCallback(lambda res:
2738                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2739         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2740         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2741         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2742         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2743         return d
2744
2745     def test_POST_mkdir_initial_children_mdmf(self):
2746         (newkids, caps) = self._create_initial_children()
2747         d = self.POST2(self.public_url +
2748                        "/foo?t=mkdir-with-children&name=newdir&format=mdmf",
2749                        simplejson.dumps(newkids))
2750         d.addCallback(lambda res:
2751                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2752         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2753         d.addCallback(lambda node:
2754             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2755         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2756         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2757                        caps['filecap1'])
2758         return d
2759
2760     # XXX: Duplication.
2761     def test_POST_mkdir_initial_children_sdmf(self):
2762         (newkids, caps) = self._create_initial_children()
2763         d = self.POST2(self.public_url +
2764                        "/foo?t=mkdir-with-children&name=newdir&format=sdmf",
2765                        simplejson.dumps(newkids))
2766         d.addCallback(lambda res:
2767                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2768         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2769         d.addCallback(lambda node:
2770             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2771         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2772         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2773                        caps['filecap1'])
2774         return d
2775
2776     def test_POST_mkdir_initial_children_bad_format(self):
2777         (newkids, caps) = self._create_initial_children()
2778         return self.shouldHTTPError("POST_mkdir_initial_children_bad_format",
2779                                     400, "Bad Request", "Unknown format: foo",
2780                                     self.POST, self.public_url + \
2781                                     "/foo?t=mkdir-with-children&name=newdir&format=foo",
2782                                     simplejson.dumps(newkids))
2783
2784     def test_POST_mkdir_immutable(self):
2785         (newkids, caps) = self._create_immutable_children()
2786         d = self.POST2(self.public_url +
2787                        "/foo?t=mkdir-immutable&name=newdir",
2788                        simplejson.dumps(newkids))
2789         d.addCallback(lambda res:
2790                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2791         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2792         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2793         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2794         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2795         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2796         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2797         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2798         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2799         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2800         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2801         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2802         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2803         return d
2804
2805     def test_POST_mkdir_immutable_bad(self):
2806         (newkids, caps) = self._create_initial_children()
2807         d = self.shouldFail2(error.Error, "POST_mkdir_immutable_bad",
2808                              "400 Bad Request",
2809                              "needed to be immutable but was not",
2810                              self.POST2,
2811                              self.public_url +
2812                              "/foo?t=mkdir-immutable&name=newdir",
2813                              simplejson.dumps(newkids))
2814         return d
2815
2816     def test_POST_mkdir_2(self):
2817         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
2818         d.addCallback(lambda res:
2819                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2820         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2821         d.addCallback(self.failUnlessNodeKeysAre, [])
2822         return d
2823
2824     def test_POST_mkdirs_2(self):
2825         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
2826         d.addCallback(lambda res:
2827                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
2828         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
2829         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
2830         d.addCallback(self.failUnlessNodeKeysAre, [])
2831         return d
2832
2833     def test_POST_mkdir_no_parentdir_noredirect(self):
2834         d = self.POST("/uri?t=mkdir")
2835         def _after_mkdir(res):
2836             uri.DirectoryURI.init_from_string(res)
2837         d.addCallback(_after_mkdir)
2838         return d
2839
2840     def test_POST_mkdir_no_parentdir_noredirect_mdmf(self):
2841         d = self.POST("/uri?t=mkdir&format=mdmf")
2842         def _after_mkdir(res):
2843             u = uri.from_string(res)
2844             # Check that this is an MDMF writecap
2845             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
2846         d.addCallback(_after_mkdir)
2847         return d
2848
2849     def test_POST_mkdir_no_parentdir_noredirect_sdmf(self):
2850         d = self.POST("/uri?t=mkdir&format=sdmf")
2851         def _after_mkdir(res):
2852             u = uri.from_string(res)
2853             self.failUnlessIsInstance(u, uri.DirectoryURI)
2854         d.addCallback(_after_mkdir)
2855         return d
2856
2857     def test_POST_mkdir_no_parentdir_noredirect_bad_format(self):
2858         return self.shouldHTTPError("POST_mkdir_no_parentdir_noredirect_bad_format",
2859                                     400, "Bad Request", "Unknown format: foo",
2860                                     self.POST, self.public_url +
2861                                     "/uri?t=mkdir&format=foo")
2862
2863     def test_POST_mkdir_no_parentdir_noredirect2(self):
2864         # make sure form-based arguments (as on the welcome page) still work
2865         d = self.POST("/uri", t="mkdir")
2866         def _after_mkdir(res):
2867             uri.DirectoryURI.init_from_string(res)
2868         d.addCallback(_after_mkdir)
2869         d.addErrback(self.explain_web_error)
2870         return d
2871
2872     def test_POST_mkdir_no_parentdir_redirect(self):
2873         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
2874         d.addBoth(self.shouldRedirect, None, statuscode='303')
2875         def _check_target(target):
2876             target = urllib.unquote(target)
2877             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2878         d.addCallback(_check_target)
2879         return d
2880
2881     def test_POST_mkdir_no_parentdir_redirect2(self):
2882         d = self.POST("/uri", t="mkdir", redirect_to_result="true")
2883         d.addBoth(self.shouldRedirect, None, statuscode='303')
2884         def _check_target(target):
2885             target = urllib.unquote(target)
2886             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2887         d.addCallback(_check_target)
2888         d.addErrback(self.explain_web_error)
2889         return d
2890
2891     def _make_readonly(self, u):
2892         ro_uri = uri.from_string(u).get_readonly()
2893         if ro_uri is None:
2894             return None
2895         return ro_uri.to_string()
2896
2897     def _create_initial_children(self):
2898         contents, n, filecap1 = self.makefile(12)
2899         md1 = {"metakey1": "metavalue1"}
2900         filecap2 = make_mutable_file_uri()
2901         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
2902         filecap3 = node3.get_readonly_uri()
2903         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
2904         dircap = DirectoryNode(node4, None, None).get_uri()
2905         mdmfcap = make_mutable_file_uri(mdmf=True)
2906         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2907         emptydircap = "URI:DIR2-LIT:"
2908         newkids = {u"child-imm":        ["filenode", {"rw_uri": filecap1,
2909                                                       "ro_uri": self._make_readonly(filecap1),
2910                                                       "metadata": md1, }],
2911                    u"child-mutable":    ["filenode", {"rw_uri": filecap2,
2912                                                       "ro_uri": self._make_readonly(filecap2)}],
2913                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
2914                    u"unknownchild-rw":  ["unknown",  {"rw_uri": unknown_rwcap,
2915                                                       "ro_uri": unknown_rocap}],
2916                    u"unknownchild-ro":  ["unknown",  {"ro_uri": unknown_rocap}],
2917                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2918                    u"dirchild":         ["dirnode",  {"rw_uri": dircap,
2919                                                       "ro_uri": self._make_readonly(dircap)}],
2920                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2921                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2922                    u"child-mutable-mdmf": ["filenode", {"rw_uri": mdmfcap,
2923                                                         "ro_uri": self._make_readonly(mdmfcap)}],
2924                    }
2925         return newkids, {'filecap1': filecap1,
2926                          'filecap2': filecap2,
2927                          'filecap3': filecap3,
2928                          'unknown_rwcap': unknown_rwcap,
2929                          'unknown_rocap': unknown_rocap,
2930                          'unknown_immcap': unknown_immcap,
2931                          'dircap': dircap,
2932                          'litdircap': litdircap,
2933                          'emptydircap': emptydircap,
2934                          'mdmfcap': mdmfcap}
2935
2936     def _create_immutable_children(self):
2937         contents, n, filecap1 = self.makefile(12)
2938         md1 = {"metakey1": "metavalue1"}
2939         tnode = create_chk_filenode("immutable directory contents\n"*10,
2940                                     self.get_all_contents())
2941         dnode = DirectoryNode(tnode, None, None)
2942         assert not dnode.is_mutable()
2943         immdircap = dnode.get_uri()
2944         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2945         emptydircap = "URI:DIR2-LIT:"
2946         newkids = {u"child-imm":        ["filenode", {"ro_uri": filecap1,
2947                                                       "metadata": md1, }],
2948                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2949                    u"dirchild-imm":     ["dirnode",  {"ro_uri": immdircap}],
2950                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2951                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2952                    }
2953         return newkids, {'filecap1': filecap1,
2954                          'unknown_immcap': unknown_immcap,
2955                          'immdircap': immdircap,
2956                          'litdircap': litdircap,
2957                          'emptydircap': emptydircap}
2958
2959     def test_POST_mkdir_no_parentdir_initial_children(self):
2960         (newkids, caps) = self._create_initial_children()
2961         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
2962         def _after_mkdir(res):
2963             self.failUnless(res.startswith("URI:DIR"), res)
2964             n = self.s.create_node_from_uri(res)
2965             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2966             d2.addCallback(lambda ign:
2967                            self.failUnlessROChildURIIs(n, u"child-imm",
2968                                                        caps['filecap1']))
2969             d2.addCallback(lambda ign:
2970                            self.failUnlessRWChildURIIs(n, u"child-mutable",
2971                                                        caps['filecap2']))
2972             d2.addCallback(lambda ign:
2973                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
2974                                                        caps['filecap3']))
2975             d2.addCallback(lambda ign:
2976                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
2977                                                        caps['unknown_rwcap']))
2978             d2.addCallback(lambda ign:
2979                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
2980                                                        caps['unknown_rocap']))
2981             d2.addCallback(lambda ign:
2982                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2983                                                        caps['unknown_immcap']))
2984             d2.addCallback(lambda ign:
2985                            self.failUnlessRWChildURIIs(n, u"dirchild",
2986                                                        caps['dircap']))
2987             return d2
2988         d.addCallback(_after_mkdir)
2989         return d
2990
2991     def test_POST_mkdir_no_parentdir_unexpected_children(self):
2992         # the regular /uri?t=mkdir operation is specified to ignore its body.
2993         # Only t=mkdir-with-children pays attention to it.
2994         (newkids, caps) = self._create_initial_children()
2995         d = self.shouldHTTPError("POST_mkdir_no_parentdir_unexpected_children",
2996                                  400, "Bad Request",
2997                                  "t=mkdir does not accept children=, "
2998                                  "try t=mkdir-with-children instead",
2999                                  self.POST2, "/uri?t=mkdir", # without children
3000                                  simplejson.dumps(newkids))
3001         return d
3002
3003     def test_POST_noparent_bad(self):
3004         d = self.shouldHTTPError("POST_noparent_bad",
3005                                  400, "Bad Request",
3006                                  "/uri accepts only PUT, PUT?t=mkdir, "
3007                                  "POST?t=upload, and POST?t=mkdir",
3008                                  self.POST, "/uri?t=bogus")
3009         return d
3010
3011     def test_POST_mkdir_no_parentdir_immutable(self):
3012         (newkids, caps) = self._create_immutable_children()
3013         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
3014         def _after_mkdir(res):
3015             self.failUnless(res.startswith("URI:DIR"), res)
3016             n = self.s.create_node_from_uri(res)
3017             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3018             d2.addCallback(lambda ign:
3019                            self.failUnlessROChildURIIs(n, u"child-imm",
3020                                                           caps['filecap1']))
3021             d2.addCallback(lambda ign:
3022                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3023                                                           caps['unknown_immcap']))
3024             d2.addCallback(lambda ign:
3025                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
3026                                                           caps['immdircap']))
3027             d2.addCallback(lambda ign:
3028                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
3029                                                           caps['litdircap']))
3030             d2.addCallback(lambda ign:
3031                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
3032                                                           caps['emptydircap']))
3033             return d2
3034         d.addCallback(_after_mkdir)
3035         return d
3036
3037     def test_POST_mkdir_no_parentdir_immutable_bad(self):
3038         (newkids, caps) = self._create_initial_children()
3039         d = self.shouldFail2(error.Error,
3040                              "test_POST_mkdir_no_parentdir_immutable_bad",
3041                              "400 Bad Request",
3042                              "needed to be immutable but was not",
3043                              self.POST2,
3044                              "/uri?t=mkdir-immutable",
3045                              simplejson.dumps(newkids))
3046         return d
3047
3048     def test_welcome_page_mkdir_button(self):
3049         # Fetch the welcome page.
3050         d = self.GET("/")
3051         def _after_get_welcome_page(res):
3052             MKDIR_BUTTON_RE = re.compile(
3053                 '<form action="([^"]*)" method="post".*?'
3054                 '<input type="hidden" name="t" value="([^"]*)" />'
3055                 '<input type="hidden" name="([^"]*)" value="([^"]*)" />'
3056                 '<input type="submit" value="Create a directory" />',
3057                 re.I)
3058             mo = MKDIR_BUTTON_RE.search(res)
3059             formaction = mo.group(1)
3060             formt = mo.group(2)
3061             formaname = mo.group(3)
3062             formavalue = mo.group(4)
3063             return (formaction, formt, formaname, formavalue)
3064         d.addCallback(_after_get_welcome_page)
3065         def _after_parse_form(res):
3066             (formaction, formt, formaname, formavalue) = res
3067             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
3068         d.addCallback(_after_parse_form)
3069         d.addBoth(self.shouldRedirect, None, statuscode='303')
3070         return d
3071
3072     def test_POST_mkdir_replace(self): # return value?
3073         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
3074         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3075         d.addCallback(self.failUnlessNodeKeysAre, [])
3076         return d
3077
3078     def test_POST_mkdir_no_replace_queryarg(self): # return value?
3079         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
3080         d.addBoth(self.shouldFail, error.Error,
3081                   "POST_mkdir_no_replace_queryarg",
3082                   "409 Conflict",
3083                   "There was already a child by that name, and you asked me "
3084                   "to not replace it")
3085         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3086         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3087         return d
3088
3089     def test_POST_mkdir_no_replace_field(self): # return value?
3090         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
3091                       replace="false")
3092         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
3093                   "409 Conflict",
3094                   "There was already a child by that name, and you asked me "
3095                   "to not replace it")
3096         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3097         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3098         return d
3099
3100     def test_POST_mkdir_whendone_field(self):
3101         d = self.POST(self.public_url + "/foo",
3102                       t="mkdir", name="newdir", when_done="/THERE")
3103         d.addBoth(self.shouldRedirect, "/THERE")
3104         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3105         d.addCallback(self.failUnlessNodeKeysAre, [])
3106         return d
3107
3108     def test_POST_mkdir_whendone_queryarg(self):
3109         d = self.POST(self.public_url + "/foo?when_done=/THERE",
3110                       t="mkdir", name="newdir")
3111         d.addBoth(self.shouldRedirect, "/THERE")
3112         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3113         d.addCallback(self.failUnlessNodeKeysAre, [])
3114         return d
3115
3116     def test_POST_bad_t(self):
3117         d = self.shouldFail2(error.Error, "POST_bad_t",
3118                              "400 Bad Request",
3119                              "POST to a directory with bad t=BOGUS",
3120                              self.POST, self.public_url + "/foo", t="BOGUS")
3121         return d
3122
3123     def test_POST_set_children(self, command_name="set_children"):
3124         contents9, n9, newuri9 = self.makefile(9)
3125         contents10, n10, newuri10 = self.makefile(10)
3126         contents11, n11, newuri11 = self.makefile(11)
3127
3128         reqbody = """{
3129                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
3130                                                 "size": 0,
3131                                                 "metadata": {
3132                                                   "ctime": 1002777696.7564139,
3133                                                   "mtime": 1002777696.7564139
3134                                                  }
3135                                                } ],
3136                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
3137                                                 "size": 1,
3138                                                 "metadata": {
3139                                                   "ctime": 1002777696.7564139,
3140                                                   "mtime": 1002777696.7564139
3141                                                  }
3142                                                } ],
3143                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
3144                                                 "size": 2,
3145                                                 "metadata": {
3146                                                   "ctime": 1002777696.7564139,
3147                                                   "mtime": 1002777696.7564139
3148                                                  }
3149                                                } ]
3150                     }""" % (newuri9, newuri10, newuri11)
3151
3152         url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
3153
3154         d = client.getPage(url, method="POST", postdata=reqbody)
3155         def _then(res):
3156             self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
3157             self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
3158             self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
3159
3160         d.addCallback(_then)
3161         d.addErrback(self.dump_error)
3162         return d
3163
3164     def test_POST_set_children_with_hyphen(self):
3165         return self.test_POST_set_children(command_name="set-children")
3166
3167     def test_POST_link_uri(self):
3168         contents, n, newuri = self.makefile(8)
3169         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
3170         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
3171         d.addCallback(lambda res:
3172                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3173                                                       contents))
3174         return d
3175
3176     def test_POST_link_uri_replace(self):
3177         contents, n, newuri = self.makefile(8)
3178         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
3179         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
3180         d.addCallback(lambda res:
3181                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3182                                                       contents))
3183         return d
3184
3185     def test_POST_link_uri_unknown_bad(self):
3186         d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
3187         d.addBoth(self.shouldFail, error.Error,
3188                   "POST_link_uri_unknown_bad",
3189                   "400 Bad Request",
3190                   "unknown cap in a write slot")
3191         return d
3192
3193     def test_POST_link_uri_unknown_ro_good(self):
3194         d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
3195         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
3196         return d
3197
3198     def test_POST_link_uri_unknown_imm_good(self):
3199         d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
3200         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
3201         return d
3202
3203     def test_POST_link_uri_no_replace_queryarg(self):
3204         contents, n, newuri = self.makefile(8)
3205         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
3206                       name="bar.txt", uri=newuri)
3207         d.addBoth(self.shouldFail, error.Error,
3208                   "POST_link_uri_no_replace_queryarg",
3209                   "409 Conflict",
3210                   "There was already a child by that name, and you asked me "
3211                   "to not replace it")
3212         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3213         d.addCallback(self.failUnlessIsBarDotTxt)
3214         return d
3215
3216     def test_POST_link_uri_no_replace_field(self):
3217         contents, n, newuri = self.makefile(8)
3218         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
3219                       name="bar.txt", uri=newuri)
3220         d.addBoth(self.shouldFail, error.Error,
3221                   "POST_link_uri_no_replace_field",
3222                   "409 Conflict",
3223                   "There was already a child by that name, and you asked me "
3224                   "to not replace it")
3225         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3226         d.addCallback(self.failUnlessIsBarDotTxt)
3227         return d
3228
3229     def test_POST_delete(self, command_name='delete'):
3230         d = self._foo_node.list()
3231         def _check_before(children):
3232             self.failUnlessIn(u"bar.txt", children)
3233         d.addCallback(_check_before)
3234         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t=command_name, name="bar.txt"))
3235         d.addCallback(lambda res: self._foo_node.list())
3236         def _check_after(children):
3237             self.failIfIn(u"bar.txt", children)
3238         d.addCallback(_check_after)
3239         return d
3240
3241     def test_POST_unlink(self):
3242         return self.test_POST_delete(command_name='unlink')
3243
3244     def test_POST_rename_file(self):
3245         d = self.POST(self.public_url + "/foo", t="rename",
3246                       from_name="bar.txt", to_name='wibble.txt')
3247         d.addCallback(lambda res:
3248                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3249         d.addCallback(lambda res:
3250                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
3251         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
3252         d.addCallback(self.failUnlessIsBarDotTxt)
3253         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
3254         d.addCallback(self.failUnlessIsBarJSON)
3255         return d
3256
3257     def test_POST_rename_file_redundant(self):
3258         d = self.POST(self.public_url + "/foo", t="rename",
3259                       from_name="bar.txt", to_name='bar.txt')
3260         d.addCallback(lambda res:
3261                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3262         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3263         d.addCallback(self.failUnlessIsBarDotTxt)
3264         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3265         d.addCallback(self.failUnlessIsBarJSON)
3266         return d
3267
3268     def test_POST_rename_file_replace(self):
3269         # rename a file and replace a directory with it
3270         d = self.POST(self.public_url + "/foo", t="rename",
3271                       from_name="bar.txt", to_name='empty')
3272         d.addCallback(lambda res:
3273                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3274         d.addCallback(lambda res:
3275                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
3276         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
3277         d.addCallback(self.failUnlessIsBarDotTxt)
3278         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3279         d.addCallback(self.failUnlessIsBarJSON)
3280         return d
3281
3282     def test_POST_rename_file_no_replace_queryarg(self):
3283         # rename a file and replace a directory with it
3284         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
3285                       from_name="bar.txt", to_name='empty')
3286         d.addBoth(self.shouldFail, error.Error,
3287                   "POST_rename_file_no_replace_queryarg",
3288                   "409 Conflict",
3289                   "There was already a child by that name, and you asked me "
3290                   "to not replace it")
3291         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3292         d.addCallback(self.failUnlessIsEmptyJSON)
3293         return d
3294
3295     def test_POST_rename_file_no_replace_field(self):
3296         # rename a file and replace a directory with it
3297         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
3298                       from_name="bar.txt", to_name='empty')
3299         d.addBoth(self.shouldFail, error.Error,
3300                   "POST_rename_file_no_replace_field",
3301                   "409 Conflict",
3302                   "There was already a child by that name, and you asked me "
3303                   "to not replace it")
3304         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3305         d.addCallback(self.failUnlessIsEmptyJSON)
3306         return d
3307
3308     def failUnlessIsEmptyJSON(self, res):
3309         data = simplejson.loads(res)
3310         self.failUnlessEqual(data[0], "dirnode", data)
3311         self.failUnlessReallyEqual(len(data[1]["children"]), 0)
3312
3313     def test_POST_rename_file_slash_fail(self):
3314         d = self.POST(self.public_url + "/foo", t="rename",
3315                       from_name="bar.txt", to_name='kirk/spock.txt')
3316         d.addBoth(self.shouldFail, error.Error,
3317                   "test_POST_rename_file_slash_fail",
3318                   "400 Bad Request",
3319                   "to_name= may not contain a slash",
3320                   )
3321         d.addCallback(lambda res:
3322                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3323         return d
3324
3325     def test_POST_rename_dir(self):
3326         d = self.POST(self.public_url, t="rename",
3327                       from_name="foo", to_name='plunk')
3328         d.addCallback(lambda res:
3329                       self.failIfNodeHasChild(self.public_root, u"foo"))
3330         d.addCallback(lambda res:
3331                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
3332         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
3333         d.addCallback(self.failUnlessIsFooJSON)
3334         return d
3335
3336     def test_POST_move_file(self):
3337         d = self.POST(self.public_url + "/foo", t="move",
3338                       from_name="bar.txt", to_dir="sub")
3339         d.addCallback(lambda res:
3340                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3341         d.addCallback(lambda res:
3342                       self.failUnlessNodeHasChild(self._sub_node, u"bar.txt"))
3343         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3344         d.addCallback(self.failUnlessIsBarDotTxt)
3345         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3346         d.addCallback(self.failUnlessIsBarJSON)
3347         return d
3348
3349     def test_POST_move_file_new_name(self):
3350         d = self.POST(self.public_url + "/foo", t="move",
3351                       from_name="bar.txt", to_name="wibble.txt", to_dir="sub")
3352         d.addCallback(lambda res:
3353                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3354         d.addCallback(lambda res:
3355                       self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3356         d.addCallback(lambda res:
3357                       self.failUnlessNodeHasChild(self._sub_node, u"wibble.txt"))
3358         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt"))
3359         d.addCallback(self.failUnlessIsBarDotTxt)
3360         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt?t=json"))
3361         d.addCallback(self.failUnlessIsBarJSON)
3362         return d
3363
3364     def test_POST_move_file_replace(self):
3365         d = self.POST(self.public_url + "/foo", t="move",
3366                       from_name="bar.txt", to_name="baz.txt", to_dir="sub")
3367         d.addCallback(lambda res:
3368                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3369         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3370         d.addCallback(self.failUnlessIsBarDotTxt)
3371         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt?t=json"))
3372         d.addCallback(self.failUnlessIsBarJSON)
3373         return d
3374
3375     def test_POST_move_file_no_replace(self):
3376         d = self.shouldFail2(error.Error, "POST_move_file_no_replace",
3377                              "409 Conflict",
3378                              "There was already a child by that name, and you asked me to not replace it",
3379                              self.POST, self.public_url + "/foo", t="move",
3380                              replace="false", from_name="bar.txt",
3381                              to_name="baz.txt", to_dir="sub")
3382         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3383         d.addCallback(self.failUnlessIsBarDotTxt)
3384         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3385         d.addCallback(self.failUnlessIsBarJSON)
3386         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3387         d.addCallback(self.failUnlessIsSubBazDotTxt)
3388         return d
3389
3390     def test_POST_move_file_slash_fail(self):
3391         d = self.shouldFail2(error.Error, "test_POST_rename_file_slash_fail",
3392                              "400 Bad Request",
3393                              "to_name= may not contain a slash",
3394                              self.POST, self.public_url + "/foo", t="move",
3395                              from_name="bar.txt",
3396                              to_name="slash/fail.txt", to_dir="sub")
3397         d.addCallback(lambda res:
3398                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3399         d.addCallback(lambda res:
3400                       self.failIfNodeHasChild(self._sub_node, u"slash/fail.txt"))
3401         d.addCallback(lambda ign:
3402                       self.shouldFail2(error.Error,
3403                                        "test_POST_rename_file_slash_fail2",
3404                                        "400 Bad Request",
3405                                        "from_name= may not contain a slash",
3406                                        self.POST, self.public_url + "/foo",
3407                                        t="move",
3408                                        from_name="nope/bar.txt",
3409                                        to_name="fail.txt", to_dir="sub"))
3410         return d
3411
3412     def test_POST_move_file_no_target(self):
3413         d = self.shouldFail2(error.Error, "POST_move_file_no_target",
3414                              "400 Bad Request",
3415                              "move requires from_name and to_dir",
3416                              self.POST, self.public_url + "/foo", t="move",
3417                              from_name="bar.txt", to_name="baz.txt")
3418         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3419         d.addCallback(self.failUnlessIsBarDotTxt)
3420         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3421         d.addCallback(self.failUnlessIsBarJSON)
3422         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3423         d.addCallback(self.failUnlessIsBazDotTxt)
3424         return d
3425
3426     def test_POST_move_file_bad_target_type(self):
3427         d = self.shouldFail2(error.Error, "test_POST_move_file_bad_target_type",
3428                              "400 Bad Request", "invalid target_type parameter",
3429                              self.POST,
3430                              self.public_url + "/foo", t="move",
3431                              target_type="*D", from_name="bar.txt",
3432                              to_dir="sub")
3433         return d
3434
3435     def test_POST_move_file_multi_level(self):
3436         d = self.POST(self.public_url + "/foo/sub/level2?t=mkdir", "")
3437         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t="move",
3438                       from_name="bar.txt", to_dir="sub/level2"))
3439         d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3440         d.addCallback(lambda res: self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3441         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt"))
3442         d.addCallback(self.failUnlessIsBarDotTxt)
3443         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt?t=json"))
3444         d.addCallback(self.failUnlessIsBarJSON)
3445         return d
3446
3447     def test_POST_move_file_to_uri(self):
3448         d = self.POST(self.public_url + "/foo", t="move", target_type="uri",
3449                       from_name="bar.txt", to_dir=self._sub_uri)
3450         d.addCallback(lambda res:
3451                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3452         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3453         d.addCallback(self.failUnlessIsBarDotTxt)
3454         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3455         d.addCallback(self.failUnlessIsBarJSON)
3456         return d
3457
3458     def test_POST_move_file_to_nonexist_dir(self):
3459         d = self.shouldFail2(error.Error, "POST_move_file_to_nonexist_dir",
3460                             "404 Not Found", "No such child: nopechucktesta",
3461                             self.POST, self.public_url + "/foo", t="move",
3462                             from_name="bar.txt", to_dir="nopechucktesta")
3463         return d
3464
3465     def test_POST_move_file_into_file(self):
3466         d = self.shouldFail2(error.Error, "POST_move_file_into_file",
3467                              "400 Bad Request", "to_dir is not a directory",
3468                              self.POST, self.public_url + "/foo", t="move",
3469                              from_name="bar.txt", to_dir="baz.txt")
3470         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3471         d.addCallback(self.failUnlessIsBazDotTxt)
3472         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3473         d.addCallback(self.failUnlessIsBarDotTxt)
3474         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3475         d.addCallback(self.failUnlessIsBarJSON)
3476         return d
3477
3478     def test_POST_move_file_to_bad_uri(self):
3479         d =  self.shouldFail2(error.Error, "POST_move_file_to_bad_uri",
3480                               "400 Bad Request", "to_dir is not a directory",
3481                               self.POST, self.public_url + "/foo", t="move",
3482                               from_name="bar.txt", target_type="uri",
3483                               to_dir="URI:DIR2:mn5jlyjnrjeuydyswlzyui72i:rmneifcj6k6sycjljjhj3f6majsq2zqffydnnul5hfa4j577arma")
3484         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3485         d.addCallback(self.failUnlessIsBarDotTxt)
3486         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3487         d.addCallback(self.failUnlessIsBarJSON)
3488         return d
3489
3490     def test_POST_move_dir(self):
3491         d = self.POST(self.public_url + "/foo", t="move",
3492                       from_name="bar.txt", to_dir="empty")
3493         d.addCallback(lambda res: self.POST(self.public_url + "/foo",
3494                       t="move", from_name="empty", to_dir="sub"))
3495         d.addCallback(lambda res:
3496                       self.failIfNodeHasChild(self._foo_node, u"empty"))
3497         d.addCallback(lambda res:
3498                       self.failUnlessNodeHasChild(self._sub_node, u"empty"))
3499         d.addCallback(lambda res:
3500                       self._sub_node.get_child_at_path(u"empty"))
3501         d.addCallback(lambda node:
3502                       self.failUnlessNodeHasChild(node, u"bar.txt"))
3503         d.addCallback(lambda res:
3504                       self.GET(self.public_url + "/foo/sub/empty/bar.txt"))
3505         d.addCallback(self.failUnlessIsBarDotTxt)
3506         return d
3507
3508     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
3509         """ If target is not None then the redirection has to go to target.  If
3510         statuscode is not None then the redirection has to be accomplished with
3511         that HTTP status code."""
3512         if not isinstance(res, failure.Failure):
3513             to_where = (target is None) and "somewhere" or ("to " + target)
3514             self.fail("%s: we were expecting to get redirected %s, not get an"
3515                       " actual page: %s" % (which, to_where, res))
3516         res.trap(error.PageRedirect)
3517         if statuscode is not None:
3518             self.failUnlessReallyEqual(res.value.status, statuscode,
3519                                        "%s: not a redirect" % which)
3520         if target is not None:
3521             # the PageRedirect does not seem to capture the uri= query arg
3522             # properly, so we can't check for it.
3523             realtarget = self.webish_url + target
3524             self.failUnlessReallyEqual(res.value.location, realtarget,
3525                                        "%s: wrong target" % which)
3526         return res.value.location
3527
3528     def test_GET_URI_form(self):
3529         base = "/uri?uri=%s" % self._bar_txt_uri
3530         # this is supposed to give us a redirect to /uri/$URI, plus arguments
3531         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
3532         d = self.GET(base)
3533         d.addBoth(self.shouldRedirect, targetbase)
3534         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
3535         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
3536         d.addCallback(lambda res: self.GET(base+"&t=json"))
3537         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
3538         d.addCallback(self.log, "about to get file by uri")
3539         d.addCallback(lambda res: self.GET(base, followRedirect=True))
3540         d.addCallback(self.failUnlessIsBarDotTxt)
3541         d.addCallback(self.log, "got file by uri, about to get dir by uri")
3542         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
3543                                            followRedirect=True))
3544         d.addCallback(self.failUnlessIsFooJSON)
3545         d.addCallback(self.log, "got dir by uri")
3546
3547         return d
3548
3549     def test_GET_URI_form_bad(self):
3550         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
3551                              "400 Bad Request", "GET /uri requires uri=",
3552                              self.GET, "/uri")
3553         return d
3554
3555     def test_GET_rename_form(self):
3556         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
3557                      followRedirect=True)
3558         def _check(res):
3559             self.failUnlessIn('name="when_done" value="."', res)
3560             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
3561             self.failUnlessIn(FAVICON_MARKUP, res)
3562         d.addCallback(_check)
3563         return d
3564
3565     def log(self, res, msg):
3566         #print "MSG: %s  RES: %s" % (msg, res)
3567         log.msg(msg)
3568         return res
3569
3570     def test_GET_URI_URL(self):
3571         base = "/uri/%s" % self._bar_txt_uri
3572         d = self.GET(base)
3573         d.addCallback(self.failUnlessIsBarDotTxt)
3574         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
3575         d.addCallback(self.failUnlessIsBarDotTxt)
3576         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
3577         d.addCallback(self.failUnlessIsBarDotTxt)
3578         return d
3579
3580     def test_GET_URI_URL_dir(self):
3581         base = "/uri/%s?t=json" % self._foo_uri
3582         d = self.GET(base)
3583         d.addCallback(self.failUnlessIsFooJSON)
3584         return d
3585
3586     def test_GET_URI_URL_missing(self):
3587         base = "/uri/%s" % self._bad_file_uri
3588         d = self.shouldHTTPError("test_GET_URI_URL_missing",
3589                                  http.GONE, None, "NotEnoughSharesError",
3590                                  self.GET, base)
3591         # TODO: how can we exercise both sides of WebDownloadTarget.fail
3592         # here? we must arrange for a download to fail after target.open()
3593         # has been called, and then inspect the response to see that it is
3594         # shorter than we expected.
3595         return d
3596
3597     def test_PUT_DIRURL_uri(self):
3598         d = self.s.create_dirnode()
3599         def _made_dir(dn):
3600             new_uri = dn.get_uri()
3601             # replace /foo with a new (empty) directory
3602             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
3603             d.addCallback(lambda res:
3604                           self.failUnlessReallyEqual(res.strip(), new_uri))
3605             d.addCallback(lambda res:
3606                           self.failUnlessRWChildURIIs(self.public_root,
3607                                                       u"foo",
3608                                                       new_uri))
3609             return d
3610         d.addCallback(_made_dir)
3611         return d
3612
3613     def test_PUT_DIRURL_uri_noreplace(self):
3614         d = self.s.create_dirnode()
3615         def _made_dir(dn):
3616             new_uri = dn.get_uri()
3617             # replace /foo with a new (empty) directory, but ask that
3618             # replace=false, so it should fail
3619             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
3620                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
3621                                  self.PUT,
3622                                  self.public_url + "/foo?t=uri&replace=false",
3623                                  new_uri)
3624             d.addCallback(lambda res:
3625                           self.failUnlessRWChildURIIs(self.public_root,
3626                                                       u"foo",
3627                                                       self._foo_uri))
3628             return d
3629         d.addCallback(_made_dir)
3630         return d
3631
3632     def test_PUT_DIRURL_bad_t(self):
3633         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
3634                              "400 Bad Request", "PUT to a directory",
3635                              self.PUT, self.public_url + "/foo?t=BOGUS", "")
3636         d.addCallback(lambda res:
3637                       self.failUnlessRWChildURIIs(self.public_root,
3638                                                   u"foo",
3639                                                   self._foo_uri))
3640         return d
3641
3642     def test_PUT_NEWFILEURL_uri(self):
3643         contents, n, new_uri = self.makefile(8)
3644         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
3645         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3646         d.addCallback(lambda res:
3647                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3648                                                       contents))
3649         return d
3650
3651     def test_PUT_NEWFILEURL_mdmf(self):
3652         new_contents = self.NEWFILE_CONTENTS * 300000
3653         d = self.PUT(self.public_url + \
3654                      "/foo/mdmf.txt?format=mdmf",
3655                      new_contents)
3656         d.addCallback(lambda ignored:
3657             self.GET(self.public_url + "/foo/mdmf.txt?t=json"))
3658         def _got_json(json):
3659             data = simplejson.loads(json)
3660             data = data[1]
3661             self.failUnlessIn("format", data)
3662             self.failUnlessEqual(data["format"], "MDMF")
3663             self.failUnless(data['rw_uri'].startswith("URI:MDMF"))
3664             self.failUnless(data['ro_uri'].startswith("URI:MDMF"))
3665         d.addCallback(_got_json)
3666         return d
3667
3668     def test_PUT_NEWFILEURL_sdmf(self):
3669         new_contents = self.NEWFILE_CONTENTS * 300000
3670         d = self.PUT(self.public_url + \
3671                      "/foo/sdmf.txt?format=sdmf",
3672                      new_contents)
3673         d.addCallback(lambda ignored:
3674             self.GET(self.public_url + "/foo/sdmf.txt?t=json"))
3675         def _got_json(json):
3676             data = simplejson.loads(json)
3677             data = data[1]
3678             self.failUnlessIn("format", data)
3679             self.failUnlessEqual(data["format"], "SDMF")
3680         d.addCallback(_got_json)
3681         return d
3682
3683     def test_PUT_NEWFILEURL_bad_format(self):
3684        new_contents = self.NEWFILE_CONTENTS * 300000
3685        return self.shouldHTTPError("PUT_NEWFILEURL_bad_format",
3686                                    400, "Bad Request", "Unknown format: foo",
3687                                    self.PUT, self.public_url + \
3688                                    "/foo/foo.txt?format=foo",
3689                                    new_contents)
3690
3691     def test_PUT_NEWFILEURL_uri_replace(self):
3692         contents, n, new_uri = self.makefile(8)
3693         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
3694         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3695         d.addCallback(lambda res:
3696                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3697                                                       contents))
3698         return d
3699
3700     def test_PUT_NEWFILEURL_uri_no_replace(self):
3701         contents, n, new_uri = self.makefile(8)
3702         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
3703         d.addBoth(self.shouldFail, error.Error,
3704                   "PUT_NEWFILEURL_uri_no_replace",
3705                   "409 Conflict",
3706                   "There was already a child by that name, and you asked me "
3707                   "to not replace it")
3708         return d
3709
3710     def test_PUT_NEWFILEURL_uri_unknown_bad(self):
3711         d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
3712         d.addBoth(self.shouldFail, error.Error,
3713                   "POST_put_uri_unknown_bad",
3714                   "400 Bad Request",
3715                   "unknown cap in a write slot")
3716         return d
3717
3718     def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
3719         d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
3720         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3721                       u"put-future-ro.txt")
3722         return d
3723
3724     def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
3725         d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
3726         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3727                       u"put-future-imm.txt")
3728         return d
3729
3730     def test_PUT_NEWFILE_URI(self):
3731         file_contents = "New file contents here\n"
3732         d = self.PUT("/uri", file_contents)
3733         def _check(uri):
3734             assert isinstance(uri, str), uri
3735             self.failUnlessIn(uri, self.get_all_contents())
3736             self.failUnlessReallyEqual(self.get_all_contents()[uri],
3737                                        file_contents)
3738             return self.GET("/uri/%s" % uri)
3739         d.addCallback(_check)
3740         def _check2(res):
3741             self.failUnlessReallyEqual(res, file_contents)
3742         d.addCallback(_check2)
3743         return d
3744
3745     def test_PUT_NEWFILE_URI_not_mutable(self):
3746         file_contents = "New file contents here\n"
3747         d = self.PUT("/uri?mutable=false", file_contents)
3748         def _check(uri):
3749             assert isinstance(uri, str), uri
3750             self.failUnlessIn(uri, self.get_all_contents())
3751             self.failUnlessReallyEqual(self.get_all_contents()[uri],
3752                                        file_contents)
3753             return self.GET("/uri/%s" % uri)
3754         d.addCallback(_check)
3755         def _check2(res):
3756             self.failUnlessReallyEqual(res, file_contents)
3757         d.addCallback(_check2)
3758         return d
3759
3760     def test_PUT_NEWFILE_URI_only_PUT(self):
3761         d = self.PUT("/uri?t=bogus", "")
3762         d.addBoth(self.shouldFail, error.Error,
3763                   "PUT_NEWFILE_URI_only_PUT",
3764                   "400 Bad Request",
3765                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
3766         return d
3767
3768     def test_PUT_NEWFILE_URI_mutable(self):
3769         file_contents = "New file contents here\n"
3770         d = self.PUT("/uri?mutable=true", file_contents)
3771         def _check1(filecap):
3772             filecap = filecap.strip()
3773             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
3774             self.filecap = filecap
3775             u = uri.WriteableSSKFileURI.init_from_string(filecap)
3776             self.failUnlessIn(u.get_storage_index(), self.get_all_contents())
3777             n = self.s.create_node_from_uri(filecap)
3778             return n.download_best_version()
3779         d.addCallback(_check1)
3780         def _check2(data):
3781             self.failUnlessReallyEqual(data, file_contents)
3782             return self.GET("/uri/%s" % urllib.quote(self.filecap))
3783         d.addCallback(_check2)
3784         def _check3(res):
3785             self.failUnlessReallyEqual(res, file_contents)
3786         d.addCallback(_check3)
3787         return d
3788
3789     def test_PUT_mkdir(self):
3790         d = self.PUT("/uri?t=mkdir", "")
3791         def _check(uri):
3792             n = self.s.create_node_from_uri(uri.strip())
3793             d2 = self.failUnlessNodeKeysAre(n, [])
3794             d2.addCallback(lambda res:
3795                            self.GET("/uri/%s?t=json" % uri))
3796             return d2
3797         d.addCallback(_check)
3798         d.addCallback(self.failUnlessIsEmptyJSON)
3799         return d
3800
3801     def test_PUT_mkdir_mdmf(self):
3802         d = self.PUT("/uri?t=mkdir&format=mdmf", "")
3803         def _got(res):
3804             u = uri.from_string(res)
3805             # Check that this is an MDMF writecap
3806             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
3807         d.addCallback(_got)
3808         return d
3809
3810     def test_PUT_mkdir_sdmf(self):
3811         d = self.PUT("/uri?t=mkdir&format=sdmf", "")
3812         def _got(res):
3813             u = uri.from_string(res)
3814             self.failUnlessIsInstance(u, uri.DirectoryURI)
3815         d.addCallback(_got)
3816         return d
3817
3818     def test_PUT_mkdir_bad_format(self):
3819         return self.shouldHTTPError("PUT_mkdir_bad_format",
3820                                     400, "Bad Request", "Unknown format: foo",
3821                                     self.PUT, "/uri?t=mkdir&format=foo",
3822                                     "")
3823
3824     def test_POST_check(self):
3825         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
3826         def _done(res):
3827             # this returns a string form of the results, which are probably
3828             # None since we're using fake filenodes.
3829             # TODO: verify that the check actually happened, by changing
3830             # FakeCHKFileNode to count how many times .check() has been
3831             # called.
3832             pass
3833         d.addCallback(_done)
3834         return d
3835
3836
3837     def test_PUT_update_at_offset(self):
3838         file_contents = "test file" * 100000 # about 900 KiB
3839         d = self.PUT("/uri?mutable=true", file_contents)
3840         def _then(filecap):
3841             self.filecap = filecap
3842             new_data = file_contents[:100]
3843             new = "replaced and so on"
3844             new_data += new
3845             new_data += file_contents[len(new_data):]
3846             assert len(new_data) == len(file_contents)
3847             self.new_data = new_data
3848         d.addCallback(_then)
3849         d.addCallback(lambda ignored:
3850             self.PUT("/uri/%s?replace=True&offset=100" % self.filecap,
3851                      "replaced and so on"))
3852         def _get_data(filecap):
3853             n = self.s.create_node_from_uri(filecap)
3854             return n.download_best_version()
3855         d.addCallback(_get_data)
3856         d.addCallback(lambda results:
3857             self.failUnlessEqual(results, self.new_data))
3858         # Now try appending things to the file
3859         d.addCallback(lambda ignored:
3860             self.PUT("/uri/%s?offset=%d" % (self.filecap, len(self.new_data)),
3861                      "puppies" * 100))
3862         d.addCallback(_get_data)
3863         d.addCallback(lambda results:
3864             self.failUnlessEqual(results, self.new_data + ("puppies" * 100)))
3865         # and try replacing the beginning of the file
3866         d.addCallback(lambda ignored:
3867             self.PUT("/uri/%s?offset=0" % self.filecap, "begin"))
3868         d.addCallback(_get_data)
3869         d.addCallback(lambda results:
3870             self.failUnlessEqual(results, "begin"+self.new_data[len("begin"):]+("puppies"*100)))
3871         return d
3872
3873     def test_PUT_update_at_invalid_offset(self):
3874         file_contents = "test file" * 100000 # about 900 KiB
3875         d = self.PUT("/uri?mutable=true", file_contents)
3876         def _then(filecap):
3877             self.filecap = filecap
3878         d.addCallback(_then)
3879         # Negative offsets should cause an error.
3880         d.addCallback(lambda ignored:
3881             self.shouldHTTPError("PUT_update_at_invalid_offset",
3882                                  400, "Bad Request",
3883                                  "Invalid offset",
3884                                  self.PUT,
3885                                  "/uri/%s?offset=-1" % self.filecap,
3886                                  "foo"))
3887         return d
3888
3889     def test_PUT_update_at_offset_immutable(self):
3890         file_contents = "Test file" * 100000
3891         d = self.PUT("/uri", file_contents)
3892         def _then(filecap):
3893             self.filecap = filecap
3894         d.addCallback(_then)
3895         d.addCallback(lambda ignored:
3896             self.shouldHTTPError("PUT_update_at_offset_immutable",
3897                                  400, "Bad Request",
3898                                  "immutable",
3899                                  self.PUT,
3900                                  "/uri/%s?offset=50" % self.filecap,
3901                                  "foo"))
3902         return d
3903
3904
3905     def test_bad_method(self):
3906         url = self.webish_url + self.public_url + "/foo/bar.txt"
3907         d = self.shouldHTTPError("bad_method",
3908                                  501, "Not Implemented",
3909                                  "I don't know how to treat a BOGUS request.",
3910                                  client.getPage, url, method="BOGUS")
3911         return d
3912
3913     def test_short_url(self):
3914         url = self.webish_url + "/uri"
3915         d = self.shouldHTTPError("short_url", 501, "Not Implemented",
3916                                  "I don't know how to treat a DELETE request.",
3917                                  client.getPage, url, method="DELETE")
3918         return d
3919
3920     def test_ophandle_bad(self):
3921         url = self.webish_url + "/operations/bogus?t=status"
3922         d = self.shouldHTTPError("ophandle_bad", 404, "404 Not Found",
3923                                  "unknown/expired handle 'bogus'",
3924                                  client.getPage, url)
3925         return d
3926
3927     def test_ophandle_cancel(self):
3928         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
3929                       followRedirect=True)
3930         d.addCallback(lambda ignored:
3931                       self.GET("/operations/128?t=status&output=JSON"))
3932         def _check1(res):
3933             data = simplejson.loads(res)
3934             self.failUnless("finished" in data, res)
3935             monitor = self.ws.root.child_operations.handles["128"][0]
3936             d = self.POST("/operations/128?t=cancel&output=JSON")
3937             def _check2(res):
3938                 data = simplejson.loads(res)
3939                 self.failUnless("finished" in data, res)
3940                 # t=cancel causes the handle to be forgotten
3941                 self.failUnless(monitor.is_cancelled())
3942             d.addCallback(_check2)
3943             return d
3944         d.addCallback(_check1)
3945         d.addCallback(lambda ignored:
3946                       self.shouldHTTPError("ophandle_cancel",
3947                                            404, "404 Not Found",
3948                                            "unknown/expired handle '128'",
3949                                            self.GET,
3950                                            "/operations/128?t=status&output=JSON"))
3951         return d
3952
3953     def test_ophandle_retainfor(self):
3954         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
3955                       followRedirect=True)
3956         d.addCallback(lambda ignored:
3957                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
3958         def _check1(res):
3959             data = simplejson.loads(res)
3960             self.failUnless("finished" in data, res)
3961         d.addCallback(_check1)
3962         # the retain-for=0 will cause the handle to be expired very soon
3963         d.addCallback(lambda ign:
3964             self.clock.advance(2.0))
3965         d.addCallback(lambda ignored:
3966                       self.shouldHTTPError("ophandle_retainfor",
3967                                            404, "404 Not Found",
3968                                            "unknown/expired handle '129'",
3969                                            self.GET,
3970                                            "/operations/129?t=status&output=JSON"))
3971         return d
3972
3973     def test_ophandle_release_after_complete(self):
3974         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
3975                       followRedirect=True)
3976         d.addCallback(self.wait_for_operation, "130")
3977         d.addCallback(lambda ignored:
3978                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
3979         # the release-after-complete=true will cause the handle to be expired
3980         d.addCallback(lambda ignored:
3981                       self.shouldHTTPError("ophandle_release_after_complete",
3982                                            404, "404 Not Found",
3983                                            "unknown/expired handle '130'",
3984                                            self.GET,
3985                                            "/operations/130?t=status&output=JSON"))
3986         return d
3987
3988     def test_uncollected_ophandle_expiration(self):
3989         # uncollected ophandles should expire after 4 days
3990         def _make_uncollected_ophandle(ophandle):
3991             d = self.POST(self.public_url +
3992                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3993                           followRedirect=False)
3994             # When we start the operation, the webapi server will want
3995             # to redirect us to the page for the ophandle, so we get
3996             # confirmation that the operation has started. If the
3997             # manifest operation has finished by the time we get there,
3998             # following that redirect (by setting followRedirect=True
3999             # above) has the side effect of collecting the ophandle that
4000             # we've just created, which means that we can't use the
4001             # ophandle to test the uncollected timeout anymore. So,
4002             # instead, catch the 302 here and don't follow it.
4003             d.addBoth(self.should302, "uncollected_ophandle_creation")
4004             return d
4005         # Create an ophandle, don't collect it, then advance the clock by
4006         # 4 days - 1 second and make sure that the ophandle is still there.
4007         d = _make_uncollected_ophandle(131)
4008         d.addCallback(lambda ign:
4009             self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
4010         d.addCallback(lambda ign:
4011             self.GET("/operations/131?t=status&output=JSON"))
4012         def _check1(res):
4013             data = simplejson.loads(res)
4014             self.failUnless("finished" in data, res)
4015         d.addCallback(_check1)
4016         # Create an ophandle, don't collect it, then try to collect it
4017         # after 4 days. It should be gone.
4018         d.addCallback(lambda ign:
4019             _make_uncollected_ophandle(132))
4020         d.addCallback(lambda ign:
4021             self.clock.advance(96*60*60))
4022         d.addCallback(lambda ign:
4023             self.shouldHTTPError("uncollected_ophandle_expired_after_100_hours",
4024                                  404, "404 Not Found",
4025                                  "unknown/expired handle '132'",
4026                                  self.GET,
4027                                  "/operations/132?t=status&output=JSON"))
4028         return d
4029
4030     def test_collected_ophandle_expiration(self):
4031         # collected ophandles should expire after 1 day
4032         def _make_collected_ophandle(ophandle):
4033             d = self.POST(self.public_url +
4034                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
4035                           followRedirect=True)
4036             # By following the initial redirect, we collect the ophandle
4037             # we've just created.
4038             return d
4039         # Create a collected ophandle, then collect it after 23 hours
4040         # and 59 seconds to make sure that it is still there.
4041         d = _make_collected_ophandle(133)
4042         d.addCallback(lambda ign:
4043             self.clock.advance((24*60*60) - 1))
4044         d.addCallback(lambda ign:
4045             self.GET("/operations/133?t=status&output=JSON"))
4046         def _check1(res):
4047             data = simplejson.loads(res)
4048             self.failUnless("finished" in data, res)
4049         d.addCallback(_check1)
4050         # Create another uncollected ophandle, then try to collect it
4051         # after 24 hours to make sure that it is gone.
4052         d.addCallback(lambda ign:
4053             _make_collected_ophandle(134))
4054         d.addCallback(lambda ign:
4055             self.clock.advance(24*60*60))
4056         d.addCallback(lambda ign:
4057             self.shouldHTTPError("collected_ophandle_expired_after_1_day",
4058                                  404, "404 Not Found",
4059                                  "unknown/expired handle '134'",
4060                                  self.GET,
4061                                  "/operations/134?t=status&output=JSON"))
4062         return d
4063
4064     def test_incident(self):
4065         d = self.POST("/report_incident", details="eek")
4066         def _done(res):
4067             self.failIfIn("<html>", res)
4068             self.failUnlessIn("Thank you for your report!", res)
4069         d.addCallback(_done)
4070         return d
4071
4072     def test_static(self):
4073         webdir = os.path.join(self.staticdir, "subdir")
4074         fileutil.make_dirs(webdir)
4075         f = open(os.path.join(webdir, "hello.txt"), "wb")
4076         f.write("hello")
4077         f.close()
4078
4079         d = self.GET("/static/subdir/hello.txt")
4080         def _check(res):
4081             self.failUnlessReallyEqual(res, "hello")
4082         d.addCallback(_check)
4083         return d
4084
4085
4086 class IntroducerWeb(unittest.TestCase):
4087     def setUp(self):
4088         self.node = None
4089
4090     def tearDown(self):
4091         d = defer.succeed(None)
4092         if self.node:
4093             d.addCallback(lambda ign: self.node.stopService())
4094         d.addCallback(flushEventualQueue)
4095         return d
4096
4097     def test_welcome(self):
4098         basedir = "web.IntroducerWeb.test_welcome"
4099         os.mkdir(basedir)
4100         fileutil.write(os.path.join(basedir, "tahoe.cfg"), "[node]\nweb.port = tcp:0\n")
4101         self.node = IntroducerNode(basedir)
4102         self.ws = self.node.getServiceNamed("webish")
4103
4104         d = fireEventually(None)
4105         d.addCallback(lambda ign: self.node.startService())
4106         d.addCallback(lambda ign: self.node.when_tub_ready())
4107
4108         d.addCallback(lambda ign: self.GET("/"))
4109         def _check(res):
4110             self.failUnlessIn('Welcome to the Tahoe-LAFS Introducer', res)
4111             self.failUnlessIn(FAVICON_MARKUP, res)
4112         d.addCallback(_check)
4113         return d
4114
4115     def GET(self, urlpath, followRedirect=False, return_response=False,
4116             **kwargs):
4117         # if return_response=True, this fires with (data, statuscode,
4118         # respheaders) instead of just data.
4119         assert not isinstance(urlpath, unicode)
4120         url = self.ws.getURL().rstrip('/') + urlpath
4121         factory = HTTPClientGETFactory(url, method="GET",
4122                                        followRedirect=followRedirect, **kwargs)
4123         reactor.connectTCP("localhost", self.ws.getPortnum(), factory)
4124         d = factory.deferred
4125         def _got_data(data):
4126             return (data, factory.status, factory.response_headers)
4127         if return_response:
4128             d.addCallback(_got_data)
4129         return factory.deferred
4130
4131
4132 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4133     def test_load_file(self):
4134         # This will raise an exception unless a well-formed XML file is found under that name.
4135         common.getxmlfile('directory.xhtml').load()
4136
4137     def test_parse_replace_arg(self):
4138         self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
4139         self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
4140         self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
4141                                    "only-files")
4142         self.shouldFail(AssertionError, "test_parse_replace_arg", "",
4143                         common.parse_replace_arg, "only_fles")
4144
4145     def test_abbreviate_time(self):
4146         self.failUnlessReallyEqual(common.abbreviate_time(None), "")
4147         self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
4148         self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
4149         self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
4150         self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
4151         self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
4152
4153     def test_compute_rate(self):
4154         self.failUnlessReallyEqual(common.compute_rate(None, None), None)
4155         self.failUnlessReallyEqual(common.compute_rate(None, 1), None)
4156         self.failUnlessReallyEqual(common.compute_rate(250000, None), None)
4157         self.failUnlessReallyEqual(common.compute_rate(250000, 0), None)
4158         self.failUnlessReallyEqual(common.compute_rate(250000, 10), 25000.0)
4159         self.failUnlessReallyEqual(common.compute_rate(0, 10), 0.0)
4160         self.shouldFail(AssertionError, "test_compute_rate", "",
4161                         common.compute_rate, -100, 10)
4162         self.shouldFail(AssertionError, "test_compute_rate", "",
4163                         common.compute_rate, 100, -10)
4164
4165         # Sanity check
4166         rate = common.compute_rate(10*1000*1000, 1)
4167         self.failUnlessReallyEqual(common.abbreviate_rate(rate), "10.00MBps")
4168
4169     def test_abbreviate_rate(self):
4170         self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
4171         self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
4172         self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
4173         self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
4174
4175     def test_abbreviate_size(self):
4176         self.failUnlessReallyEqual(common.abbreviate_size(None), "")
4177         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
4178         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
4179         self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
4180         self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
4181
4182     def test_plural(self):
4183         def convert(s):
4184             return "%d second%s" % (s, status.plural(s))
4185         self.failUnlessReallyEqual(convert(0), "0 seconds")
4186         self.failUnlessReallyEqual(convert(1), "1 second")
4187         self.failUnlessReallyEqual(convert(2), "2 seconds")
4188         def convert2(s):
4189             return "has share%s: %s" % (status.plural(s), ",".join(s))
4190         self.failUnlessReallyEqual(convert2([]), "has shares: ")
4191         self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
4192         self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
4193
4194
4195 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4196
4197     def CHECK(self, ign, which, args, clientnum=0):
4198         fileurl = self.fileurls[which]
4199         url = fileurl + "?" + args
4200         return self.GET(url, method="POST", clientnum=clientnum)
4201
4202     def test_filecheck(self):
4203         self.basedir = "web/Grid/filecheck"
4204         self.set_up_grid()
4205         c0 = self.g.clients[0]
4206         self.uris = {}
4207         DATA = "data" * 100
4208         d = c0.upload(upload.Data(DATA, convergence=""))
4209         def _stash_uri(ur, which):
4210             self.uris[which] = ur.get_uri()
4211         d.addCallback(_stash_uri, "good")
4212         d.addCallback(lambda ign:
4213                       c0.upload(upload.Data(DATA+"1", convergence="")))
4214         d.addCallback(_stash_uri, "sick")
4215         d.addCallback(lambda ign:
4216                       c0.upload(upload.Data(DATA+"2", convergence="")))
4217         d.addCallback(_stash_uri, "dead")
4218         def _stash_mutable_uri(n, which):
4219             self.uris[which] = n.get_uri()
4220             assert isinstance(self.uris[which], str)
4221         d.addCallback(lambda ign:
4222             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4223         d.addCallback(_stash_mutable_uri, "corrupt")
4224         d.addCallback(lambda ign:
4225                       c0.upload(upload.Data("literal", convergence="")))
4226         d.addCallback(_stash_uri, "small")
4227         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
4228         d.addCallback(_stash_mutable_uri, "smalldir")
4229
4230         def _compute_fileurls(ignored):
4231             self.fileurls = {}
4232             for which in self.uris:
4233                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4234         d.addCallback(_compute_fileurls)
4235
4236         def _clobber_shares(ignored):
4237             good_shares = self.find_uri_shares(self.uris["good"])
4238             self.failUnlessReallyEqual(len(good_shares), 10)
4239             sick_shares = self.find_uri_shares(self.uris["sick"])
4240             os.unlink(sick_shares[0][2])
4241             dead_shares = self.find_uri_shares(self.uris["dead"])
4242             for i in range(1, 10):
4243                 os.unlink(dead_shares[i][2])
4244             c_shares = self.find_uri_shares(self.uris["corrupt"])
4245             cso = CorruptShareOptions()
4246             cso.stdout = StringIO()
4247             cso.parseOptions([c_shares[0][2]])
4248             corrupt_share(cso)
4249         d.addCallback(_clobber_shares)
4250
4251         d.addCallback(self.CHECK, "good", "t=check")
4252         def _got_html_good(res):
4253             self.failUnlessIn("Healthy", res)
4254             self.failIfIn("Not Healthy", res)
4255             self.failUnlessIn(FAVICON_MARKUP, res)
4256         d.addCallback(_got_html_good)
4257         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
4258         def _got_html_good_return_to(res):
4259             self.failUnlessIn("Healthy", res)
4260             self.failIfIn("Not Healthy", res)
4261             self.failUnlessIn('<a href="somewhere">Return to file', res)
4262         d.addCallback(_got_html_good_return_to)
4263         d.addCallback(self.CHECK, "good", "t=check&output=json")
4264         def _got_json_good(res):
4265             r = simplejson.loads(res)
4266             self.failUnlessEqual(r["summary"], "Healthy")
4267             self.failUnless(r["results"]["healthy"])
4268             self.failIf(r["results"]["needs-rebalancing"])
4269             self.failUnless(r["results"]["recoverable"])
4270         d.addCallback(_got_json_good)
4271
4272         d.addCallback(self.CHECK, "small", "t=check")
4273         def _got_html_small(res):
4274             self.failUnlessIn("Literal files are always healthy", res)
4275             self.failIfIn("Not Healthy", res)
4276         d.addCallback(_got_html_small)
4277         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
4278         def _got_html_small_return_to(res):
4279             self.failUnlessIn("Literal files are always healthy", res)
4280             self.failIfIn("Not Healthy", res)
4281             self.failUnlessIn('<a href="somewhere">Return to file', res)
4282         d.addCallback(_got_html_small_return_to)
4283         d.addCallback(self.CHECK, "small", "t=check&output=json")
4284         def _got_json_small(res):
4285             r = simplejson.loads(res)
4286             self.failUnlessEqual(r["storage-index"], "")
4287             self.failUnless(r["results"]["healthy"])
4288         d.addCallback(_got_json_small)
4289
4290         d.addCallback(self.CHECK, "smalldir", "t=check")
4291         def _got_html_smalldir(res):
4292             self.failUnlessIn("Literal files are always healthy", res)
4293             self.failIfIn("Not Healthy", res)
4294         d.addCallback(_got_html_smalldir)
4295         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
4296         def _got_json_smalldir(res):
4297             r = simplejson.loads(res)
4298             self.failUnlessEqual(r["storage-index"], "")
4299             self.failUnless(r["results"]["healthy"])
4300         d.addCallback(_got_json_smalldir)
4301
4302         d.addCallback(self.CHECK, "sick", "t=check")
4303         def _got_html_sick(res):
4304             self.failUnlessIn("Not Healthy", res)
4305         d.addCallback(_got_html_sick)
4306         d.addCallback(self.CHECK, "sick", "t=check&output=json")
4307         def _got_json_sick(res):
4308             r = simplejson.loads(res)
4309             self.failUnlessEqual(r["summary"],
4310                                  "Not Healthy: 9 shares (enc 3-of-10)")
4311             self.failIf(r["results"]["healthy"])
4312             self.failIf(r["results"]["needs-rebalancing"])
4313             self.failUnless(r["results"]["recoverable"])
4314         d.addCallback(_got_json_sick)
4315
4316         d.addCallback(self.CHECK, "dead", "t=check")
4317         def _got_html_dead(res):
4318             self.failUnlessIn("Not Healthy", res)
4319         d.addCallback(_got_html_dead)
4320         d.addCallback(self.CHECK, "dead", "t=check&output=json")
4321         def _got_json_dead(res):
4322             r = simplejson.loads(res)
4323             self.failUnlessEqual(r["summary"],
4324                                  "Not Healthy: 1 shares (enc 3-of-10)")
4325             self.failIf(r["results"]["healthy"])
4326             self.failIf(r["results"]["needs-rebalancing"])
4327             self.failIf(r["results"]["recoverable"])
4328         d.addCallback(_got_json_dead)
4329
4330         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
4331         def _got_html_corrupt(res):
4332             self.failUnlessIn("Not Healthy! : Unhealthy", res)
4333         d.addCallback(_got_html_corrupt)
4334         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
4335         def _got_json_corrupt(res):
4336             r = simplejson.loads(res)
4337             self.failUnlessIn("Unhealthy: 9 shares (enc 3-of-10)", r["summary"])
4338             self.failIf(r["results"]["healthy"])
4339             self.failUnless(r["results"]["recoverable"])
4340             self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
4341             self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
4342         d.addCallback(_got_json_corrupt)
4343
4344         d.addErrback(self.explain_web_error)
4345         return d
4346
4347     def test_repair_html(self):
4348         self.basedir = "web/Grid/repair_html"
4349         self.set_up_grid()
4350         c0 = self.g.clients[0]
4351         self.uris = {}
4352         DATA = "data" * 100
4353         d = c0.upload(upload.Data(DATA, convergence=""))
4354         def _stash_uri(ur, which):
4355             self.uris[which] = ur.get_uri()
4356         d.addCallback(_stash_uri, "good")
4357         d.addCallback(lambda ign:
4358                       c0.upload(upload.Data(DATA+"1", convergence="")))
4359         d.addCallback(_stash_uri, "sick")
4360         d.addCallback(lambda ign:
4361                       c0.upload(upload.Data(DATA+"2", convergence="")))
4362         d.addCallback(_stash_uri, "dead")
4363         def _stash_mutable_uri(n, which):
4364             self.uris[which] = n.get_uri()
4365             assert isinstance(self.uris[which], str)
4366         d.addCallback(lambda ign:
4367             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4368         d.addCallback(_stash_mutable_uri, "corrupt")
4369
4370         def _compute_fileurls(ignored):
4371             self.fileurls = {}
4372             for which in self.uris:
4373                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4374         d.addCallback(_compute_fileurls)
4375
4376         def _clobber_shares(ignored):
4377             good_shares = self.find_uri_shares(self.uris["good"])
4378             self.failUnlessReallyEqual(len(good_shares), 10)
4379             sick_shares = self.find_uri_shares(self.uris["sick"])
4380             os.unlink(sick_shares[0][2])
4381             dead_shares = self.find_uri_shares(self.uris["dead"])
4382             for i in range(1, 10):
4383                 os.unlink(dead_shares[i][2])
4384             c_shares = self.find_uri_shares(self.uris["corrupt"])
4385             cso = CorruptShareOptions()
4386             cso.stdout = StringIO()
4387             cso.parseOptions([c_shares[0][2]])
4388             corrupt_share(cso)
4389         d.addCallback(_clobber_shares)
4390
4391         d.addCallback(self.CHECK, "good", "t=check&repair=true")
4392         def _got_html_good(res):
4393             self.failUnlessIn("Healthy", res)
4394             self.failIfIn("Not Healthy", res)
4395             self.failUnlessIn("No repair necessary", res)
4396             self.failUnlessIn(FAVICON_MARKUP, res)
4397         d.addCallback(_got_html_good)
4398
4399         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
4400         def _got_html_sick(res):
4401             self.failUnlessIn("Healthy : healthy", res)
4402             self.failIfIn("Not Healthy", res)
4403             self.failUnlessIn("Repair successful", res)
4404         d.addCallback(_got_html_sick)
4405
4406         # repair of a dead file will fail, of course, but it isn't yet
4407         # clear how this should be reported. Right now it shows up as
4408         # a "410 Gone".
4409         #
4410         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
4411         #def _got_html_dead(res):
4412         #    print res
4413         #    self.failUnlessIn("Healthy : healthy", res)
4414         #    self.failIfIn("Not Healthy", res)
4415         #    self.failUnlessIn("No repair necessary", res)
4416         #d.addCallback(_got_html_dead)
4417
4418         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
4419         def _got_html_corrupt(res):
4420             self.failUnlessIn("Healthy : Healthy", res)
4421             self.failIfIn("Not Healthy", res)
4422             self.failUnlessIn("Repair successful", res)
4423         d.addCallback(_got_html_corrupt)
4424
4425         d.addErrback(self.explain_web_error)
4426         return d
4427
4428     def test_repair_json(self):
4429         self.basedir = "web/Grid/repair_json"
4430         self.set_up_grid()
4431         c0 = self.g.clients[0]
4432         self.uris = {}
4433         DATA = "data" * 100
4434         d = c0.upload(upload.Data(DATA+"1", convergence=""))
4435         def _stash_uri(ur, which):
4436             self.uris[which] = ur.get_uri()
4437         d.addCallback(_stash_uri, "sick")
4438
4439         def _compute_fileurls(ignored):
4440             self.fileurls = {}
4441             for which in self.uris:
4442                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4443         d.addCallback(_compute_fileurls)
4444
4445         def _clobber_shares(ignored):
4446             sick_shares = self.find_uri_shares(self.uris["sick"])
4447             os.unlink(sick_shares[0][2])
4448         d.addCallback(_clobber_shares)
4449
4450         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
4451         def _got_json_sick(res):
4452             r = simplejson.loads(res)
4453             self.failUnlessReallyEqual(r["repair-attempted"], True)
4454             self.failUnlessReallyEqual(r["repair-successful"], True)
4455             self.failUnlessEqual(r["pre-repair-results"]["summary"],
4456                                  "Not Healthy: 9 shares (enc 3-of-10)")
4457             self.failIf(r["pre-repair-results"]["results"]["healthy"])
4458             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
4459             self.failUnless(r["post-repair-results"]["results"]["healthy"])
4460         d.addCallback(_got_json_sick)
4461
4462         d.addErrback(self.explain_web_error)
4463         return d
4464
4465     def test_unknown(self, immutable=False):
4466         self.basedir = "web/Grid/unknown"
4467         if immutable:
4468             self.basedir = "web/Grid/unknown-immutable"
4469
4470         self.set_up_grid()
4471         c0 = self.g.clients[0]
4472         self.uris = {}
4473         self.fileurls = {}
4474
4475         # the future cap format may contain slashes, which must be tolerated
4476         expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
4477                                                            safe="")
4478
4479         if immutable:
4480             name = u"future-imm"
4481             future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
4482             d = c0.create_immutable_dirnode({name: (future_node, {})})
4483         else:
4484             name = u"future"
4485             future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4486             d = c0.create_dirnode()
4487
4488         def _stash_root_and_create_file(n):
4489             self.rootnode = n
4490             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
4491             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
4492             if not immutable:
4493                 return self.rootnode.set_node(name, future_node)
4494         d.addCallback(_stash_root_and_create_file)
4495
4496         # make sure directory listing tolerates unknown nodes
4497         d.addCallback(lambda ign: self.GET(self.rooturl))
4498         def _check_directory_html(res, expected_type_suffix):
4499             pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
4500                                   '<td>%s</td>' % (expected_type_suffix, str(name)),
4501                                  re.DOTALL)
4502             self.failUnless(re.search(pattern, res), res)
4503             # find the More Info link for name, should be relative
4504             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4505             info_url = mo.group(1)
4506             self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
4507         if immutable:
4508             d.addCallback(_check_directory_html, "-IMM")
4509         else:
4510             d.addCallback(_check_directory_html, "")
4511
4512         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4513         def _check_directory_json(res, expect_rw_uri):
4514             data = simplejson.loads(res)
4515             self.failUnlessEqual(data[0], "dirnode")
4516             f = data[1]["children"][name]
4517             self.failUnlessEqual(f[0], "unknown")
4518             if expect_rw_uri:
4519                 self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data)
4520             else:
4521                 self.failIfIn("rw_uri", f[1])
4522             if immutable:
4523                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
4524             else:
4525                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
4526             self.failUnlessIn("metadata", f[1])
4527         d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
4528
4529         def _check_info(res, expect_rw_uri, expect_ro_uri):
4530             self.failUnlessIn("Object Type: <span>unknown</span>", res)
4531             if expect_rw_uri:
4532                 self.failUnlessIn(unknown_rwcap, res)
4533             if expect_ro_uri:
4534                 if immutable:
4535                     self.failUnlessIn(unknown_immcap, res)
4536                 else:
4537                     self.failUnlessIn(unknown_rocap, res)
4538             else:
4539                 self.failIfIn(unknown_rocap, res)
4540             self.failIfIn("Raw data as", res)
4541             self.failIfIn("Directory writecap", res)
4542             self.failIfIn("Checker Operations", res)
4543             self.failIfIn("Mutable File Operations", res)
4544             self.failIfIn("Directory Operations", res)
4545
4546         # FIXME: these should have expect_rw_uri=not immutable; I don't know
4547         # why they fail. Possibly related to ticket #922.
4548
4549         d.addCallback(lambda ign: self.GET(expected_info_url))
4550         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
4551         d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
4552         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
4553
4554         def _check_json(res, expect_rw_uri):
4555             data = simplejson.loads(res)
4556             self.failUnlessEqual(data[0], "unknown")
4557             if expect_rw_uri:
4558                 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data)
4559             else:
4560                 self.failIfIn("rw_uri", data[1])
4561
4562             if immutable:
4563                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data)
4564                 self.failUnlessReallyEqual(data[1]["mutable"], False)
4565             elif expect_rw_uri:
4566                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4567                 self.failUnlessReallyEqual(data[1]["mutable"], True)
4568             else:
4569                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4570                 self.failIfIn("mutable", data[1])
4571
4572             # TODO: check metadata contents
4573             self.failUnlessIn("metadata", data[1])
4574
4575         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
4576         d.addCallback(_check_json, expect_rw_uri=not immutable)
4577
4578         # and make sure that a read-only version of the directory can be
4579         # rendered too. This version will not have unknown_rwcap, whether
4580         # or not future_node was immutable.
4581         d.addCallback(lambda ign: self.GET(self.rourl))
4582         if immutable:
4583             d.addCallback(_check_directory_html, "-IMM")
4584         else:
4585             d.addCallback(_check_directory_html, "-RO")
4586
4587         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
4588         d.addCallback(_check_directory_json, expect_rw_uri=False)
4589
4590         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
4591         d.addCallback(_check_json, expect_rw_uri=False)
4592
4593         # TODO: check that getting t=info from the Info link in the ro directory
4594         # works, and does not include the writecap URI.
4595         return d
4596
4597     def test_immutable_unknown(self):
4598         return self.test_unknown(immutable=True)
4599
4600     def test_mutant_dirnodes_are_omitted(self):
4601         self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
4602
4603         self.set_up_grid()
4604         c = self.g.clients[0]
4605         nm = c.nodemaker
4606         self.uris = {}
4607         self.fileurls = {}
4608
4609         lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
4610         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
4611         mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
4612
4613         # This method tests mainly dirnode, but we'd have to duplicate code in order to
4614         # test the dirnode and web layers separately.
4615
4616         # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
4617         # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
4618         # When the directory is read, the mutants should be silently disposed of, leaving
4619         # their lonely sibling.
4620         # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
4621         # because immutable directories don't have a writecap and therefore that field
4622         # isn't (and can't be) decrypted.
4623         # TODO: The field still exists in the netstring. Technically we should check what
4624         # happens if something is put there (_unpack_contents should raise ValueError),
4625         # but that can wait.
4626
4627         lonely_child = nm.create_from_cap(lonely_uri)
4628         mutant_ro_child = nm.create_from_cap(mut_read_uri)
4629         mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
4630
4631         def _by_hook_or_by_crook():
4632             return True
4633         for n in [mutant_ro_child, mutant_write_in_ro_child]:
4634             n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
4635
4636         mutant_write_in_ro_child.get_write_uri    = lambda: None
4637         mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
4638
4639         kids = {u"lonely":      (lonely_child, {}),
4640                 u"ro":          (mutant_ro_child, {}),
4641                 u"write-in-ro": (mutant_write_in_ro_child, {}),
4642                 }
4643         d = c.create_immutable_dirnode(kids)
4644
4645         def _created(dn):
4646             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
4647             self.failIf(dn.is_mutable())
4648             self.failUnless(dn.is_readonly())
4649             # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
4650             self.failIf(hasattr(dn._node, 'get_writekey'))
4651             rep = str(dn)
4652             self.failUnlessIn("RO-IMM", rep)
4653             cap = dn.get_cap()
4654             self.failUnlessIn("CHK", cap.to_string())
4655             self.cap = cap
4656             self.rootnode = dn
4657             self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
4658             return download_to_data(dn._node)
4659         d.addCallback(_created)
4660
4661         def _check_data(data):
4662             # Decode the netstring representation of the directory to check that all children
4663             # are present. This is a bit of an abstraction violation, but there's not really
4664             # any other way to do it given that the real DirectoryNode._unpack_contents would
4665             # strip the mutant children out (which is what we're trying to test, later).
4666             position = 0
4667             numkids = 0
4668             while position < len(data):
4669                 entries, position = split_netstring(data, 1, position)
4670                 entry = entries[0]
4671                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
4672                 name = name_utf8.decode("utf-8")
4673                 self.failUnlessEqual(rwcapdata, "")
4674                 self.failUnlessIn(name, kids)
4675                 (expected_child, ign) = kids[name]
4676                 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
4677                 numkids += 1
4678
4679             self.failUnlessReallyEqual(numkids, 3)
4680             return self.rootnode.list()
4681         d.addCallback(_check_data)
4682
4683         # Now when we use the real directory listing code, the mutants should be absent.
4684         def _check_kids(children):
4685             self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
4686             lonely_node, lonely_metadata = children[u"lonely"]
4687
4688             self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
4689             self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
4690         d.addCallback(_check_kids)
4691
4692         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
4693         d.addCallback(lambda n: n.list())
4694         d.addCallback(_check_kids)  # again with dirnode recreated from cap
4695
4696         # Make sure the lonely child can be listed in HTML...
4697         d.addCallback(lambda ign: self.GET(self.rooturl))
4698         def _check_html(res):
4699             self.failIfIn("URI:SSK", res)
4700             get_lonely = "".join([r'<td>FILE</td>',
4701                                   r'\s+<td>',
4702                                   r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
4703                                   r'</td>',
4704                                   r'\s+<td align="right">%d</td>' % len("one"),
4705                                   ])
4706             self.failUnless(re.search(get_lonely, res), res)
4707
4708             # find the More Info link for name, should be relative
4709             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4710             info_url = mo.group(1)
4711             self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
4712         d.addCallback(_check_html)
4713
4714         # ... and in JSON.
4715         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4716         def _check_json(res):
4717             data = simplejson.loads(res)
4718             self.failUnlessEqual(data[0], "dirnode")
4719             listed_children = data[1]["children"]
4720             self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
4721             ll_type, ll_data = listed_children[u"lonely"]
4722             self.failUnlessEqual(ll_type, "filenode")
4723             self.failIfIn("rw_uri", ll_data)
4724             self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
4725         d.addCallback(_check_json)
4726         return d
4727
4728     def test_deep_check(self):
4729         self.basedir = "web/Grid/deep_check"
4730         self.set_up_grid()
4731         c0 = self.g.clients[0]
4732         self.uris = {}
4733         self.fileurls = {}
4734         DATA = "data" * 100
4735         d = c0.create_dirnode()
4736         def _stash_root_and_create_file(n):
4737             self.rootnode = n
4738             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4739             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4740         d.addCallback(_stash_root_and_create_file)
4741         def _stash_uri(fn, which):
4742             self.uris[which] = fn.get_uri()
4743             return fn
4744         d.addCallback(_stash_uri, "good")
4745         d.addCallback(lambda ign:
4746                       self.rootnode.add_file(u"small",
4747                                              upload.Data("literal",
4748                                                         convergence="")))
4749         d.addCallback(_stash_uri, "small")
4750         d.addCallback(lambda ign:
4751                       self.rootnode.add_file(u"sick",
4752                                              upload.Data(DATA+"1",
4753                                                         convergence="")))
4754         d.addCallback(_stash_uri, "sick")
4755
4756         # this tests that deep-check and stream-manifest will ignore
4757         # UnknownNode instances. Hopefully this will also cover deep-stats.
4758         future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4759         d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
4760
4761         def _clobber_shares(ignored):
4762             self.delete_shares_numbered(self.uris["sick"], [0,1])
4763         d.addCallback(_clobber_shares)
4764
4765         # root
4766         # root/good
4767         # root/small
4768         # root/sick
4769         # root/future
4770
4771         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4772         def _done(res):
4773             try:
4774                 units = [simplejson.loads(line)
4775                          for line in res.splitlines()
4776                          if line]
4777             except ValueError:
4778                 print "response is:", res
4779                 print "undecodeable line was '%s'" % line
4780                 raise
4781             self.failUnlessReallyEqual(len(units), 5+1)
4782             # should be parent-first
4783             u0 = units[0]
4784             self.failUnlessEqual(u0["path"], [])
4785             self.failUnlessEqual(u0["type"], "directory")
4786             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4787             u0cr = u0["check-results"]
4788             self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
4789
4790             ugood = [u for u in units
4791                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4792             self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
4793             ugoodcr = ugood["check-results"]
4794             self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
4795
4796             stats = units[-1]
4797             self.failUnlessEqual(stats["type"], "stats")
4798             s = stats["stats"]
4799             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4800             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4801             self.failUnlessReallyEqual(s["count-directories"], 1)
4802             self.failUnlessReallyEqual(s["count-unknown"], 1)
4803         d.addCallback(_done)
4804
4805         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4806         def _check_manifest(res):
4807             self.failUnless(res.endswith("\n"))
4808             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
4809             self.failUnlessReallyEqual(len(units), 5+1)
4810             self.failUnlessEqual(units[-1]["type"], "stats")
4811             first = units[0]
4812             self.failUnlessEqual(first["path"], [])
4813             self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri())
4814             self.failUnlessEqual(first["type"], "directory")
4815             stats = units[-1]["stats"]
4816             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4817             self.failUnlessReallyEqual(stats["count-literal-files"], 1)
4818             self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
4819             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4820             self.failUnlessReallyEqual(stats["count-unknown"], 1)
4821         d.addCallback(_check_manifest)
4822
4823         # now add root/subdir and root/subdir/grandchild, then make subdir
4824         # unrecoverable, then see what happens
4825
4826         d.addCallback(lambda ign:
4827                       self.rootnode.create_subdirectory(u"subdir"))
4828         d.addCallback(_stash_uri, "subdir")
4829         d.addCallback(lambda subdir_node:
4830                       subdir_node.add_file(u"grandchild",
4831                                            upload.Data(DATA+"2",
4832                                                        convergence="")))
4833         d.addCallback(_stash_uri, "grandchild")
4834
4835         d.addCallback(lambda ign:
4836                       self.delete_shares_numbered(self.uris["subdir"],
4837                                                   range(1, 10)))
4838
4839         # root
4840         # root/good
4841         # root/small
4842         # root/sick
4843         # root/future
4844         # root/subdir [unrecoverable]
4845         # root/subdir/grandchild
4846
4847         # how should a streaming-JSON API indicate fatal error?
4848         # answer: emit ERROR: instead of a JSON string
4849
4850         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4851         def _check_broken_manifest(res):
4852             lines = res.splitlines()
4853             error_lines = [i
4854                            for (i,line) in enumerate(lines)
4855                            if line.startswith("ERROR:")]
4856             if not error_lines:
4857                 self.fail("no ERROR: in output: %s" % (res,))
4858             first_error = error_lines[0]
4859             error_line = lines[first_error]
4860             error_msg = lines[first_error+1:]
4861             error_msg_s = "\n".join(error_msg) + "\n"
4862             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4863                               error_line)
4864             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4865             units = [simplejson.loads(line) for line in lines[:first_error]]
4866             self.failUnlessReallyEqual(len(units), 6) # includes subdir
4867             last_unit = units[-1]
4868             self.failUnlessEqual(last_unit["path"], ["subdir"])
4869         d.addCallback(_check_broken_manifest)
4870
4871         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4872         def _check_broken_deepcheck(res):
4873             lines = res.splitlines()
4874             error_lines = [i
4875                            for (i,line) in enumerate(lines)
4876                            if line.startswith("ERROR:")]
4877             if not error_lines:
4878                 self.fail("no ERROR: in output: %s" % (res,))
4879             first_error = error_lines[0]
4880             error_line = lines[first_error]
4881             error_msg = lines[first_error+1:]
4882             error_msg_s = "\n".join(error_msg) + "\n"
4883             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4884                               error_line)
4885             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4886             units = [simplejson.loads(line) for line in lines[:first_error]]
4887             self.failUnlessReallyEqual(len(units), 6) # includes subdir
4888             last_unit = units[-1]
4889             self.failUnlessEqual(last_unit["path"], ["subdir"])
4890             r = last_unit["check-results"]["results"]
4891             self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
4892             self.failUnlessReallyEqual(r["count-shares-good"], 1)
4893             self.failUnlessReallyEqual(r["recoverable"], False)
4894         d.addCallback(_check_broken_deepcheck)
4895
4896         d.addErrback(self.explain_web_error)
4897         return d
4898
4899     def test_deep_check_and_repair(self):
4900         self.basedir = "web/Grid/deep_check_and_repair"
4901         self.set_up_grid()
4902         c0 = self.g.clients[0]
4903         self.uris = {}
4904         self.fileurls = {}
4905         DATA = "data" * 100
4906         d = c0.create_dirnode()
4907         def _stash_root_and_create_file(n):
4908             self.rootnode = n
4909             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4910             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4911         d.addCallback(_stash_root_and_create_file)
4912         def _stash_uri(fn, which):
4913             self.uris[which] = fn.get_uri()
4914         d.addCallback(_stash_uri, "good")
4915         d.addCallback(lambda ign:
4916                       self.rootnode.add_file(u"small",
4917                                              upload.Data("literal",
4918                                                         convergence="")))
4919         d.addCallback(_stash_uri, "small")
4920         d.addCallback(lambda ign:
4921                       self.rootnode.add_file(u"sick",
4922                                              upload.Data(DATA+"1",
4923                                                         convergence="")))
4924         d.addCallback(_stash_uri, "sick")
4925         #d.addCallback(lambda ign:
4926         #              self.rootnode.add_file(u"dead",
4927         #                                     upload.Data(DATA+"2",
4928         #                                                convergence="")))
4929         #d.addCallback(_stash_uri, "dead")
4930
4931         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
4932         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
4933         #d.addCallback(_stash_uri, "corrupt")
4934
4935         def _clobber_shares(ignored):
4936             good_shares = self.find_uri_shares(self.uris["good"])
4937             self.failUnlessReallyEqual(len(good_shares), 10)
4938             sick_shares = self.find_uri_shares(self.uris["sick"])
4939             os.unlink(sick_shares[0][2])
4940             #dead_shares = self.find_uri_shares(self.uris["dead"])
4941             #for i in range(1, 10):
4942             #    os.unlink(dead_shares[i][2])
4943
4944             #c_shares = self.find_uri_shares(self.uris["corrupt"])
4945             #cso = CorruptShareOptions()
4946             #cso.stdout = StringIO()
4947             #cso.parseOptions([c_shares[0][2]])
4948             #corrupt_share(cso)
4949         d.addCallback(_clobber_shares)
4950
4951         # root
4952         # root/good   CHK, 10 shares
4953         # root/small  LIT
4954         # root/sick   CHK, 9 shares
4955
4956         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
4957         def _done(res):
4958             units = [simplejson.loads(line)
4959                      for line in res.splitlines()
4960                      if line]
4961             self.failUnlessReallyEqual(len(units), 4+1)
4962             # should be parent-first
4963             u0 = units[0]
4964             self.failUnlessEqual(u0["path"], [])
4965             self.failUnlessEqual(u0["type"], "directory")
4966             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4967             u0crr = u0["check-and-repair-results"]
4968             self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
4969             self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
4970
4971             ugood = [u for u in units
4972                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4973             self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
4974             ugoodcrr = ugood["check-and-repair-results"]
4975             self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
4976             self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
4977
4978             usick = [u for u in units
4979                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
4980             self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"])
4981             usickcrr = usick["check-and-repair-results"]
4982             self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
4983             self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
4984             self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
4985             self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
4986
4987             stats = units[-1]
4988             self.failUnlessEqual(stats["type"], "stats")
4989             s = stats["stats"]
4990             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4991             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4992             self.failUnlessReallyEqual(s["count-directories"], 1)
4993         d.addCallback(_done)
4994
4995         d.addErrback(self.explain_web_error)
4996         return d
4997
4998     def _count_leases(self, ignored, which):
4999         u = self.uris[which]
5000         shares = self.find_uri_shares(u)
5001         lease_counts = []
5002         for shnum, serverid, fn in shares:
5003             sf = get_share_file(fn)
5004             num_leases = len(list(sf.get_leases()))
5005             lease_counts.append( (fn, num_leases) )
5006         return lease_counts
5007
5008     def _assert_leasecount(self, lease_counts, expected):
5009         for (fn, num_leases) in lease_counts:
5010             if num_leases != expected:
5011                 self.fail("expected %d leases, have %d, on %s" %
5012                           (expected, num_leases, fn))
5013
5014     def test_add_lease(self):
5015         self.basedir = "web/Grid/add_lease"
5016         self.set_up_grid(num_clients=2)
5017         c0 = self.g.clients[0]
5018         self.uris = {}
5019         DATA = "data" * 100
5020         d = c0.upload(upload.Data(DATA, convergence=""))
5021         def _stash_uri(ur, which):
5022             self.uris[which] = ur.get_uri()
5023         d.addCallback(_stash_uri, "one")
5024         d.addCallback(lambda ign:
5025                       c0.upload(upload.Data(DATA+"1", convergence="")))
5026         d.addCallback(_stash_uri, "two")
5027         def _stash_mutable_uri(n, which):
5028             self.uris[which] = n.get_uri()
5029             assert isinstance(self.uris[which], str)
5030         d.addCallback(lambda ign:
5031             c0.create_mutable_file(publish.MutableData(DATA+"2")))
5032         d.addCallback(_stash_mutable_uri, "mutable")
5033
5034         def _compute_fileurls(ignored):
5035             self.fileurls = {}
5036             for which in self.uris:
5037                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
5038         d.addCallback(_compute_fileurls)
5039
5040         d.addCallback(self._count_leases, "one")
5041         d.addCallback(self._assert_leasecount, 1)
5042         d.addCallback(self._count_leases, "two")
5043         d.addCallback(self._assert_leasecount, 1)
5044         d.addCallback(self._count_leases, "mutable")
5045         d.addCallback(self._assert_leasecount, 1)
5046
5047         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
5048         def _got_html_good(res):
5049             self.failUnlessIn("Healthy", res)
5050             self.failIfIn("Not Healthy", res)
5051         d.addCallback(_got_html_good)
5052
5053         d.addCallback(self._count_leases, "one")
5054         d.addCallback(self._assert_leasecount, 1)
5055         d.addCallback(self._count_leases, "two")
5056         d.addCallback(self._assert_leasecount, 1)
5057         d.addCallback(self._count_leases, "mutable")
5058         d.addCallback(self._assert_leasecount, 1)
5059
5060         # this CHECK uses the original client, which uses the same
5061         # lease-secrets, so it will just renew the original lease
5062         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
5063         d.addCallback(_got_html_good)
5064
5065         d.addCallback(self._count_leases, "one")
5066         d.addCallback(self._assert_leasecount, 1)
5067         d.addCallback(self._count_leases, "two")
5068         d.addCallback(self._assert_leasecount, 1)
5069         d.addCallback(self._count_leases, "mutable")
5070         d.addCallback(self._assert_leasecount, 1)
5071
5072         # this CHECK uses an alternate client, which adds a second lease
5073         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
5074         d.addCallback(_got_html_good)
5075
5076         d.addCallback(self._count_leases, "one")
5077         d.addCallback(self._assert_leasecount, 2)
5078         d.addCallback(self._count_leases, "two")
5079         d.addCallback(self._assert_leasecount, 1)
5080         d.addCallback(self._count_leases, "mutable")
5081         d.addCallback(self._assert_leasecount, 1)
5082
5083         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
5084         d.addCallback(_got_html_good)
5085
5086         d.addCallback(self._count_leases, "one")
5087         d.addCallback(self._assert_leasecount, 2)
5088         d.addCallback(self._count_leases, "two")
5089         d.addCallback(self._assert_leasecount, 1)
5090         d.addCallback(self._count_leases, "mutable")
5091         d.addCallback(self._assert_leasecount, 1)
5092
5093         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
5094                       clientnum=1)
5095         d.addCallback(_got_html_good)
5096
5097         d.addCallback(self._count_leases, "one")
5098         d.addCallback(self._assert_leasecount, 2)
5099         d.addCallback(self._count_leases, "two")
5100         d.addCallback(self._assert_leasecount, 1)
5101         d.addCallback(self._count_leases, "mutable")
5102         d.addCallback(self._assert_leasecount, 2)
5103
5104         d.addErrback(self.explain_web_error)
5105         return d
5106
5107     def test_deep_add_lease(self):
5108         self.basedir = "web/Grid/deep_add_lease"
5109         self.set_up_grid(num_clients=2)
5110         c0 = self.g.clients[0]
5111         self.uris = {}
5112         self.fileurls = {}
5113         DATA = "data" * 100
5114         d = c0.create_dirnode()
5115         def _stash_root_and_create_file(n):
5116             self.rootnode = n
5117             self.uris["root"] = n.get_uri()
5118             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5119             return n.add_file(u"one", upload.Data(DATA, convergence=""))
5120         d.addCallback(_stash_root_and_create_file)
5121         def _stash_uri(fn, which):
5122             self.uris[which] = fn.get_uri()
5123         d.addCallback(_stash_uri, "one")
5124         d.addCallback(lambda ign:
5125                       self.rootnode.add_file(u"small",
5126                                              upload.Data("literal",
5127                                                         convergence="")))
5128         d.addCallback(_stash_uri, "small")
5129
5130         d.addCallback(lambda ign:
5131             c0.create_mutable_file(publish.MutableData("mutable")))
5132         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
5133         d.addCallback(_stash_uri, "mutable")
5134
5135         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
5136         def _done(res):
5137             units = [simplejson.loads(line)
5138                      for line in res.splitlines()
5139                      if line]
5140             # root, one, small, mutable,   stats
5141             self.failUnlessReallyEqual(len(units), 4+1)
5142         d.addCallback(_done)
5143
5144         d.addCallback(self._count_leases, "root")
5145         d.addCallback(self._assert_leasecount, 1)
5146         d.addCallback(self._count_leases, "one")
5147         d.addCallback(self._assert_leasecount, 1)
5148         d.addCallback(self._count_leases, "mutable")
5149         d.addCallback(self._assert_leasecount, 1)
5150
5151         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
5152         d.addCallback(_done)
5153
5154         d.addCallback(self._count_leases, "root")
5155         d.addCallback(self._assert_leasecount, 1)
5156         d.addCallback(self._count_leases, "one")
5157         d.addCallback(self._assert_leasecount, 1)
5158         d.addCallback(self._count_leases, "mutable")
5159         d.addCallback(self._assert_leasecount, 1)
5160
5161         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
5162                       clientnum=1)
5163         d.addCallback(_done)
5164
5165         d.addCallback(self._count_leases, "root")
5166         d.addCallback(self._assert_leasecount, 2)
5167         d.addCallback(self._count_leases, "one")
5168         d.addCallback(self._assert_leasecount, 2)
5169         d.addCallback(self._count_leases, "mutable")
5170         d.addCallback(self._assert_leasecount, 2)
5171
5172         d.addErrback(self.explain_web_error)
5173         return d
5174
5175
5176     def test_exceptions(self):
5177         self.basedir = "web/Grid/exceptions"
5178         self.set_up_grid(num_clients=1, num_servers=2)
5179         c0 = self.g.clients[0]
5180         c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 2
5181         self.fileurls = {}
5182         DATA = "data" * 100
5183         d = c0.create_dirnode()
5184         def _stash_root(n):
5185             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5186             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
5187             return n
5188         d.addCallback(_stash_root)
5189         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
5190         def _stash_bad(ur):
5191             self.fileurls["1share"] = "uri/" + urllib.quote(ur.get_uri())
5192             self.delete_shares_numbered(ur.get_uri(), range(1,10))
5193
5194             u = uri.from_string(ur.get_uri())
5195             u.key = testutil.flip_bit(u.key, 0)
5196             baduri = u.to_string()
5197             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
5198         d.addCallback(_stash_bad)
5199         d.addCallback(lambda ign: c0.create_dirnode())
5200         def _mangle_dirnode_1share(n):
5201             u = n.get_uri()
5202             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
5203             self.fileurls["dir-1share-json"] = url + "?t=json"
5204             self.delete_shares_numbered(u, range(1,10))
5205         d.addCallback(_mangle_dirnode_1share)
5206         d.addCallback(lambda ign: c0.create_dirnode())
5207         def _mangle_dirnode_0share(n):
5208             u = n.get_uri()
5209             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
5210             self.fileurls["dir-0share-json"] = url + "?t=json"
5211             self.delete_shares_numbered(u, range(0,10))
5212         d.addCallback(_mangle_dirnode_0share)
5213
5214         # NotEnoughSharesError should be reported sensibly, with a
5215         # text/plain explanation of the problem, and perhaps some
5216         # information on which shares *could* be found.
5217
5218         d.addCallback(lambda ignored:
5219                       self.shouldHTTPError("GET unrecoverable",
5220                                            410, "Gone", "NoSharesError",
5221                                            self.GET, self.fileurls["0shares"]))
5222         def _check_zero_shares(body):
5223             self.failIfIn("<html>", body)
5224             body = " ".join(body.strip().split())
5225             exp = ("NoSharesError: no shares could be found. "
5226                    "Zero shares usually indicates a corrupt URI, or that "
5227                    "no servers were connected, but it might also indicate "
5228                    "severe corruption. You should perform a filecheck on "
5229                    "this object to learn more. The full error message is: "
5230                    "no shares (need 3). Last failure: None")
5231             self.failUnlessReallyEqual(exp, body)
5232         d.addCallback(_check_zero_shares)
5233
5234
5235         d.addCallback(lambda ignored:
5236                       self.shouldHTTPError("GET 1share",
5237                                            410, "Gone", "NotEnoughSharesError",
5238                                            self.GET, self.fileurls["1share"]))
5239         def _check_one_share(body):
5240             self.failIfIn("<html>", body)
5241             body = " ".join(body.strip().split())
5242             msgbase = ("NotEnoughSharesError: This indicates that some "
5243                        "servers were unavailable, or that shares have been "
5244                        "lost to server departure, hard drive failure, or disk "
5245                        "corruption. You should perform a filecheck on "
5246                        "this object to learn more. The full error message is:"
5247                        )
5248             msg1 = msgbase + (" ran out of shares:"
5249                               " complete=sh0"
5250                               " pending="
5251                               " overdue= unused= need 3. Last failure: None")
5252             msg2 = msgbase + (" ran out of shares:"
5253                               " complete="
5254                               " pending=Share(sh0-on-xgru5)"
5255                               " overdue= unused= need 3. Last failure: None")
5256             self.failUnless(body == msg1 or body == msg2, body)
5257         d.addCallback(_check_one_share)
5258
5259         d.addCallback(lambda ignored:
5260                       self.shouldHTTPError("GET imaginary",
5261                                            404, "Not Found", None,
5262                                            self.GET, self.fileurls["imaginary"]))
5263         def _missing_child(body):
5264             self.failUnlessIn("No such child: imaginary", body)
5265         d.addCallback(_missing_child)
5266
5267         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
5268         def _check_0shares_dir_html(body):
5269             self.failUnlessIn("<html>", body)
5270             # we should see the regular page, but without the child table or
5271             # the dirops forms
5272             body = " ".join(body.strip().split())
5273             self.failUnlessIn('href="?t=info">More info on this directory',
5274                               body)
5275             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5276                    "could not be retrieved, because there were insufficient "
5277                    "good shares. This might indicate that no servers were "
5278                    "connected, insufficient servers were connected, the URI "
5279                    "was corrupt, or that shares have been lost due to server "
5280                    "departure, hard drive failure, or disk corruption. You "
5281                    "should perform a filecheck on this object to learn more.")
5282             self.failUnlessIn(exp, body)
5283             self.failUnlessIn("No upload forms: directory is unreadable", body)
5284         d.addCallback(_check_0shares_dir_html)
5285
5286         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
5287         def _check_1shares_dir_html(body):
5288             # at some point, we'll split UnrecoverableFileError into 0-shares
5289             # and some-shares like we did for immutable files (since there
5290             # are different sorts of advice to offer in each case). For now,
5291             # they present the same way.
5292             self.failUnlessIn("<html>", body)
5293             body = " ".join(body.strip().split())
5294             self.failUnlessIn('href="?t=info">More info on this directory',
5295                               body)
5296             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5297                    "could not be retrieved, because there were insufficient "
5298                    "good shares. This might indicate that no servers were "
5299                    "connected, insufficient servers were connected, the URI "
5300                    "was corrupt, or that shares have been lost due to server "
5301                    "departure, hard drive failure, or disk corruption. You "
5302                    "should perform a filecheck on this object to learn more.")
5303             self.failUnlessIn(exp, body)
5304             self.failUnlessIn("No upload forms: directory is unreadable", body)
5305         d.addCallback(_check_1shares_dir_html)
5306
5307         d.addCallback(lambda ignored:
5308                       self.shouldHTTPError("GET dir-0share-json",
5309                                            410, "Gone", "UnrecoverableFileError",
5310                                            self.GET,
5311                                            self.fileurls["dir-0share-json"]))
5312         def _check_unrecoverable_file(body):
5313             self.failIfIn("<html>", body)
5314             body = " ".join(body.strip().split())
5315             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5316                    "could not be retrieved, because there were insufficient "
5317                    "good shares. This might indicate that no servers were "
5318                    "connected, insufficient servers were connected, the URI "
5319                    "was corrupt, or that shares have been lost due to server "
5320                    "departure, hard drive failure, or disk corruption. You "
5321                    "should perform a filecheck on this object to learn more.")
5322             self.failUnlessReallyEqual(exp, body)
5323         d.addCallback(_check_unrecoverable_file)
5324
5325         d.addCallback(lambda ignored:
5326                       self.shouldHTTPError("GET dir-1share-json",
5327                                            410, "Gone", "UnrecoverableFileError",
5328                                            self.GET,
5329                                            self.fileurls["dir-1share-json"]))
5330         d.addCallback(_check_unrecoverable_file)
5331
5332         d.addCallback(lambda ignored:
5333                       self.shouldHTTPError("GET imaginary",
5334                                            404, "Not Found", None,
5335                                            self.GET, self.fileurls["imaginary"]))
5336
5337         # attach a webapi child that throws a random error, to test how it
5338         # gets rendered.
5339         w = c0.getServiceNamed("webish")
5340         w.root.putChild("ERRORBOOM", ErrorBoom())
5341
5342         # "Accept: */*" :        should get a text/html stack trace
5343         # "Accept: text/plain" : should get a text/plain stack trace
5344         # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
5345         # no Accept header:      should get a text/html stack trace
5346
5347         d.addCallback(lambda ignored:
5348                       self.shouldHTTPError("GET errorboom_html",
5349                                            500, "Internal Server Error", None,
5350                                            self.GET, "ERRORBOOM",
5351                                            headers={"accept": "*/*"}))
5352         def _internal_error_html1(body):
5353             self.failUnlessIn("<html>", "expected HTML, not '%s'" % body)
5354         d.addCallback(_internal_error_html1)
5355
5356         d.addCallback(lambda ignored:
5357                       self.shouldHTTPError("GET errorboom_text",
5358                                            500, "Internal Server Error", None,
5359                                            self.GET, "ERRORBOOM",
5360                                            headers={"accept": "text/plain"}))
5361         def _internal_error_text2(body):
5362             self.failIfIn("<html>", body)
5363             self.failUnless(body.startswith("Traceback "), body)
5364         d.addCallback(_internal_error_text2)
5365
5366         CLI_accepts = "text/plain, application/octet-stream"
5367         d.addCallback(lambda ignored:
5368                       self.shouldHTTPError("GET errorboom_text",
5369                                            500, "Internal Server Error", None,
5370                                            self.GET, "ERRORBOOM",
5371                                            headers={"accept": CLI_accepts}))
5372         def _internal_error_text3(body):
5373             self.failIfIn("<html>", body)
5374             self.failUnless(body.startswith("Traceback "), body)
5375         d.addCallback(_internal_error_text3)
5376
5377         d.addCallback(lambda ignored:
5378                       self.shouldHTTPError("GET errorboom_text",
5379                                            500, "Internal Server Error", None,
5380                                            self.GET, "ERRORBOOM"))
5381         def _internal_error_html4(body):
5382             self.failUnlessIn("<html>", body)
5383         d.addCallback(_internal_error_html4)
5384
5385         def _flush_errors(res):
5386             # Trial: please ignore the CompletelyUnhandledError in the logs
5387             self.flushLoggedErrors(CompletelyUnhandledError)
5388             return res
5389         d.addBoth(_flush_errors)
5390
5391         return d
5392
5393     def test_blacklist(self):
5394         # download from a blacklisted URI, get an error
5395         self.basedir = "web/Grid/blacklist"
5396         self.set_up_grid()
5397         c0 = self.g.clients[0]
5398         c0_basedir = c0.basedir
5399         fn = os.path.join(c0_basedir, "access.blacklist")
5400         self.uris = {}
5401         DATA = "off-limits " * 50
5402
5403         d = c0.upload(upload.Data(DATA, convergence=""))
5404         def _stash_uri_and_create_dir(ur):
5405             self.uri = ur.get_uri()
5406             self.url = "uri/"+self.uri
5407             u = uri.from_string_filenode(self.uri)
5408             self.si = u.get_storage_index()
5409             childnode = c0.create_node_from_uri(self.uri, None)
5410             return c0.create_dirnode({u"blacklisted.txt": (childnode,{}) })
5411         d.addCallback(_stash_uri_and_create_dir)
5412         def _stash_dir(node):
5413             self.dir_node = node
5414             self.dir_uri = node.get_uri()
5415             self.dir_url = "uri/"+self.dir_uri
5416         d.addCallback(_stash_dir)
5417         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5418         def _check_dir_html(body):
5419             self.failUnlessIn("<html>", body)
5420             self.failUnlessIn("blacklisted.txt</a>", body)
5421         d.addCallback(_check_dir_html)
5422         d.addCallback(lambda ign: self.GET(self.url))
5423         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5424
5425         def _blacklist(ign):
5426             f = open(fn, "w")
5427             f.write(" # this is a comment\n")
5428             f.write(" \n")
5429             f.write("\n") # also exercise blank lines
5430             f.write("%s %s\n" % (base32.b2a(self.si), "off-limits to you"))
5431             f.close()
5432             # clients should be checking the blacklist each time, so we don't
5433             # need to restart the client
5434         d.addCallback(_blacklist)
5435         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_uri",
5436                                                        403, "Forbidden",
5437                                                        "Access Prohibited: off-limits",
5438                                                        self.GET, self.url))
5439
5440         # We should still be able to list the parent directory, in HTML...
5441         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5442         def _check_dir_html2(body):
5443             self.failUnlessIn("<html>", body)
5444             self.failUnlessIn("blacklisted.txt</strike>", body)
5445         d.addCallback(_check_dir_html2)
5446
5447         # ... and in JSON (used by CLI).
5448         d.addCallback(lambda ign: self.GET(self.dir_url+"?t=json", followRedirect=True))
5449         def _check_dir_json(res):
5450             data = simplejson.loads(res)
5451             self.failUnless(isinstance(data, list), data)
5452             self.failUnlessEqual(data[0], "dirnode")
5453             self.failUnless(isinstance(data[1], dict), data)
5454             self.failUnlessIn("children", data[1])
5455             self.failUnlessIn("blacklisted.txt", data[1]["children"])
5456             childdata = data[1]["children"]["blacklisted.txt"]
5457             self.failUnless(isinstance(childdata, list), data)
5458             self.failUnlessEqual(childdata[0], "filenode")
5459             self.failUnless(isinstance(childdata[1], dict), data)
5460         d.addCallback(_check_dir_json)
5461
5462         def _unblacklist(ign):
5463             open(fn, "w").close()
5464             # the Blacklist object watches mtime to tell when the file has
5465             # changed, but on windows this test will run faster than the
5466             # filesystem's mtime resolution. So we edit Blacklist.last_mtime
5467             # to force a reload.
5468             self.g.clients[0].blacklist.last_mtime -= 2.0
5469         d.addCallback(_unblacklist)
5470
5471         # now a read should work
5472         d.addCallback(lambda ign: self.GET(self.url))
5473         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5474
5475         # read again to exercise the blacklist-is-unchanged logic
5476         d.addCallback(lambda ign: self.GET(self.url))
5477         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5478
5479         # now add a blacklisted directory, and make sure files under it are
5480         # refused too
5481         def _add_dir(ign):
5482             childnode = c0.create_node_from_uri(self.uri, None)
5483             return c0.create_dirnode({u"child": (childnode,{}) })
5484         d.addCallback(_add_dir)
5485         def _get_dircap(dn):
5486             self.dir_si_b32 = base32.b2a(dn.get_storage_index())
5487             self.dir_url_base = "uri/"+dn.get_write_uri()
5488             self.dir_url_json1 = "uri/"+dn.get_write_uri()+"?t=json"
5489             self.dir_url_json2 = "uri/"+dn.get_write_uri()+"/?t=json"
5490             self.dir_url_json_ro = "uri/"+dn.get_readonly_uri()+"/?t=json"
5491             self.child_url = "uri/"+dn.get_readonly_uri()+"/child"
5492         d.addCallback(_get_dircap)
5493         d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True))
5494         d.addCallback(lambda body: self.failUnlessIn("<html>", body))
5495         d.addCallback(lambda ign: self.GET(self.dir_url_json1))
5496         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5497         d.addCallback(lambda ign: self.GET(self.dir_url_json2))
5498         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5499         d.addCallback(lambda ign: self.GET(self.dir_url_json_ro))
5500         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5501         d.addCallback(lambda ign: self.GET(self.child_url))
5502         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5503
5504         def _block_dir(ign):
5505             f = open(fn, "w")
5506             f.write("%s %s\n" % (self.dir_si_b32, "dir-off-limits to you"))
5507             f.close()
5508             self.g.clients[0].blacklist.last_mtime -= 2.0
5509         d.addCallback(_block_dir)
5510         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir base",
5511                                                        403, "Forbidden",
5512                                                        "Access Prohibited: dir-off-limits",
5513                                                        self.GET, self.dir_url_base))
5514         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json1",
5515                                                        403, "Forbidden",
5516                                                        "Access Prohibited: dir-off-limits",
5517                                                        self.GET, self.dir_url_json1))
5518         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json2",
5519                                                        403, "Forbidden",
5520                                                        "Access Prohibited: dir-off-limits",
5521                                                        self.GET, self.dir_url_json2))
5522         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json_ro",
5523                                                        403, "Forbidden",
5524                                                        "Access Prohibited: dir-off-limits",
5525                                                        self.GET, self.dir_url_json_ro))
5526         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir child",
5527                                                        403, "Forbidden",
5528                                                        "Access Prohibited: dir-off-limits",
5529                                                        self.GET, self.child_url))
5530         return d
5531
5532
5533 class CompletelyUnhandledError(Exception):
5534     pass
5535 class ErrorBoom(rend.Page):
5536     def beforeRender(self, ctx):
5537         raise CompletelyUnhandledError("whoops")