src/allmydata/test/test_web.py

   1 import os.path, re, urllib, time
   2 import simplejson
   3 from StringIO import StringIO
   4
   5 from twisted.application import service
   6 from twisted.trial import unittest
   7 from twisted.internet import defer, reactor
   8 from twisted.internet.task import Clock
   9 from twisted.web import client, error, http, html
  10 from twisted.python import failure, log
  11
  12 from foolscap.api import fireEventually, flushEventualQueue
  13
  14 from nevow.util import escapeToXML
  15 from nevow import rend
  16
  17 from allmydata import interfaces, uri, webish, dirnode
  18 from allmydata.storage.shares import get_share_file
  19 from allmydata.storage_client import StorageFarmBroker, StubServer
  20 from allmydata.immutable import upload
  21 from allmydata.immutable.downloader.status import DownloadStatus
  22 from allmydata.dirnode import DirectoryNode
  23 from allmydata.nodemaker import NodeMaker
  24 from allmydata.unknown import UnknownNode
  25 from allmydata.web import status, common
  26 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  27 from allmydata.util import fileutil, base32, hashutil
  28 from allmydata.util.consumer import download_to_data
  29 from allmydata.util.netstring import split_netstring
  30 from allmydata.util.encodingutil import to_str
  31 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  32      create_chk_filenode, WebErrorMixin, ShouldFailMixin, \
  33      make_mutable_file_uri, create_mutable_filenode
  34 from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
  35 from allmydata.mutable import servermap, publish, retrieve
  36 import allmydata.test.common_util as testutil
  37 from allmydata.test.no_network import GridTestMixin
  38 from allmydata.test.common_web import HTTPClientGETFactory, \
  39      HTTPClientHEADFactory
  40 from allmydata.client import Client, SecretHolder
  41 from allmydata.introducer import IntroducerNode
  42
  43 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  44 # create a webserver that works against them
  45
  46 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  47
  48 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
  49 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
  50 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
  51
  52 FAVICON_MARKUP = '<link href="/icon.png" rel="shortcut icon" />'
  53
  54
  55 class FakeStatsProvider:
  56     def get_stats(self):
  57         stats = {'stats': {}, 'counters': {}}
  58         return stats
  59
  60 class FakeNodeMaker(NodeMaker):
  61     encoding_params = {
  62         'k': 3,
  63         'n': 10,
  64         'happy': 7,
  65         'max_segment_size':128*1024 # 1024=KiB
  66     }
  67     def _create_lit(self, cap):
  68         return FakeCHKFileNode(cap, self.all_contents)
  69     def _create_immutable(self, cap):
  70         return FakeCHKFileNode(cap, self.all_contents)
  71     def _create_mutable(self, cap):
  72         return FakeMutableFileNode(None, None,
  73                                    self.encoding_params, None,
  74                                    self.all_contents).init_from_cap(cap)
  75     def create_mutable_file(self, contents="", keysize=None,
  76                             version=SDMF_VERSION):
  77         n = FakeMutableFileNode(None, None, self.encoding_params, None,
  78                                 self.all_contents)
  79         return n.create(contents, version=version)
  80
  81 class FakeUploader(service.Service):
  82     name = "uploader"
  83     def upload(self, uploadable):
  84         d = uploadable.get_size()
  85         d.addCallback(lambda size: uploadable.read(size))
  86         def _got_data(datav):
  87             data = "".join(datav)
  88             n = create_chk_filenode(data, self.all_contents)
  89             ur = upload.UploadResults(file_size=len(data),
  90                                       ciphertext_fetched=0,
  91                                       preexisting_shares=0,
  92                                       pushed_shares=10,
  93                                       sharemap={},
  94                                       servermap={},
  95                                       timings={},
  96                                       uri_extension_data={},
  97                                       uri_extension_hash="fake",
  98                                       verifycapstr="fakevcap")
  99             ur.set_uri(n.get_uri())
 100             return ur
 101         d.addCallback(_got_data)
 102         return d
 103     def get_helper_info(self):
 104         return (None, False)
 105
 106 def build_one_ds():
 107     ds = DownloadStatus("storage_index", 1234)
 108     now = time.time()
 109
 110     serverA = StubServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
 111     serverB = StubServer(hashutil.tagged_hash("foo", "serverid_b")[:20])
 112     storage_index = hashutil.storage_index_hash("SI")
 113     e0 = ds.add_segment_request(0, now)
 114     e0.activate(now+0.5)
 115     e0.deliver(now+1, 0, 100, 0.5) # when, start,len, decodetime
 116     e1 = ds.add_segment_request(1, now+2)
 117     e1.error(now+3)
 118     # two outstanding requests
 119     e2 = ds.add_segment_request(2, now+4)
 120     e3 = ds.add_segment_request(3, now+5)
 121     del e2,e3 # hush pyflakes
 122
 123     # simulate a segment which gets delivered faster than a system clock tick (ticket #1166)
 124     e = ds.add_segment_request(4, now)
 125     e.activate(now)
 126     e.deliver(now, 0, 140, 0.5)
 127
 128     e = ds.add_dyhb_request(serverA, now)
 129     e.finished([1,2], now+1)
 130     e = ds.add_dyhb_request(serverB, now+2) # left unfinished
 131
 132     e = ds.add_read_event(0, 120, now)
 133     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 134     e.finished(now+1)
 135     e = ds.add_read_event(120, 30, now+2) # left unfinished
 136
 137     e = ds.add_block_request(serverA, 1, 100, 20, now)
 138     e.finished(20, now+1)
 139     e = ds.add_block_request(serverB, 1, 120, 30, now+1) # left unfinished
 140
 141     # make sure that add_read_event() can come first too
 142     ds1 = DownloadStatus(storage_index, 1234)
 143     e = ds1.add_read_event(0, 120, now)
 144     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 145     e.finished(now+1)
 146
 147     return ds
 148
 149 class FakeHistory:
 150     _all_upload_status = [upload.UploadStatus()]
 151     _all_download_status = [build_one_ds()]
 152     _all_mapupdate_statuses = [servermap.UpdateStatus()]
 153     _all_publish_statuses = [publish.PublishStatus()]
 154     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
 155
 156     def list_all_upload_statuses(self):
 157         return self._all_upload_status
 158     def list_all_download_statuses(self):
 159         return self._all_download_status
 160     def list_all_mapupdate_statuses(self):
 161         return self._all_mapupdate_statuses
 162     def list_all_publish_statuses(self):
 163         return self._all_publish_statuses
 164     def list_all_retrieve_statuses(self):
 165         return self._all_retrieve_statuses
 166     def list_all_helper_statuses(self):
 167         return []
 168
 169 class FakeClient(Client):
 170     def __init__(self):
 171         # don't upcall to Client.__init__, since we only want to initialize a
 172         # minimal subset
 173         service.MultiService.__init__(self)
 174         self.all_contents = {}
 175         self.nodeid = "fake_nodeid"
 176         self.nickname = "fake_nickname"
 177         self.introducer_furl = "None"
 178         self.stats_provider = FakeStatsProvider()
 179         self._secret_holder = SecretHolder("lease secret", "convergence secret")
 180         self.helper = None
 181         self.convergence = "some random string"
 182         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 183         self.introducer_client = None
 184         self.history = FakeHistory()
 185         self.uploader = FakeUploader()
 186         self.uploader.all_contents = self.all_contents
 187         self.uploader.setServiceParent(self)
 188         self.blacklist = None
 189         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 190                                        self.uploader, None,
 191                                        None, None, None)
 192         self.nodemaker.all_contents = self.all_contents
 193         self.mutable_file_default = SDMF_VERSION
 194
 195     def startService(self):
 196         return service.MultiService.startService(self)
 197     def stopService(self):
 198         return service.MultiService.stopService(self)
 199
 200     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 201
 202 class WebMixin(object):
 203     def setUp(self):
 204         self.s = FakeClient()
 205         self.s.startService()
 206         self.staticdir = self.mktemp()
 207         self.clock = Clock()
 208         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
 209                                       clock=self.clock)
 210         self.ws.setServiceParent(self.s)
 211         self.webish_port = self.ws.getPortnum()
 212         self.webish_url = self.ws.getURL()
 213         assert self.webish_url.endswith("/")
 214         self.webish_url = self.webish_url[:-1] # these tests add their own /
 215
 216         l = [ self.s.create_dirnode() for x in range(6) ]
 217         d = defer.DeferredList(l)
 218         def _then(res):
 219             self.public_root = res[0][1]
 220             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 221             self.public_url = "/uri/" + self.public_root.get_uri()
 222             self.private_root = res[1][1]
 223
 224             foo = res[2][1]
 225             self._foo_node = foo
 226             self._foo_uri = foo.get_uri()
 227             self._foo_readonly_uri = foo.get_readonly_uri()
 228             self._foo_verifycap = foo.get_verify_cap().to_string()
 229             # NOTE: we ignore the deferred on all set_uri() calls, because we
 230             # know the fake nodes do these synchronously
 231             self.public_root.set_uri(u"foo", foo.get_uri(),
 232                                      foo.get_readonly_uri())
 233
 234             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 235             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 236             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 237
 238             # sdmf
 239             # XXX: Do we ever use this?
 240             self.BAZ_CONTENTS, n, self._baz_txt_uri, self._baz_txt_readonly_uri = self.makefile_mutable(0)
 241
 242             foo.set_uri(u"baz.txt", self._baz_txt_uri, self._baz_txt_readonly_uri)
 243
 244             # mdmf
 245             self.QUUX_CONTENTS, n, self._quux_txt_uri, self._quux_txt_readonly_uri = self.makefile_mutable(0, mdmf=True)
 246             assert self._quux_txt_uri.startswith("URI:MDMF")
 247             foo.set_uri(u"quux.txt", self._quux_txt_uri, self._quux_txt_readonly_uri)
 248
 249             foo.set_uri(u"empty", res[3][1].get_uri(),
 250                         res[3][1].get_readonly_uri())
 251             sub_uri = res[4][1].get_uri()
 252             self._sub_uri = sub_uri
 253             foo.set_uri(u"sub", sub_uri, sub_uri)
 254             sub = self.s.create_node_from_uri(sub_uri)
 255             self._sub_node = sub
 256
 257             _ign, n, blocking_uri = self.makefile(1)
 258             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 259
 260             # filenode to test for html encoding issues
 261             self._htmlname_unicode = u"<&weirdly'named\"file>>>_<iframe />.txt"
 262             self._htmlname_raw = self._htmlname_unicode.encode('utf-8')
 263             self._htmlname_urlencoded = urllib.quote(self._htmlname_raw, '')
 264             self._htmlname_escaped = escapeToXML(self._htmlname_raw)
 265             self._htmlname_escaped_attr = html.escape(self._htmlname_raw)
 266             self._htmlname_escaped_double = escapeToXML(html.escape(self._htmlname_raw))
 267             self.HTMLNAME_CONTENTS, n, self._htmlname_txt_uri = self.makefile(0)
 268             foo.set_uri(self._htmlname_unicode, self._htmlname_txt_uri, self._htmlname_txt_uri)
 269
 270             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 271             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 272             # still think of it as an umlaut
 273             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 274
 275             self.SUBBAZ_CONTENTS, n, baz_file = self.makefile(2)
 276             self._baz_file_uri = baz_file
 277             sub.set_uri(u"baz.txt", baz_file, baz_file)
 278
 279             _ign, n, self._bad_file_uri = self.makefile(3)
 280             # this uri should not be downloadable
 281             del self.s.all_contents[self._bad_file_uri]
 282
 283             rodir = res[5][1]
 284             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 285                                      rodir.get_readonly_uri())
 286             rodir.set_uri(u"nor", baz_file, baz_file)
 287
 288             # public/
 289             # public/foo/
 290             # public/foo/bar.txt
 291             # public/foo/baz.txt
 292             # public/foo/quux.txt
 293             # public/foo/blockingfile
 294             # public/foo/<&weirdly'named\"file>>>_<iframe />.txt
 295             # public/foo/empty/
 296             # public/foo/sub/
 297             # public/foo/sub/baz.txt
 298             # public/reedownlee/
 299             # public/reedownlee/nor
 300             self.NEWFILE_CONTENTS = "newfile contents\n"
 301
 302             return foo.get_metadata_for(u"bar.txt")
 303         d.addCallback(_then)
 304         def _got_metadata(metadata):
 305             self._bar_txt_metadata = metadata
 306         d.addCallback(_got_metadata)
 307         return d
 308
 309     def get_all_contents(self):
 310         return self.s.all_contents
 311
 312     def makefile(self, number):
 313         contents = "contents of file %s\n" % number
 314         n = create_chk_filenode(contents, self.get_all_contents())
 315         return contents, n, n.get_uri()
 316
 317     def makefile_mutable(self, number, mdmf=False):
 318         contents = "contents of mutable file %s\n" % number
 319         n = create_mutable_filenode(contents, mdmf, self.s.all_contents)
 320         return contents, n, n.get_uri(), n.get_readonly_uri()
 321
 322     def tearDown(self):
 323         return self.s.stopService()
 324
 325     def failUnlessIsBarDotTxt(self, res):
 326         self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
 327
 328     def failUnlessIsQuuxDotTxt(self, res):
 329         self.failUnlessReallyEqual(res, self.QUUX_CONTENTS, res)
 330
 331     def failUnlessIsBazDotTxt(self, res):
 332         self.failUnlessReallyEqual(res, self.BAZ_CONTENTS, res)
 333
 334     def failUnlessIsSubBazDotTxt(self, res):
 335         self.failUnlessReallyEqual(res, self.SUBBAZ_CONTENTS, res)
 336
 337     def failUnlessIsBarJSON(self, res):
 338         data = simplejson.loads(res)
 339         self.failUnless(isinstance(data, list))
 340         self.failUnlessEqual(data[0], "filenode")
 341         self.failUnless(isinstance(data[1], dict))
 342         self.failIf(data[1]["mutable"])
 343         self.failIfIn("rw_uri", data[1]) # immutable
 344         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
 345         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
 346         self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
 347
 348     def failUnlessIsQuuxJSON(self, res, readonly=False):
 349         data = simplejson.loads(res)
 350         self.failUnless(isinstance(data, list))
 351         self.failUnlessEqual(data[0], "filenode")
 352         self.failUnless(isinstance(data[1], dict))
 353         metadata = data[1]
 354         return self.failUnlessIsQuuxDotTxtMetadata(metadata, readonly)
 355
 356     def failUnlessIsQuuxDotTxtMetadata(self, metadata, readonly):
 357         self.failUnless(metadata['mutable'])
 358         if readonly:
 359             self.failIfIn("rw_uri", metadata)
 360         else:
 361             self.failUnlessIn("rw_uri", metadata)
 362             self.failUnlessEqual(metadata['rw_uri'], self._quux_txt_uri)
 363         self.failUnlessIn("ro_uri", metadata)
 364         self.failUnlessEqual(metadata['ro_uri'], self._quux_txt_readonly_uri)
 365         self.failUnlessReallyEqual(metadata['size'], len(self.QUUX_CONTENTS))
 366
 367     def failUnlessIsFooJSON(self, res):
 368         data = simplejson.loads(res)
 369         self.failUnless(isinstance(data, list))
 370         self.failUnlessEqual(data[0], "dirnode", res)
 371         self.failUnless(isinstance(data[1], dict))
 372         self.failUnless(data[1]["mutable"])
 373         self.failUnlessIn("rw_uri", data[1]) # mutable
 374         self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
 375         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
 376         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
 377
 378         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 379         self.failUnlessEqual(kidnames,
 380                              [self._htmlname_unicode, u"bar.txt", u"baz.txt",
 381                               u"blockingfile", u"empty", u"n\u00fc.txt", u"quux.txt", u"sub"])
 382         kids = dict( [(unicode(name),value)
 383                       for (name,value)
 384                       in data[1]["children"].iteritems()] )
 385         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 386         self.failUnlessIn("metadata", kids[u"sub"][1])
 387         self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
 388         tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
 389         self.failUnlessIn("linkcrtime", tahoe_md)
 390         self.failUnlessIn("linkmotime", tahoe_md)
 391         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 392         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 393         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri)
 394         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]),
 395                                    self._bar_txt_verifycap)
 396         self.failUnlessIn("metadata", kids[u"bar.txt"][1])
 397         self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
 398         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
 399                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
 400         self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
 401                                    self._bar_txt_uri)
 402         self.failUnlessIn("quux.txt", kids)
 403         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["rw_uri"]),
 404                                    self._quux_txt_uri)
 405         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["ro_uri"]),
 406                                    self._quux_txt_readonly_uri)
 407
 408     def GET(self, urlpath, followRedirect=False, return_response=False,
 409             **kwargs):
 410         # if return_response=True, this fires with (data, statuscode,
 411         # respheaders) instead of just data.
 412         assert not isinstance(urlpath, unicode)
 413         url = self.webish_url + urlpath
 414         factory = HTTPClientGETFactory(url, method="GET",
 415                                        followRedirect=followRedirect, **kwargs)
 416         reactor.connectTCP("localhost", self.webish_port, factory)
 417         d = factory.deferred
 418         def _got_data(data):
 419             return (data, factory.status, factory.response_headers)
 420         if return_response:
 421             d.addCallback(_got_data)
 422         return factory.deferred
 423
 424     def HEAD(self, urlpath, return_response=False, **kwargs):
 425         # this requires some surgery, because twisted.web.client doesn't want
 426         # to give us back the response headers.
 427         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 428         reactor.connectTCP("localhost", self.webish_port, factory)
 429         d = factory.deferred
 430         def _got_data(data):
 431             return (data, factory.status, factory.response_headers)
 432         if return_response:
 433             d.addCallback(_got_data)
 434         return factory.deferred
 435
 436     def PUT(self, urlpath, data, **kwargs):
 437         url = self.webish_url + urlpath
 438         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 439
 440     def DELETE(self, urlpath):
 441         url = self.webish_url + urlpath
 442         return client.getPage(url, method="DELETE")
 443
 444     def POST(self, urlpath, followRedirect=False, **fields):
 445         sepbase = "boogabooga"
 446         sep = "--" + sepbase
 447         form = []
 448         form.append(sep)
 449         form.append('Content-Disposition: form-data; name="_charset"')
 450         form.append('')
 451         form.append('UTF-8')
 452         form.append(sep)
 453         for name, value in fields.iteritems():
 454             if isinstance(value, tuple):
 455                 filename, value = value
 456                 form.append('Content-Disposition: form-data; name="%s"; '
 457                             'filename="%s"' % (name, filename.encode("utf-8")))
 458             else:
 459                 form.append('Content-Disposition: form-data; name="%s"' % name)
 460             form.append('')
 461             if isinstance(value, unicode):
 462                 value = value.encode("utf-8")
 463             else:
 464                 value = str(value)
 465             assert isinstance(value, str)
 466             form.append(value)
 467             form.append(sep)
 468         form[-1] += "--"
 469         body = ""
 470         headers = {}
 471         if fields:
 472             body = "\r\n".join(form) + "\r\n"
 473             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 474         return self.POST2(urlpath, body, headers, followRedirect)
 475
 476     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 477         url = self.webish_url + urlpath
 478         return client.getPage(url, method="POST", postdata=body,
 479                               headers=headers, followRedirect=followRedirect)
 480
 481     def shouldFail(self, res, expected_failure, which,
 482                    substring=None, response_substring=None):
 483         if isinstance(res, failure.Failure):
 484             res.trap(expected_failure)
 485             if substring:
 486                 self.failUnlessIn(substring, str(res), which)
 487             if response_substring:
 488                 self.failUnlessIn(response_substring, res.value.response, which)
 489         else:
 490             self.fail("%s was supposed to raise %s, not get '%s'" %
 491                       (which, expected_failure, res))
 492
 493     def shouldFail2(self, expected_failure, which, substring,
 494                     response_substring,
 495                     callable, *args, **kwargs):
 496         assert substring is None or isinstance(substring, str)
 497         assert response_substring is None or isinstance(response_substring, str)
 498         d = defer.maybeDeferred(callable, *args, **kwargs)
 499         def done(res):
 500             if isinstance(res, failure.Failure):
 501                 res.trap(expected_failure)
 502                 if substring:
 503                     self.failUnlessIn(substring, str(res),
 504                                       "'%s' not in '%s' for test '%s'" % \
 505                                       (substring, str(res), which))
 506                 if response_substring:
 507                     self.failUnlessIn(response_substring, res.value.response,
 508                                       "'%s' not in '%s' for test '%s'" % \
 509                                       (response_substring, res.value.response,
 510                                        which))
 511             else:
 512                 self.fail("%s was supposed to raise %s, not get '%s'" %
 513                           (which, expected_failure, res))
 514         d.addBoth(done)
 515         return d
 516
 517     def should404(self, res, which):
 518         if isinstance(res, failure.Failure):
 519             res.trap(error.Error)
 520             self.failUnlessReallyEqual(res.value.status, "404")
 521         else:
 522             self.fail("%s was supposed to Error(404), not get '%s'" %
 523                       (which, res))
 524
 525     def should302(self, res, which):
 526         if isinstance(res, failure.Failure):
 527             res.trap(error.Error)
 528             self.failUnlessReallyEqual(res.value.status, "302")
 529         else:
 530             self.fail("%s was supposed to Error(302), not get '%s'" %
 531                       (which, res))
 532
 533
 534 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
 535     def test_create(self):
 536         pass
 537
 538     def test_welcome(self):
 539         d = self.GET("/")
 540         def _check(res):
 541             self.failUnlessIn('Welcome to Tahoe-LAFS', res)
 542             self.failUnlessIn(FAVICON_MARKUP, res)
 543             self.failUnlessIn('href="https://tahoe-lafs.org/"', res)
 544
 545             self.s.basedir = 'web/test_welcome'
 546             fileutil.make_dirs("web/test_welcome")
 547             fileutil.make_dirs("web/test_welcome/private")
 548             return self.GET("/")
 549         d.addCallback(_check)
 550         return d
 551
 552     def test_status(self):
 553         h = self.s.get_history()
 554         dl_num = h.list_all_download_statuses()[0].get_counter()
 555         ul_num = h.list_all_upload_statuses()[0].get_counter()
 556         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 557         pub_num = h.list_all_publish_statuses()[0].get_counter()
 558         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 559         d = self.GET("/status", followRedirect=True)
 560         def _check(res):
 561             self.failUnlessIn('Upload and Download Status', res)
 562             self.failUnlessIn('"down-%d"' % dl_num, res)
 563             self.failUnlessIn('"up-%d"' % ul_num, res)
 564             self.failUnlessIn('"mapupdate-%d"' % mu_num, res)
 565             self.failUnlessIn('"publish-%d"' % pub_num, res)
 566             self.failUnlessIn('"retrieve-%d"' % ret_num, res)
 567         d.addCallback(_check)
 568         d.addCallback(lambda res: self.GET("/status/?t=json"))
 569         def _check_json(res):
 570             data = simplejson.loads(res)
 571             self.failUnless(isinstance(data, dict))
 572             #active = data["active"]
 573             # TODO: test more. We need a way to fake an active operation
 574             # here.
 575         d.addCallback(_check_json)
 576
 577         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 578         def _check_dl(res):
 579             self.failUnlessIn("File Download Status", res)
 580         d.addCallback(_check_dl)
 581         d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
 582         def _check_dl_json(res):
 583             data = simplejson.loads(res)
 584             self.failUnless(isinstance(data, dict))
 585             self.failUnlessIn("read", data)
 586             self.failUnlessEqual(data["read"][0]["length"], 120)
 587             self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
 588             self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
 589             self.failUnlessEqual(data["segment"][2]["finish_time"], None)
 590             phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
 591             cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
 592             # serverids[] keys are strings, since that's what JSON does, but
 593             # we'd really like them to be ints
 594             self.failUnlessEqual(data["serverids"]["0"], "phwrsjte")
 595             self.failUnless(data["serverids"].has_key("1"),
 596                             str(data["serverids"]))
 597             self.failUnlessEqual(data["serverids"]["1"], "cmpuvkjm",
 598                                  str(data["serverids"]))
 599             self.failUnlessEqual(data["server_info"][phwr_id]["short"],
 600                                  "phwrsjte")
 601             self.failUnlessEqual(data["server_info"][cmpu_id]["short"],
 602                                  "cmpuvkjm")
 603             self.failUnlessIn("dyhb", data)
 604             self.failUnlessIn("misc", data)
 605         d.addCallback(_check_dl_json)
 606         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 607         def _check_ul(res):
 608             self.failUnlessIn("File Upload Status", res)
 609         d.addCallback(_check_ul)
 610         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 611         def _check_mapupdate(res):
 612             self.failUnlessIn("Mutable File Servermap Update Status", res)
 613         d.addCallback(_check_mapupdate)
 614         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 615         def _check_publish(res):
 616             self.failUnlessIn("Mutable File Publish Status", res)
 617         d.addCallback(_check_publish)
 618         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 619         def _check_retrieve(res):
 620             self.failUnlessIn("Mutable File Retrieve Status", res)
 621         d.addCallback(_check_retrieve)
 622
 623         return d
 624
 625     def test_status_numbers(self):
 626         drrm = status.DownloadResultsRendererMixin()
 627         self.failUnlessReallyEqual(drrm.render_time(None, None), "")
 628         self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
 629         self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
 630         self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
 631         self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
 632         self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
 633         self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 634         self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
 635         self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
 636
 637         urrm = status.UploadResultsRendererMixin()
 638         self.failUnlessReallyEqual(urrm.render_time(None, None), "")
 639         self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
 640         self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
 641         self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
 642         self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
 643         self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
 644         self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 645         self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
 646         self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
 647
 648     def test_GET_FILEURL(self):
 649         d = self.GET(self.public_url + "/foo/bar.txt")
 650         d.addCallback(self.failUnlessIsBarDotTxt)
 651         return d
 652
 653     def test_GET_FILEURL_range(self):
 654         headers = {"range": "bytes=1-10"}
 655         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 656                      return_response=True)
 657         def _got((res, status, headers)):
 658             self.failUnlessReallyEqual(int(status), 206)
 659             self.failUnless(headers.has_key("content-range"))
 660             self.failUnlessReallyEqual(headers["content-range"][0],
 661                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 662             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
 663         d.addCallback(_got)
 664         return d
 665
 666     def test_GET_FILEURL_partial_range(self):
 667         headers = {"range": "bytes=5-"}
 668         length  = len(self.BAR_CONTENTS)
 669         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 670                      return_response=True)
 671         def _got((res, status, headers)):
 672             self.failUnlessReallyEqual(int(status), 206)
 673             self.failUnless(headers.has_key("content-range"))
 674             self.failUnlessReallyEqual(headers["content-range"][0],
 675                                        "bytes 5-%d/%d" % (length-1, length))
 676             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
 677         d.addCallback(_got)
 678         return d
 679
 680     def test_GET_FILEURL_partial_end_range(self):
 681         headers = {"range": "bytes=-5"}
 682         length  = len(self.BAR_CONTENTS)
 683         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 684                      return_response=True)
 685         def _got((res, status, headers)):
 686             self.failUnlessReallyEqual(int(status), 206)
 687             self.failUnless(headers.has_key("content-range"))
 688             self.failUnlessReallyEqual(headers["content-range"][0],
 689                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 690             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
 691         d.addCallback(_got)
 692         return d
 693
 694     def test_GET_FILEURL_partial_range_overrun(self):
 695         headers = {"range": "bytes=100-200"}
 696         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
 697                              "416 Requested Range not satisfiable",
 698                              "First beyond end of file",
 699                              self.GET, self.public_url + "/foo/bar.txt",
 700                              headers=headers)
 701         return d
 702
 703     def test_HEAD_FILEURL_range(self):
 704         headers = {"range": "bytes=1-10"}
 705         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 706                      return_response=True)
 707         def _got((res, status, headers)):
 708             self.failUnlessReallyEqual(res, "")
 709             self.failUnlessReallyEqual(int(status), 206)
 710             self.failUnless(headers.has_key("content-range"))
 711             self.failUnlessReallyEqual(headers["content-range"][0],
 712                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 713         d.addCallback(_got)
 714         return d
 715
 716     def test_HEAD_FILEURL_partial_range(self):
 717         headers = {"range": "bytes=5-"}
 718         length  = len(self.BAR_CONTENTS)
 719         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 720                      return_response=True)
 721         def _got((res, status, headers)):
 722             self.failUnlessReallyEqual(int(status), 206)
 723             self.failUnless(headers.has_key("content-range"))
 724             self.failUnlessReallyEqual(headers["content-range"][0],
 725                                        "bytes 5-%d/%d" % (length-1, length))
 726         d.addCallback(_got)
 727         return d
 728
 729     def test_HEAD_FILEURL_partial_end_range(self):
 730         headers = {"range": "bytes=-5"}
 731         length  = len(self.BAR_CONTENTS)
 732         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 733                      return_response=True)
 734         def _got((res, status, headers)):
 735             self.failUnlessReallyEqual(int(status), 206)
 736             self.failUnless(headers.has_key("content-range"))
 737             self.failUnlessReallyEqual(headers["content-range"][0],
 738                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 739         d.addCallback(_got)
 740         return d
 741
 742     def test_HEAD_FILEURL_partial_range_overrun(self):
 743         headers = {"range": "bytes=100-200"}
 744         d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
 745                              "416 Requested Range not satisfiable",
 746                              "",
 747                              self.HEAD, self.public_url + "/foo/bar.txt",
 748                              headers=headers)
 749         return d
 750
 751     def test_GET_FILEURL_range_bad(self):
 752         headers = {"range": "BOGUS=fizbop-quarnak"}
 753         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 754                      return_response=True)
 755         def _got((res, status, headers)):
 756             self.failUnlessReallyEqual(int(status), 200)
 757             self.failUnless(not headers.has_key("content-range"))
 758             self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
 759         d.addCallback(_got)
 760         return d
 761
 762     def test_HEAD_FILEURL(self):
 763         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 764         def _got((res, status, headers)):
 765             self.failUnlessReallyEqual(res, "")
 766             self.failUnlessReallyEqual(headers["content-length"][0],
 767                                        str(len(self.BAR_CONTENTS)))
 768             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
 769         d.addCallback(_got)
 770         return d
 771
 772     def test_GET_FILEURL_named(self):
 773         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 774         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 775         d = self.GET(base + "/@@name=/blah.txt")
 776         d.addCallback(self.failUnlessIsBarDotTxt)
 777         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 778         d.addCallback(self.failUnlessIsBarDotTxt)
 779         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 780         d.addCallback(self.failUnlessIsBarDotTxt)
 781         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 782         d.addCallback(self.failUnlessIsBarDotTxt)
 783         save_url = base + "?save=true&filename=blah.txt"
 784         d.addCallback(lambda res: self.GET(save_url))
 785         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 786         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 787         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 788         u_url = base + "?save=true&filename=" + u_fn_e
 789         d.addCallback(lambda res: self.GET(u_url))
 790         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 791         return d
 792
 793     def test_PUT_FILEURL_named_bad(self):
 794         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 795         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 796                              "400 Bad Request",
 797                              "/file can only be used with GET or HEAD",
 798                              self.PUT, base + "/@@name=/blah.txt", "")
 799         return d
 800
 801
 802     def test_GET_DIRURL_named_bad(self):
 803         base = "/file/%s" % urllib.quote(self._foo_uri)
 804         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 805                              "400 Bad Request",
 806                              "is not a file-cap",
 807                              self.GET, base + "/@@name=/blah.txt")
 808         return d
 809
 810     def test_GET_slash_file_bad(self):
 811         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 812                              "404 Not Found",
 813                              "/file must be followed by a file-cap and a name",
 814                              self.GET, "/file")
 815         return d
 816
 817     def test_GET_unhandled_URI_named(self):
 818         contents, n, newuri = self.makefile(12)
 819         verifier_cap = n.get_verify_cap().to_string()
 820         base = "/file/%s" % urllib.quote(verifier_cap)
 821         # client.create_node_from_uri() can't handle verify-caps
 822         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 823                              "400 Bad Request", "is not a file-cap",
 824                              self.GET, base)
 825         return d
 826
 827     def test_GET_unhandled_URI(self):
 828         contents, n, newuri = self.makefile(12)
 829         verifier_cap = n.get_verify_cap().to_string()
 830         base = "/uri/%s" % urllib.quote(verifier_cap)
 831         # client.create_node_from_uri() can't handle verify-caps
 832         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 833                              "400 Bad Request",
 834                              "GET unknown URI type: can only do t=info",
 835                              self.GET, base)
 836         return d
 837
 838     def test_GET_FILE_URI(self):
 839         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 840         d = self.GET(base)
 841         d.addCallback(self.failUnlessIsBarDotTxt)
 842         return d
 843
 844     def test_GET_FILE_URI_mdmf(self):
 845         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 846         d = self.GET(base)
 847         d.addCallback(self.failUnlessIsQuuxDotTxt)
 848         return d
 849
 850     def test_GET_FILE_URI_mdmf_extensions(self):
 851         base = "/uri/%s" % urllib.quote("%s:RANDOMSTUFF" % self._quux_txt_uri)
 852         d = self.GET(base)
 853         d.addCallback(self.failUnlessIsQuuxDotTxt)
 854         return d
 855
 856     def test_GET_FILE_URI_mdmf_readonly(self):
 857         base = "/uri/%s" % urllib.quote(self._quux_txt_readonly_uri)
 858         d = self.GET(base)
 859         d.addCallback(self.failUnlessIsQuuxDotTxt)
 860         return d
 861
 862     def test_GET_FILE_URI_badchild(self):
 863         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 864         errmsg = "Files have no children, certainly not named 'boguschild'"
 865         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 866                              "400 Bad Request", errmsg,
 867                              self.GET, base)
 868         return d
 869
 870     def test_PUT_FILE_URI_badchild(self):
 871         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 872         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 873         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 874                              "400 Bad Request", errmsg,
 875                              self.PUT, base, "")
 876         return d
 877
 878     def test_PUT_FILE_URI_mdmf(self):
 879         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 880         self._quux_new_contents = "new_contents"
 881         d = self.GET(base)
 882         d.addCallback(lambda res:
 883             self.failUnlessIsQuuxDotTxt(res))
 884         d.addCallback(lambda ignored:
 885             self.PUT(base, self._quux_new_contents))
 886         d.addCallback(lambda ignored:
 887             self.GET(base))
 888         d.addCallback(lambda res:
 889             self.failUnlessReallyEqual(res, self._quux_new_contents))
 890         return d
 891
 892     def test_PUT_FILE_URI_mdmf_extensions(self):
 893         base = "/uri/%s" % urllib.quote("%s:EXTENSIONSTUFF" % self._quux_txt_uri)
 894         self._quux_new_contents = "new_contents"
 895         d = self.GET(base)
 896         d.addCallback(lambda res: self.failUnlessIsQuuxDotTxt(res))
 897         d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
 898         d.addCallback(lambda ignored: self.GET(base))
 899         d.addCallback(lambda res: self.failUnlessEqual(self._quux_new_contents,
 900                                                        res))
 901         return d
 902
 903     def test_PUT_FILE_URI_mdmf_readonly(self):
 904         # We're not allowed to PUT things to a readonly cap.
 905         base = "/uri/%s" % self._quux_txt_readonly_uri
 906         d = self.GET(base)
 907         d.addCallback(lambda res:
 908             self.failUnlessIsQuuxDotTxt(res))
 909         # What should we get here? We get a 500 error now; that's not right.
 910         d.addCallback(lambda ignored:
 911             self.shouldFail2(error.Error, "test_PUT_FILE_URI_mdmf_readonly",
 912                              "400 Bad Request", "read-only cap",
 913                              self.PUT, base, "new data"))
 914         return d
 915
 916     def test_PUT_FILE_URI_sdmf_readonly(self):
 917         # We're not allowed to put things to a readonly cap.
 918         base = "/uri/%s" % self._baz_txt_readonly_uri
 919         d = self.GET(base)
 920         d.addCallback(lambda res:
 921             self.failUnlessIsBazDotTxt(res))
 922         d.addCallback(lambda ignored:
 923             self.shouldFail2(error.Error, "test_PUT_FILE_URI_sdmf_readonly",
 924                              "400 Bad Request", "read-only cap",
 925                              self.PUT, base, "new_data"))
 926         return d
 927
 928     def test_GET_etags(self):
 929
 930         def _check_etags(uri):
 931             d1 = _get_etag(uri)
 932             d2 = _get_etag(uri, 'json')
 933             d = defer.DeferredList([d1, d2], consumeErrors=True)
 934             def _check(results):
 935                 # All deferred must succeed
 936                 self.failUnless(all([r[0] for r in results]))
 937                 # the etag for the t=json form should be just like the etag
 938                 # fo the default t='' form, but with a 'json' suffix
 939                 self.failUnlessEqual(results[0][1] + 'json', results[1][1])
 940             d.addCallback(_check)
 941             return d
 942
 943         def _get_etag(uri, t=''):
 944             targetbase = "/uri/%s?t=%s" % (urllib.quote(uri.strip()), t)
 945             d = self.GET(targetbase, return_response=True, followRedirect=True)
 946             def _just_the_etag(result):
 947                 data, response, headers = result
 948                 etag = headers['etag'][0]
 949                 if uri.startswith('URI:DIR'):
 950                     self.failUnless(etag.startswith('DIR:'), etag)
 951                 return etag
 952             return d.addCallback(_just_the_etag)
 953
 954         # Check that etags work with immutable directories
 955         (newkids, caps) = self._create_immutable_children()
 956         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
 957                       simplejson.dumps(newkids))
 958         def _stash_immdir_uri(uri):
 959             self._immdir_uri = uri
 960             return uri
 961         d.addCallback(_stash_immdir_uri)
 962         d.addCallback(_check_etags)
 963
 964         # Check that etags work with immutable files
 965         d.addCallback(lambda _: _check_etags(self._bar_txt_uri))
 966
 967         # use the ETag on GET
 968         def _check_match(ign):
 969             uri = "/uri/%s" % self._bar_txt_uri
 970             d = self.GET(uri, return_response=True)
 971             # extract the ETag
 972             d.addCallback(lambda (data, code, headers):
 973                           headers['etag'][0])
 974             # do a GET that's supposed to match the ETag
 975             d.addCallback(lambda etag:
 976                           self.GET(uri, return_response=True,
 977                                    headers={"If-None-Match": etag}))
 978             # make sure it short-circuited (304 instead of 200)
 979             d.addCallback(lambda (data, code, headers):
 980                           self.failUnlessEqual(int(code), http.NOT_MODIFIED))
 981             return d
 982         d.addCallback(_check_match)
 983
 984         def _no_etag(uri, t):
 985             target = "/uri/%s?t=%s" % (uri, t)
 986             d = self.GET(target, return_response=True, followRedirect=True)
 987             d.addCallback(lambda (data, code, headers):
 988                           self.failIf("etag" in headers, target))
 989             return d
 990         def _yes_etag(uri, t):
 991             target = "/uri/%s?t=%s" % (uri, t)
 992             d = self.GET(target, return_response=True, followRedirect=True)
 993             d.addCallback(lambda (data, code, headers):
 994                           self.failUnless("etag" in headers, target))
 995             return d
 996
 997         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, ""))
 998         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "json"))
 999         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "uri"))
1000         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "readonly-uri"))
1001         d.addCallback(lambda ign: _no_etag(self._bar_txt_uri, "info"))
1002
1003         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, ""))
1004         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "json"))
1005         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "uri"))
1006         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "readonly-uri"))
1007         d.addCallback(lambda ign: _no_etag(self._immdir_uri, "info"))
1008         d.addCallback(lambda ign: _no_etag(self._immdir_uri, "rename-form"))
1009
1010         return d
1011
1012     # TODO: version of this with a Unicode filename
1013     def test_GET_FILEURL_save(self):
1014         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
1015                      return_response=True)
1016         def _got((res, statuscode, headers)):
1017             content_disposition = headers["content-disposition"][0]
1018             self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
1019             self.failUnlessIsBarDotTxt(res)
1020         d.addCallback(_got)
1021         return d
1022
1023     def test_GET_FILEURL_missing(self):
1024         d = self.GET(self.public_url + "/foo/missing")
1025         d.addBoth(self.should404, "test_GET_FILEURL_missing")
1026         return d
1027
1028     def test_GET_FILEURL_info_mdmf(self):
1029         d = self.GET("/uri/%s?t=info" % self._quux_txt_uri)
1030         def _got(res):
1031             self.failUnlessIn("mutable file (mdmf)", res)
1032             self.failUnlessIn(self._quux_txt_uri, res)
1033             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1034         d.addCallback(_got)
1035         return d
1036
1037     def test_GET_FILEURL_info_mdmf_readonly(self):
1038         d = self.GET("/uri/%s?t=info" % self._quux_txt_readonly_uri)
1039         def _got(res):
1040             self.failUnlessIn("mutable file (mdmf)", res)
1041             self.failIfIn(self._quux_txt_uri, res)
1042             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1043         d.addCallback(_got)
1044         return d
1045
1046     def test_GET_FILEURL_info_sdmf(self):
1047         d = self.GET("/uri/%s?t=info" % self._baz_txt_uri)
1048         def _got(res):
1049             self.failUnlessIn("mutable file (sdmf)", res)
1050             self.failUnlessIn(self._baz_txt_uri, res)
1051         d.addCallback(_got)
1052         return d
1053
1054     def test_GET_FILEURL_info_mdmf_extensions(self):
1055         d = self.GET("/uri/%s:STUFF?t=info" % self._quux_txt_uri)
1056         def _got(res):
1057             self.failUnlessIn("mutable file (mdmf)", res)
1058             self.failUnlessIn(self._quux_txt_uri, res)
1059             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1060         d.addCallback(_got)
1061         return d
1062
1063     def test_PUT_overwrite_only_files(self):
1064         # create a directory, put a file in that directory.
1065         contents, n, filecap = self.makefile(8)
1066         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
1067         d.addCallback(lambda res:
1068             self.PUT(self.public_url + "/foo/dir/file1.txt",
1069                      self.NEWFILE_CONTENTS))
1070         # try to overwrite the file with replace=only-files
1071         # (this should work)
1072         d.addCallback(lambda res:
1073             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
1074                      filecap))
1075         d.addCallback(lambda res:
1076             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
1077                  "There was already a child by that name, and you asked me "
1078                  "to not replace it",
1079                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
1080                  filecap))
1081         return d
1082
1083     def test_PUT_NEWFILEURL(self):
1084         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
1085         # TODO: we lose the response code, so we can't check this
1086         #self.failUnlessReallyEqual(responsecode, 201)
1087         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1088         d.addCallback(lambda res:
1089                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1090                                                       self.NEWFILE_CONTENTS))
1091         return d
1092
1093     def test_PUT_NEWFILEURL_not_mutable(self):
1094         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
1095                      self.NEWFILE_CONTENTS)
1096         # TODO: we lose the response code, so we can't check this
1097         #self.failUnlessReallyEqual(responsecode, 201)
1098         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1099         d.addCallback(lambda res:
1100                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1101                                                       self.NEWFILE_CONTENTS))
1102         return d
1103
1104     def test_PUT_NEWFILEURL_unlinked_mdmf(self):
1105         # this should get us a few segments of an MDMF mutable file,
1106         # which we can then test for.
1107         contents = self.NEWFILE_CONTENTS * 300000
1108         d = self.PUT("/uri?format=mdmf",
1109                      contents)
1110         def _got_filecap(filecap):
1111             self.failUnless(filecap.startswith("URI:MDMF"))
1112             return filecap
1113         d.addCallback(_got_filecap)
1114         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1115         d.addCallback(lambda json: self.failUnlessIn("MDMF", json))
1116         return d
1117
1118     def test_PUT_NEWFILEURL_unlinked_sdmf(self):
1119         contents = self.NEWFILE_CONTENTS * 300000
1120         d = self.PUT("/uri?format=sdmf",
1121                      contents)
1122         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1123         d.addCallback(lambda json: self.failUnlessIn("SDMF", json))
1124         return d
1125
1126     def test_PUT_NEWFILEURL_unlinked_bad_format(self):
1127         contents = self.NEWFILE_CONTENTS * 300000
1128         return self.shouldHTTPError("PUT_NEWFILEURL_unlinked_bad_format",
1129                                     400, "Bad Request", "Unknown format: foo",
1130                                     self.PUT, "/uri?format=foo",
1131                                     contents)
1132
1133     def test_PUT_NEWFILEURL_range_bad(self):
1134         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
1135         target = self.public_url + "/foo/new.txt"
1136         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
1137                              "501 Not Implemented",
1138                              "Content-Range in PUT not yet supported",
1139                              # (and certainly not for immutable files)
1140                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
1141                              headers=headers)
1142         d.addCallback(lambda res:
1143                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
1144         return d
1145
1146     def test_PUT_NEWFILEURL_mutable(self):
1147         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1148                      self.NEWFILE_CONTENTS)
1149         # TODO: we lose the response code, so we can't check this
1150         #self.failUnlessReallyEqual(responsecode, 201)
1151         def _check_uri(res):
1152             u = uri.from_string_mutable_filenode(res)
1153             self.failUnless(u.is_mutable())
1154             self.failIf(u.is_readonly())
1155             return res
1156         d.addCallback(_check_uri)
1157         d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
1158         d.addCallback(lambda res:
1159                       self.failUnlessMutableChildContentsAre(self._foo_node,
1160                                                              u"new.txt",
1161                                                              self.NEWFILE_CONTENTS))
1162         return d
1163
1164     def test_PUT_NEWFILEURL_mutable_toobig(self):
1165         # It is okay to upload large mutable files, so we should be able
1166         # to do that.
1167         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1168                      "b" * (self.s.MUTABLE_SIZELIMIT + 1))
1169         return d
1170
1171     def test_PUT_NEWFILEURL_replace(self):
1172         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
1173         # TODO: we lose the response code, so we can't check this
1174         #self.failUnlessReallyEqual(responsecode, 200)
1175         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
1176         d.addCallback(lambda res:
1177                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1178                                                       self.NEWFILE_CONTENTS))
1179         return d
1180
1181     def test_PUT_NEWFILEURL_bad_t(self):
1182         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
1183                              "PUT to a file: bad t=bogus",
1184                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
1185                              "contents")
1186         return d
1187
1188     def test_PUT_NEWFILEURL_no_replace(self):
1189         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
1190                      self.NEWFILE_CONTENTS)
1191         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
1192                   "409 Conflict",
1193                   "There was already a child by that name, and you asked me "
1194                   "to not replace it")
1195         return d
1196
1197     def test_PUT_NEWFILEURL_mkdirs(self):
1198         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
1199         fn = self._foo_node
1200         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
1201         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
1202         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
1203         d.addCallback(lambda res:
1204                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
1205                                                       self.NEWFILE_CONTENTS))
1206         return d
1207
1208     def test_PUT_NEWFILEURL_blocked(self):
1209         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
1210                      self.NEWFILE_CONTENTS)
1211         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
1212                   "409 Conflict",
1213                   "Unable to create directory 'blockingfile': a file was in the way")
1214         return d
1215
1216     def test_PUT_NEWFILEURL_emptyname(self):
1217         # an empty pathname component (i.e. a double-slash) is disallowed
1218         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
1219                              "400 Bad Request",
1220                              "The webapi does not allow empty pathname components",
1221                              self.PUT, self.public_url + "/foo//new.txt", "")
1222         return d
1223
1224     def test_DELETE_FILEURL(self):
1225         d = self.DELETE(self.public_url + "/foo/bar.txt")
1226         d.addCallback(lambda res:
1227                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1228         return d
1229
1230     def test_DELETE_FILEURL_missing(self):
1231         d = self.DELETE(self.public_url + "/foo/missing")
1232         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
1233         return d
1234
1235     def test_DELETE_FILEURL_missing2(self):
1236         d = self.DELETE(self.public_url + "/missing/missing")
1237         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
1238         return d
1239
1240     def failUnlessHasBarDotTxtMetadata(self, res):
1241         data = simplejson.loads(res)
1242         self.failUnless(isinstance(data, list))
1243         self.failUnlessIn("metadata", data[1])
1244         self.failUnlessIn("tahoe", data[1]["metadata"])
1245         self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
1246         self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
1247         self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
1248                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
1249
1250     def test_GET_FILEURL_json(self):
1251         # twisted.web.http.parse_qs ignores any query args without an '=', so
1252         # I can't do "GET /path?json", I have to do "GET /path/t=json"
1253         # instead. This may make it tricky to emulate the S3 interface
1254         # completely.
1255         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
1256         def _check1(data):
1257             self.failUnlessIsBarJSON(data)
1258             self.failUnlessHasBarDotTxtMetadata(data)
1259             return
1260         d.addCallback(_check1)
1261         return d
1262
1263     def test_GET_FILEURL_json_mutable_type(self):
1264         # The JSON should include format, which says whether the
1265         # file is SDMF or MDMF
1266         d = self.PUT("/uri?format=mdmf",
1267                      self.NEWFILE_CONTENTS * 300000)
1268         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1269         def _got_json(json, version):
1270             data = simplejson.loads(json)
1271             assert "filenode" == data[0]
1272             data = data[1]
1273             assert isinstance(data, dict)
1274
1275             self.failUnlessIn("format", data)
1276             self.failUnlessEqual(data["format"], version)
1277
1278         d.addCallback(_got_json, "MDMF")
1279         # Now make an SDMF file and check that it is reported correctly.
1280         d.addCallback(lambda ignored:
1281             self.PUT("/uri?format=sdmf",
1282                       self.NEWFILE_CONTENTS * 300000))
1283         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1284         d.addCallback(_got_json, "SDMF")
1285         return d
1286
1287     def test_GET_FILEURL_json_mdmf(self):
1288         d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
1289         d.addCallback(self.failUnlessIsQuuxJSON)
1290         return d
1291
1292     def test_GET_FILEURL_json_missing(self):
1293         d = self.GET(self.public_url + "/foo/missing?json")
1294         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
1295         return d
1296
1297     def test_GET_FILEURL_uri(self):
1298         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
1299         def _check(res):
1300             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1301         d.addCallback(_check)
1302         d.addCallback(lambda res:
1303                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
1304         def _check2(res):
1305             # for now, for files, uris and readonly-uris are the same
1306             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1307         d.addCallback(_check2)
1308         return d
1309
1310     def test_GET_FILEURL_badtype(self):
1311         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
1312                                  "bad t=bogus",
1313                                  self.GET,
1314                                  self.public_url + "/foo/bar.txt?t=bogus")
1315         return d
1316
1317     def test_CSS_FILE(self):
1318         d = self.GET("/tahoe.css", followRedirect=True)
1319         def _check(res):
1320             CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
1321             self.failUnless(CSS_STYLE.search(res), res)
1322         d.addCallback(_check)
1323         return d
1324
1325     def test_GET_FILEURL_uri_missing(self):
1326         d = self.GET(self.public_url + "/foo/missing?t=uri")
1327         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
1328         return d
1329
1330     def _check_upload_and_mkdir_forms(self, html):
1331         # We should have a form to create a file, with radio buttons that allow
1332         # the user to toggle whether it is a CHK/LIT (default), SDMF, or MDMF file.
1333         self.failUnlessIn('name="t" value="upload"', html)
1334         self.failUnlessIn('input checked="checked" type="radio" id="upload-chk" value="chk" name="format"', html)
1335         self.failUnlessIn('input type="radio" id="upload-sdmf" value="sdmf" name="format"', html)
1336         self.failUnlessIn('input type="radio" id="upload-mdmf" value="mdmf" name="format"', html)
1337
1338         # We should also have the ability to create a mutable directory, with
1339         # radio buttons that allow the user to toggle whether it is an SDMF (default)
1340         # or MDMF directory.
1341         self.failUnlessIn('name="t" value="mkdir"', html)
1342         self.failUnlessIn('input checked="checked" type="radio" id="mkdir-sdmf" value="sdmf" name="format"', html)
1343         self.failUnlessIn('input type="radio" id="mkdir-mdmf" value="mdmf" name="format"', html)
1344
1345         self.failUnlessIn(FAVICON_MARKUP, html)
1346
1347     def test_GET_DIRECTORY_html(self):
1348         d = self.GET(self.public_url + "/foo", followRedirect=True)
1349         def _check(html):
1350             self.failUnlessIn('<div class="toolbar-item"><a href="../../..">Return to Welcome page</a></div>', html)
1351             self._check_upload_and_mkdir_forms(html)
1352             self.failUnlessIn("quux", html)
1353         d.addCallback(_check)
1354         return d
1355
1356     def test_GET_DIRECTORY_html_filenode_encoding(self):
1357         d = self.GET(self.public_url + "/foo", followRedirect=True)
1358         def _check(html):
1359             # Check if encoded entries are there
1360             self.failUnlessIn('@@named=/' + self._htmlname_urlencoded + '">'
1361                               + self._htmlname_escaped + '</a>', html)
1362             self.failUnlessIn('value="' + self._htmlname_escaped_attr + '"', html)
1363             self.failIfIn(self._htmlname_escaped_double, html)
1364             # Make sure that Nevow escaping actually works by checking for unsafe characters
1365             # and that '&' is escaped.
1366             for entity in '<>':
1367                 self.failUnlessIn(entity, self._htmlname_raw)
1368                 self.failIfIn(entity, self._htmlname_escaped)
1369             self.failUnlessIn('&', re.sub(r'&(amp|lt|gt|quot|apos);', '', self._htmlname_raw))
1370             self.failIfIn('&', re.sub(r'&(amp|lt|gt|quot|apos);', '', self._htmlname_escaped))
1371         d.addCallback(_check)
1372         return d
1373
1374     def test_GET_root_html(self):
1375         d = self.GET("/")
1376         d.addCallback(self._check_upload_and_mkdir_forms)
1377         return d
1378
1379     def test_GET_DIRURL(self):
1380         # the addSlash means we get a redirect here
1381         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1382         ROOT = "../../.."
1383         d = self.GET(self.public_url + "/foo", followRedirect=True)
1384         def _check(res):
1385             self.failUnlessIn('<a href="%s">Return to Welcome page' % ROOT, res)
1386
1387             # the FILE reference points to a URI, but it should end in bar.txt
1388             bar_url = ("%s/file/%s/@@named=/bar.txt" %
1389                        (ROOT, urllib.quote(self._bar_txt_uri)))
1390             get_bar = "".join([r'<td>FILE</td>',
1391                                r'\s+<td>',
1392                                r'<a href="%s">bar.txt</a>' % bar_url,
1393                                r'</td>',
1394                                r'\s+<td align="right">%d</td>' % len(self.BAR_CONTENTS),
1395                                ])
1396             self.failUnless(re.search(get_bar, res), res)
1397             for label in ['unlink', 'rename/move']:
1398                 for line in res.split("\n"):
1399                     # find the line that contains the relevant button for bar.txt
1400                     if ("form action" in line and
1401                         ('value="%s"' % (label,)) in line and
1402                         'value="bar.txt"' in line):
1403                         # the form target should use a relative URL
1404                         foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1405                         self.failUnlessIn('action="%s"' % foo_url, line)
1406                         # and the when_done= should too
1407                         #done_url = urllib.quote(???)
1408                         #self.failUnlessIn('name="when_done" value="%s"' % done_url, line)
1409
1410                         # 'unlink' needs to use POST because it directly has a side effect
1411                         if label == 'unlink':
1412                             self.failUnlessIn('method="post"', line)
1413                         break
1414                 else:
1415                     self.fail("unable to find '%s bar.txt' line" % (label,))
1416
1417             # the DIR reference just points to a URI
1418             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1419             get_sub = ((r'<td>DIR</td>')
1420                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1421             self.failUnless(re.search(get_sub, res), res)
1422         d.addCallback(_check)
1423
1424         # look at a readonly directory
1425         d.addCallback(lambda res:
1426                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
1427         def _check2(res):
1428             self.failUnlessIn("(read-only)", res)
1429             self.failIfIn("Upload a file", res)
1430         d.addCallback(_check2)
1431
1432         # and at a directory that contains a readonly directory
1433         d.addCallback(lambda res:
1434                       self.GET(self.public_url, followRedirect=True))
1435         def _check3(res):
1436             self.failUnless(re.search('<td>DIR-RO</td>'
1437                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1438         d.addCallback(_check3)
1439
1440         # and an empty directory
1441         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1442         def _check4(res):
1443             self.failUnlessIn("directory is empty", res)
1444             MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
1445             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1446         d.addCallback(_check4)
1447
1448         # and at a literal directory
1449         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1450         d.addCallback(lambda res:
1451                       self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1452         def _check5(res):
1453             self.failUnlessIn('(immutable)', res)
1454             self.failUnless(re.search('<td>FILE</td>'
1455                                       r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1456         d.addCallback(_check5)
1457         return d
1458
1459     def test_GET_DIRURL_badtype(self):
1460         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1461                                  400, "Bad Request",
1462                                  "bad t=bogus",
1463                                  self.GET,
1464                                  self.public_url + "/foo?t=bogus")
1465         return d
1466
1467     def test_GET_DIRURL_json(self):
1468         d = self.GET(self.public_url + "/foo?t=json")
1469         d.addCallback(self.failUnlessIsFooJSON)
1470         return d
1471
1472     def test_GET_DIRURL_json_format(self):
1473         d = self.PUT(self.public_url + \
1474                      "/foo/sdmf.txt?format=sdmf",
1475                      self.NEWFILE_CONTENTS * 300000)
1476         d.addCallback(lambda ignored:
1477             self.PUT(self.public_url + \
1478                      "/foo/mdmf.txt?format=mdmf",
1479                      self.NEWFILE_CONTENTS * 300000))
1480         # Now we have an MDMF and SDMF file in the directory. If we GET
1481         # its JSON, we should see their encodings.
1482         d.addCallback(lambda ignored:
1483             self.GET(self.public_url + "/foo?t=json"))
1484         def _got_json(json):
1485             data = simplejson.loads(json)
1486             assert data[0] == "dirnode"
1487
1488             data = data[1]
1489             kids = data['children']
1490
1491             mdmf_data = kids['mdmf.txt'][1]
1492             self.failUnlessIn("format", mdmf_data)
1493             self.failUnlessEqual(mdmf_data["format"], "MDMF")
1494
1495             sdmf_data = kids['sdmf.txt'][1]
1496             self.failUnlessIn("format", sdmf_data)
1497             self.failUnlessEqual(sdmf_data["format"], "SDMF")
1498         d.addCallback(_got_json)
1499         return d
1500
1501
1502     def test_POST_DIRURL_manifest_no_ophandle(self):
1503         d = self.shouldFail2(error.Error,
1504                              "test_POST_DIRURL_manifest_no_ophandle",
1505                              "400 Bad Request",
1506                              "slow operation requires ophandle=",
1507                              self.POST, self.public_url, t="start-manifest")
1508         return d
1509
1510     def test_POST_DIRURL_manifest(self):
1511         d = defer.succeed(None)
1512         def getman(ignored, output):
1513             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1514                           followRedirect=True)
1515             d.addCallback(self.wait_for_operation, "125")
1516             d.addCallback(self.get_operation_results, "125", output)
1517             return d
1518         d.addCallback(getman, None)
1519         def _got_html(manifest):
1520             self.failUnlessIn("Manifest of SI=", manifest)
1521             self.failUnlessIn("<td>sub</td>", manifest)
1522             self.failUnlessIn(self._sub_uri, manifest)
1523             self.failUnlessIn("<td>sub/baz.txt</td>", manifest)
1524             self.failUnlessIn(FAVICON_MARKUP, manifest)
1525         d.addCallback(_got_html)
1526
1527         # both t=status and unadorned GET should be identical
1528         d.addCallback(lambda res: self.GET("/operations/125"))
1529         d.addCallback(_got_html)
1530
1531         d.addCallback(getman, "html")
1532         d.addCallback(_got_html)
1533         d.addCallback(getman, "text")
1534         def _got_text(manifest):
1535             self.failUnlessIn("\nsub " + self._sub_uri + "\n", manifest)
1536             self.failUnlessIn("\nsub/baz.txt URI:CHK:", manifest)
1537         d.addCallback(_got_text)
1538         d.addCallback(getman, "JSON")
1539         def _got_json(res):
1540             data = res["manifest"]
1541             got = {}
1542             for (path_list, cap) in data:
1543                 got[tuple(path_list)] = cap
1544             self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
1545             self.failUnlessIn((u"sub", u"baz.txt"), got)
1546             self.failUnlessIn("finished", res)
1547             self.failUnlessIn("origin", res)
1548             self.failUnlessIn("storage-index", res)
1549             self.failUnlessIn("verifycaps", res)
1550             self.failUnlessIn("stats", res)
1551         d.addCallback(_got_json)
1552         return d
1553
1554     def test_POST_DIRURL_deepsize_no_ophandle(self):
1555         d = self.shouldFail2(error.Error,
1556                              "test_POST_DIRURL_deepsize_no_ophandle",
1557                              "400 Bad Request",
1558                              "slow operation requires ophandle=",
1559                              self.POST, self.public_url, t="start-deep-size")
1560         return d
1561
1562     def test_POST_DIRURL_deepsize(self):
1563         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1564                       followRedirect=True)
1565         d.addCallback(self.wait_for_operation, "126")
1566         d.addCallback(self.get_operation_results, "126", "json")
1567         def _got_json(data):
1568             self.failUnlessReallyEqual(data["finished"], True)
1569             size = data["size"]
1570             self.failUnless(size > 1000)
1571         d.addCallback(_got_json)
1572         d.addCallback(self.get_operation_results, "126", "text")
1573         def _got_text(res):
1574             mo = re.search(r'^size: (\d+)$', res, re.M)
1575             self.failUnless(mo, res)
1576             size = int(mo.group(1))
1577             # with directories, the size varies.
1578             self.failUnless(size > 1000)
1579         d.addCallback(_got_text)
1580         return d
1581
1582     def test_POST_DIRURL_deepstats_no_ophandle(self):
1583         d = self.shouldFail2(error.Error,
1584                              "test_POST_DIRURL_deepstats_no_ophandle",
1585                              "400 Bad Request",
1586                              "slow operation requires ophandle=",
1587                              self.POST, self.public_url, t="start-deep-stats")
1588         return d
1589
1590     def test_POST_DIRURL_deepstats(self):
1591         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1592                       followRedirect=True)
1593         d.addCallback(self.wait_for_operation, "127")
1594         d.addCallback(self.get_operation_results, "127", "json")
1595         def _got_json(stats):
1596             expected = {"count-immutable-files": 4,
1597                         "count-mutable-files": 2,
1598                         "count-literal-files": 0,
1599                         "count-files": 6,
1600                         "count-directories": 3,
1601                         "size-immutable-files": 76,
1602                         "size-literal-files": 0,
1603                         #"size-directories": 1912, # varies
1604                         #"largest-directory": 1590,
1605                         "largest-directory-children": 8,
1606                         "largest-immutable-file": 19,
1607                         }
1608             for k,v in expected.iteritems():
1609                 self.failUnlessReallyEqual(stats[k], v,
1610                                            "stats[%s] was %s, not %s" %
1611                                            (k, stats[k], v))
1612             self.failUnlessReallyEqual(stats["size-files-histogram"],
1613                                        [ [11, 31, 4] ])
1614         d.addCallback(_got_json)
1615         return d
1616
1617     def test_POST_DIRURL_stream_manifest(self):
1618         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1619         def _check(res):
1620             self.failUnless(res.endswith("\n"))
1621             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1622             self.failUnlessReallyEqual(len(units), 10)
1623             self.failUnlessEqual(units[-1]["type"], "stats")
1624             first = units[0]
1625             self.failUnlessEqual(first["path"], [])
1626             self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri)
1627             self.failUnlessEqual(first["type"], "directory")
1628             baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0]
1629             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1630             self.failIfEqual(baz["storage-index"], None)
1631             self.failIfEqual(baz["verifycap"], None)
1632             self.failIfEqual(baz["repaircap"], None)
1633             # XXX: Add quux and baz to this test.
1634             return
1635         d.addCallback(_check)
1636         return d
1637
1638     def test_GET_DIRURL_uri(self):
1639         d = self.GET(self.public_url + "/foo?t=uri")
1640         def _check(res):
1641             self.failUnlessReallyEqual(to_str(res), self._foo_uri)
1642         d.addCallback(_check)
1643         return d
1644
1645     def test_GET_DIRURL_readonly_uri(self):
1646         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1647         def _check(res):
1648             self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri)
1649         d.addCallback(_check)
1650         return d
1651
1652     def test_PUT_NEWDIRURL(self):
1653         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1654         d.addCallback(lambda res:
1655                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1656         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1657         d.addCallback(self.failUnlessNodeKeysAre, [])
1658         return d
1659
1660     def test_PUT_NEWDIRURL_mdmf(self):
1661         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1662         d.addCallback(lambda res:
1663                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1664         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1665         d.addCallback(lambda node:
1666             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1667         return d
1668
1669     def test_PUT_NEWDIRURL_sdmf(self):
1670         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=sdmf",
1671                      "")
1672         d.addCallback(lambda res:
1673                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1674         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1675         d.addCallback(lambda node:
1676             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1677         return d
1678
1679     def test_PUT_NEWDIRURL_bad_format(self):
1680         return self.shouldHTTPError("PUT_NEWDIRURL_bad_format",
1681                                     400, "Bad Request", "Unknown format: foo",
1682                                     self.PUT, self.public_url +
1683                                     "/foo/newdir=?t=mkdir&format=foo", "")
1684
1685     def test_POST_NEWDIRURL(self):
1686         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1687         d.addCallback(lambda res:
1688                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1689         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1690         d.addCallback(self.failUnlessNodeKeysAre, [])
1691         return d
1692
1693     def test_POST_NEWDIRURL_mdmf(self):
1694         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1695         d.addCallback(lambda res:
1696                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1697         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1698         d.addCallback(lambda node:
1699             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1700         return d
1701
1702     def test_POST_NEWDIRURL_sdmf(self):
1703         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=sdmf", "")
1704         d.addCallback(lambda res:
1705             self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1706         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1707         d.addCallback(lambda node:
1708             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1709         return d
1710
1711     def test_POST_NEWDIRURL_bad_format(self):
1712         return self.shouldHTTPError("POST_NEWDIRURL_bad_format",
1713                                     400, "Bad Request", "Unknown format: foo",
1714                                     self.POST2, self.public_url + \
1715                                     "/foo/newdir?t=mkdir&format=foo", "")
1716
1717     def test_POST_NEWDIRURL_emptyname(self):
1718         # an empty pathname component (i.e. a double-slash) is disallowed
1719         d = self.shouldFail2(error.Error, "POST_NEWDIRURL_emptyname",
1720                              "400 Bad Request",
1721                              "The webapi does not allow empty pathname components, i.e. a double slash",
1722                              self.POST, self.public_url + "//?t=mkdir")
1723         return d
1724
1725     def _do_POST_NEWDIRURL_initial_children_test(self, version=None):
1726         (newkids, caps) = self._create_initial_children()
1727         query = "/foo/newdir?t=mkdir-with-children"
1728         if version == MDMF_VERSION:
1729             query += "&format=mdmf"
1730         elif version == SDMF_VERSION:
1731             query += "&format=sdmf"
1732         else:
1733             version = SDMF_VERSION # for later
1734         d = self.POST2(self.public_url + query,
1735                        simplejson.dumps(newkids))
1736         def _check(uri):
1737             n = self.s.create_node_from_uri(uri.strip())
1738             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1739             self.failUnlessEqual(n._node.get_version(), version)
1740             d2.addCallback(lambda ign:
1741                            self.failUnlessROChildURIIs(n, u"child-imm",
1742                                                        caps['filecap1']))
1743             d2.addCallback(lambda ign:
1744                            self.failUnlessRWChildURIIs(n, u"child-mutable",
1745                                                        caps['filecap2']))
1746             d2.addCallback(lambda ign:
1747                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1748                                                        caps['filecap3']))
1749             d2.addCallback(lambda ign:
1750                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1751                                                        caps['unknown_rocap']))
1752             d2.addCallback(lambda ign:
1753                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1754                                                        caps['unknown_rwcap']))
1755             d2.addCallback(lambda ign:
1756                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1757                                                        caps['unknown_immcap']))
1758             d2.addCallback(lambda ign:
1759                            self.failUnlessRWChildURIIs(n, u"dirchild",
1760                                                        caps['dircap']))
1761             d2.addCallback(lambda ign:
1762                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1763                                                        caps['litdircap']))
1764             d2.addCallback(lambda ign:
1765                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1766                                                        caps['emptydircap']))
1767             return d2
1768         d.addCallback(_check)
1769         d.addCallback(lambda res:
1770                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1771         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1772         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1773         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1774         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1775         return d
1776
1777     def test_POST_NEWDIRURL_initial_children(self):
1778         return self._do_POST_NEWDIRURL_initial_children_test()
1779
1780     def test_POST_NEWDIRURL_initial_children_mdmf(self):
1781         return self._do_POST_NEWDIRURL_initial_children_test(MDMF_VERSION)
1782
1783     def test_POST_NEWDIRURL_initial_children_sdmf(self):
1784         return self._do_POST_NEWDIRURL_initial_children_test(SDMF_VERSION)
1785
1786     def test_POST_NEWDIRURL_initial_children_bad_format(self):
1787         (newkids, caps) = self._create_initial_children()
1788         return self.shouldHTTPError("POST_NEWDIRURL_initial_children_bad_format",
1789                                     400, "Bad Request", "Unknown format: foo",
1790                                     self.POST2, self.public_url + \
1791                                     "/foo/newdir?t=mkdir-with-children&format=foo",
1792                                     simplejson.dumps(newkids))
1793
1794     def test_POST_NEWDIRURL_immutable(self):
1795         (newkids, caps) = self._create_immutable_children()
1796         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1797                        simplejson.dumps(newkids))
1798         def _check(uri):
1799             n = self.s.create_node_from_uri(uri.strip())
1800             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1801             d2.addCallback(lambda ign:
1802                            self.failUnlessROChildURIIs(n, u"child-imm",
1803                                                        caps['filecap1']))
1804             d2.addCallback(lambda ign:
1805                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1806                                                        caps['unknown_immcap']))
1807             d2.addCallback(lambda ign:
1808                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
1809                                                        caps['immdircap']))
1810             d2.addCallback(lambda ign:
1811                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1812                                                        caps['litdircap']))
1813             d2.addCallback(lambda ign:
1814                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1815                                                        caps['emptydircap']))
1816             return d2
1817         d.addCallback(_check)
1818         d.addCallback(lambda res:
1819                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1820         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1821         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1822         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1823         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1824         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1825         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1826         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1827         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
1828         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1829         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
1830         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1831         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
1832         d.addErrback(self.explain_web_error)
1833         return d
1834
1835     def test_POST_NEWDIRURL_immutable_bad(self):
1836         (newkids, caps) = self._create_initial_children()
1837         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
1838                              "400 Bad Request",
1839                              "needed to be immutable but was not",
1840                              self.POST2,
1841                              self.public_url + "/foo/newdir?t=mkdir-immutable",
1842                              simplejson.dumps(newkids))
1843         return d
1844
1845     def test_PUT_NEWDIRURL_exists(self):
1846         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1847         d.addCallback(lambda res:
1848                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1849         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1850         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1851         return d
1852
1853     def test_PUT_NEWDIRURL_blocked(self):
1854         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1855                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1856                              self.PUT,
1857                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1858         d.addCallback(lambda res:
1859                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1860         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1861         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1862         return d
1863
1864     def test_PUT_NEWDIRURL_mkdirs(self):
1865         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1866         d.addCallback(lambda res:
1867                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1868         d.addCallback(lambda res:
1869                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1870         d.addCallback(lambda res:
1871                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1872         d.addCallback(self.failUnlessNodeKeysAre, [])
1873         return d
1874
1875     def test_PUT_NEWDIRURL_mkdirs_mdmf(self):
1876         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=mdmf", "")
1877         d.addCallback(lambda ignored:
1878             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1879         d.addCallback(lambda ignored:
1880             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1881         d.addCallback(lambda ignored:
1882             self._foo_node.get_child_at_path(u"subdir"))
1883         def _got_subdir(subdir):
1884             # XXX: What we want?
1885             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1886             self.failUnlessNodeHasChild(subdir, u"newdir")
1887             return subdir.get_child_at_path(u"newdir")
1888         d.addCallback(_got_subdir)
1889         d.addCallback(lambda newdir:
1890             self.failUnlessEqual(newdir._node.get_version(), MDMF_VERSION))
1891         return d
1892
1893     def test_PUT_NEWDIRURL_mkdirs_sdmf(self):
1894         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=sdmf", "")
1895         d.addCallback(lambda ignored:
1896             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1897         d.addCallback(lambda ignored:
1898             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1899         d.addCallback(lambda ignored:
1900             self._foo_node.get_child_at_path(u"subdir"))
1901         def _got_subdir(subdir):
1902             # XXX: What we want?
1903             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1904             self.failUnlessNodeHasChild(subdir, u"newdir")
1905             return subdir.get_child_at_path(u"newdir")
1906         d.addCallback(_got_subdir)
1907         d.addCallback(lambda newdir:
1908             self.failUnlessEqual(newdir._node.get_version(), SDMF_VERSION))
1909         return d
1910
1911     def test_PUT_NEWDIRURL_mkdirs_bad_format(self):
1912         return self.shouldHTTPError("PUT_NEWDIRURL_mkdirs_bad_format",
1913                                     400, "Bad Request", "Unknown format: foo",
1914                                     self.PUT, self.public_url + \
1915                                     "/foo/subdir/newdir?t=mkdir&format=foo",
1916                                     "")
1917
1918     def test_DELETE_DIRURL(self):
1919         d = self.DELETE(self.public_url + "/foo")
1920         d.addCallback(lambda res:
1921                       self.failIfNodeHasChild(self.public_root, u"foo"))
1922         return d
1923
1924     def test_DELETE_DIRURL_missing(self):
1925         d = self.DELETE(self.public_url + "/foo/missing")
1926         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1927         d.addCallback(lambda res:
1928                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1929         return d
1930
1931     def test_DELETE_DIRURL_missing2(self):
1932         d = self.DELETE(self.public_url + "/missing")
1933         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1934         return d
1935
1936     def dump_root(self):
1937         print "NODEWALK"
1938         w = webish.DirnodeWalkerMixin()
1939         def visitor(childpath, childnode, metadata):
1940             print childpath
1941         d = w.walk(self.public_root, visitor)
1942         return d
1943
1944     def failUnlessNodeKeysAre(self, node, expected_keys):
1945         for k in expected_keys:
1946             assert isinstance(k, unicode)
1947         d = node.list()
1948         def _check(children):
1949             self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
1950         d.addCallback(_check)
1951         return d
1952     def failUnlessNodeHasChild(self, node, name):
1953         assert isinstance(name, unicode)
1954         d = node.list()
1955         def _check(children):
1956             self.failUnlessIn(name, children)
1957         d.addCallback(_check)
1958         return d
1959     def failIfNodeHasChild(self, node, name):
1960         assert isinstance(name, unicode)
1961         d = node.list()
1962         def _check(children):
1963             self.failIfIn(name, children)
1964         d.addCallback(_check)
1965         return d
1966
1967     def failUnlessChildContentsAre(self, node, name, expected_contents):
1968         assert isinstance(name, unicode)
1969         d = node.get_child_at_path(name)
1970         d.addCallback(lambda node: download_to_data(node))
1971         def _check(contents):
1972             self.failUnlessReallyEqual(contents, expected_contents)
1973         d.addCallback(_check)
1974         return d
1975
1976     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1977         assert isinstance(name, unicode)
1978         d = node.get_child_at_path(name)
1979         d.addCallback(lambda node: node.download_best_version())
1980         def _check(contents):
1981             self.failUnlessReallyEqual(contents, expected_contents)
1982         d.addCallback(_check)
1983         return d
1984
1985     def failUnlessRWChildURIIs(self, node, name, expected_uri):
1986         assert isinstance(name, unicode)
1987         d = node.get_child_at_path(name)
1988         def _check(child):
1989             self.failUnless(child.is_unknown() or not child.is_readonly())
1990             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1991             self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
1992             expected_ro_uri = self._make_readonly(expected_uri)
1993             if expected_ro_uri:
1994                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1995         d.addCallback(_check)
1996         return d
1997
1998     def failUnlessROChildURIIs(self, node, name, expected_uri):
1999         assert isinstance(name, unicode)
2000         d = node.get_child_at_path(name)
2001         def _check(child):
2002             self.failUnless(child.is_unknown() or child.is_readonly())
2003             self.failUnlessReallyEqual(child.get_write_uri(), None)
2004             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
2005             self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
2006         d.addCallback(_check)
2007         return d
2008
2009     def failUnlessURIMatchesRWChild(self, got_uri, node, name):
2010         assert isinstance(name, unicode)
2011         d = node.get_child_at_path(name)
2012         def _check(child):
2013             self.failUnless(child.is_unknown() or not child.is_readonly())
2014             self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
2015             self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
2016             expected_ro_uri = self._make_readonly(got_uri)
2017             if expected_ro_uri:
2018                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
2019         d.addCallback(_check)
2020         return d
2021
2022     def failUnlessURIMatchesROChild(self, got_uri, node, name):
2023         assert isinstance(name, unicode)
2024         d = node.get_child_at_path(name)
2025         def _check(child):
2026             self.failUnless(child.is_unknown() or child.is_readonly())
2027             self.failUnlessReallyEqual(child.get_write_uri(), None)
2028             self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
2029             self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
2030         d.addCallback(_check)
2031         return d
2032
2033     def failUnlessCHKURIHasContents(self, got_uri, contents):
2034         self.failUnless(self.get_all_contents()[got_uri] == contents)
2035
2036     def test_POST_upload(self):
2037         d = self.POST(self.public_url + "/foo", t="upload",
2038                       file=("new.txt", self.NEWFILE_CONTENTS))
2039         fn = self._foo_node
2040         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2041         d.addCallback(lambda res:
2042                       self.failUnlessChildContentsAre(fn, u"new.txt",
2043                                                       self.NEWFILE_CONTENTS))
2044         return d
2045
2046     def test_POST_upload_unicode(self):
2047         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2048         d = self.POST(self.public_url + "/foo", t="upload",
2049                       file=(filename, self.NEWFILE_CONTENTS))
2050         fn = self._foo_node
2051         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2052         d.addCallback(lambda res:
2053                       self.failUnlessChildContentsAre(fn, filename,
2054                                                       self.NEWFILE_CONTENTS))
2055         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2056         d.addCallback(lambda res: self.GET(target_url))
2057         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2058                                                                   self.NEWFILE_CONTENTS,
2059                                                                   contents))
2060         return d
2061
2062     def test_POST_upload_unicode_named(self):
2063         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2064         d = self.POST(self.public_url + "/foo", t="upload",
2065                       name=filename,
2066                       file=("overridden", self.NEWFILE_CONTENTS))
2067         fn = self._foo_node
2068         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2069         d.addCallback(lambda res:
2070                       self.failUnlessChildContentsAre(fn, filename,
2071                                                       self.NEWFILE_CONTENTS))
2072         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2073         d.addCallback(lambda res: self.GET(target_url))
2074         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2075                                                                   self.NEWFILE_CONTENTS,
2076                                                                   contents))
2077         return d
2078
2079     def test_POST_upload_no_link(self):
2080         d = self.POST("/uri", t="upload",
2081                       file=("new.txt", self.NEWFILE_CONTENTS))
2082         def _check_upload_results(page):
2083             # this should be a page which describes the results of the upload
2084             # that just finished.
2085             self.failUnlessIn("Upload Results:", page)
2086             self.failUnlessIn("URI:", page)
2087             uri_re = re.compile("URI: <tt><span>(.*)</span>")
2088             mo = uri_re.search(page)
2089             self.failUnless(mo, page)
2090             new_uri = mo.group(1)
2091             return new_uri
2092         d.addCallback(_check_upload_results)
2093         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
2094         return d
2095
2096     def test_POST_upload_no_link_whendone(self):
2097         d = self.POST("/uri", t="upload", when_done="/",
2098                       file=("new.txt", self.NEWFILE_CONTENTS))
2099         d.addBoth(self.shouldRedirect, "/")
2100         return d
2101
2102     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
2103         d = defer.maybeDeferred(callable, *args, **kwargs)
2104         def done(res):
2105             if isinstance(res, failure.Failure):
2106                 res.trap(error.PageRedirect)
2107                 statuscode = res.value.status
2108                 target = res.value.location
2109                 return checker(statuscode, target)
2110             self.fail("%s: callable was supposed to redirect, not return '%s'"
2111                       % (which, res))
2112         d.addBoth(done)
2113         return d
2114
2115     def test_POST_upload_no_link_whendone_results(self):
2116         def check(statuscode, target):
2117             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2118             self.failUnless(target.startswith(self.webish_url), target)
2119             return client.getPage(target, method="GET")
2120         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
2121                                  check,
2122                                  self.POST, "/uri", t="upload",
2123                                  when_done="/uri/%(uri)s",
2124                                  file=("new.txt", self.NEWFILE_CONTENTS))
2125         d.addCallback(lambda res:
2126                       self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
2127         return d
2128
2129     def test_POST_upload_no_link_mutable(self):
2130         d = self.POST("/uri", t="upload", mutable="true",
2131                       file=("new.txt", self.NEWFILE_CONTENTS))
2132         def _check(filecap):
2133             filecap = filecap.strip()
2134             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2135             self.filecap = filecap
2136             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2137             self.failUnlessIn(u.get_storage_index(), self.get_all_contents())
2138             n = self.s.create_node_from_uri(filecap)
2139             return n.download_best_version()
2140         d.addCallback(_check)
2141         def _check2(data):
2142             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2143             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2144         d.addCallback(_check2)
2145         def _check3(data):
2146             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2147             return self.GET("/file/%s" % urllib.quote(self.filecap))
2148         d.addCallback(_check3)
2149         def _check4(data):
2150             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2151         d.addCallback(_check4)
2152         return d
2153
2154     def test_POST_upload_no_link_mutable_toobig(self):
2155         # The SDMF size limit is no longer in place, so we should be
2156         # able to upload mutable files that are as large as we want them
2157         # to be.
2158         d = self.POST("/uri", t="upload", mutable="true",
2159                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2160         return d
2161
2162
2163     def test_POST_upload_format_unlinked(self):
2164         def _check_upload_unlinked(ign, format, uri_prefix):
2165             filename = format + ".txt"
2166             d = self.POST("/uri?t=upload&format=" + format,
2167                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2168             def _got_results(results):
2169                 if format.upper() in ("SDMF", "MDMF"):
2170                     # webapi.rst says this returns a filecap
2171                     filecap = results
2172                 else:
2173                     # for immutable, it returns an "upload results page", and
2174                     # the filecap is buried inside
2175                     line = [l for l in results.split("\n") if "URI: " in l][0]
2176                     mo = re.search(r'<span>([^<]+)</span>', line)
2177                     filecap = mo.group(1)
2178                 self.failUnless(filecap.startswith(uri_prefix),
2179                                 (uri_prefix, filecap))
2180                 return self.GET("/uri/%s?t=json" % filecap)
2181             d.addCallback(_got_results)
2182             def _got_json(json):
2183                 data = simplejson.loads(json)
2184                 data = data[1]
2185                 self.failUnlessIn("format", data)
2186                 self.failUnlessEqual(data["format"], format.upper())
2187             d.addCallback(_got_json)
2188             return d
2189         d = defer.succeed(None)
2190         d.addCallback(_check_upload_unlinked, "chk", "URI:CHK")
2191         d.addCallback(_check_upload_unlinked, "CHK", "URI:CHK")
2192         d.addCallback(_check_upload_unlinked, "sdmf", "URI:SSK")
2193         d.addCallback(_check_upload_unlinked, "mdmf", "URI:MDMF")
2194         return d
2195
2196     def test_POST_upload_bad_format_unlinked(self):
2197         return self.shouldHTTPError("POST_upload_bad_format_unlinked",
2198                                     400, "Bad Request", "Unknown format: foo",
2199                                     self.POST,
2200                                     "/uri?t=upload&format=foo",
2201                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2202
2203     def test_POST_upload_format(self):
2204         def _check_upload(ign, format, uri_prefix, fn=None):
2205             filename = format + ".txt"
2206             d = self.POST(self.public_url +
2207                           "/foo?t=upload&format=" + format,
2208                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2209             def _got_filecap(filecap):
2210                 if fn is not None:
2211                     filenameu = unicode(filename)
2212                     self.failUnlessURIMatchesRWChild(filecap, fn, filenameu)
2213                 self.failUnless(filecap.startswith(uri_prefix))
2214                 return self.GET(self.public_url + "/foo/%s?t=json" % filename)
2215             d.addCallback(_got_filecap)
2216             def _got_json(json):
2217                 data = simplejson.loads(json)
2218                 data = data[1]
2219                 self.failUnlessIn("format", data)
2220                 self.failUnlessEqual(data["format"], format.upper())
2221             d.addCallback(_got_json)
2222             return d
2223
2224         d = defer.succeed(None)
2225         d.addCallback(_check_upload, "chk", "URI:CHK")
2226         d.addCallback(_check_upload, "sdmf", "URI:SSK", self._foo_node)
2227         d.addCallback(_check_upload, "mdmf", "URI:MDMF")
2228         d.addCallback(_check_upload, "MDMF", "URI:MDMF")
2229         return d
2230
2231     def test_POST_upload_bad_format(self):
2232         return self.shouldHTTPError("POST_upload_bad_format",
2233                                     400, "Bad Request", "Unknown format: foo",
2234                                     self.POST, self.public_url + \
2235                                     "/foo?t=upload&format=foo",
2236                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2237
2238     def test_POST_upload_mutable(self):
2239         # this creates a mutable file
2240         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
2241                       file=("new.txt", self.NEWFILE_CONTENTS))
2242         fn = self._foo_node
2243         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2244         d.addCallback(lambda res:
2245                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2246                                                              self.NEWFILE_CONTENTS))
2247         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2248         def _got(newnode):
2249             self.failUnless(IMutableFileNode.providedBy(newnode))
2250             self.failUnless(newnode.is_mutable())
2251             self.failIf(newnode.is_readonly())
2252             self._mutable_node = newnode
2253             self._mutable_uri = newnode.get_uri()
2254         d.addCallback(_got)
2255
2256         # now upload it again and make sure that the URI doesn't change
2257         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
2258         d.addCallback(lambda res:
2259                       self.POST(self.public_url + "/foo", t="upload",
2260                                 mutable="true",
2261                                 file=("new.txt", NEWER_CONTENTS)))
2262         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2263         d.addCallback(lambda res:
2264                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2265                                                              NEWER_CONTENTS))
2266         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2267         def _got2(newnode):
2268             self.failUnless(IMutableFileNode.providedBy(newnode))
2269             self.failUnless(newnode.is_mutable())
2270             self.failIf(newnode.is_readonly())
2271             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2272         d.addCallback(_got2)
2273
2274         # upload a second time, using PUT instead of POST
2275         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
2276         d.addCallback(lambda res:
2277                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
2278         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2279         d.addCallback(lambda res:
2280                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2281                                                              NEW2_CONTENTS))
2282
2283         # finally list the directory, since mutable files are displayed
2284         # slightly differently
2285
2286         d.addCallback(lambda res:
2287                       self.GET(self.public_url + "/foo/",
2288                                followRedirect=True))
2289         def _check_page(res):
2290             # TODO: assert more about the contents
2291             self.failUnlessIn("SSK", res)
2292             return res
2293         d.addCallback(_check_page)
2294
2295         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2296         def _got3(newnode):
2297             self.failUnless(IMutableFileNode.providedBy(newnode))
2298             self.failUnless(newnode.is_mutable())
2299             self.failIf(newnode.is_readonly())
2300             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2301         d.addCallback(_got3)
2302
2303         # look at the JSON form of the enclosing directory
2304         d.addCallback(lambda res:
2305                       self.GET(self.public_url + "/foo/?t=json",
2306                                followRedirect=True))
2307         def _check_page_json(res):
2308             parsed = simplejson.loads(res)
2309             self.failUnlessEqual(parsed[0], "dirnode")
2310             children = dict( [(unicode(name),value)
2311                               for (name,value)
2312                               in parsed[1]["children"].iteritems()] )
2313             self.failUnlessIn(u"new.txt", children)
2314             new_json = children[u"new.txt"]
2315             self.failUnlessEqual(new_json[0], "filenode")
2316             self.failUnless(new_json[1]["mutable"])
2317             self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri)
2318             ro_uri = self._mutable_node.get_readonly().to_string()
2319             self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri)
2320         d.addCallback(_check_page_json)
2321
2322         # and the JSON form of the file
2323         d.addCallback(lambda res:
2324                       self.GET(self.public_url + "/foo/new.txt?t=json"))
2325         def _check_file_json(res):
2326             parsed = simplejson.loads(res)
2327             self.failUnlessEqual(parsed[0], "filenode")
2328             self.failUnless(parsed[1]["mutable"])
2329             self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri)
2330             ro_uri = self._mutable_node.get_readonly().to_string()
2331             self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri)
2332         d.addCallback(_check_file_json)
2333
2334         # and look at t=uri and t=readonly-uri
2335         d.addCallback(lambda res:
2336                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
2337         d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
2338         d.addCallback(lambda res:
2339                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
2340         def _check_ro_uri(res):
2341             ro_uri = self._mutable_node.get_readonly().to_string()
2342             self.failUnlessReallyEqual(res, ro_uri)
2343         d.addCallback(_check_ro_uri)
2344
2345         # make sure we can get to it from /uri/URI
2346         d.addCallback(lambda res:
2347                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
2348         d.addCallback(lambda res:
2349                       self.failUnlessReallyEqual(res, NEW2_CONTENTS))
2350
2351         # and that HEAD computes the size correctly
2352         d.addCallback(lambda res:
2353                       self.HEAD(self.public_url + "/foo/new.txt",
2354                                 return_response=True))
2355         def _got_headers((res, status, headers)):
2356             self.failUnlessReallyEqual(res, "")
2357             self.failUnlessReallyEqual(headers["content-length"][0],
2358                                        str(len(NEW2_CONTENTS)))
2359             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
2360         d.addCallback(_got_headers)
2361
2362         # make sure that outdated size limits aren't enforced anymore.
2363         d.addCallback(lambda ignored:
2364             self.POST(self.public_url + "/foo", t="upload",
2365                       mutable="true",
2366                       file=("new.txt",
2367                             "b" * (self.s.MUTABLE_SIZELIMIT+1))))
2368         d.addErrback(self.dump_error)
2369         return d
2370
2371     def test_POST_upload_mutable_toobig(self):
2372         # SDMF had a size limti that was removed a while ago. MDMF has
2373         # never had a size limit. Test to make sure that we do not
2374         # encounter errors when trying to upload large mutable files,
2375         # since there should be no coded prohibitions regarding large
2376         # mutable files.
2377         d = self.POST(self.public_url + "/foo",
2378                       t="upload", mutable="true",
2379                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2380         return d
2381
2382     def dump_error(self, f):
2383         # if the web server returns an error code (like 400 Bad Request),
2384         # web.client.getPage puts the HTTP response body into the .response
2385         # attribute of the exception object that it gives back. It does not
2386         # appear in the Failure's repr(), so the ERROR that trial displays
2387         # will be rather terse and unhelpful. addErrback this method to the
2388         # end of your chain to get more information out of these errors.
2389         if f.check(error.Error):
2390             print "web.error.Error:"
2391             print f
2392             print f.value.response
2393         return f
2394
2395     def test_POST_upload_replace(self):
2396         d = self.POST(self.public_url + "/foo", t="upload",
2397                       file=("bar.txt", self.NEWFILE_CONTENTS))
2398         fn = self._foo_node
2399         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
2400         d.addCallback(lambda res:
2401                       self.failUnlessChildContentsAre(fn, u"bar.txt",
2402                                                       self.NEWFILE_CONTENTS))
2403         return d
2404
2405     def test_POST_upload_no_replace_ok(self):
2406         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2407                       file=("new.txt", self.NEWFILE_CONTENTS))
2408         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
2409         d.addCallback(lambda res: self.failUnlessReallyEqual(res,
2410                                                              self.NEWFILE_CONTENTS))
2411         return d
2412
2413     def test_POST_upload_no_replace_queryarg(self):
2414         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2415                       file=("bar.txt", self.NEWFILE_CONTENTS))
2416         d.addBoth(self.shouldFail, error.Error,
2417                   "POST_upload_no_replace_queryarg",
2418                   "409 Conflict",
2419                   "There was already a child by that name, and you asked me "
2420                   "to not replace it")
2421         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2422         d.addCallback(self.failUnlessIsBarDotTxt)
2423         return d
2424
2425     def test_POST_upload_no_replace_field(self):
2426         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
2427                       file=("bar.txt", self.NEWFILE_CONTENTS))
2428         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
2429                   "409 Conflict",
2430                   "There was already a child by that name, and you asked me "
2431                   "to not replace it")
2432         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2433         d.addCallback(self.failUnlessIsBarDotTxt)
2434         return d
2435
2436     def test_POST_upload_whendone(self):
2437         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
2438                       file=("new.txt", self.NEWFILE_CONTENTS))
2439         d.addBoth(self.shouldRedirect, "/THERE")
2440         fn = self._foo_node
2441         d.addCallback(lambda res:
2442                       self.failUnlessChildContentsAre(fn, u"new.txt",
2443                                                       self.NEWFILE_CONTENTS))
2444         return d
2445
2446     def test_POST_upload_named(self):
2447         fn = self._foo_node
2448         d = self.POST(self.public_url + "/foo", t="upload",
2449                       name="new.txt", file=self.NEWFILE_CONTENTS)
2450         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2451         d.addCallback(lambda res:
2452                       self.failUnlessChildContentsAre(fn, u"new.txt",
2453                                                       self.NEWFILE_CONTENTS))
2454         return d
2455
2456     def test_POST_upload_named_badfilename(self):
2457         d = self.POST(self.public_url + "/foo", t="upload",
2458                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
2459         d.addBoth(self.shouldFail, error.Error,
2460                   "test_POST_upload_named_badfilename",
2461                   "400 Bad Request",
2462                   "name= may not contain a slash",
2463                   )
2464         # make sure that nothing was added
2465         d.addCallback(lambda res:
2466                       self.failUnlessNodeKeysAre(self._foo_node,
2467                                                  [self._htmlname_unicode,
2468                                                   u"bar.txt", u"baz.txt", u"blockingfile",
2469                                                   u"empty", u"n\u00fc.txt", u"quux.txt",
2470                                                   u"sub"]))
2471         return d
2472
2473     def test_POST_FILEURL_check(self):
2474         bar_url = self.public_url + "/foo/bar.txt"
2475         d = self.POST(bar_url, t="check")
2476         def _check(res):
2477             self.failUnlessIn("Healthy :", res)
2478         d.addCallback(_check)
2479         redir_url = "http://allmydata.org/TARGET"
2480         def _check2(statuscode, target):
2481             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2482             self.failUnlessReallyEqual(target, redir_url)
2483         d.addCallback(lambda res:
2484                       self.shouldRedirect2("test_POST_FILEURL_check",
2485                                            _check2,
2486                                            self.POST, bar_url,
2487                                            t="check",
2488                                            when_done=redir_url))
2489         d.addCallback(lambda res:
2490                       self.POST(bar_url, t="check", return_to=redir_url))
2491         def _check3(res):
2492             self.failUnlessIn("Healthy :", res)
2493             self.failUnlessIn("Return to file", res)
2494             self.failUnlessIn(redir_url, res)
2495         d.addCallback(_check3)
2496
2497         d.addCallback(lambda res:
2498                       self.POST(bar_url, t="check", output="JSON"))
2499         def _check_json(res):
2500             data = simplejson.loads(res)
2501             self.failUnlessIn("storage-index", data)
2502             self.failUnless(data["results"]["healthy"])
2503         d.addCallback(_check_json)
2504
2505         return d
2506
2507     def test_POST_FILEURL_check_and_repair(self):
2508         bar_url = self.public_url + "/foo/bar.txt"
2509         d = self.POST(bar_url, t="check", repair="true")
2510         def _check(res):
2511             self.failUnlessIn("Healthy :", res)
2512         d.addCallback(_check)
2513         redir_url = "http://allmydata.org/TARGET"
2514         def _check2(statuscode, target):
2515             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2516             self.failUnlessReallyEqual(target, redir_url)
2517         d.addCallback(lambda res:
2518                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
2519                                            _check2,
2520                                            self.POST, bar_url,
2521                                            t="check", repair="true",
2522                                            when_done=redir_url))
2523         d.addCallback(lambda res:
2524                       self.POST(bar_url, t="check", return_to=redir_url))
2525         def _check3(res):
2526             self.failUnlessIn("Healthy :", res)
2527             self.failUnlessIn("Return to file", res)
2528             self.failUnlessIn(redir_url, res)
2529         d.addCallback(_check3)
2530         return d
2531
2532     def test_POST_DIRURL_check(self):
2533         foo_url = self.public_url + "/foo/"
2534         d = self.POST(foo_url, t="check")
2535         def _check(res):
2536             self.failUnlessIn("Healthy :", res)
2537         d.addCallback(_check)
2538         redir_url = "http://allmydata.org/TARGET"
2539         def _check2(statuscode, target):
2540             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2541             self.failUnlessReallyEqual(target, redir_url)
2542         d.addCallback(lambda res:
2543                       self.shouldRedirect2("test_POST_DIRURL_check",
2544                                            _check2,
2545                                            self.POST, foo_url,
2546                                            t="check",
2547                                            when_done=redir_url))
2548         d.addCallback(lambda res:
2549                       self.POST(foo_url, t="check", return_to=redir_url))
2550         def _check3(res):
2551             self.failUnlessIn("Healthy :", res)
2552             self.failUnlessIn("Return to file/directory", res)
2553             self.failUnlessIn(redir_url, res)
2554         d.addCallback(_check3)
2555
2556         d.addCallback(lambda res:
2557                       self.POST(foo_url, t="check", output="JSON"))
2558         def _check_json(res):
2559             data = simplejson.loads(res)
2560             self.failUnlessIn("storage-index", data)
2561             self.failUnless(data["results"]["healthy"])
2562         d.addCallback(_check_json)
2563
2564         return d
2565
2566     def test_POST_DIRURL_check_and_repair(self):
2567         foo_url = self.public_url + "/foo/"
2568         d = self.POST(foo_url, t="check", repair="true")
2569         def _check(res):
2570             self.failUnlessIn("Healthy :", res)
2571         d.addCallback(_check)
2572         redir_url = "http://allmydata.org/TARGET"
2573         def _check2(statuscode, target):
2574             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2575             self.failUnlessReallyEqual(target, redir_url)
2576         d.addCallback(lambda res:
2577                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2578                                            _check2,
2579                                            self.POST, foo_url,
2580                                            t="check", repair="true",
2581                                            when_done=redir_url))
2582         d.addCallback(lambda res:
2583                       self.POST(foo_url, t="check", return_to=redir_url))
2584         def _check3(res):
2585             self.failUnlessIn("Healthy :", res)
2586             self.failUnlessIn("Return to file/directory", res)
2587             self.failUnlessIn(redir_url, res)
2588         d.addCallback(_check3)
2589         return d
2590
2591     def test_POST_FILEURL_mdmf_check(self):
2592         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2593         d = self.POST(quux_url, t="check")
2594         def _check(res):
2595             self.failUnlessIn("Healthy", res)
2596         d.addCallback(_check)
2597         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2598         d.addCallback(lambda ignored:
2599                       self.POST(quux_extension_url, t="check"))
2600         d.addCallback(_check)
2601         return d
2602
2603     def test_POST_FILEURL_mdmf_check_and_repair(self):
2604         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2605         d = self.POST(quux_url, t="check", repair="true")
2606         def _check(res):
2607             self.failUnlessIn("Healthy", res)
2608         d.addCallback(_check)
2609         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2610         d.addCallback(lambda ignored:
2611                       self.POST(quux_extension_url, t="check", repair="true"))
2612         d.addCallback(_check)
2613         return d
2614
2615     def wait_for_operation(self, ignored, ophandle):
2616         url = "/operations/" + ophandle
2617         url += "?t=status&output=JSON"
2618         d = self.GET(url)
2619         def _got(res):
2620             data = simplejson.loads(res)
2621             if not data["finished"]:
2622                 d = self.stall(delay=1.0)
2623                 d.addCallback(self.wait_for_operation, ophandle)
2624                 return d
2625             return data
2626         d.addCallback(_got)
2627         return d
2628
2629     def get_operation_results(self, ignored, ophandle, output=None):
2630         url = "/operations/" + ophandle
2631         url += "?t=status"
2632         if output:
2633             url += "&output=" + output
2634         d = self.GET(url)
2635         def _got(res):
2636             if output and output.lower() == "json":
2637                 return simplejson.loads(res)
2638             return res
2639         d.addCallback(_got)
2640         return d
2641
2642     def test_POST_DIRURL_deepcheck_no_ophandle(self):
2643         d = self.shouldFail2(error.Error,
2644                              "test_POST_DIRURL_deepcheck_no_ophandle",
2645                              "400 Bad Request",
2646                              "slow operation requires ophandle=",
2647                              self.POST, self.public_url, t="start-deep-check")
2648         return d
2649
2650     def test_POST_DIRURL_deepcheck(self):
2651         def _check_redirect(statuscode, target):
2652             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2653             self.failUnless(target.endswith("/operations/123"))
2654         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2655                                  self.POST, self.public_url,
2656                                  t="start-deep-check", ophandle="123")
2657         d.addCallback(self.wait_for_operation, "123")
2658         def _check_json(data):
2659             self.failUnlessReallyEqual(data["finished"], True)
2660             self.failUnlessReallyEqual(data["count-objects-checked"], 11)
2661             self.failUnlessReallyEqual(data["count-objects-healthy"], 11)
2662         d.addCallback(_check_json)
2663         d.addCallback(self.get_operation_results, "123", "html")
2664         def _check_html(res):
2665             self.failUnlessIn("Objects Checked: <span>11</span>", res)
2666             self.failUnlessIn("Objects Healthy: <span>11</span>", res)
2667             self.failUnlessIn(FAVICON_MARKUP, res)
2668         d.addCallback(_check_html)
2669
2670         d.addCallback(lambda res:
2671                       self.GET("/operations/123/"))
2672         d.addCallback(_check_html) # should be the same as without the slash
2673
2674         d.addCallback(lambda res:
2675                       self.shouldFail2(error.Error, "one", "404 Not Found",
2676                                        "No detailed results for SI bogus",
2677                                        self.GET, "/operations/123/bogus"))
2678
2679         foo_si = self._foo_node.get_storage_index()
2680         foo_si_s = base32.b2a(foo_si)
2681         d.addCallback(lambda res:
2682                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2683         def _check_foo_json(res):
2684             data = simplejson.loads(res)
2685             self.failUnlessEqual(data["storage-index"], foo_si_s)
2686             self.failUnless(data["results"]["healthy"])
2687         d.addCallback(_check_foo_json)
2688         return d
2689
2690     def test_POST_DIRURL_deepcheck_and_repair(self):
2691         d = self.POST(self.public_url, t="start-deep-check", repair="true",
2692                       ophandle="124", output="json", followRedirect=True)
2693         d.addCallback(self.wait_for_operation, "124")
2694         def _check_json(data):
2695             self.failUnlessReallyEqual(data["finished"], True)
2696             self.failUnlessReallyEqual(data["count-objects-checked"], 11)
2697             self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 11)
2698             self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2699             self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2700             self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2701             self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2702             self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2703             self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 11)
2704             self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2705             self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2706         d.addCallback(_check_json)
2707         d.addCallback(self.get_operation_results, "124", "html")
2708         def _check_html(res):
2709             self.failUnlessIn("Objects Checked: <span>11</span>", res)
2710
2711             self.failUnlessIn("Objects Healthy (before repair): <span>11</span>", res)
2712             self.failUnlessIn("Objects Unhealthy (before repair): <span>0</span>", res)
2713             self.failUnlessIn("Corrupt Shares (before repair): <span>0</span>", res)
2714
2715             self.failUnlessIn("Repairs Attempted: <span>0</span>", res)
2716             self.failUnlessIn("Repairs Successful: <span>0</span>", res)
2717             self.failUnlessIn("Repairs Unsuccessful: <span>0</span>", res)
2718
2719             self.failUnlessIn("Objects Healthy (after repair): <span>11</span>", res)
2720             self.failUnlessIn("Objects Unhealthy (after repair): <span>0</span>", res)
2721             self.failUnlessIn("Corrupt Shares (after repair): <span>0</span>", res)
2722
2723             self.failUnlessIn(FAVICON_MARKUP, res)
2724         d.addCallback(_check_html)
2725         return d
2726
2727     def test_POST_FILEURL_bad_t(self):
2728         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2729                              "POST to file: bad t=bogus",
2730                              self.POST, self.public_url + "/foo/bar.txt",
2731                              t="bogus")
2732         return d
2733
2734     def test_POST_mkdir(self): # return value?
2735         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2736         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2737         d.addCallback(self.failUnlessNodeKeysAre, [])
2738         return d
2739
2740     def test_POST_mkdir_mdmf(self):
2741         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=mdmf")
2742         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2743         d.addCallback(lambda node:
2744             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2745         return d
2746
2747     def test_POST_mkdir_sdmf(self):
2748         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=sdmf")
2749         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2750         d.addCallback(lambda node:
2751             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2752         return d
2753
2754     def test_POST_mkdir_bad_format(self):
2755         return self.shouldHTTPError("POST_mkdir_bad_format",
2756                                     400, "Bad Request", "Unknown format: foo",
2757                                     self.POST, self.public_url +
2758                                     "/foo?t=mkdir&name=newdir&format=foo")
2759
2760     def test_POST_mkdir_initial_children(self):
2761         (newkids, caps) = self._create_initial_children()
2762         d = self.POST2(self.public_url +
2763                        "/foo?t=mkdir-with-children&name=newdir",
2764                        simplejson.dumps(newkids))
2765         d.addCallback(lambda res:
2766                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2767         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2768         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2769         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2770         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2771         return d
2772
2773     def test_POST_mkdir_initial_children_mdmf(self):
2774         (newkids, caps) = self._create_initial_children()
2775         d = self.POST2(self.public_url +
2776                        "/foo?t=mkdir-with-children&name=newdir&format=mdmf",
2777                        simplejson.dumps(newkids))
2778         d.addCallback(lambda res:
2779                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2780         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2781         d.addCallback(lambda node:
2782             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2783         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2784         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2785                        caps['filecap1'])
2786         return d
2787
2788     # XXX: Duplication.
2789     def test_POST_mkdir_initial_children_sdmf(self):
2790         (newkids, caps) = self._create_initial_children()
2791         d = self.POST2(self.public_url +
2792                        "/foo?t=mkdir-with-children&name=newdir&format=sdmf",
2793                        simplejson.dumps(newkids))
2794         d.addCallback(lambda res:
2795                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2796         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2797         d.addCallback(lambda node:
2798             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2799         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2800         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2801                        caps['filecap1'])
2802         return d
2803
2804     def test_POST_mkdir_initial_children_bad_format(self):
2805         (newkids, caps) = self._create_initial_children()
2806         return self.shouldHTTPError("POST_mkdir_initial_children_bad_format",
2807                                     400, "Bad Request", "Unknown format: foo",
2808                                     self.POST, self.public_url + \
2809                                     "/foo?t=mkdir-with-children&name=newdir&format=foo",
2810                                     simplejson.dumps(newkids))
2811
2812     def test_POST_mkdir_immutable(self):
2813         (newkids, caps) = self._create_immutable_children()
2814         d = self.POST2(self.public_url +
2815                        "/foo?t=mkdir-immutable&name=newdir",
2816                        simplejson.dumps(newkids))
2817         d.addCallback(lambda res:
2818                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2819         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2820         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2821         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2822         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2823         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2824         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2825         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2826         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2827         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2828         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2829         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2830         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2831         return d
2832
2833     def test_POST_mkdir_immutable_bad(self):
2834         (newkids, caps) = self._create_initial_children()
2835         d = self.shouldFail2(error.Error, "POST_mkdir_immutable_bad",
2836                              "400 Bad Request",
2837                              "needed to be immutable but was not",
2838                              self.POST2,
2839                              self.public_url +
2840                              "/foo?t=mkdir-immutable&name=newdir",
2841                              simplejson.dumps(newkids))
2842         return d
2843
2844     def test_POST_mkdir_2(self):
2845         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
2846         d.addCallback(lambda res:
2847                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2848         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2849         d.addCallback(self.failUnlessNodeKeysAre, [])
2850         return d
2851
2852     def test_POST_mkdirs_2(self):
2853         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
2854         d.addCallback(lambda res:
2855                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
2856         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
2857         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
2858         d.addCallback(self.failUnlessNodeKeysAre, [])
2859         return d
2860
2861     def test_POST_mkdir_no_parentdir_noredirect(self):
2862         d = self.POST("/uri?t=mkdir")
2863         def _after_mkdir(res):
2864             uri.DirectoryURI.init_from_string(res)
2865         d.addCallback(_after_mkdir)
2866         return d
2867
2868     def test_POST_mkdir_no_parentdir_noredirect_mdmf(self):
2869         d = self.POST("/uri?t=mkdir&format=mdmf")
2870         def _after_mkdir(res):
2871             u = uri.from_string(res)
2872             # Check that this is an MDMF writecap
2873             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
2874         d.addCallback(_after_mkdir)
2875         return d
2876
2877     def test_POST_mkdir_no_parentdir_noredirect_sdmf(self):
2878         d = self.POST("/uri?t=mkdir&format=sdmf")
2879         def _after_mkdir(res):
2880             u = uri.from_string(res)
2881             self.failUnlessIsInstance(u, uri.DirectoryURI)
2882         d.addCallback(_after_mkdir)
2883         return d
2884
2885     def test_POST_mkdir_no_parentdir_noredirect_bad_format(self):
2886         return self.shouldHTTPError("POST_mkdir_no_parentdir_noredirect_bad_format",
2887                                     400, "Bad Request", "Unknown format: foo",
2888                                     self.POST, self.public_url +
2889                                     "/uri?t=mkdir&format=foo")
2890
2891     def test_POST_mkdir_no_parentdir_noredirect2(self):
2892         # make sure form-based arguments (as on the welcome page) still work
2893         d = self.POST("/uri", t="mkdir")
2894         def _after_mkdir(res):
2895             uri.DirectoryURI.init_from_string(res)
2896         d.addCallback(_after_mkdir)
2897         d.addErrback(self.explain_web_error)
2898         return d
2899
2900     def test_POST_mkdir_no_parentdir_redirect(self):
2901         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
2902         d.addBoth(self.shouldRedirect, None, statuscode='303')
2903         def _check_target(target):
2904             target = urllib.unquote(target)
2905             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2906         d.addCallback(_check_target)
2907         return d
2908
2909     def test_POST_mkdir_no_parentdir_redirect2(self):
2910         d = self.POST("/uri", t="mkdir", redirect_to_result="true")
2911         d.addBoth(self.shouldRedirect, None, statuscode='303')
2912         def _check_target(target):
2913             target = urllib.unquote(target)
2914             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2915         d.addCallback(_check_target)
2916         d.addErrback(self.explain_web_error)
2917         return d
2918
2919     def _make_readonly(self, u):
2920         ro_uri = uri.from_string(u).get_readonly()
2921         if ro_uri is None:
2922             return None
2923         return ro_uri.to_string()
2924
2925     def _create_initial_children(self):
2926         contents, n, filecap1 = self.makefile(12)
2927         md1 = {"metakey1": "metavalue1"}
2928         filecap2 = make_mutable_file_uri()
2929         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
2930         filecap3 = node3.get_readonly_uri()
2931         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
2932         dircap = DirectoryNode(node4, None, None).get_uri()
2933         mdmfcap = make_mutable_file_uri(mdmf=True)
2934         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2935         emptydircap = "URI:DIR2-LIT:"
2936         newkids = {u"child-imm":        ["filenode", {"rw_uri": filecap1,
2937                                                       "ro_uri": self._make_readonly(filecap1),
2938                                                       "metadata": md1, }],
2939                    u"child-mutable":    ["filenode", {"rw_uri": filecap2,
2940                                                       "ro_uri": self._make_readonly(filecap2)}],
2941                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
2942                    u"unknownchild-rw":  ["unknown",  {"rw_uri": unknown_rwcap,
2943                                                       "ro_uri": unknown_rocap}],
2944                    u"unknownchild-ro":  ["unknown",  {"ro_uri": unknown_rocap}],
2945                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2946                    u"dirchild":         ["dirnode",  {"rw_uri": dircap,
2947                                                       "ro_uri": self._make_readonly(dircap)}],
2948                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2949                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2950                    u"child-mutable-mdmf": ["filenode", {"rw_uri": mdmfcap,
2951                                                         "ro_uri": self._make_readonly(mdmfcap)}],
2952                    }
2953         return newkids, {'filecap1': filecap1,
2954                          'filecap2': filecap2,
2955                          'filecap3': filecap3,
2956                          'unknown_rwcap': unknown_rwcap,
2957                          'unknown_rocap': unknown_rocap,
2958                          'unknown_immcap': unknown_immcap,
2959                          'dircap': dircap,
2960                          'litdircap': litdircap,
2961                          'emptydircap': emptydircap,
2962                          'mdmfcap': mdmfcap}
2963
2964     def _create_immutable_children(self):
2965         contents, n, filecap1 = self.makefile(12)
2966         md1 = {"metakey1": "metavalue1"}
2967         tnode = create_chk_filenode("immutable directory contents\n"*10,
2968                                     self.get_all_contents())
2969         dnode = DirectoryNode(tnode, None, None)
2970         assert not dnode.is_mutable()
2971         immdircap = dnode.get_uri()
2972         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2973         emptydircap = "URI:DIR2-LIT:"
2974         newkids = {u"child-imm":        ["filenode", {"ro_uri": filecap1,
2975                                                       "metadata": md1, }],
2976                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2977                    u"dirchild-imm":     ["dirnode",  {"ro_uri": immdircap}],
2978                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2979                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2980                    }
2981         return newkids, {'filecap1': filecap1,
2982                          'unknown_immcap': unknown_immcap,
2983                          'immdircap': immdircap,
2984                          'litdircap': litdircap,
2985                          'emptydircap': emptydircap}
2986
2987     def test_POST_mkdir_no_parentdir_initial_children(self):
2988         (newkids, caps) = self._create_initial_children()
2989         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
2990         def _after_mkdir(res):
2991             self.failUnless(res.startswith("URI:DIR"), res)
2992             n = self.s.create_node_from_uri(res)
2993             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2994             d2.addCallback(lambda ign:
2995                            self.failUnlessROChildURIIs(n, u"child-imm",
2996                                                        caps['filecap1']))
2997             d2.addCallback(lambda ign:
2998                            self.failUnlessRWChildURIIs(n, u"child-mutable",
2999                                                        caps['filecap2']))
3000             d2.addCallback(lambda ign:
3001                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
3002                                                        caps['filecap3']))
3003             d2.addCallback(lambda ign:
3004                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
3005                                                        caps['unknown_rwcap']))
3006             d2.addCallback(lambda ign:
3007                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
3008                                                        caps['unknown_rocap']))
3009             d2.addCallback(lambda ign:
3010                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3011                                                        caps['unknown_immcap']))
3012             d2.addCallback(lambda ign:
3013                            self.failUnlessRWChildURIIs(n, u"dirchild",
3014                                                        caps['dircap']))
3015             return d2
3016         d.addCallback(_after_mkdir)
3017         return d
3018
3019     def test_POST_mkdir_no_parentdir_unexpected_children(self):
3020         # the regular /uri?t=mkdir operation is specified to ignore its body.
3021         # Only t=mkdir-with-children pays attention to it.
3022         (newkids, caps) = self._create_initial_children()
3023         d = self.shouldHTTPError("POST_mkdir_no_parentdir_unexpected_children",
3024                                  400, "Bad Request",
3025                                  "t=mkdir does not accept children=, "
3026                                  "try t=mkdir-with-children instead",
3027                                  self.POST2, "/uri?t=mkdir", # without children
3028                                  simplejson.dumps(newkids))
3029         return d
3030
3031     def test_POST_noparent_bad(self):
3032         d = self.shouldHTTPError("POST_noparent_bad",
3033                                  400, "Bad Request",
3034                                  "/uri accepts only PUT, PUT?t=mkdir, "
3035                                  "POST?t=upload, and POST?t=mkdir",
3036                                  self.POST, "/uri?t=bogus")
3037         return d
3038
3039     def test_POST_mkdir_no_parentdir_immutable(self):
3040         (newkids, caps) = self._create_immutable_children()
3041         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
3042         def _after_mkdir(res):
3043             self.failUnless(res.startswith("URI:DIR"), res)
3044             n = self.s.create_node_from_uri(res)
3045             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3046             d2.addCallback(lambda ign:
3047                            self.failUnlessROChildURIIs(n, u"child-imm",
3048                                                           caps['filecap1']))
3049             d2.addCallback(lambda ign:
3050                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3051                                                           caps['unknown_immcap']))
3052             d2.addCallback(lambda ign:
3053                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
3054                                                           caps['immdircap']))
3055             d2.addCallback(lambda ign:
3056                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
3057                                                           caps['litdircap']))
3058             d2.addCallback(lambda ign:
3059                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
3060                                                           caps['emptydircap']))
3061             return d2
3062         d.addCallback(_after_mkdir)
3063         return d
3064
3065     def test_POST_mkdir_no_parentdir_immutable_bad(self):
3066         (newkids, caps) = self._create_initial_children()
3067         d = self.shouldFail2(error.Error,
3068                              "test_POST_mkdir_no_parentdir_immutable_bad",
3069                              "400 Bad Request",
3070                              "needed to be immutable but was not",
3071                              self.POST2,
3072                              "/uri?t=mkdir-immutable",
3073                              simplejson.dumps(newkids))
3074         return d
3075
3076     def test_welcome_page_mkdir_button(self):
3077         # Fetch the welcome page.
3078         d = self.GET("/")
3079         def _after_get_welcome_page(res):
3080             MKDIR_BUTTON_RE = re.compile(
3081                 '<form action="([^"]*)" method="post".*?'
3082                 '<input type="hidden" name="t" value="([^"]*)" />'
3083                 '<input type="hidden" name="([^"]*)" value="([^"]*)" />'
3084                 '<input type="submit" value="Create a directory" />',
3085                 re.I)
3086             mo = MKDIR_BUTTON_RE.search(res)
3087             formaction = mo.group(1)
3088             formt = mo.group(2)
3089             formaname = mo.group(3)
3090             formavalue = mo.group(4)
3091             return (formaction, formt, formaname, formavalue)
3092         d.addCallback(_after_get_welcome_page)
3093         def _after_parse_form(res):
3094             (formaction, formt, formaname, formavalue) = res
3095             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
3096         d.addCallback(_after_parse_form)
3097         d.addBoth(self.shouldRedirect, None, statuscode='303')
3098         return d
3099
3100     def test_POST_mkdir_replace(self): # return value?
3101         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
3102         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3103         d.addCallback(self.failUnlessNodeKeysAre, [])
3104         return d
3105
3106     def test_POST_mkdir_no_replace_queryarg(self): # return value?
3107         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
3108         d.addBoth(self.shouldFail, error.Error,
3109                   "POST_mkdir_no_replace_queryarg",
3110                   "409 Conflict",
3111                   "There was already a child by that name, and you asked me "
3112                   "to not replace it")
3113         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3114         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3115         return d
3116
3117     def test_POST_mkdir_no_replace_field(self): # return value?
3118         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
3119                       replace="false")
3120         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
3121                   "409 Conflict",
3122                   "There was already a child by that name, and you asked me "
3123                   "to not replace it")
3124         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3125         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3126         return d
3127
3128     def test_POST_mkdir_whendone_field(self):
3129         d = self.POST(self.public_url + "/foo",
3130                       t="mkdir", name="newdir", when_done="/THERE")
3131         d.addBoth(self.shouldRedirect, "/THERE")
3132         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3133         d.addCallback(self.failUnlessNodeKeysAre, [])
3134         return d
3135
3136     def test_POST_mkdir_whendone_queryarg(self):
3137         d = self.POST(self.public_url + "/foo?when_done=/THERE",
3138                       t="mkdir", name="newdir")
3139         d.addBoth(self.shouldRedirect, "/THERE")
3140         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3141         d.addCallback(self.failUnlessNodeKeysAre, [])
3142         return d
3143
3144     def test_POST_bad_t(self):
3145         d = self.shouldFail2(error.Error, "POST_bad_t",
3146                              "400 Bad Request",
3147                              "POST to a directory with bad t=BOGUS",
3148                              self.POST, self.public_url + "/foo", t="BOGUS")
3149         return d
3150
3151     def test_POST_set_children(self, command_name="set_children"):
3152         contents9, n9, newuri9 = self.makefile(9)
3153         contents10, n10, newuri10 = self.makefile(10)
3154         contents11, n11, newuri11 = self.makefile(11)
3155
3156         reqbody = """{
3157                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
3158                                                 "size": 0,
3159                                                 "metadata": {
3160                                                   "ctime": 1002777696.7564139,
3161                                                   "mtime": 1002777696.7564139
3162                                                  }
3163                                                } ],
3164                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
3165                                                 "size": 1,
3166                                                 "metadata": {
3167                                                   "ctime": 1002777696.7564139,
3168                                                   "mtime": 1002777696.7564139
3169                                                  }
3170                                                } ],
3171                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
3172                                                 "size": 2,
3173                                                 "metadata": {
3174                                                   "ctime": 1002777696.7564139,
3175                                                   "mtime": 1002777696.7564139
3176                                                  }
3177                                                } ]
3178                     }""" % (newuri9, newuri10, newuri11)
3179
3180         url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
3181
3182         d = client.getPage(url, method="POST", postdata=reqbody)
3183         def _then(res):
3184             self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
3185             self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
3186             self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
3187
3188         d.addCallback(_then)
3189         d.addErrback(self.dump_error)
3190         return d
3191
3192     def test_POST_set_children_with_hyphen(self):
3193         return self.test_POST_set_children(command_name="set-children")
3194
3195     def test_POST_link_uri(self):
3196         contents, n, newuri = self.makefile(8)
3197         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
3198         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
3199         d.addCallback(lambda res:
3200                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3201                                                       contents))
3202         return d
3203
3204     def test_POST_link_uri_replace(self):
3205         contents, n, newuri = self.makefile(8)
3206         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
3207         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
3208         d.addCallback(lambda res:
3209                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3210                                                       contents))
3211         return d
3212
3213     def test_POST_link_uri_unknown_bad(self):
3214         d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
3215         d.addBoth(self.shouldFail, error.Error,
3216                   "POST_link_uri_unknown_bad",
3217                   "400 Bad Request",
3218                   "unknown cap in a write slot")
3219         return d
3220
3221     def test_POST_link_uri_unknown_ro_good(self):
3222         d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
3223         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
3224         return d
3225
3226     def test_POST_link_uri_unknown_imm_good(self):
3227         d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
3228         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
3229         return d
3230
3231     def test_POST_link_uri_no_replace_queryarg(self):
3232         contents, n, newuri = self.makefile(8)
3233         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
3234                       name="bar.txt", uri=newuri)
3235         d.addBoth(self.shouldFail, error.Error,
3236                   "POST_link_uri_no_replace_queryarg",
3237                   "409 Conflict",
3238                   "There was already a child by that name, and you asked me "
3239                   "to not replace it")
3240         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3241         d.addCallback(self.failUnlessIsBarDotTxt)
3242         return d
3243
3244     def test_POST_link_uri_no_replace_field(self):
3245         contents, n, newuri = self.makefile(8)
3246         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
3247                       name="bar.txt", uri=newuri)
3248         d.addBoth(self.shouldFail, error.Error,
3249                   "POST_link_uri_no_replace_field",
3250                   "409 Conflict",
3251                   "There was already a child by that name, and you asked me "
3252                   "to not replace it")
3253         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3254         d.addCallback(self.failUnlessIsBarDotTxt)
3255         return d
3256
3257     def test_POST_delete(self, command_name='delete'):
3258         d = self._foo_node.list()
3259         def _check_before(children):
3260             self.failUnlessIn(u"bar.txt", children)
3261         d.addCallback(_check_before)
3262         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t=command_name, name="bar.txt"))
3263         d.addCallback(lambda res: self._foo_node.list())
3264         def _check_after(children):
3265             self.failIfIn(u"bar.txt", children)
3266         d.addCallback(_check_after)
3267         return d
3268
3269     def test_POST_unlink(self):
3270         return self.test_POST_delete(command_name='unlink')
3271
3272     def test_POST_rename_file(self):
3273         d = self.POST(self.public_url + "/foo", t="rename",
3274                       from_name="bar.txt", to_name='wibble.txt')
3275         d.addCallback(lambda res:
3276                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3277         d.addCallback(lambda res:
3278                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
3279         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
3280         d.addCallback(self.failUnlessIsBarDotTxt)
3281         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
3282         d.addCallback(self.failUnlessIsBarJSON)
3283         return d
3284
3285     def test_POST_rename_file_redundant(self):
3286         d = self.POST(self.public_url + "/foo", t="rename",
3287                       from_name="bar.txt", to_name='bar.txt')
3288         d.addCallback(lambda res:
3289                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3290         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3291         d.addCallback(self.failUnlessIsBarDotTxt)
3292         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3293         d.addCallback(self.failUnlessIsBarJSON)
3294         return d
3295
3296     def test_POST_rename_file_replace(self):
3297         # rename a file and replace a directory with it
3298         d = self.POST(self.public_url + "/foo", t="rename",
3299                       from_name="bar.txt", to_name='empty')
3300         d.addCallback(lambda res:
3301                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3302         d.addCallback(lambda res:
3303                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
3304         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
3305         d.addCallback(self.failUnlessIsBarDotTxt)
3306         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3307         d.addCallback(self.failUnlessIsBarJSON)
3308         return d
3309
3310     def test_POST_rename_file_no_replace_queryarg(self):
3311         # rename a file and replace a directory with it
3312         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
3313                       from_name="bar.txt", to_name='empty')
3314         d.addBoth(self.shouldFail, error.Error,
3315                   "POST_rename_file_no_replace_queryarg",
3316                   "409 Conflict",
3317                   "There was already a child by that name, and you asked me "
3318                   "to not replace it")
3319         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3320         d.addCallback(self.failUnlessIsEmptyJSON)
3321         return d
3322
3323     def test_POST_rename_file_no_replace_field(self):
3324         # rename a file and replace a directory with it
3325         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
3326                       from_name="bar.txt", to_name='empty')
3327         d.addBoth(self.shouldFail, error.Error,
3328                   "POST_rename_file_no_replace_field",
3329                   "409 Conflict",
3330                   "There was already a child by that name, and you asked me "
3331                   "to not replace it")
3332         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3333         d.addCallback(self.failUnlessIsEmptyJSON)
3334         return d
3335
3336     def failUnlessIsEmptyJSON(self, res):
3337         data = simplejson.loads(res)
3338         self.failUnlessEqual(data[0], "dirnode", data)
3339         self.failUnlessReallyEqual(len(data[1]["children"]), 0)
3340
3341     def test_POST_rename_file_slash_fail(self):
3342         d = self.POST(self.public_url + "/foo", t="rename",
3343                       from_name="bar.txt", to_name='kirk/spock.txt')
3344         d.addBoth(self.shouldFail, error.Error,
3345                   "test_POST_rename_file_slash_fail",
3346                   "400 Bad Request",
3347                   "to_name= may not contain a slash",
3348                   )
3349         d.addCallback(lambda res:
3350                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3351         return d
3352
3353     def test_POST_rename_dir(self):
3354         d = self.POST(self.public_url, t="rename",
3355                       from_name="foo", to_name='plunk')
3356         d.addCallback(lambda res:
3357                       self.failIfNodeHasChild(self.public_root, u"foo"))
3358         d.addCallback(lambda res:
3359                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
3360         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
3361         d.addCallback(self.failUnlessIsFooJSON)
3362         return d
3363
3364     def test_POST_move_file(self):
3365         d = self.POST(self.public_url + "/foo", t="move",
3366                       from_name="bar.txt", to_dir="sub")
3367         d.addCallback(lambda res:
3368                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3369         d.addCallback(lambda res:
3370                       self.failUnlessNodeHasChild(self._sub_node, u"bar.txt"))
3371         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3372         d.addCallback(self.failUnlessIsBarDotTxt)
3373         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3374         d.addCallback(self.failUnlessIsBarJSON)
3375         return d
3376
3377     def test_POST_move_file_new_name(self):
3378         d = self.POST(self.public_url + "/foo", t="move",
3379                       from_name="bar.txt", to_name="wibble.txt", to_dir="sub")
3380         d.addCallback(lambda res:
3381                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3382         d.addCallback(lambda res:
3383                       self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3384         d.addCallback(lambda res:
3385                       self.failUnlessNodeHasChild(self._sub_node, u"wibble.txt"))
3386         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt"))
3387         d.addCallback(self.failUnlessIsBarDotTxt)
3388         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt?t=json"))
3389         d.addCallback(self.failUnlessIsBarJSON)
3390         return d
3391
3392     def test_POST_move_file_replace(self):
3393         d = self.POST(self.public_url + "/foo", t="move",
3394                       from_name="bar.txt", to_name="baz.txt", to_dir="sub")
3395         d.addCallback(lambda res:
3396                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3397         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3398         d.addCallback(self.failUnlessIsBarDotTxt)
3399         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt?t=json"))
3400         d.addCallback(self.failUnlessIsBarJSON)
3401         return d
3402
3403     def test_POST_move_file_no_replace(self):
3404         d = self.shouldFail2(error.Error, "POST_move_file_no_replace",
3405                              "409 Conflict",
3406                              "There was already a child by that name, and you asked me to not replace it",
3407                              self.POST, self.public_url + "/foo", t="move",
3408                              replace="false", from_name="bar.txt",
3409                              to_name="baz.txt", to_dir="sub")
3410         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3411         d.addCallback(self.failUnlessIsBarDotTxt)
3412         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3413         d.addCallback(self.failUnlessIsBarJSON)
3414         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3415         d.addCallback(self.failUnlessIsSubBazDotTxt)
3416         return d
3417
3418     def test_POST_move_file_slash_fail(self):
3419         d = self.shouldFail2(error.Error, "test_POST_rename_file_slash_fail",
3420                              "400 Bad Request",
3421                              "to_name= may not contain a slash",
3422                              self.POST, self.public_url + "/foo", t="move",
3423                              from_name="bar.txt",
3424                              to_name="slash/fail.txt", to_dir="sub")
3425         d.addCallback(lambda res:
3426                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3427         d.addCallback(lambda res:
3428                       self.failIfNodeHasChild(self._sub_node, u"slash/fail.txt"))
3429         d.addCallback(lambda ign:
3430                       self.shouldFail2(error.Error,
3431                                        "test_POST_rename_file_slash_fail2",
3432                                        "400 Bad Request",
3433                                        "from_name= may not contain a slash",
3434                                        self.POST, self.public_url + "/foo",
3435                                        t="move",
3436                                        from_name="nope/bar.txt",
3437                                        to_name="fail.txt", to_dir="sub"))
3438         return d
3439
3440     def test_POST_move_file_no_target(self):
3441         d = self.shouldFail2(error.Error, "POST_move_file_no_target",
3442                              "400 Bad Request",
3443                              "move requires from_name and to_dir",
3444                              self.POST, self.public_url + "/foo", t="move",
3445                              from_name="bar.txt", to_name="baz.txt")
3446         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3447         d.addCallback(self.failUnlessIsBarDotTxt)
3448         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3449         d.addCallback(self.failUnlessIsBarJSON)
3450         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3451         d.addCallback(self.failUnlessIsBazDotTxt)
3452         return d
3453
3454     def test_POST_move_file_bad_target_type(self):
3455         d = self.shouldFail2(error.Error, "test_POST_move_file_bad_target_type",
3456                              "400 Bad Request", "invalid target_type parameter",
3457                              self.POST,
3458                              self.public_url + "/foo", t="move",
3459                              target_type="*D", from_name="bar.txt",
3460                              to_dir="sub")
3461         return d
3462
3463     def test_POST_move_file_multi_level(self):
3464         d = self.POST(self.public_url + "/foo/sub/level2?t=mkdir", "")
3465         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t="move",
3466                       from_name="bar.txt", to_dir="sub/level2"))
3467         d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3468         d.addCallback(lambda res: self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3469         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt"))
3470         d.addCallback(self.failUnlessIsBarDotTxt)
3471         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt?t=json"))
3472         d.addCallback(self.failUnlessIsBarJSON)
3473         return d
3474
3475     def test_POST_move_file_to_uri(self):
3476         d = self.POST(self.public_url + "/foo", t="move", target_type="uri",
3477                       from_name="bar.txt", to_dir=self._sub_uri)
3478         d.addCallback(lambda res:
3479                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3480         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3481         d.addCallback(self.failUnlessIsBarDotTxt)
3482         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3483         d.addCallback(self.failUnlessIsBarJSON)
3484         return d
3485
3486     def test_POST_move_file_to_nonexist_dir(self):
3487         d = self.shouldFail2(error.Error, "POST_move_file_to_nonexist_dir",
3488                             "404 Not Found", "No such child: nopechucktesta",
3489                             self.POST, self.public_url + "/foo", t="move",
3490                             from_name="bar.txt", to_dir="nopechucktesta")
3491         return d
3492
3493     def test_POST_move_file_into_file(self):
3494         d = self.shouldFail2(error.Error, "POST_move_file_into_file",
3495                              "400 Bad Request", "to_dir is not a directory",
3496                              self.POST, self.public_url + "/foo", t="move",
3497                              from_name="bar.txt", to_dir="baz.txt")
3498         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3499         d.addCallback(self.failUnlessIsBazDotTxt)
3500         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3501         d.addCallback(self.failUnlessIsBarDotTxt)
3502         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3503         d.addCallback(self.failUnlessIsBarJSON)
3504         return d
3505
3506     def test_POST_move_file_to_bad_uri(self):
3507         d =  self.shouldFail2(error.Error, "POST_move_file_to_bad_uri",
3508                               "400 Bad Request", "to_dir is not a directory",
3509                               self.POST, self.public_url + "/foo", t="move",
3510                               from_name="bar.txt", target_type="uri",
3511                               to_dir="URI:DIR2:mn5jlyjnrjeuydyswlzyui72i:rmneifcj6k6sycjljjhj3f6majsq2zqffydnnul5hfa4j577arma")
3512         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3513         d.addCallback(self.failUnlessIsBarDotTxt)
3514         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3515         d.addCallback(self.failUnlessIsBarJSON)
3516         return d
3517
3518     def test_POST_move_dir(self):
3519         d = self.POST(self.public_url + "/foo", t="move",
3520                       from_name="bar.txt", to_dir="empty")
3521         d.addCallback(lambda res: self.POST(self.public_url + "/foo",
3522                       t="move", from_name="empty", to_dir="sub"))
3523         d.addCallback(lambda res:
3524                       self.failIfNodeHasChild(self._foo_node, u"empty"))
3525         d.addCallback(lambda res:
3526                       self.failUnlessNodeHasChild(self._sub_node, u"empty"))
3527         d.addCallback(lambda res:
3528                       self._sub_node.get_child_at_path(u"empty"))
3529         d.addCallback(lambda node:
3530                       self.failUnlessNodeHasChild(node, u"bar.txt"))
3531         d.addCallback(lambda res:
3532                       self.GET(self.public_url + "/foo/sub/empty/bar.txt"))
3533         d.addCallback(self.failUnlessIsBarDotTxt)
3534         return d
3535
3536     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
3537         """ If target is not None then the redirection has to go to target.  If
3538         statuscode is not None then the redirection has to be accomplished with
3539         that HTTP status code."""
3540         if not isinstance(res, failure.Failure):
3541             to_where = (target is None) and "somewhere" or ("to " + target)
3542             self.fail("%s: we were expecting to get redirected %s, not get an"
3543                       " actual page: %s" % (which, to_where, res))
3544         res.trap(error.PageRedirect)
3545         if statuscode is not None:
3546             self.failUnlessReallyEqual(res.value.status, statuscode,
3547                                        "%s: not a redirect" % which)
3548         if target is not None:
3549             # the PageRedirect does not seem to capture the uri= query arg
3550             # properly, so we can't check for it.
3551             realtarget = self.webish_url + target
3552             self.failUnlessReallyEqual(res.value.location, realtarget,
3553                                        "%s: wrong target" % which)
3554         return res.value.location
3555
3556     def test_GET_URI_form(self):
3557         base = "/uri?uri=%s" % self._bar_txt_uri
3558         # this is supposed to give us a redirect to /uri/$URI, plus arguments
3559         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
3560         d = self.GET(base)
3561         d.addBoth(self.shouldRedirect, targetbase)
3562         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
3563         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
3564         d.addCallback(lambda res: self.GET(base+"&t=json"))
3565         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
3566         d.addCallback(self.log, "about to get file by uri")
3567         d.addCallback(lambda res: self.GET(base, followRedirect=True))
3568         d.addCallback(self.failUnlessIsBarDotTxt)
3569         d.addCallback(self.log, "got file by uri, about to get dir by uri")
3570         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
3571                                            followRedirect=True))
3572         d.addCallback(self.failUnlessIsFooJSON)
3573         d.addCallback(self.log, "got dir by uri")
3574
3575         return d
3576
3577     def test_GET_URI_form_bad(self):
3578         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
3579                              "400 Bad Request", "GET /uri requires uri=",
3580                              self.GET, "/uri")
3581         return d
3582
3583     def test_GET_rename_form(self):
3584         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
3585                      followRedirect=True)
3586         def _check(res):
3587             self.failUnlessIn('name="when_done" value="."', res)
3588             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
3589             self.failUnlessIn(FAVICON_MARKUP, res)
3590         d.addCallback(_check)
3591         return d
3592
3593     def log(self, res, msg):
3594         #print "MSG: %s  RES: %s" % (msg, res)
3595         log.msg(msg)
3596         return res
3597
3598     def test_GET_URI_URL(self):
3599         base = "/uri/%s" % self._bar_txt_uri
3600         d = self.GET(base)
3601         d.addCallback(self.failUnlessIsBarDotTxt)
3602         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
3603         d.addCallback(self.failUnlessIsBarDotTxt)
3604         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
3605         d.addCallback(self.failUnlessIsBarDotTxt)
3606         return d
3607
3608     def test_GET_URI_URL_dir(self):
3609         base = "/uri/%s?t=json" % self._foo_uri
3610         d = self.GET(base)
3611         d.addCallback(self.failUnlessIsFooJSON)
3612         return d
3613
3614     def test_GET_URI_URL_missing(self):
3615         base = "/uri/%s" % self._bad_file_uri
3616         d = self.shouldHTTPError("test_GET_URI_URL_missing",
3617                                  http.GONE, None, "NotEnoughSharesError",
3618                                  self.GET, base)
3619         # TODO: how can we exercise both sides of WebDownloadTarget.fail
3620         # here? we must arrange for a download to fail after target.open()
3621         # has been called, and then inspect the response to see that it is
3622         # shorter than we expected.
3623         return d
3624
3625     def test_PUT_DIRURL_uri(self):
3626         d = self.s.create_dirnode()
3627         def _made_dir(dn):
3628             new_uri = dn.get_uri()
3629             # replace /foo with a new (empty) directory
3630             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
3631             d.addCallback(lambda res:
3632                           self.failUnlessReallyEqual(res.strip(), new_uri))
3633             d.addCallback(lambda res:
3634                           self.failUnlessRWChildURIIs(self.public_root,
3635                                                       u"foo",
3636                                                       new_uri))
3637             return d
3638         d.addCallback(_made_dir)
3639         return d
3640
3641     def test_PUT_DIRURL_uri_noreplace(self):
3642         d = self.s.create_dirnode()
3643         def _made_dir(dn):
3644             new_uri = dn.get_uri()
3645             # replace /foo with a new (empty) directory, but ask that
3646             # replace=false, so it should fail
3647             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
3648                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
3649                                  self.PUT,
3650                                  self.public_url + "/foo?t=uri&replace=false",
3651                                  new_uri)
3652             d.addCallback(lambda res:
3653                           self.failUnlessRWChildURIIs(self.public_root,
3654                                                       u"foo",
3655                                                       self._foo_uri))
3656             return d
3657         d.addCallback(_made_dir)
3658         return d
3659
3660     def test_PUT_DIRURL_bad_t(self):
3661         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
3662                              "400 Bad Request", "PUT to a directory",
3663                              self.PUT, self.public_url + "/foo?t=BOGUS", "")
3664         d.addCallback(lambda res:
3665                       self.failUnlessRWChildURIIs(self.public_root,
3666                                                   u"foo",
3667                                                   self._foo_uri))
3668         return d
3669
3670     def test_PUT_NEWFILEURL_uri(self):
3671         contents, n, new_uri = self.makefile(8)
3672         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
3673         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3674         d.addCallback(lambda res:
3675                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3676                                                       contents))
3677         return d
3678
3679     def test_PUT_NEWFILEURL_mdmf(self):
3680         new_contents = self.NEWFILE_CONTENTS * 300000
3681         d = self.PUT(self.public_url + \
3682                      "/foo/mdmf.txt?format=mdmf",
3683                      new_contents)
3684         d.addCallback(lambda ignored:
3685             self.GET(self.public_url + "/foo/mdmf.txt?t=json"))
3686         def _got_json(json):
3687             data = simplejson.loads(json)
3688             data = data[1]
3689             self.failUnlessIn("format", data)
3690             self.failUnlessEqual(data["format"], "MDMF")
3691             self.failUnless(data['rw_uri'].startswith("URI:MDMF"))
3692             self.failUnless(data['ro_uri'].startswith("URI:MDMF"))
3693         d.addCallback(_got_json)
3694         return d
3695
3696     def test_PUT_NEWFILEURL_sdmf(self):
3697         new_contents = self.NEWFILE_CONTENTS * 300000
3698         d = self.PUT(self.public_url + \
3699                      "/foo/sdmf.txt?format=sdmf",
3700                      new_contents)
3701         d.addCallback(lambda ignored:
3702             self.GET(self.public_url + "/foo/sdmf.txt?t=json"))
3703         def _got_json(json):
3704             data = simplejson.loads(json)
3705             data = data[1]
3706             self.failUnlessIn("format", data)
3707             self.failUnlessEqual(data["format"], "SDMF")
3708         d.addCallback(_got_json)
3709         return d
3710
3711     def test_PUT_NEWFILEURL_bad_format(self):
3712         new_contents = self.NEWFILE_CONTENTS * 300000
3713         return self.shouldHTTPError("PUT_NEWFILEURL_bad_format",
3714                                     400, "Bad Request", "Unknown format: foo",
3715                                     self.PUT, self.public_url + \
3716                                     "/foo/foo.txt?format=foo",
3717                                     new_contents)
3718
3719     def test_PUT_NEWFILEURL_uri_replace(self):
3720         contents, n, new_uri = self.makefile(8)
3721         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
3722         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3723         d.addCallback(lambda res:
3724                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3725                                                       contents))
3726         return d
3727
3728     def test_PUT_NEWFILEURL_uri_no_replace(self):
3729         contents, n, new_uri = self.makefile(8)
3730         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
3731         d.addBoth(self.shouldFail, error.Error,
3732                   "PUT_NEWFILEURL_uri_no_replace",
3733                   "409 Conflict",
3734                   "There was already a child by that name, and you asked me "
3735                   "to not replace it")
3736         return d
3737
3738     def test_PUT_NEWFILEURL_uri_unknown_bad(self):
3739         d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
3740         d.addBoth(self.shouldFail, error.Error,
3741                   "POST_put_uri_unknown_bad",
3742                   "400 Bad Request",
3743                   "unknown cap in a write slot")
3744         return d
3745
3746     def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
3747         d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
3748         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3749                       u"put-future-ro.txt")
3750         return d
3751
3752     def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
3753         d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
3754         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3755                       u"put-future-imm.txt")
3756         return d
3757
3758     def test_PUT_NEWFILE_URI(self):
3759         file_contents = "New file contents here\n"
3760         d = self.PUT("/uri", file_contents)
3761         def _check(uri):
3762             assert isinstance(uri, str), uri
3763             self.failUnlessIn(uri, self.get_all_contents())
3764             self.failUnlessReallyEqual(self.get_all_contents()[uri],
3765                                        file_contents)
3766             return self.GET("/uri/%s" % uri)
3767         d.addCallback(_check)
3768         def _check2(res):
3769             self.failUnlessReallyEqual(res, file_contents)
3770         d.addCallback(_check2)
3771         return d
3772
3773     def test_PUT_NEWFILE_URI_not_mutable(self):
3774         file_contents = "New file contents here\n"
3775         d = self.PUT("/uri?mutable=false", file_contents)
3776         def _check(uri):
3777             assert isinstance(uri, str), uri
3778             self.failUnlessIn(uri, self.get_all_contents())
3779             self.failUnlessReallyEqual(self.get_all_contents()[uri],
3780                                        file_contents)
3781             return self.GET("/uri/%s" % uri)
3782         d.addCallback(_check)
3783         def _check2(res):
3784             self.failUnlessReallyEqual(res, file_contents)
3785         d.addCallback(_check2)
3786         return d
3787
3788     def test_PUT_NEWFILE_URI_only_PUT(self):
3789         d = self.PUT("/uri?t=bogus", "")
3790         d.addBoth(self.shouldFail, error.Error,
3791                   "PUT_NEWFILE_URI_only_PUT",
3792                   "400 Bad Request",
3793                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
3794         return d
3795
3796     def test_PUT_NEWFILE_URI_mutable(self):
3797         file_contents = "New file contents here\n"
3798         d = self.PUT("/uri?mutable=true", file_contents)
3799         def _check1(filecap):
3800             filecap = filecap.strip()
3801             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
3802             self.filecap = filecap
3803             u = uri.WriteableSSKFileURI.init_from_string(filecap)
3804             self.failUnlessIn(u.get_storage_index(), self.get_all_contents())
3805             n = self.s.create_node_from_uri(filecap)
3806             return n.download_best_version()
3807         d.addCallback(_check1)
3808         def _check2(data):
3809             self.failUnlessReallyEqual(data, file_contents)
3810             return self.GET("/uri/%s" % urllib.quote(self.filecap))
3811         d.addCallback(_check2)
3812         def _check3(res):
3813             self.failUnlessReallyEqual(res, file_contents)
3814         d.addCallback(_check3)
3815         return d
3816
3817     def test_PUT_mkdir(self):
3818         d = self.PUT("/uri?t=mkdir", "")
3819         def _check(uri):
3820             n = self.s.create_node_from_uri(uri.strip())
3821             d2 = self.failUnlessNodeKeysAre(n, [])
3822             d2.addCallback(lambda res:
3823                            self.GET("/uri/%s?t=json" % uri))
3824             return d2
3825         d.addCallback(_check)
3826         d.addCallback(self.failUnlessIsEmptyJSON)
3827         return d
3828
3829     def test_PUT_mkdir_mdmf(self):
3830         d = self.PUT("/uri?t=mkdir&format=mdmf", "")
3831         def _got(res):
3832             u = uri.from_string(res)
3833             # Check that this is an MDMF writecap
3834             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
3835         d.addCallback(_got)
3836         return d
3837
3838     def test_PUT_mkdir_sdmf(self):
3839         d = self.PUT("/uri?t=mkdir&format=sdmf", "")
3840         def _got(res):
3841             u = uri.from_string(res)
3842             self.failUnlessIsInstance(u, uri.DirectoryURI)
3843         d.addCallback(_got)
3844         return d
3845
3846     def test_PUT_mkdir_bad_format(self):
3847         return self.shouldHTTPError("PUT_mkdir_bad_format",
3848                                     400, "Bad Request", "Unknown format: foo",
3849                                     self.PUT, "/uri?t=mkdir&format=foo",
3850                                     "")
3851
3852     def test_POST_check(self):
3853         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
3854         def _done(res):
3855             # this returns a string form of the results, which are probably
3856             # None since we're using fake filenodes.
3857             # TODO: verify that the check actually happened, by changing
3858             # FakeCHKFileNode to count how many times .check() has been
3859             # called.
3860             pass
3861         d.addCallback(_done)
3862         return d
3863
3864
3865     def test_PUT_update_at_offset(self):
3866         file_contents = "test file" * 100000 # about 900 KiB
3867         d = self.PUT("/uri?mutable=true", file_contents)
3868         def _then(filecap):
3869             self.filecap = filecap
3870             new_data = file_contents[:100]
3871             new = "replaced and so on"
3872             new_data += new
3873             new_data += file_contents[len(new_data):]
3874             assert len(new_data) == len(file_contents)
3875             self.new_data = new_data
3876         d.addCallback(_then)
3877         d.addCallback(lambda ignored:
3878             self.PUT("/uri/%s?replace=True&offset=100" % self.filecap,
3879                      "replaced and so on"))
3880         def _get_data(filecap):
3881             n = self.s.create_node_from_uri(filecap)
3882             return n.download_best_version()
3883         d.addCallback(_get_data)
3884         d.addCallback(lambda results:
3885             self.failUnlessEqual(results, self.new_data))
3886         # Now try appending things to the file
3887         d.addCallback(lambda ignored:
3888             self.PUT("/uri/%s?offset=%d" % (self.filecap, len(self.new_data)),
3889                      "puppies" * 100))
3890         d.addCallback(_get_data)
3891         d.addCallback(lambda results:
3892             self.failUnlessEqual(results, self.new_data + ("puppies" * 100)))
3893         # and try replacing the beginning of the file
3894         d.addCallback(lambda ignored:
3895             self.PUT("/uri/%s?offset=0" % self.filecap, "begin"))
3896         d.addCallback(_get_data)
3897         d.addCallback(lambda results:
3898             self.failUnlessEqual(results, "begin"+self.new_data[len("begin"):]+("puppies"*100)))
3899         return d
3900
3901     def test_PUT_update_at_invalid_offset(self):
3902         file_contents = "test file" * 100000 # about 900 KiB
3903         d = self.PUT("/uri?mutable=true", file_contents)
3904         def _then(filecap):
3905             self.filecap = filecap
3906         d.addCallback(_then)
3907         # Negative offsets should cause an error.
3908         d.addCallback(lambda ignored:
3909             self.shouldHTTPError("PUT_update_at_invalid_offset",
3910                                  400, "Bad Request",
3911                                  "Invalid offset",
3912                                  self.PUT,
3913                                  "/uri/%s?offset=-1" % self.filecap,
3914                                  "foo"))
3915         return d
3916
3917     def test_PUT_update_at_offset_immutable(self):
3918         file_contents = "Test file" * 100000
3919         d = self.PUT("/uri", file_contents)
3920         def _then(filecap):
3921             self.filecap = filecap
3922         d.addCallback(_then)
3923         d.addCallback(lambda ignored:
3924             self.shouldHTTPError("PUT_update_at_offset_immutable",
3925                                  400, "Bad Request",
3926                                  "immutable",
3927                                  self.PUT,
3928                                  "/uri/%s?offset=50" % self.filecap,
3929                                  "foo"))
3930         return d
3931
3932
3933     def test_bad_method(self):
3934         url = self.webish_url + self.public_url + "/foo/bar.txt"
3935         d = self.shouldHTTPError("bad_method",
3936                                  501, "Not Implemented",
3937                                  "I don't know how to treat a BOGUS request.",
3938                                  client.getPage, url, method="BOGUS")
3939         return d
3940
3941     def test_short_url(self):
3942         url = self.webish_url + "/uri"
3943         d = self.shouldHTTPError("short_url", 501, "Not Implemented",
3944                                  "I don't know how to treat a DELETE request.",
3945                                  client.getPage, url, method="DELETE")
3946         return d
3947
3948     def test_ophandle_bad(self):
3949         url = self.webish_url + "/operations/bogus?t=status"
3950         d = self.shouldHTTPError("ophandle_bad", 404, "404 Not Found",
3951                                  "unknown/expired handle 'bogus'",
3952                                  client.getPage, url)
3953         return d
3954
3955     def test_ophandle_cancel(self):
3956         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
3957                       followRedirect=True)
3958         d.addCallback(lambda ignored:
3959                       self.GET("/operations/128?t=status&output=JSON"))
3960         def _check1(res):
3961             data = simplejson.loads(res)
3962             self.failUnless("finished" in data, res)
3963             monitor = self.ws.root.child_operations.handles["128"][0]
3964             d = self.POST("/operations/128?t=cancel&output=JSON")
3965             def _check2(res):
3966                 data = simplejson.loads(res)
3967                 self.failUnless("finished" in data, res)
3968                 # t=cancel causes the handle to be forgotten
3969                 self.failUnless(monitor.is_cancelled())
3970             d.addCallback(_check2)
3971             return d
3972         d.addCallback(_check1)
3973         d.addCallback(lambda ignored:
3974                       self.shouldHTTPError("ophandle_cancel",
3975                                            404, "404 Not Found",
3976                                            "unknown/expired handle '128'",
3977                                            self.GET,
3978                                            "/operations/128?t=status&output=JSON"))
3979         return d
3980
3981     def test_ophandle_retainfor(self):
3982         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
3983                       followRedirect=True)
3984         d.addCallback(lambda ignored:
3985                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
3986         def _check1(res):
3987             data = simplejson.loads(res)
3988             self.failUnless("finished" in data, res)
3989         d.addCallback(_check1)
3990         # the retain-for=0 will cause the handle to be expired very soon
3991         d.addCallback(lambda ign:
3992             self.clock.advance(2.0))
3993         d.addCallback(lambda ignored:
3994                       self.shouldHTTPError("ophandle_retainfor",
3995                                            404, "404 Not Found",
3996                                            "unknown/expired handle '129'",
3997                                            self.GET,
3998                                            "/operations/129?t=status&output=JSON"))
3999         return d
4000
4001     def test_ophandle_release_after_complete(self):
4002         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
4003                       followRedirect=True)
4004         d.addCallback(self.wait_for_operation, "130")
4005         d.addCallback(lambda ignored:
4006                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
4007         # the release-after-complete=true will cause the handle to be expired
4008         d.addCallback(lambda ignored:
4009                       self.shouldHTTPError("ophandle_release_after_complete",
4010                                            404, "404 Not Found",
4011                                            "unknown/expired handle '130'",
4012                                            self.GET,
4013                                            "/operations/130?t=status&output=JSON"))
4014         return d
4015
4016     def test_uncollected_ophandle_expiration(self):
4017         # uncollected ophandles should expire after 4 days
4018         def _make_uncollected_ophandle(ophandle):
4019             d = self.POST(self.public_url +
4020                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
4021                           followRedirect=False)
4022             # When we start the operation, the webapi server will want
4023             # to redirect us to the page for the ophandle, so we get
4024             # confirmation that the operation has started. If the
4025             # manifest operation has finished by the time we get there,
4026             # following that redirect (by setting followRedirect=True
4027             # above) has the side effect of collecting the ophandle that
4028             # we've just created, which means that we can't use the
4029             # ophandle to test the uncollected timeout anymore. So,
4030             # instead, catch the 302 here and don't follow it.
4031             d.addBoth(self.should302, "uncollected_ophandle_creation")
4032             return d
4033         # Create an ophandle, don't collect it, then advance the clock by
4034         # 4 days - 1 second and make sure that the ophandle is still there.
4035         d = _make_uncollected_ophandle(131)
4036         d.addCallback(lambda ign:
4037             self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
4038         d.addCallback(lambda ign:
4039             self.GET("/operations/131?t=status&output=JSON"))
4040         def _check1(res):
4041             data = simplejson.loads(res)
4042             self.failUnless("finished" in data, res)
4043         d.addCallback(_check1)
4044         # Create an ophandle, don't collect it, then try to collect it
4045         # after 4 days. It should be gone.
4046         d.addCallback(lambda ign:
4047             _make_uncollected_ophandle(132))
4048         d.addCallback(lambda ign:
4049             self.clock.advance(96*60*60))
4050         d.addCallback(lambda ign:
4051             self.shouldHTTPError("uncollected_ophandle_expired_after_100_hours",
4052                                  404, "404 Not Found",
4053                                  "unknown/expired handle '132'",
4054                                  self.GET,
4055                                  "/operations/132?t=status&output=JSON"))
4056         return d
4057
4058     def test_collected_ophandle_expiration(self):
4059         # collected ophandles should expire after 1 day
4060         def _make_collected_ophandle(ophandle):
4061             d = self.POST(self.public_url +
4062                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
4063                           followRedirect=True)
4064             # By following the initial redirect, we collect the ophandle
4065             # we've just created.
4066             return d
4067         # Create a collected ophandle, then collect it after 23 hours
4068         # and 59 seconds to make sure that it is still there.
4069         d = _make_collected_ophandle(133)
4070         d.addCallback(lambda ign:
4071             self.clock.advance((24*60*60) - 1))
4072         d.addCallback(lambda ign:
4073             self.GET("/operations/133?t=status&output=JSON"))
4074         def _check1(res):
4075             data = simplejson.loads(res)
4076             self.failUnless("finished" in data, res)
4077         d.addCallback(_check1)
4078         # Create another uncollected ophandle, then try to collect it
4079         # after 24 hours to make sure that it is gone.
4080         d.addCallback(lambda ign:
4081             _make_collected_ophandle(134))
4082         d.addCallback(lambda ign:
4083             self.clock.advance(24*60*60))
4084         d.addCallback(lambda ign:
4085             self.shouldHTTPError("collected_ophandle_expired_after_1_day",
4086                                  404, "404 Not Found",
4087                                  "unknown/expired handle '134'",
4088                                  self.GET,
4089                                  "/operations/134?t=status&output=JSON"))
4090         return d
4091
4092     def test_incident(self):
4093         d = self.POST("/report_incident", details="eek")
4094         def _done(res):
4095             self.failIfIn("<html>", res)
4096             self.failUnlessIn("Thank you for your report!", res)
4097         d.addCallback(_done)
4098         return d
4099
4100     def test_static(self):
4101         webdir = os.path.join(self.staticdir, "subdir")
4102         fileutil.make_dirs(webdir)
4103         f = open(os.path.join(webdir, "hello.txt"), "wb")
4104         f.write("hello")
4105         f.close()
4106
4107         d = self.GET("/static/subdir/hello.txt")
4108         def _check(res):
4109             self.failUnlessReallyEqual(res, "hello")
4110         d.addCallback(_check)
4111         return d
4112
4113
4114 class IntroducerWeb(unittest.TestCase):
4115     def setUp(self):
4116         self.node = None
4117
4118     def tearDown(self):
4119         d = defer.succeed(None)
4120         if self.node:
4121             d.addCallback(lambda ign: self.node.stopService())
4122         d.addCallback(flushEventualQueue)
4123         return d
4124
4125     def test_welcome(self):
4126         basedir = "web.IntroducerWeb.test_welcome"
4127         os.mkdir(basedir)
4128         fileutil.write(os.path.join(basedir, "tahoe.cfg"), "[node]\nweb.port = tcp:0\n")
4129         self.node = IntroducerNode(basedir)
4130         self.ws = self.node.getServiceNamed("webish")
4131
4132         d = fireEventually(None)
4133         d.addCallback(lambda ign: self.node.startService())
4134         d.addCallback(lambda ign: self.node.when_tub_ready())
4135
4136         d.addCallback(lambda ign: self.GET("/"))
4137         def _check(res):
4138             self.failUnlessIn('Welcome to the Tahoe-LAFS Introducer', res)
4139             self.failUnlessIn(FAVICON_MARKUP, res)
4140         d.addCallback(_check)
4141         return d
4142
4143     def GET(self, urlpath, followRedirect=False, return_response=False,
4144             **kwargs):
4145         # if return_response=True, this fires with (data, statuscode,
4146         # respheaders) instead of just data.
4147         assert not isinstance(urlpath, unicode)
4148         url = self.ws.getURL().rstrip('/') + urlpath
4149         factory = HTTPClientGETFactory(url, method="GET",
4150                                        followRedirect=followRedirect, **kwargs)
4151         reactor.connectTCP("localhost", self.ws.getPortnum(), factory)
4152         d = factory.deferred
4153         def _got_data(data):
4154             return (data, factory.status, factory.response_headers)
4155         if return_response:
4156             d.addCallback(_got_data)
4157         return factory.deferred
4158
4159
4160 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4161     def test_load_file(self):
4162         # This will raise an exception unless a well-formed XML file is found under that name.
4163         common.getxmlfile('directory.xhtml').load()
4164
4165     def test_parse_replace_arg(self):
4166         self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
4167         self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
4168         self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
4169                                    "only-files")
4170         self.shouldFail(AssertionError, "test_parse_replace_arg", "",
4171                         common.parse_replace_arg, "only_fles")
4172
4173     def test_abbreviate_time(self):
4174         self.failUnlessReallyEqual(common.abbreviate_time(None), "")
4175         self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
4176         self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
4177         self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
4178         self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
4179         self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
4180
4181     def test_compute_rate(self):
4182         self.failUnlessReallyEqual(common.compute_rate(None, None), None)
4183         self.failUnlessReallyEqual(common.compute_rate(None, 1), None)
4184         self.failUnlessReallyEqual(common.compute_rate(250000, None), None)
4185         self.failUnlessReallyEqual(common.compute_rate(250000, 0), None)
4186         self.failUnlessReallyEqual(common.compute_rate(250000, 10), 25000.0)
4187         self.failUnlessReallyEqual(common.compute_rate(0, 10), 0.0)
4188         self.shouldFail(AssertionError, "test_compute_rate", "",
4189                         common.compute_rate, -100, 10)
4190         self.shouldFail(AssertionError, "test_compute_rate", "",
4191                         common.compute_rate, 100, -10)
4192
4193         # Sanity check
4194         rate = common.compute_rate(10*1000*1000, 1)
4195         self.failUnlessReallyEqual(common.abbreviate_rate(rate), "10.00MBps")
4196
4197     def test_abbreviate_rate(self):
4198         self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
4199         self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
4200         self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
4201         self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
4202
4203     def test_abbreviate_size(self):
4204         self.failUnlessReallyEqual(common.abbreviate_size(None), "")
4205         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
4206         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
4207         self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
4208         self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
4209
4210     def test_plural(self):
4211         def convert(s):
4212             return "%d second%s" % (s, status.plural(s))
4213         self.failUnlessReallyEqual(convert(0), "0 seconds")
4214         self.failUnlessReallyEqual(convert(1), "1 second")
4215         self.failUnlessReallyEqual(convert(2), "2 seconds")
4216         def convert2(s):
4217             return "has share%s: %s" % (status.plural(s), ",".join(s))
4218         self.failUnlessReallyEqual(convert2([]), "has shares: ")
4219         self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
4220         self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
4221
4222
4223 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4224
4225     def CHECK(self, ign, which, args, clientnum=0):
4226         fileurl = self.fileurls[which]
4227         url = fileurl + "?" + args
4228         return self.GET(url, method="POST", clientnum=clientnum)
4229
4230     def test_filecheck(self):
4231         self.basedir = "web/Grid/filecheck"
4232         self.set_up_grid()
4233         c0 = self.g.clients[0]
4234         self.uris = {}
4235         DATA = "data" * 100
4236         d = c0.upload(upload.Data(DATA, convergence=""))
4237         def _stash_uri(ur, which):
4238             self.uris[which] = ur.get_uri()
4239         d.addCallback(_stash_uri, "good")
4240         d.addCallback(lambda ign:
4241                       c0.upload(upload.Data(DATA+"1", convergence="")))
4242         d.addCallback(_stash_uri, "sick")
4243         d.addCallback(lambda ign:
4244                       c0.upload(upload.Data(DATA+"2", convergence="")))
4245         d.addCallback(_stash_uri, "dead")
4246         def _stash_mutable_uri(n, which):
4247             self.uris[which] = n.get_uri()
4248             assert isinstance(self.uris[which], str)
4249         d.addCallback(lambda ign:
4250             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4251         d.addCallback(_stash_mutable_uri, "corrupt")
4252         d.addCallback(lambda ign:
4253                       c0.upload(upload.Data("literal", convergence="")))
4254         d.addCallback(_stash_uri, "small")
4255         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
4256         d.addCallback(_stash_mutable_uri, "smalldir")
4257
4258         def _compute_fileurls(ignored):
4259             self.fileurls = {}
4260             for which in self.uris:
4261                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4262         d.addCallback(_compute_fileurls)
4263
4264         def _clobber_shares(ignored):
4265             good_shares = self.find_uri_shares(self.uris["good"])
4266             self.failUnlessReallyEqual(len(good_shares), 10)
4267             sick_shares = self.find_uri_shares(self.uris["sick"])
4268             os.unlink(sick_shares[0][2])
4269             dead_shares = self.find_uri_shares(self.uris["dead"])
4270             for i in range(1, 10):
4271                 os.unlink(dead_shares[i][2])
4272             c_shares = self.find_uri_shares(self.uris["corrupt"])
4273             cso = CorruptShareOptions()
4274             cso.stdout = StringIO()
4275             cso.parseOptions([c_shares[0][2]])
4276             corrupt_share(cso)
4277         d.addCallback(_clobber_shares)
4278
4279         d.addCallback(self.CHECK, "good", "t=check")
4280         def _got_html_good(res):
4281             self.failUnlessIn("Healthy", res)
4282             self.failIfIn("Not Healthy", res)
4283             self.failUnlessIn(FAVICON_MARKUP, res)
4284         d.addCallback(_got_html_good)
4285         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
4286         def _got_html_good_return_to(res):
4287             self.failUnlessIn("Healthy", res)
4288             self.failIfIn("Not Healthy", res)
4289             self.failUnlessIn('<a href="somewhere">Return to file', res)
4290         d.addCallback(_got_html_good_return_to)
4291         d.addCallback(self.CHECK, "good", "t=check&output=json")
4292         def _got_json_good(res):
4293             r = simplejson.loads(res)
4294             self.failUnlessEqual(r["summary"], "Healthy")
4295             self.failUnless(r["results"]["healthy"])
4296             self.failIf(r["results"]["needs-rebalancing"])
4297             self.failUnless(r["results"]["recoverable"])
4298         d.addCallback(_got_json_good)
4299
4300         d.addCallback(self.CHECK, "small", "t=check")
4301         def _got_html_small(res):
4302             self.failUnlessIn("Literal files are always healthy", res)
4303             self.failIfIn("Not Healthy", res)
4304         d.addCallback(_got_html_small)
4305         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
4306         def _got_html_small_return_to(res):
4307             self.failUnlessIn("Literal files are always healthy", res)
4308             self.failIfIn("Not Healthy", res)
4309             self.failUnlessIn('<a href="somewhere">Return to file', res)
4310         d.addCallback(_got_html_small_return_to)
4311         d.addCallback(self.CHECK, "small", "t=check&output=json")
4312         def _got_json_small(res):
4313             r = simplejson.loads(res)
4314             self.failUnlessEqual(r["storage-index"], "")
4315             self.failUnless(r["results"]["healthy"])
4316         d.addCallback(_got_json_small)
4317
4318         d.addCallback(self.CHECK, "smalldir", "t=check")
4319         def _got_html_smalldir(res):
4320             self.failUnlessIn("Literal files are always healthy", res)
4321             self.failIfIn("Not Healthy", res)
4322         d.addCallback(_got_html_smalldir)
4323         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
4324         def _got_json_smalldir(res):
4325             r = simplejson.loads(res)
4326             self.failUnlessEqual(r["storage-index"], "")
4327             self.failUnless(r["results"]["healthy"])
4328         d.addCallback(_got_json_smalldir)
4329
4330         d.addCallback(self.CHECK, "sick", "t=check")
4331         def _got_html_sick(res):
4332             self.failUnlessIn("Not Healthy", res)
4333         d.addCallback(_got_html_sick)
4334         d.addCallback(self.CHECK, "sick", "t=check&output=json")
4335         def _got_json_sick(res):
4336             r = simplejson.loads(res)
4337             self.failUnlessEqual(r["summary"],
4338                                  "Not Healthy: 9 shares (enc 3-of-10)")
4339             self.failIf(r["results"]["healthy"])
4340             self.failIf(r["results"]["needs-rebalancing"])
4341             self.failUnless(r["results"]["recoverable"])
4342         d.addCallback(_got_json_sick)
4343
4344         d.addCallback(self.CHECK, "dead", "t=check")
4345         def _got_html_dead(res):
4346             self.failUnlessIn("Not Healthy", res)
4347         d.addCallback(_got_html_dead)
4348         d.addCallback(self.CHECK, "dead", "t=check&output=json")
4349         def _got_json_dead(res):
4350             r = simplejson.loads(res)
4351             self.failUnlessEqual(r["summary"],
4352                                  "Not Healthy: 1 shares (enc 3-of-10)")
4353             self.failIf(r["results"]["healthy"])
4354             self.failIf(r["results"]["needs-rebalancing"])
4355             self.failIf(r["results"]["recoverable"])
4356         d.addCallback(_got_json_dead)
4357
4358         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
4359         def _got_html_corrupt(res):
4360             self.failUnlessIn("Not Healthy! : Unhealthy", res)
4361         d.addCallback(_got_html_corrupt)
4362         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
4363         def _got_json_corrupt(res):
4364             r = simplejson.loads(res)
4365             self.failUnlessIn("Unhealthy: 9 shares (enc 3-of-10)", r["summary"])
4366             self.failIf(r["results"]["healthy"])
4367             self.failUnless(r["results"]["recoverable"])
4368             self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
4369             self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
4370         d.addCallback(_got_json_corrupt)
4371
4372         d.addErrback(self.explain_web_error)
4373         return d
4374
4375     def test_repair_html(self):
4376         self.basedir = "web/Grid/repair_html"
4377         self.set_up_grid()
4378         c0 = self.g.clients[0]
4379         self.uris = {}
4380         DATA = "data" * 100
4381         d = c0.upload(upload.Data(DATA, convergence=""))
4382         def _stash_uri(ur, which):
4383             self.uris[which] = ur.get_uri()
4384         d.addCallback(_stash_uri, "good")
4385         d.addCallback(lambda ign:
4386                       c0.upload(upload.Data(DATA+"1", convergence="")))
4387         d.addCallback(_stash_uri, "sick")
4388         d.addCallback(lambda ign:
4389                       c0.upload(upload.Data(DATA+"2", convergence="")))
4390         d.addCallback(_stash_uri, "dead")
4391         def _stash_mutable_uri(n, which):
4392             self.uris[which] = n.get_uri()
4393             assert isinstance(self.uris[which], str)
4394         d.addCallback(lambda ign:
4395             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4396         d.addCallback(_stash_mutable_uri, "corrupt")
4397
4398         def _compute_fileurls(ignored):
4399             self.fileurls = {}
4400             for which in self.uris:
4401                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4402         d.addCallback(_compute_fileurls)
4403
4404         def _clobber_shares(ignored):
4405             good_shares = self.find_uri_shares(self.uris["good"])
4406             self.failUnlessReallyEqual(len(good_shares), 10)
4407             sick_shares = self.find_uri_shares(self.uris["sick"])
4408             os.unlink(sick_shares[0][2])
4409             dead_shares = self.find_uri_shares(self.uris["dead"])
4410             for i in range(1, 10):
4411                 os.unlink(dead_shares[i][2])
4412             c_shares = self.find_uri_shares(self.uris["corrupt"])
4413             cso = CorruptShareOptions()
4414             cso.stdout = StringIO()
4415             cso.parseOptions([c_shares[0][2]])
4416             corrupt_share(cso)
4417         d.addCallback(_clobber_shares)
4418
4419         d.addCallback(self.CHECK, "good", "t=check&repair=true")
4420         def _got_html_good(res):
4421             self.failUnlessIn("Healthy", res)
4422             self.failIfIn("Not Healthy", res)
4423             self.failUnlessIn("No repair necessary", res)
4424             self.failUnlessIn(FAVICON_MARKUP, res)
4425         d.addCallback(_got_html_good)
4426
4427         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
4428         def _got_html_sick(res):
4429             self.failUnlessIn("Healthy : healthy", res)
4430             self.failIfIn("Not Healthy", res)
4431             self.failUnlessIn("Repair successful", res)
4432         d.addCallback(_got_html_sick)
4433
4434         # repair of a dead file will fail, of course, but it isn't yet
4435         # clear how this should be reported. Right now it shows up as
4436         # a "410 Gone".
4437         #
4438         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
4439         #def _got_html_dead(res):
4440         #    print res
4441         #    self.failUnlessIn("Healthy : healthy", res)
4442         #    self.failIfIn("Not Healthy", res)
4443         #    self.failUnlessIn("No repair necessary", res)
4444         #d.addCallback(_got_html_dead)
4445
4446         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
4447         def _got_html_corrupt(res):
4448             self.failUnlessIn("Healthy : Healthy", res)
4449             self.failIfIn("Not Healthy", res)
4450             self.failUnlessIn("Repair successful", res)
4451         d.addCallback(_got_html_corrupt)
4452
4453         d.addErrback(self.explain_web_error)
4454         return d
4455
4456     def test_repair_json(self):
4457         self.basedir = "web/Grid/repair_json"
4458         self.set_up_grid()
4459         c0 = self.g.clients[0]
4460         self.uris = {}
4461         DATA = "data" * 100
4462         d = c0.upload(upload.Data(DATA+"1", convergence=""))
4463         def _stash_uri(ur, which):
4464             self.uris[which] = ur.get_uri()
4465         d.addCallback(_stash_uri, "sick")
4466
4467         def _compute_fileurls(ignored):
4468             self.fileurls = {}
4469             for which in self.uris:
4470                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4471         d.addCallback(_compute_fileurls)
4472
4473         def _clobber_shares(ignored):
4474             sick_shares = self.find_uri_shares(self.uris["sick"])
4475             os.unlink(sick_shares[0][2])
4476         d.addCallback(_clobber_shares)
4477
4478         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
4479         def _got_json_sick(res):
4480             r = simplejson.loads(res)
4481             self.failUnlessReallyEqual(r["repair-attempted"], True)
4482             self.failUnlessReallyEqual(r["repair-successful"], True)
4483             self.failUnlessEqual(r["pre-repair-results"]["summary"],
4484                                  "Not Healthy: 9 shares (enc 3-of-10)")
4485             self.failIf(r["pre-repair-results"]["results"]["healthy"])
4486             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
4487             self.failUnless(r["post-repair-results"]["results"]["healthy"])
4488         d.addCallback(_got_json_sick)
4489
4490         d.addErrback(self.explain_web_error)
4491         return d
4492
4493     def test_unknown(self, immutable=False):
4494         self.basedir = "web/Grid/unknown"
4495         if immutable:
4496             self.basedir = "web/Grid/unknown-immutable"
4497
4498         self.set_up_grid()
4499         c0 = self.g.clients[0]
4500         self.uris = {}
4501         self.fileurls = {}
4502
4503         # the future cap format may contain slashes, which must be tolerated
4504         expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
4505                                                            safe="")
4506
4507         if immutable:
4508             name = u"future-imm"
4509             future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
4510             d = c0.create_immutable_dirnode({name: (future_node, {})})
4511         else:
4512             name = u"future"
4513             future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4514             d = c0.create_dirnode()
4515
4516         def _stash_root_and_create_file(n):
4517             self.rootnode = n
4518             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
4519             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
4520             if not immutable:
4521                 return self.rootnode.set_node(name, future_node)
4522         d.addCallback(_stash_root_and_create_file)
4523
4524         # make sure directory listing tolerates unknown nodes
4525         d.addCallback(lambda ign: self.GET(self.rooturl))
4526         def _check_directory_html(res, expected_type_suffix):
4527             pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
4528                                   '<td>%s</td>' % (expected_type_suffix, str(name)),
4529                                  re.DOTALL)
4530             self.failUnless(re.search(pattern, res), res)
4531             # find the More Info link for name, should be relative
4532             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4533             info_url = mo.group(1)
4534             self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
4535         if immutable:
4536             d.addCallback(_check_directory_html, "-IMM")
4537         else:
4538             d.addCallback(_check_directory_html, "")
4539
4540         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4541         def _check_directory_json(res, expect_rw_uri):
4542             data = simplejson.loads(res)
4543             self.failUnlessEqual(data[0], "dirnode")
4544             f = data[1]["children"][name]
4545             self.failUnlessEqual(f[0], "unknown")
4546             if expect_rw_uri:
4547                 self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data)
4548             else:
4549                 self.failIfIn("rw_uri", f[1])
4550             if immutable:
4551                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
4552             else:
4553                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
4554             self.failUnlessIn("metadata", f[1])
4555         d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
4556
4557         def _check_info(res, expect_rw_uri, expect_ro_uri):
4558             self.failUnlessIn("Object Type: <span>unknown</span>", res)
4559             if expect_rw_uri:
4560                 self.failUnlessIn(unknown_rwcap, res)
4561             if expect_ro_uri:
4562                 if immutable:
4563                     self.failUnlessIn(unknown_immcap, res)
4564                 else:
4565                     self.failUnlessIn(unknown_rocap, res)
4566             else:
4567                 self.failIfIn(unknown_rocap, res)
4568             self.failIfIn("Raw data as", res)
4569             self.failIfIn("Directory writecap", res)
4570             self.failIfIn("Checker Operations", res)
4571             self.failIfIn("Mutable File Operations", res)
4572             self.failIfIn("Directory Operations", res)
4573
4574         # FIXME: these should have expect_rw_uri=not immutable; I don't know
4575         # why they fail. Possibly related to ticket #922.
4576
4577         d.addCallback(lambda ign: self.GET(expected_info_url))
4578         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
4579         d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
4580         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
4581
4582         def _check_json(res, expect_rw_uri):
4583             data = simplejson.loads(res)
4584             self.failUnlessEqual(data[0], "unknown")
4585             if expect_rw_uri:
4586                 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data)
4587             else:
4588                 self.failIfIn("rw_uri", data[1])
4589
4590             if immutable:
4591                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data)
4592                 self.failUnlessReallyEqual(data[1]["mutable"], False)
4593             elif expect_rw_uri:
4594                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4595                 self.failUnlessReallyEqual(data[1]["mutable"], True)
4596             else:
4597                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4598                 self.failIfIn("mutable", data[1])
4599
4600             # TODO: check metadata contents
4601             self.failUnlessIn("metadata", data[1])
4602
4603         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
4604         d.addCallback(_check_json, expect_rw_uri=not immutable)
4605
4606         # and make sure that a read-only version of the directory can be
4607         # rendered too. This version will not have unknown_rwcap, whether
4608         # or not future_node was immutable.
4609         d.addCallback(lambda ign: self.GET(self.rourl))
4610         if immutable:
4611             d.addCallback(_check_directory_html, "-IMM")
4612         else:
4613             d.addCallback(_check_directory_html, "-RO")
4614
4615         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
4616         d.addCallback(_check_directory_json, expect_rw_uri=False)
4617
4618         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
4619         d.addCallback(_check_json, expect_rw_uri=False)
4620
4621         # TODO: check that getting t=info from the Info link in the ro directory
4622         # works, and does not include the writecap URI.
4623         return d
4624
4625     def test_immutable_unknown(self):
4626         return self.test_unknown(immutable=True)
4627
4628     def test_mutant_dirnodes_are_omitted(self):
4629         self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
4630
4631         self.set_up_grid()
4632         c = self.g.clients[0]
4633         nm = c.nodemaker
4634         self.uris = {}
4635         self.fileurls = {}
4636
4637         lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
4638         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
4639         mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
4640
4641         # This method tests mainly dirnode, but we'd have to duplicate code in order to
4642         # test the dirnode and web layers separately.
4643
4644         # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
4645         # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
4646         # When the directory is read, the mutants should be silently disposed of, leaving
4647         # their lonely sibling.
4648         # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
4649         # because immutable directories don't have a writecap and therefore that field
4650         # isn't (and can't be) decrypted.
4651         # TODO: The field still exists in the netstring. Technically we should check what
4652         # happens if something is put there (_unpack_contents should raise ValueError),
4653         # but that can wait.
4654
4655         lonely_child = nm.create_from_cap(lonely_uri)
4656         mutant_ro_child = nm.create_from_cap(mut_read_uri)
4657         mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
4658
4659         def _by_hook_or_by_crook():
4660             return True
4661         for n in [mutant_ro_child, mutant_write_in_ro_child]:
4662             n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
4663
4664         mutant_write_in_ro_child.get_write_uri    = lambda: None
4665         mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
4666
4667         kids = {u"lonely":      (lonely_child, {}),
4668                 u"ro":          (mutant_ro_child, {}),
4669                 u"write-in-ro": (mutant_write_in_ro_child, {}),
4670                 }
4671         d = c.create_immutable_dirnode(kids)
4672
4673         def _created(dn):
4674             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
4675             self.failIf(dn.is_mutable())
4676             self.failUnless(dn.is_readonly())
4677             # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
4678             self.failIf(hasattr(dn._node, 'get_writekey'))
4679             rep = str(dn)
4680             self.failUnlessIn("RO-IMM", rep)
4681             cap = dn.get_cap()
4682             self.failUnlessIn("CHK", cap.to_string())
4683             self.cap = cap
4684             self.rootnode = dn
4685             self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
4686             return download_to_data(dn._node)
4687         d.addCallback(_created)
4688
4689         def _check_data(data):
4690             # Decode the netstring representation of the directory to check that all children
4691             # are present. This is a bit of an abstraction violation, but there's not really
4692             # any other way to do it given that the real DirectoryNode._unpack_contents would
4693             # strip the mutant children out (which is what we're trying to test, later).
4694             position = 0
4695             numkids = 0
4696             while position < len(data):
4697                 entries, position = split_netstring(data, 1, position)
4698                 entry = entries[0]
4699                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
4700                 name = name_utf8.decode("utf-8")
4701                 self.failUnlessEqual(rwcapdata, "")
4702                 self.failUnlessIn(name, kids)
4703                 (expected_child, ign) = kids[name]
4704                 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
4705                 numkids += 1
4706
4707             self.failUnlessReallyEqual(numkids, 3)
4708             return self.rootnode.list()
4709         d.addCallback(_check_data)
4710
4711         # Now when we use the real directory listing code, the mutants should be absent.
4712         def _check_kids(children):
4713             self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
4714             lonely_node, lonely_metadata = children[u"lonely"]
4715
4716             self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
4717             self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
4718         d.addCallback(_check_kids)
4719
4720         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
4721         d.addCallback(lambda n: n.list())
4722         d.addCallback(_check_kids)  # again with dirnode recreated from cap
4723
4724         # Make sure the lonely child can be listed in HTML...
4725         d.addCallback(lambda ign: self.GET(self.rooturl))
4726         def _check_html(res):
4727             self.failIfIn("URI:SSK", res)
4728             get_lonely = "".join([r'<td>FILE</td>',
4729                                   r'\s+<td>',
4730                                   r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
4731                                   r'</td>',
4732                                   r'\s+<td align="right">%d</td>' % len("one"),
4733                                   ])
4734             self.failUnless(re.search(get_lonely, res), res)
4735
4736             # find the More Info link for name, should be relative
4737             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4738             info_url = mo.group(1)
4739             self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
4740         d.addCallback(_check_html)
4741
4742         # ... and in JSON.
4743         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4744         def _check_json(res):
4745             data = simplejson.loads(res)
4746             self.failUnlessEqual(data[0], "dirnode")
4747             listed_children = data[1]["children"]
4748             self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
4749             ll_type, ll_data = listed_children[u"lonely"]
4750             self.failUnlessEqual(ll_type, "filenode")
4751             self.failIfIn("rw_uri", ll_data)
4752             self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
4753         d.addCallback(_check_json)
4754         return d
4755
4756     def test_deep_check(self):
4757         self.basedir = "web/Grid/deep_check"
4758         self.set_up_grid()
4759         c0 = self.g.clients[0]
4760         self.uris = {}
4761         self.fileurls = {}
4762         DATA = "data" * 100
4763         d = c0.create_dirnode()
4764         def _stash_root_and_create_file(n):
4765             self.rootnode = n
4766             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4767             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4768         d.addCallback(_stash_root_and_create_file)
4769         def _stash_uri(fn, which):
4770             self.uris[which] = fn.get_uri()
4771             return fn
4772         d.addCallback(_stash_uri, "good")
4773         d.addCallback(lambda ign:
4774                       self.rootnode.add_file(u"small",
4775                                              upload.Data("literal",
4776                                                         convergence="")))
4777         d.addCallback(_stash_uri, "small")
4778         d.addCallback(lambda ign:
4779                       self.rootnode.add_file(u"sick",
4780                                              upload.Data(DATA+"1",
4781                                                         convergence="")))
4782         d.addCallback(_stash_uri, "sick")
4783
4784         # this tests that deep-check and stream-manifest will ignore
4785         # UnknownNode instances. Hopefully this will also cover deep-stats.
4786         future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4787         d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
4788
4789         def _clobber_shares(ignored):
4790             self.delete_shares_numbered(self.uris["sick"], [0,1])
4791         d.addCallback(_clobber_shares)
4792
4793         # root
4794         # root/good
4795         # root/small
4796         # root/sick
4797         # root/future
4798
4799         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4800         def _done(res):
4801             try:
4802                 units = [simplejson.loads(line)
4803                          for line in res.splitlines()
4804                          if line]
4805             except ValueError:
4806                 print "response is:", res
4807                 print "undecodeable line was '%s'" % line
4808                 raise
4809             self.failUnlessReallyEqual(len(units), 5+1)
4810             # should be parent-first
4811             u0 = units[0]
4812             self.failUnlessEqual(u0["path"], [])
4813             self.failUnlessEqual(u0["type"], "directory")
4814             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4815             u0cr = u0["check-results"]
4816             self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
4817
4818             ugood = [u for u in units
4819                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4820             self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
4821             ugoodcr = ugood["check-results"]
4822             self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
4823
4824             stats = units[-1]
4825             self.failUnlessEqual(stats["type"], "stats")
4826             s = stats["stats"]
4827             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4828             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4829             self.failUnlessReallyEqual(s["count-directories"], 1)
4830             self.failUnlessReallyEqual(s["count-unknown"], 1)
4831         d.addCallback(_done)
4832
4833         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4834         def _check_manifest(res):
4835             self.failUnless(res.endswith("\n"))
4836             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
4837             self.failUnlessReallyEqual(len(units), 5+1)
4838             self.failUnlessEqual(units[-1]["type"], "stats")
4839             first = units[0]
4840             self.failUnlessEqual(first["path"], [])
4841             self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri())
4842             self.failUnlessEqual(first["type"], "directory")
4843             stats = units[-1]["stats"]
4844             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4845             self.failUnlessReallyEqual(stats["count-literal-files"], 1)
4846             self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
4847             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4848             self.failUnlessReallyEqual(stats["count-unknown"], 1)
4849         d.addCallback(_check_manifest)
4850
4851         # now add root/subdir and root/subdir/grandchild, then make subdir
4852         # unrecoverable, then see what happens
4853
4854         d.addCallback(lambda ign:
4855                       self.rootnode.create_subdirectory(u"subdir"))
4856         d.addCallback(_stash_uri, "subdir")
4857         d.addCallback(lambda subdir_node:
4858                       subdir_node.add_file(u"grandchild",
4859                                            upload.Data(DATA+"2",
4860                                                        convergence="")))
4861         d.addCallback(_stash_uri, "grandchild")
4862
4863         d.addCallback(lambda ign:
4864                       self.delete_shares_numbered(self.uris["subdir"],
4865                                                   range(1, 10)))
4866
4867         # root
4868         # root/good
4869         # root/small
4870         # root/sick
4871         # root/future
4872         # root/subdir [unrecoverable]
4873         # root/subdir/grandchild
4874
4875         # how should a streaming-JSON API indicate fatal error?
4876         # answer: emit ERROR: instead of a JSON string
4877
4878         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4879         def _check_broken_manifest(res):
4880             lines = res.splitlines()
4881             error_lines = [i
4882                            for (i,line) in enumerate(lines)
4883                            if line.startswith("ERROR:")]
4884             if not error_lines:
4885                 self.fail("no ERROR: in output: %s" % (res,))
4886             first_error = error_lines[0]
4887             error_line = lines[first_error]
4888             error_msg = lines[first_error+1:]
4889             error_msg_s = "\n".join(error_msg) + "\n"
4890             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4891                               error_line)
4892             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4893             units = [simplejson.loads(line) for line in lines[:first_error]]
4894             self.failUnlessReallyEqual(len(units), 6) # includes subdir
4895             last_unit = units[-1]
4896             self.failUnlessEqual(last_unit["path"], ["subdir"])
4897         d.addCallback(_check_broken_manifest)
4898
4899         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4900         def _check_broken_deepcheck(res):
4901             lines = res.splitlines()
4902             error_lines = [i
4903                            for (i,line) in enumerate(lines)
4904                            if line.startswith("ERROR:")]
4905             if not error_lines:
4906                 self.fail("no ERROR: in output: %s" % (res,))
4907             first_error = error_lines[0]
4908             error_line = lines[first_error]
4909             error_msg = lines[first_error+1:]
4910             error_msg_s = "\n".join(error_msg) + "\n"
4911             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4912                               error_line)
4913             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4914             units = [simplejson.loads(line) for line in lines[:first_error]]
4915             self.failUnlessReallyEqual(len(units), 6) # includes subdir
4916             last_unit = units[-1]
4917             self.failUnlessEqual(last_unit["path"], ["subdir"])
4918             r = last_unit["check-results"]["results"]
4919             self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
4920             self.failUnlessReallyEqual(r["count-shares-good"], 1)
4921             self.failUnlessReallyEqual(r["recoverable"], False)
4922         d.addCallback(_check_broken_deepcheck)
4923
4924         d.addErrback(self.explain_web_error)
4925         return d
4926
4927     def test_deep_check_and_repair(self):
4928         self.basedir = "web/Grid/deep_check_and_repair"
4929         self.set_up_grid()
4930         c0 = self.g.clients[0]
4931         self.uris = {}
4932         self.fileurls = {}
4933         DATA = "data" * 100
4934         d = c0.create_dirnode()
4935         def _stash_root_and_create_file(n):
4936             self.rootnode = n
4937             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4938             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4939         d.addCallback(_stash_root_and_create_file)
4940         def _stash_uri(fn, which):
4941             self.uris[which] = fn.get_uri()
4942         d.addCallback(_stash_uri, "good")
4943         d.addCallback(lambda ign:
4944                       self.rootnode.add_file(u"small",
4945                                              upload.Data("literal",
4946                                                         convergence="")))
4947         d.addCallback(_stash_uri, "small")
4948         d.addCallback(lambda ign:
4949                       self.rootnode.add_file(u"sick",
4950                                              upload.Data(DATA+"1",
4951                                                         convergence="")))
4952         d.addCallback(_stash_uri, "sick")
4953         #d.addCallback(lambda ign:
4954         #              self.rootnode.add_file(u"dead",
4955         #                                     upload.Data(DATA+"2",
4956         #                                                convergence="")))
4957         #d.addCallback(_stash_uri, "dead")
4958
4959         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
4960         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
4961         #d.addCallback(_stash_uri, "corrupt")
4962
4963         def _clobber_shares(ignored):
4964             good_shares = self.find_uri_shares(self.uris["good"])
4965             self.failUnlessReallyEqual(len(good_shares), 10)
4966             sick_shares = self.find_uri_shares(self.uris["sick"])
4967             os.unlink(sick_shares[0][2])
4968             #dead_shares = self.find_uri_shares(self.uris["dead"])
4969             #for i in range(1, 10):
4970             #    os.unlink(dead_shares[i][2])
4971
4972             #c_shares = self.find_uri_shares(self.uris["corrupt"])
4973             #cso = CorruptShareOptions()
4974             #cso.stdout = StringIO()
4975             #cso.parseOptions([c_shares[0][2]])
4976             #corrupt_share(cso)
4977         d.addCallback(_clobber_shares)
4978
4979         # root
4980         # root/good   CHK, 10 shares
4981         # root/small  LIT
4982         # root/sick   CHK, 9 shares
4983
4984         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
4985         def _done(res):
4986             units = [simplejson.loads(line)
4987                      for line in res.splitlines()
4988                      if line]
4989             self.failUnlessReallyEqual(len(units), 4+1)
4990             # should be parent-first
4991             u0 = units[0]
4992             self.failUnlessEqual(u0["path"], [])
4993             self.failUnlessEqual(u0["type"], "directory")
4994             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4995             u0crr = u0["check-and-repair-results"]
4996             self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
4997             self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
4998
4999             ugood = [u for u in units
5000                      if u["type"] == "file" and u["path"] == [u"good"]][0]
5001             self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
5002             ugoodcrr = ugood["check-and-repair-results"]
5003             self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
5004             self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
5005
5006             usick = [u for u in units
5007                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
5008             self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"])
5009             usickcrr = usick["check-and-repair-results"]
5010             self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
5011             self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
5012             self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
5013             self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
5014
5015             stats = units[-1]
5016             self.failUnlessEqual(stats["type"], "stats")
5017             s = stats["stats"]
5018             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
5019             self.failUnlessReallyEqual(s["count-literal-files"], 1)
5020             self.failUnlessReallyEqual(s["count-directories"], 1)
5021         d.addCallback(_done)
5022
5023         d.addErrback(self.explain_web_error)
5024         return d
5025
5026     def _count_leases(self, ignored, which):
5027         u = self.uris[which]
5028         shares = self.find_uri_shares(u)
5029         lease_counts = []
5030         for shnum, serverid, fn in shares:
5031             sf = get_share_file(fn)
5032             num_leases = len(list(sf.get_leases()))
5033             lease_counts.append( (fn, num_leases) )
5034         return lease_counts
5035
5036     def _assert_leasecount(self, lease_counts, expected):
5037         for (fn, num_leases) in lease_counts:
5038             if num_leases != expected:
5039                 self.fail("expected %d leases, have %d, on %s" %
5040                           (expected, num_leases, fn))
5041
5042     def test_add_lease(self):
5043         self.basedir = "web/Grid/add_lease"
5044         self.set_up_grid(num_clients=2)
5045         c0 = self.g.clients[0]
5046         self.uris = {}
5047         DATA = "data" * 100
5048         d = c0.upload(upload.Data(DATA, convergence=""))
5049         def _stash_uri(ur, which):
5050             self.uris[which] = ur.get_uri()
5051         d.addCallback(_stash_uri, "one")
5052         d.addCallback(lambda ign:
5053                       c0.upload(upload.Data(DATA+"1", convergence="")))
5054         d.addCallback(_stash_uri, "two")
5055         def _stash_mutable_uri(n, which):
5056             self.uris[which] = n.get_uri()
5057             assert isinstance(self.uris[which], str)
5058         d.addCallback(lambda ign:
5059             c0.create_mutable_file(publish.MutableData(DATA+"2")))
5060         d.addCallback(_stash_mutable_uri, "mutable")
5061
5062         def _compute_fileurls(ignored):
5063             self.fileurls = {}
5064             for which in self.uris:
5065                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
5066         d.addCallback(_compute_fileurls)
5067
5068         d.addCallback(self._count_leases, "one")
5069         d.addCallback(self._assert_leasecount, 1)
5070         d.addCallback(self._count_leases, "two")
5071         d.addCallback(self._assert_leasecount, 1)
5072         d.addCallback(self._count_leases, "mutable")
5073         d.addCallback(self._assert_leasecount, 1)
5074
5075         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
5076         def _got_html_good(res):
5077             self.failUnlessIn("Healthy", res)
5078             self.failIfIn("Not Healthy", res)
5079         d.addCallback(_got_html_good)
5080
5081         d.addCallback(self._count_leases, "one")
5082         d.addCallback(self._assert_leasecount, 1)
5083         d.addCallback(self._count_leases, "two")
5084         d.addCallback(self._assert_leasecount, 1)
5085         d.addCallback(self._count_leases, "mutable")
5086         d.addCallback(self._assert_leasecount, 1)
5087
5088         # this CHECK uses the original client, which uses the same
5089         # lease-secrets, so it will just renew the original lease
5090         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
5091         d.addCallback(_got_html_good)
5092
5093         d.addCallback(self._count_leases, "one")
5094         d.addCallback(self._assert_leasecount, 1)
5095         d.addCallback(self._count_leases, "two")
5096         d.addCallback(self._assert_leasecount, 1)
5097         d.addCallback(self._count_leases, "mutable")
5098         d.addCallback(self._assert_leasecount, 1)
5099
5100         # this CHECK uses an alternate client, which adds a second lease
5101         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
5102         d.addCallback(_got_html_good)
5103
5104         d.addCallback(self._count_leases, "one")
5105         d.addCallback(self._assert_leasecount, 2)
5106         d.addCallback(self._count_leases, "two")
5107         d.addCallback(self._assert_leasecount, 1)
5108         d.addCallback(self._count_leases, "mutable")
5109         d.addCallback(self._assert_leasecount, 1)
5110
5111         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
5112         d.addCallback(_got_html_good)
5113
5114         d.addCallback(self._count_leases, "one")
5115         d.addCallback(self._assert_leasecount, 2)
5116         d.addCallback(self._count_leases, "two")
5117         d.addCallback(self._assert_leasecount, 1)
5118         d.addCallback(self._count_leases, "mutable")
5119         d.addCallback(self._assert_leasecount, 1)
5120
5121         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
5122                       clientnum=1)
5123         d.addCallback(_got_html_good)
5124
5125         d.addCallback(self._count_leases, "one")
5126         d.addCallback(self._assert_leasecount, 2)
5127         d.addCallback(self._count_leases, "two")
5128         d.addCallback(self._assert_leasecount, 1)
5129         d.addCallback(self._count_leases, "mutable")
5130         d.addCallback(self._assert_leasecount, 2)
5131
5132         d.addErrback(self.explain_web_error)
5133         return d
5134
5135     def test_deep_add_lease(self):
5136         self.basedir = "web/Grid/deep_add_lease"
5137         self.set_up_grid(num_clients=2)
5138         c0 = self.g.clients[0]
5139         self.uris = {}
5140         self.fileurls = {}
5141         DATA = "data" * 100
5142         d = c0.create_dirnode()
5143         def _stash_root_and_create_file(n):
5144             self.rootnode = n
5145             self.uris["root"] = n.get_uri()
5146             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5147             return n.add_file(u"one", upload.Data(DATA, convergence=""))
5148         d.addCallback(_stash_root_and_create_file)
5149         def _stash_uri(fn, which):
5150             self.uris[which] = fn.get_uri()
5151         d.addCallback(_stash_uri, "one")
5152         d.addCallback(lambda ign:
5153                       self.rootnode.add_file(u"small",
5154                                              upload.Data("literal",
5155                                                         convergence="")))
5156         d.addCallback(_stash_uri, "small")
5157
5158         d.addCallback(lambda ign:
5159             c0.create_mutable_file(publish.MutableData("mutable")))
5160         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
5161         d.addCallback(_stash_uri, "mutable")
5162
5163         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
5164         def _done(res):
5165             units = [simplejson.loads(line)
5166                      for line in res.splitlines()
5167                      if line]
5168             # root, one, small, mutable,   stats
5169             self.failUnlessReallyEqual(len(units), 4+1)
5170         d.addCallback(_done)
5171
5172         d.addCallback(self._count_leases, "root")
5173         d.addCallback(self._assert_leasecount, 1)
5174         d.addCallback(self._count_leases, "one")
5175         d.addCallback(self._assert_leasecount, 1)
5176         d.addCallback(self._count_leases, "mutable")
5177         d.addCallback(self._assert_leasecount, 1)
5178
5179         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
5180         d.addCallback(_done)
5181
5182         d.addCallback(self._count_leases, "root")
5183         d.addCallback(self._assert_leasecount, 1)
5184         d.addCallback(self._count_leases, "one")
5185         d.addCallback(self._assert_leasecount, 1)
5186         d.addCallback(self._count_leases, "mutable")
5187         d.addCallback(self._assert_leasecount, 1)
5188
5189         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
5190                       clientnum=1)
5191         d.addCallback(_done)
5192
5193         d.addCallback(self._count_leases, "root")
5194         d.addCallback(self._assert_leasecount, 2)
5195         d.addCallback(self._count_leases, "one")
5196         d.addCallback(self._assert_leasecount, 2)
5197         d.addCallback(self._count_leases, "mutable")
5198         d.addCallback(self._assert_leasecount, 2)
5199
5200         d.addErrback(self.explain_web_error)
5201         return d
5202
5203
5204     def test_exceptions(self):
5205         self.basedir = "web/Grid/exceptions"
5206         self.set_up_grid(num_clients=1, num_servers=2)
5207         c0 = self.g.clients[0]
5208         c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 2
5209         self.fileurls = {}
5210         DATA = "data" * 100
5211         d = c0.create_dirnode()
5212         def _stash_root(n):
5213             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5214             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
5215             return n
5216         d.addCallback(_stash_root)
5217         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
5218         def _stash_bad(ur):
5219             self.fileurls["1share"] = "uri/" + urllib.quote(ur.get_uri())
5220             self.delete_shares_numbered(ur.get_uri(), range(1,10))
5221
5222             u = uri.from_string(ur.get_uri())
5223             u.key = testutil.flip_bit(u.key, 0)
5224             baduri = u.to_string()
5225             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
5226         d.addCallback(_stash_bad)
5227         d.addCallback(lambda ign: c0.create_dirnode())
5228         def _mangle_dirnode_1share(n):
5229             u = n.get_uri()
5230             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
5231             self.fileurls["dir-1share-json"] = url + "?t=json"
5232             self.delete_shares_numbered(u, range(1,10))
5233         d.addCallback(_mangle_dirnode_1share)
5234         d.addCallback(lambda ign: c0.create_dirnode())
5235         def _mangle_dirnode_0share(n):
5236             u = n.get_uri()
5237             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
5238             self.fileurls["dir-0share-json"] = url + "?t=json"
5239             self.delete_shares_numbered(u, range(0,10))
5240         d.addCallback(_mangle_dirnode_0share)
5241
5242         # NotEnoughSharesError should be reported sensibly, with a
5243         # text/plain explanation of the problem, and perhaps some
5244         # information on which shares *could* be found.
5245
5246         d.addCallback(lambda ignored:
5247                       self.shouldHTTPError("GET unrecoverable",
5248                                            410, "Gone", "NoSharesError",
5249                                            self.GET, self.fileurls["0shares"]))
5250         def _check_zero_shares(body):
5251             self.failIfIn("<html>", body)
5252             body = " ".join(body.strip().split())
5253             exp = ("NoSharesError: no shares could be found. "
5254                    "Zero shares usually indicates a corrupt URI, or that "
5255                    "no servers were connected, but it might also indicate "
5256                    "severe corruption. You should perform a filecheck on "
5257                    "this object to learn more. The full error message is: "
5258                    "no shares (need 3). Last failure: None")
5259             self.failUnlessReallyEqual(exp, body)
5260         d.addCallback(_check_zero_shares)
5261
5262
5263         d.addCallback(lambda ignored:
5264                       self.shouldHTTPError("GET 1share",
5265                                            410, "Gone", "NotEnoughSharesError",
5266                                            self.GET, self.fileurls["1share"]))
5267         def _check_one_share(body):
5268             self.failIfIn("<html>", body)
5269             body = " ".join(body.strip().split())
5270             msgbase = ("NotEnoughSharesError: This indicates that some "
5271                        "servers were unavailable, or that shares have been "
5272                        "lost to server departure, hard drive failure, or disk "
5273                        "corruption. You should perform a filecheck on "
5274                        "this object to learn more. The full error message is:"
5275                        )
5276             msg1 = msgbase + (" ran out of shares:"
5277                               " complete=sh0"
5278                               " pending="
5279                               " overdue= unused= need 3. Last failure: None")
5280             msg2 = msgbase + (" ran out of shares:"
5281                               " complete="
5282                               " pending=Share(sh0-on-xgru5)"
5283                               " overdue= unused= need 3. Last failure: None")
5284             self.failUnless(body == msg1 or body == msg2, body)
5285         d.addCallback(_check_one_share)
5286
5287         d.addCallback(lambda ignored:
5288                       self.shouldHTTPError("GET imaginary",
5289                                            404, "Not Found", None,
5290                                            self.GET, self.fileurls["imaginary"]))
5291         def _missing_child(body):
5292             self.failUnlessIn("No such child: imaginary", body)
5293         d.addCallback(_missing_child)
5294
5295         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
5296         def _check_0shares_dir_html(body):
5297             self.failUnlessIn("<html>", body)
5298             # we should see the regular page, but without the child table or
5299             # the dirops forms
5300             body = " ".join(body.strip().split())
5301             self.failUnlessIn('href="?t=info">More info on this directory',
5302                               body)
5303             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5304                    "could not be retrieved, because there were insufficient "
5305                    "good shares. This might indicate that no servers were "
5306                    "connected, insufficient servers were connected, the URI "
5307                    "was corrupt, or that shares have been lost due to server "
5308                    "departure, hard drive failure, or disk corruption. You "
5309                    "should perform a filecheck on this object to learn more.")
5310             self.failUnlessIn(exp, body)
5311             self.failUnlessIn("No upload forms: directory is unreadable", body)
5312         d.addCallback(_check_0shares_dir_html)
5313
5314         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
5315         def _check_1shares_dir_html(body):
5316             # at some point, we'll split UnrecoverableFileError into 0-shares
5317             # and some-shares like we did for immutable files (since there
5318             # are different sorts of advice to offer in each case). For now,
5319             # they present the same way.
5320             self.failUnlessIn("<html>", body)
5321             body = " ".join(body.strip().split())
5322             self.failUnlessIn('href="?t=info">More info on this directory',
5323                               body)
5324             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5325                    "could not be retrieved, because there were insufficient "
5326                    "good shares. This might indicate that no servers were "
5327                    "connected, insufficient servers were connected, the URI "
5328                    "was corrupt, or that shares have been lost due to server "
5329                    "departure, hard drive failure, or disk corruption. You "
5330                    "should perform a filecheck on this object to learn more.")
5331             self.failUnlessIn(exp, body)
5332             self.failUnlessIn("No upload forms: directory is unreadable", body)
5333         d.addCallback(_check_1shares_dir_html)
5334
5335         d.addCallback(lambda ignored:
5336                       self.shouldHTTPError("GET dir-0share-json",
5337                                            410, "Gone", "UnrecoverableFileError",
5338                                            self.GET,
5339                                            self.fileurls["dir-0share-json"]))
5340         def _check_unrecoverable_file(body):
5341             self.failIfIn("<html>", body)
5342             body = " ".join(body.strip().split())
5343             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5344                    "could not be retrieved, because there were insufficient "
5345                    "good shares. This might indicate that no servers were "
5346                    "connected, insufficient servers were connected, the URI "
5347                    "was corrupt, or that shares have been lost due to server "
5348                    "departure, hard drive failure, or disk corruption. You "
5349                    "should perform a filecheck on this object to learn more.")
5350             self.failUnlessReallyEqual(exp, body)
5351         d.addCallback(_check_unrecoverable_file)
5352
5353         d.addCallback(lambda ignored:
5354                       self.shouldHTTPError("GET dir-1share-json",
5355                                            410, "Gone", "UnrecoverableFileError",
5356                                            self.GET,
5357                                            self.fileurls["dir-1share-json"]))
5358         d.addCallback(_check_unrecoverable_file)
5359
5360         d.addCallback(lambda ignored:
5361                       self.shouldHTTPError("GET imaginary",
5362                                            404, "Not Found", None,
5363                                            self.GET, self.fileurls["imaginary"]))
5364
5365         # attach a webapi child that throws a random error, to test how it
5366         # gets rendered.
5367         w = c0.getServiceNamed("webish")
5368         w.root.putChild("ERRORBOOM", ErrorBoom())
5369
5370         # "Accept: */*" :        should get a text/html stack trace
5371         # "Accept: text/plain" : should get a text/plain stack trace
5372         # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
5373         # no Accept header:      should get a text/html stack trace
5374
5375         d.addCallback(lambda ignored:
5376                       self.shouldHTTPError("GET errorboom_html",
5377                                            500, "Internal Server Error", None,
5378                                            self.GET, "ERRORBOOM",
5379                                            headers={"accept": "*/*"}))
5380         def _internal_error_html1(body):
5381             self.failUnlessIn("<html>", "expected HTML, not '%s'" % body)
5382         d.addCallback(_internal_error_html1)
5383
5384         d.addCallback(lambda ignored:
5385                       self.shouldHTTPError("GET errorboom_text",
5386                                            500, "Internal Server Error", None,
5387                                            self.GET, "ERRORBOOM",
5388                                            headers={"accept": "text/plain"}))
5389         def _internal_error_text2(body):
5390             self.failIfIn("<html>", body)
5391             self.failUnless(body.startswith("Traceback "), body)
5392         d.addCallback(_internal_error_text2)
5393
5394         CLI_accepts = "text/plain, application/octet-stream"
5395         d.addCallback(lambda ignored:
5396                       self.shouldHTTPError("GET errorboom_text",
5397                                            500, "Internal Server Error", None,
5398                                            self.GET, "ERRORBOOM",
5399                                            headers={"accept": CLI_accepts}))
5400         def _internal_error_text3(body):
5401             self.failIfIn("<html>", body)
5402             self.failUnless(body.startswith("Traceback "), body)
5403         d.addCallback(_internal_error_text3)
5404
5405         d.addCallback(lambda ignored:
5406                       self.shouldHTTPError("GET errorboom_text",
5407                                            500, "Internal Server Error", None,
5408                                            self.GET, "ERRORBOOM"))
5409         def _internal_error_html4(body):
5410             self.failUnlessIn("<html>", body)
5411         d.addCallback(_internal_error_html4)
5412
5413         def _flush_errors(res):
5414             # Trial: please ignore the CompletelyUnhandledError in the logs
5415             self.flushLoggedErrors(CompletelyUnhandledError)
5416             return res
5417         d.addBoth(_flush_errors)
5418
5419         return d
5420
5421     def test_blacklist(self):
5422         # download from a blacklisted URI, get an error
5423         self.basedir = "web/Grid/blacklist"
5424         self.set_up_grid()
5425         c0 = self.g.clients[0]
5426         c0_basedir = c0.basedir
5427         fn = os.path.join(c0_basedir, "access.blacklist")
5428         self.uris = {}
5429         DATA = "off-limits " * 50
5430
5431         d = c0.upload(upload.Data(DATA, convergence=""))
5432         def _stash_uri_and_create_dir(ur):
5433             self.uri = ur.get_uri()
5434             self.url = "uri/"+self.uri
5435             u = uri.from_string_filenode(self.uri)
5436             self.si = u.get_storage_index()
5437             childnode = c0.create_node_from_uri(self.uri, None)
5438             return c0.create_dirnode({u"blacklisted.txt": (childnode,{}) })
5439         d.addCallback(_stash_uri_and_create_dir)
5440         def _stash_dir(node):
5441             self.dir_node = node
5442             self.dir_uri = node.get_uri()
5443             self.dir_url = "uri/"+self.dir_uri
5444         d.addCallback(_stash_dir)
5445         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5446         def _check_dir_html(body):
5447             self.failUnlessIn("<html>", body)
5448             self.failUnlessIn("blacklisted.txt</a>", body)
5449         d.addCallback(_check_dir_html)
5450         d.addCallback(lambda ign: self.GET(self.url))
5451         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5452
5453         def _blacklist(ign):
5454             f = open(fn, "w")
5455             f.write(" # this is a comment\n")
5456             f.write(" \n")
5457             f.write("\n") # also exercise blank lines
5458             f.write("%s %s\n" % (base32.b2a(self.si), "off-limits to you"))
5459             f.close()
5460             # clients should be checking the blacklist each time, so we don't
5461             # need to restart the client
5462         d.addCallback(_blacklist)
5463         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_uri",
5464                                                        403, "Forbidden",
5465                                                        "Access Prohibited: off-limits",
5466                                                        self.GET, self.url))
5467
5468         # We should still be able to list the parent directory, in HTML...
5469         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5470         def _check_dir_html2(body):
5471             self.failUnlessIn("<html>", body)
5472             self.failUnlessIn("blacklisted.txt</strike>", body)
5473         d.addCallback(_check_dir_html2)
5474
5475         # ... and in JSON (used by CLI).
5476         d.addCallback(lambda ign: self.GET(self.dir_url+"?t=json", followRedirect=True))
5477         def _check_dir_json(res):
5478             data = simplejson.loads(res)
5479             self.failUnless(isinstance(data, list), data)
5480             self.failUnlessEqual(data[0], "dirnode")
5481             self.failUnless(isinstance(data[1], dict), data)
5482             self.failUnlessIn("children", data[1])
5483             self.failUnlessIn("blacklisted.txt", data[1]["children"])
5484             childdata = data[1]["children"]["blacklisted.txt"]
5485             self.failUnless(isinstance(childdata, list), data)
5486             self.failUnlessEqual(childdata[0], "filenode")
5487             self.failUnless(isinstance(childdata[1], dict), data)
5488         d.addCallback(_check_dir_json)
5489
5490         def _unblacklist(ign):
5491             open(fn, "w").close()
5492             # the Blacklist object watches mtime to tell when the file has
5493             # changed, but on windows this test will run faster than the
5494             # filesystem's mtime resolution. So we edit Blacklist.last_mtime
5495             # to force a reload.
5496             self.g.clients[0].blacklist.last_mtime -= 2.0
5497         d.addCallback(_unblacklist)
5498
5499         # now a read should work
5500         d.addCallback(lambda ign: self.GET(self.url))
5501         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5502
5503         # read again to exercise the blacklist-is-unchanged logic
5504         d.addCallback(lambda ign: self.GET(self.url))
5505         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5506
5507         # now add a blacklisted directory, and make sure files under it are
5508         # refused too
5509         def _add_dir(ign):
5510             childnode = c0.create_node_from_uri(self.uri, None)
5511             return c0.create_dirnode({u"child": (childnode,{}) })
5512         d.addCallback(_add_dir)
5513         def _get_dircap(dn):
5514             self.dir_si_b32 = base32.b2a(dn.get_storage_index())
5515             self.dir_url_base = "uri/"+dn.get_write_uri()
5516             self.dir_url_json1 = "uri/"+dn.get_write_uri()+"?t=json"
5517             self.dir_url_json2 = "uri/"+dn.get_write_uri()+"/?t=json"
5518             self.dir_url_json_ro = "uri/"+dn.get_readonly_uri()+"/?t=json"
5519             self.child_url = "uri/"+dn.get_readonly_uri()+"/child"
5520         d.addCallback(_get_dircap)
5521         d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True))
5522         d.addCallback(lambda body: self.failUnlessIn("<html>", body))
5523         d.addCallback(lambda ign: self.GET(self.dir_url_json1))
5524         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5525         d.addCallback(lambda ign: self.GET(self.dir_url_json2))
5526         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5527         d.addCallback(lambda ign: self.GET(self.dir_url_json_ro))
5528         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5529         d.addCallback(lambda ign: self.GET(self.child_url))
5530         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5531
5532         def _block_dir(ign):
5533             f = open(fn, "w")
5534             f.write("%s %s\n" % (self.dir_si_b32, "dir-off-limits to you"))
5535             f.close()
5536             self.g.clients[0].blacklist.last_mtime -= 2.0
5537         d.addCallback(_block_dir)
5538         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir base",
5539                                                        403, "Forbidden",
5540                                                        "Access Prohibited: dir-off-limits",
5541                                                        self.GET, self.dir_url_base))
5542         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json1",
5543                                                        403, "Forbidden",
5544                                                        "Access Prohibited: dir-off-limits",
5545                                                        self.GET, self.dir_url_json1))
5546         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json2",
5547                                                        403, "Forbidden",
5548                                                        "Access Prohibited: dir-off-limits",
5549                                                        self.GET, self.dir_url_json2))
5550         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json_ro",
5551                                                        403, "Forbidden",
5552                                                        "Access Prohibited: dir-off-limits",
5553                                                        self.GET, self.dir_url_json_ro))
5554         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir child",
5555                                                        403, "Forbidden",
5556                                                        "Access Prohibited: dir-off-limits",
5557                                                        self.GET, self.child_url))
5558         return d
5559
5560
5561 class CompletelyUnhandledError(Exception):
5562     pass
5563 class ErrorBoom(rend.Page):
5564     def beforeRender(self, ctx):
5565         raise CompletelyUnhandledError("whoops")