src/allmydata/test/test_web.py

   1 import os.path, re, urllib, time
   2 import simplejson
   3 from StringIO import StringIO
   4
   5 from twisted.application import service
   6 from twisted.trial import unittest
   7 from twisted.internet import defer, reactor
   8 from twisted.internet.task import Clock
   9 from twisted.web import client, error, http, html
  10 from twisted.python import failure, log
  11
  12 from foolscap.api import fireEventually, flushEventualQueue
  13
  14 from nevow.util import escapeToXML
  15 from nevow import rend
  16
  17 from allmydata import interfaces, uri, webish, dirnode
  18 from allmydata.storage.shares import get_share_file
  19 from allmydata.storage_client import StorageFarmBroker, StubServer
  20 from allmydata.immutable import upload
  21 from allmydata.immutable.downloader.status import DownloadStatus
  22 from allmydata.dirnode import DirectoryNode
  23 from allmydata.nodemaker import NodeMaker
  24 from allmydata.unknown import UnknownNode
  25 from allmydata.web import status, common
  26 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  27 from allmydata.util import fileutil, base32, hashutil
  28 from allmydata.util.consumer import download_to_data
  29 from allmydata.util.netstring import split_netstring
  30 from allmydata.util.encodingutil import to_str
  31 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  32      create_chk_filenode, WebErrorMixin, ShouldFailMixin, \
  33      make_mutable_file_uri, create_mutable_filenode
  34 from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
  35 from allmydata.mutable import servermap, publish, retrieve
  36 import allmydata.test.common_util as testutil
  37 from allmydata.test.no_network import GridTestMixin
  38 from allmydata.test.common_web import HTTPClientGETFactory, \
  39      HTTPClientHEADFactory
  40 from allmydata.client import Client, SecretHolder
  41 from allmydata.introducer import IntroducerNode
  42
  43 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  44 # create a webserver that works against them
  45
  46 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  47
  48 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
  49 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
  50 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
  51
  52 FAVICON_MARKUP = '<link href="/icon.png" rel="shortcut icon" />'
  53
  54
  55 class FakeStatsProvider:
  56     def get_stats(self):
  57         stats = {'stats': {}, 'counters': {}}
  58         return stats
  59
  60 class FakeNodeMaker(NodeMaker):
  61     encoding_params = {
  62         'k': 3,
  63         'n': 10,
  64         'happy': 7,
  65         'max_segment_size':128*1024 # 1024=KiB
  66     }
  67     def _create_lit(self, cap):
  68         return FakeCHKFileNode(cap, self.all_contents)
  69     def _create_immutable(self, cap):
  70         return FakeCHKFileNode(cap, self.all_contents)
  71     def _create_mutable(self, cap):
  72         return FakeMutableFileNode(None, None,
  73                                    self.encoding_params, None,
  74                                    self.all_contents).init_from_cap(cap)
  75     def create_mutable_file(self, contents="", keysize=None,
  76                             version=SDMF_VERSION):
  77         n = FakeMutableFileNode(None, None, self.encoding_params, None,
  78                                 self.all_contents)
  79         return n.create(contents, version=version)
  80
  81 class FakeUploader(service.Service):
  82     name = "uploader"
  83     def upload(self, uploadable):
  84         d = uploadable.get_size()
  85         d.addCallback(lambda size: uploadable.read(size))
  86         def _got_data(datav):
  87             data = "".join(datav)
  88             n = create_chk_filenode(data, self.all_contents)
  89             ur = upload.UploadResults(file_size=len(data),
  90                                       ciphertext_fetched=0,
  91                                       preexisting_shares=0,
  92                                       pushed_shares=10,
  93                                       sharemap={},
  94                                       servermap={},
  95                                       timings={},
  96                                       uri_extension_data={},
  97                                       uri_extension_hash="fake",
  98                                       verifycapstr="fakevcap")
  99             ur.set_uri(n.get_uri())
 100             return ur
 101         d.addCallback(_got_data)
 102         return d
 103     def get_helper_info(self):
 104         return (None, False)
 105
 106 def build_one_ds():
 107     ds = DownloadStatus("storage_index", 1234)
 108     now = time.time()
 109
 110     serverA = StubServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
 111     serverB = StubServer(hashutil.tagged_hash("foo", "serverid_b")[:20])
 112     storage_index = hashutil.storage_index_hash("SI")
 113     e0 = ds.add_segment_request(0, now)
 114     e0.activate(now+0.5)
 115     e0.deliver(now+1, 0, 100, 0.5) # when, start,len, decodetime
 116     e1 = ds.add_segment_request(1, now+2)
 117     e1.error(now+3)
 118     # two outstanding requests
 119     e2 = ds.add_segment_request(2, now+4)
 120     e3 = ds.add_segment_request(3, now+5)
 121     del e2,e3 # hush pyflakes
 122
 123     # simulate a segment which gets delivered faster than a system clock tick (ticket #1166)
 124     e = ds.add_segment_request(4, now)
 125     e.activate(now)
 126     e.deliver(now, 0, 140, 0.5)
 127
 128     e = ds.add_dyhb_request(serverA, now)
 129     e.finished([1,2], now+1)
 130     e = ds.add_dyhb_request(serverB, now+2) # left unfinished
 131
 132     e = ds.add_read_event(0, 120, now)
 133     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 134     e.finished(now+1)
 135     e = ds.add_read_event(120, 30, now+2) # left unfinished
 136
 137     e = ds.add_block_request(serverA, 1, 100, 20, now)
 138     e.finished(20, now+1)
 139     e = ds.add_block_request(serverB, 1, 120, 30, now+1) # left unfinished
 140
 141     # make sure that add_read_event() can come first too
 142     ds1 = DownloadStatus(storage_index, 1234)
 143     e = ds1.add_read_event(0, 120, now)
 144     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 145     e.finished(now+1)
 146
 147     return ds
 148
 149 class FakeHistory:
 150     _all_upload_status = [upload.UploadStatus()]
 151     _all_download_status = [build_one_ds()]
 152     _all_mapupdate_statuses = [servermap.UpdateStatus()]
 153     _all_publish_statuses = [publish.PublishStatus()]
 154     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
 155
 156     def list_all_upload_statuses(self):
 157         return self._all_upload_status
 158     def list_all_download_statuses(self):
 159         return self._all_download_status
 160     def list_all_mapupdate_statuses(self):
 161         return self._all_mapupdate_statuses
 162     def list_all_publish_statuses(self):
 163         return self._all_publish_statuses
 164     def list_all_retrieve_statuses(self):
 165         return self._all_retrieve_statuses
 166     def list_all_helper_statuses(self):
 167         return []
 168
 169 class FakeDisplayableServer(StubServer):
 170     def __init__(self, serverid, nickname):
 171         StubServer.__init__(self, serverid)
 172         self.announcement = {"my-version": "allmydata-tahoe-fake",
 173                              "service-name": "storage",
 174                              "nickname": nickname}
 175     def is_connected(self):
 176         return True
 177     def get_permutation_seed(self):
 178         return ""
 179     def get_remote_host(self):
 180         return ""
 181     def get_last_loss_time(self):
 182         return None
 183     def get_announcement_time(self):
 184         return None
 185     def get_announcement(self):
 186         return self.announcement
 187     def get_nickname(self):
 188         return self.announcement["nickname"]
 189
 190 class FakeBucketCounter(object):
 191     def get_state(self):
 192         return {"last-complete-bucket-count": 0}
 193     def get_progress(self):
 194         return {"estimated-time-per-cycle": 0,
 195                 "cycle-in-progress": False,
 196                 "remaining-wait-time": 0}
 197
 198 class FakeLeaseChecker(object):
 199     def __init__(self):
 200         self.expiration_enabled = False
 201         self.mode = "age"
 202         self.override_lease_duration = None
 203         self.sharetypes_to_expire = {}
 204     def get_state(self):
 205         return {"history": None}
 206     def get_progress(self):
 207         return {"estimated-time-per-cycle": 0,
 208                 "cycle-in-progress": False,
 209                 "remaining-wait-time": 0}
 210
 211 class FakeStorageServer(service.MultiService):
 212     name = 'storage'
 213     def __init__(self, nodeid, nickname):
 214         service.MultiService.__init__(self)
 215         self.my_nodeid = nodeid
 216         self.nickname = nickname
 217         self.bucket_counter = FakeBucketCounter()
 218         self.lease_checker = FakeLeaseChecker()
 219     def get_stats(self):
 220         return {"storage_server.accepting_immutable_shares": False}
 221
 222 class FakeClient(Client):
 223     def __init__(self):
 224         # don't upcall to Client.__init__, since we only want to initialize a
 225         # minimal subset
 226         service.MultiService.__init__(self)
 227         self.all_contents = {}
 228         self.nodeid = "fake_nodeid"
 229         self.nickname = u"fake_nickname \u263A"
 230         self.introducer_furl = "None"
 231         self.stats_provider = FakeStatsProvider()
 232         self._secret_holder = SecretHolder("lease secret", "convergence secret")
 233         self.helper = None
 234         self.convergence = "some random string"
 235         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 236         # fake knowledge of another server
 237         self.storage_broker.test_add_server("other_nodeid",
 238                                             FakeDisplayableServer("other_nodeid", u"other_nickname \u263B"))
 239         self.introducer_client = None
 240         self.history = FakeHistory()
 241         self.uploader = FakeUploader()
 242         self.uploader.all_contents = self.all_contents
 243         self.uploader.setServiceParent(self)
 244         self.blacklist = None
 245         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 246                                        self.uploader, None,
 247                                        None, None, None)
 248         self.nodemaker.all_contents = self.all_contents
 249         self.mutable_file_default = SDMF_VERSION
 250         self.addService(FakeStorageServer(self.nodeid, self.nickname))
 251
 252     def startService(self):
 253         return service.MultiService.startService(self)
 254     def stopService(self):
 255         return service.MultiService.stopService(self)
 256
 257     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 258
 259 class WebMixin(object):
 260     def setUp(self):
 261         self.s = FakeClient()
 262         self.s.startService()
 263         self.staticdir = self.mktemp()
 264         self.clock = Clock()
 265         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
 266                                       clock=self.clock)
 267         self.ws.setServiceParent(self.s)
 268         self.webish_port = self.ws.getPortnum()
 269         self.webish_url = self.ws.getURL()
 270         assert self.webish_url.endswith("/")
 271         self.webish_url = self.webish_url[:-1] # these tests add their own /
 272
 273         l = [ self.s.create_dirnode() for x in range(6) ]
 274         d = defer.DeferredList(l)
 275         def _then(res):
 276             self.public_root = res[0][1]
 277             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 278             self.public_url = "/uri/" + self.public_root.get_uri()
 279             self.private_root = res[1][1]
 280
 281             foo = res[2][1]
 282             self._foo_node = foo
 283             self._foo_uri = foo.get_uri()
 284             self._foo_readonly_uri = foo.get_readonly_uri()
 285             self._foo_verifycap = foo.get_verify_cap().to_string()
 286             # NOTE: we ignore the deferred on all set_uri() calls, because we
 287             # know the fake nodes do these synchronously
 288             self.public_root.set_uri(u"foo", foo.get_uri(),
 289                                      foo.get_readonly_uri())
 290
 291             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 292             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 293             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 294
 295             # sdmf
 296             # XXX: Do we ever use this?
 297             self.BAZ_CONTENTS, n, self._baz_txt_uri, self._baz_txt_readonly_uri = self.makefile_mutable(0)
 298
 299             foo.set_uri(u"baz.txt", self._baz_txt_uri, self._baz_txt_readonly_uri)
 300
 301             # mdmf
 302             self.QUUX_CONTENTS, n, self._quux_txt_uri, self._quux_txt_readonly_uri = self.makefile_mutable(0, mdmf=True)
 303             assert self._quux_txt_uri.startswith("URI:MDMF")
 304             foo.set_uri(u"quux.txt", self._quux_txt_uri, self._quux_txt_readonly_uri)
 305
 306             foo.set_uri(u"empty", res[3][1].get_uri(),
 307                         res[3][1].get_readonly_uri())
 308             sub_uri = res[4][1].get_uri()
 309             self._sub_uri = sub_uri
 310             foo.set_uri(u"sub", sub_uri, sub_uri)
 311             sub = self.s.create_node_from_uri(sub_uri)
 312             self._sub_node = sub
 313
 314             _ign, n, blocking_uri = self.makefile(1)
 315             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 316
 317             # filenode to test for html encoding issues
 318             self._htmlname_unicode = u"<&weirdly'named\"file>>>_<iframe />.txt"
 319             self._htmlname_raw = self._htmlname_unicode.encode('utf-8')
 320             self._htmlname_urlencoded = urllib.quote(self._htmlname_raw, '')
 321             self._htmlname_escaped = escapeToXML(self._htmlname_raw)
 322             self._htmlname_escaped_attr = html.escape(self._htmlname_raw)
 323             self._htmlname_escaped_double = escapeToXML(html.escape(self._htmlname_raw))
 324             self.HTMLNAME_CONTENTS, n, self._htmlname_txt_uri = self.makefile(0)
 325             foo.set_uri(self._htmlname_unicode, self._htmlname_txt_uri, self._htmlname_txt_uri)
 326
 327             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 328             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 329             # still think of it as an umlaut
 330             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 331
 332             self.SUBBAZ_CONTENTS, n, baz_file = self.makefile(2)
 333             self._baz_file_uri = baz_file
 334             sub.set_uri(u"baz.txt", baz_file, baz_file)
 335
 336             _ign, n, self._bad_file_uri = self.makefile(3)
 337             # this uri should not be downloadable
 338             del self.s.all_contents[self._bad_file_uri]
 339
 340             rodir = res[5][1]
 341             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 342                                      rodir.get_readonly_uri())
 343             rodir.set_uri(u"nor", baz_file, baz_file)
 344
 345             # public/
 346             # public/foo/
 347             # public/foo/bar.txt
 348             # public/foo/baz.txt
 349             # public/foo/quux.txt
 350             # public/foo/blockingfile
 351             # public/foo/<&weirdly'named\"file>>>_<iframe />.txt
 352             # public/foo/empty/
 353             # public/foo/sub/
 354             # public/foo/sub/baz.txt
 355             # public/reedownlee/
 356             # public/reedownlee/nor
 357             self.NEWFILE_CONTENTS = "newfile contents\n"
 358
 359             return foo.get_metadata_for(u"bar.txt")
 360         d.addCallback(_then)
 361         def _got_metadata(metadata):
 362             self._bar_txt_metadata = metadata
 363         d.addCallback(_got_metadata)
 364         return d
 365
 366     def get_all_contents(self):
 367         return self.s.all_contents
 368
 369     def makefile(self, number):
 370         contents = "contents of file %s\n" % number
 371         n = create_chk_filenode(contents, self.get_all_contents())
 372         return contents, n, n.get_uri()
 373
 374     def makefile_mutable(self, number, mdmf=False):
 375         contents = "contents of mutable file %s\n" % number
 376         n = create_mutable_filenode(contents, mdmf, self.s.all_contents)
 377         return contents, n, n.get_uri(), n.get_readonly_uri()
 378
 379     def tearDown(self):
 380         return self.s.stopService()
 381
 382     def failUnlessIsBarDotTxt(self, res):
 383         self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
 384
 385     def failUnlessIsQuuxDotTxt(self, res):
 386         self.failUnlessReallyEqual(res, self.QUUX_CONTENTS, res)
 387
 388     def failUnlessIsBazDotTxt(self, res):
 389         self.failUnlessReallyEqual(res, self.BAZ_CONTENTS, res)
 390
 391     def failUnlessIsSubBazDotTxt(self, res):
 392         self.failUnlessReallyEqual(res, self.SUBBAZ_CONTENTS, res)
 393
 394     def failUnlessIsBarJSON(self, res):
 395         data = simplejson.loads(res)
 396         self.failUnless(isinstance(data, list))
 397         self.failUnlessEqual(data[0], "filenode")
 398         self.failUnless(isinstance(data[1], dict))
 399         self.failIf(data[1]["mutable"])
 400         self.failIfIn("rw_uri", data[1]) # immutable
 401         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
 402         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
 403         self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
 404
 405     def failUnlessIsQuuxJSON(self, res, readonly=False):
 406         data = simplejson.loads(res)
 407         self.failUnless(isinstance(data, list))
 408         self.failUnlessEqual(data[0], "filenode")
 409         self.failUnless(isinstance(data[1], dict))
 410         metadata = data[1]
 411         return self.failUnlessIsQuuxDotTxtMetadata(metadata, readonly)
 412
 413     def failUnlessIsQuuxDotTxtMetadata(self, metadata, readonly):
 414         self.failUnless(metadata['mutable'])
 415         if readonly:
 416             self.failIfIn("rw_uri", metadata)
 417         else:
 418             self.failUnlessIn("rw_uri", metadata)
 419             self.failUnlessEqual(metadata['rw_uri'], self._quux_txt_uri)
 420         self.failUnlessIn("ro_uri", metadata)
 421         self.failUnlessEqual(metadata['ro_uri'], self._quux_txt_readonly_uri)
 422         self.failUnlessReallyEqual(metadata['size'], len(self.QUUX_CONTENTS))
 423
 424     def failUnlessIsFooJSON(self, res):
 425         data = simplejson.loads(res)
 426         self.failUnless(isinstance(data, list))
 427         self.failUnlessEqual(data[0], "dirnode", res)
 428         self.failUnless(isinstance(data[1], dict))
 429         self.failUnless(data[1]["mutable"])
 430         self.failUnlessIn("rw_uri", data[1]) # mutable
 431         self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
 432         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
 433         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
 434
 435         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 436         self.failUnlessEqual(kidnames,
 437                              [self._htmlname_unicode, u"bar.txt", u"baz.txt",
 438                               u"blockingfile", u"empty", u"n\u00fc.txt", u"quux.txt", u"sub"])
 439         kids = dict( [(unicode(name),value)
 440                       for (name,value)
 441                       in data[1]["children"].iteritems()] )
 442         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 443         self.failUnlessIn("metadata", kids[u"sub"][1])
 444         self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
 445         tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
 446         self.failUnlessIn("linkcrtime", tahoe_md)
 447         self.failUnlessIn("linkmotime", tahoe_md)
 448         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 449         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 450         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri)
 451         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]),
 452                                    self._bar_txt_verifycap)
 453         self.failUnlessIn("metadata", kids[u"bar.txt"][1])
 454         self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
 455         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
 456                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
 457         self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
 458                                    self._bar_txt_uri)
 459         self.failUnlessIn("quux.txt", kids)
 460         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["rw_uri"]),
 461                                    self._quux_txt_uri)
 462         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["ro_uri"]),
 463                                    self._quux_txt_readonly_uri)
 464
 465     def GET(self, urlpath, followRedirect=False, return_response=False,
 466             **kwargs):
 467         # if return_response=True, this fires with (data, statuscode,
 468         # respheaders) instead of just data.
 469         assert not isinstance(urlpath, unicode)
 470         url = self.webish_url + urlpath
 471         factory = HTTPClientGETFactory(url, method="GET",
 472                                        followRedirect=followRedirect, **kwargs)
 473         reactor.connectTCP("localhost", self.webish_port, factory)
 474         d = factory.deferred
 475         def _got_data(data):
 476             return (data, factory.status, factory.response_headers)
 477         if return_response:
 478             d.addCallback(_got_data)
 479         return factory.deferred
 480
 481     def HEAD(self, urlpath, return_response=False, **kwargs):
 482         # this requires some surgery, because twisted.web.client doesn't want
 483         # to give us back the response headers.
 484         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 485         reactor.connectTCP("localhost", self.webish_port, factory)
 486         d = factory.deferred
 487         def _got_data(data):
 488             return (data, factory.status, factory.response_headers)
 489         if return_response:
 490             d.addCallback(_got_data)
 491         return factory.deferred
 492
 493     def PUT(self, urlpath, data, **kwargs):
 494         url = self.webish_url + urlpath
 495         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 496
 497     def DELETE(self, urlpath):
 498         url = self.webish_url + urlpath
 499         return client.getPage(url, method="DELETE")
 500
 501     def POST(self, urlpath, followRedirect=False, **fields):
 502         sepbase = "boogabooga"
 503         sep = "--" + sepbase
 504         form = []
 505         form.append(sep)
 506         form.append('Content-Disposition: form-data; name="_charset"')
 507         form.append('')
 508         form.append('UTF-8')
 509         form.append(sep)
 510         for name, value in fields.iteritems():
 511             if isinstance(value, tuple):
 512                 filename, value = value
 513                 form.append('Content-Disposition: form-data; name="%s"; '
 514                             'filename="%s"' % (name, filename.encode("utf-8")))
 515             else:
 516                 form.append('Content-Disposition: form-data; name="%s"' % name)
 517             form.append('')
 518             if isinstance(value, unicode):
 519                 value = value.encode("utf-8")
 520             else:
 521                 value = str(value)
 522             assert isinstance(value, str)
 523             form.append(value)
 524             form.append(sep)
 525         form[-1] += "--"
 526         body = ""
 527         headers = {}
 528         if fields:
 529             body = "\r\n".join(form) + "\r\n"
 530             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 531         return self.POST2(urlpath, body, headers, followRedirect)
 532
 533     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 534         url = self.webish_url + urlpath
 535         return client.getPage(url, method="POST", postdata=body,
 536                               headers=headers, followRedirect=followRedirect)
 537
 538     def shouldFail(self, res, expected_failure, which,
 539                    substring=None, response_substring=None):
 540         if isinstance(res, failure.Failure):
 541             res.trap(expected_failure)
 542             if substring:
 543                 self.failUnlessIn(substring, str(res), which)
 544             if response_substring:
 545                 self.failUnlessIn(response_substring, res.value.response, which)
 546         else:
 547             self.fail("%s was supposed to raise %s, not get '%s'" %
 548                       (which, expected_failure, res))
 549
 550     def shouldFail2(self, expected_failure, which, substring,
 551                     response_substring,
 552                     callable, *args, **kwargs):
 553         assert substring is None or isinstance(substring, str)
 554         assert response_substring is None or isinstance(response_substring, str)
 555         d = defer.maybeDeferred(callable, *args, **kwargs)
 556         def done(res):
 557             if isinstance(res, failure.Failure):
 558                 res.trap(expected_failure)
 559                 if substring:
 560                     self.failUnlessIn(substring, str(res),
 561                                       "'%s' not in '%s' for test '%s'" % \
 562                                       (substring, str(res), which))
 563                 if response_substring:
 564                     self.failUnlessIn(response_substring, res.value.response,
 565                                       "'%s' not in '%s' for test '%s'" % \
 566                                       (response_substring, res.value.response,
 567                                        which))
 568             else:
 569                 self.fail("%s was supposed to raise %s, not get '%s'" %
 570                           (which, expected_failure, res))
 571         d.addBoth(done)
 572         return d
 573
 574     def should404(self, res, which):
 575         if isinstance(res, failure.Failure):
 576             res.trap(error.Error)
 577             self.failUnlessReallyEqual(res.value.status, "404")
 578         else:
 579             self.fail("%s was supposed to Error(404), not get '%s'" %
 580                       (which, res))
 581
 582     def should302(self, res, which):
 583         if isinstance(res, failure.Failure):
 584             res.trap(error.Error)
 585             self.failUnlessReallyEqual(res.value.status, "302")
 586         else:
 587             self.fail("%s was supposed to Error(302), not get '%s'" %
 588                       (which, res))
 589
 590
 591 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
 592     def test_create(self):
 593         pass
 594
 595     def test_welcome(self):
 596         d = self.GET("/")
 597         def _check(res):
 598             self.failUnlessIn('Welcome to Tahoe-LAFS', res)
 599             self.failUnlessIn(FAVICON_MARKUP, res)
 600             self.failUnlessIn('href="https://tahoe-lafs.org/"', res)
 601             res_u = res.decode('utf-8')
 602             self.failUnlessIn(u'<th>My nickname:</th> <td class="nickname mine">fake_nickname \u263A</td></tr>', res_u)
 603             self.failUnlessIn(u'<div class="nickname">other_nickname \u263B</div>', res_u)
 604
 605             self.s.basedir = 'web/test_welcome'
 606             fileutil.make_dirs("web/test_welcome")
 607             fileutil.make_dirs("web/test_welcome/private")
 608             return self.GET("/")
 609         d.addCallback(_check)
 610         return d
 611
 612     def test_storage(self):
 613         d = self.GET("/storage")
 614         def _check(res):
 615             self.failUnlessIn('Storage Server Status', res)
 616             self.failUnlessIn(FAVICON_MARKUP, res)
 617             res_u = res.decode('utf-8')
 618             self.failUnlessIn(u'<li>Server Nickname: <span class="nickname mine">fake_nickname \u263A</span></li>', res_u)
 619         d.addCallback(_check)
 620         return d
 621
 622     def test_status(self):
 623         h = self.s.get_history()
 624         dl_num = h.list_all_download_statuses()[0].get_counter()
 625         ul_num = h.list_all_upload_statuses()[0].get_counter()
 626         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 627         pub_num = h.list_all_publish_statuses()[0].get_counter()
 628         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 629         d = self.GET("/status", followRedirect=True)
 630         def _check(res):
 631             self.failUnlessIn('Upload and Download Status', res)
 632             self.failUnlessIn('"down-%d"' % dl_num, res)
 633             self.failUnlessIn('"up-%d"' % ul_num, res)
 634             self.failUnlessIn('"mapupdate-%d"' % mu_num, res)
 635             self.failUnlessIn('"publish-%d"' % pub_num, res)
 636             self.failUnlessIn('"retrieve-%d"' % ret_num, res)
 637         d.addCallback(_check)
 638         d.addCallback(lambda res: self.GET("/status/?t=json"))
 639         def _check_json(res):
 640             data = simplejson.loads(res)
 641             self.failUnless(isinstance(data, dict))
 642             #active = data["active"]
 643             # TODO: test more. We need a way to fake an active operation
 644             # here.
 645         d.addCallback(_check_json)
 646
 647         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 648         def _check_dl(res):
 649             self.failUnlessIn("File Download Status", res)
 650         d.addCallback(_check_dl)
 651         d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
 652         def _check_dl_json(res):
 653             data = simplejson.loads(res)
 654             self.failUnless(isinstance(data, dict))
 655             self.failUnlessIn("read", data)
 656             self.failUnlessEqual(data["read"][0]["length"], 120)
 657             self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
 658             self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
 659             self.failUnlessEqual(data["segment"][2]["finish_time"], None)
 660             phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
 661             cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
 662             # serverids[] keys are strings, since that's what JSON does, but
 663             # we'd really like them to be ints
 664             self.failUnlessEqual(data["serverids"]["0"], "phwrsjte")
 665             self.failUnless(data["serverids"].has_key("1"),
 666                             str(data["serverids"]))
 667             self.failUnlessEqual(data["serverids"]["1"], "cmpuvkjm",
 668                                  str(data["serverids"]))
 669             self.failUnlessEqual(data["server_info"][phwr_id]["short"],
 670                                  "phwrsjte")
 671             self.failUnlessEqual(data["server_info"][cmpu_id]["short"],
 672                                  "cmpuvkjm")
 673             self.failUnlessIn("dyhb", data)
 674             self.failUnlessIn("misc", data)
 675         d.addCallback(_check_dl_json)
 676         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 677         def _check_ul(res):
 678             self.failUnlessIn("File Upload Status", res)
 679         d.addCallback(_check_ul)
 680         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 681         def _check_mapupdate(res):
 682             self.failUnlessIn("Mutable File Servermap Update Status", res)
 683         d.addCallback(_check_mapupdate)
 684         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 685         def _check_publish(res):
 686             self.failUnlessIn("Mutable File Publish Status", res)
 687         d.addCallback(_check_publish)
 688         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 689         def _check_retrieve(res):
 690             self.failUnlessIn("Mutable File Retrieve Status", res)
 691         d.addCallback(_check_retrieve)
 692
 693         return d
 694
 695     def test_status_numbers(self):
 696         drrm = status.DownloadResultsRendererMixin()
 697         self.failUnlessReallyEqual(drrm.render_time(None, None), "")
 698         self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
 699         self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
 700         self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
 701         self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
 702         self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
 703         self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 704         self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
 705         self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
 706
 707         urrm = status.UploadResultsRendererMixin()
 708         self.failUnlessReallyEqual(urrm.render_time(None, None), "")
 709         self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
 710         self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
 711         self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
 712         self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
 713         self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
 714         self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 715         self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
 716         self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
 717
 718     def test_GET_FILEURL(self):
 719         d = self.GET(self.public_url + "/foo/bar.txt")
 720         d.addCallback(self.failUnlessIsBarDotTxt)
 721         return d
 722
 723     def test_GET_FILEURL_range(self):
 724         headers = {"range": "bytes=1-10"}
 725         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 726                      return_response=True)
 727         def _got((res, status, headers)):
 728             self.failUnlessReallyEqual(int(status), 206)
 729             self.failUnless(headers.has_key("content-range"))
 730             self.failUnlessReallyEqual(headers["content-range"][0],
 731                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 732             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
 733         d.addCallback(_got)
 734         return d
 735
 736     def test_GET_FILEURL_partial_range(self):
 737         headers = {"range": "bytes=5-"}
 738         length  = len(self.BAR_CONTENTS)
 739         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 740                      return_response=True)
 741         def _got((res, status, headers)):
 742             self.failUnlessReallyEqual(int(status), 206)
 743             self.failUnless(headers.has_key("content-range"))
 744             self.failUnlessReallyEqual(headers["content-range"][0],
 745                                        "bytes 5-%d/%d" % (length-1, length))
 746             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
 747         d.addCallback(_got)
 748         return d
 749
 750     def test_GET_FILEURL_partial_end_range(self):
 751         headers = {"range": "bytes=-5"}
 752         length  = len(self.BAR_CONTENTS)
 753         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 754                      return_response=True)
 755         def _got((res, status, headers)):
 756             self.failUnlessReallyEqual(int(status), 206)
 757             self.failUnless(headers.has_key("content-range"))
 758             self.failUnlessReallyEqual(headers["content-range"][0],
 759                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 760             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
 761         d.addCallback(_got)
 762         return d
 763
 764     def test_GET_FILEURL_partial_range_overrun(self):
 765         headers = {"range": "bytes=100-200"}
 766         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
 767                              "416 Requested Range not satisfiable",
 768                              "First beyond end of file",
 769                              self.GET, self.public_url + "/foo/bar.txt",
 770                              headers=headers)
 771         return d
 772
 773     def test_HEAD_FILEURL_range(self):
 774         headers = {"range": "bytes=1-10"}
 775         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 776                      return_response=True)
 777         def _got((res, status, headers)):
 778             self.failUnlessReallyEqual(res, "")
 779             self.failUnlessReallyEqual(int(status), 206)
 780             self.failUnless(headers.has_key("content-range"))
 781             self.failUnlessReallyEqual(headers["content-range"][0],
 782                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 783         d.addCallback(_got)
 784         return d
 785
 786     def test_HEAD_FILEURL_partial_range(self):
 787         headers = {"range": "bytes=5-"}
 788         length  = len(self.BAR_CONTENTS)
 789         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 790                      return_response=True)
 791         def _got((res, status, headers)):
 792             self.failUnlessReallyEqual(int(status), 206)
 793             self.failUnless(headers.has_key("content-range"))
 794             self.failUnlessReallyEqual(headers["content-range"][0],
 795                                        "bytes 5-%d/%d" % (length-1, length))
 796         d.addCallback(_got)
 797         return d
 798
 799     def test_HEAD_FILEURL_partial_end_range(self):
 800         headers = {"range": "bytes=-5"}
 801         length  = len(self.BAR_CONTENTS)
 802         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 803                      return_response=True)
 804         def _got((res, status, headers)):
 805             self.failUnlessReallyEqual(int(status), 206)
 806             self.failUnless(headers.has_key("content-range"))
 807             self.failUnlessReallyEqual(headers["content-range"][0],
 808                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 809         d.addCallback(_got)
 810         return d
 811
 812     def test_HEAD_FILEURL_partial_range_overrun(self):
 813         headers = {"range": "bytes=100-200"}
 814         d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
 815                              "416 Requested Range not satisfiable",
 816                              "",
 817                              self.HEAD, self.public_url + "/foo/bar.txt",
 818                              headers=headers)
 819         return d
 820
 821     def test_GET_FILEURL_range_bad(self):
 822         headers = {"range": "BOGUS=fizbop-quarnak"}
 823         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 824                      return_response=True)
 825         def _got((res, status, headers)):
 826             self.failUnlessReallyEqual(int(status), 200)
 827             self.failUnless(not headers.has_key("content-range"))
 828             self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
 829         d.addCallback(_got)
 830         return d
 831
 832     def test_HEAD_FILEURL(self):
 833         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 834         def _got((res, status, headers)):
 835             self.failUnlessReallyEqual(res, "")
 836             self.failUnlessReallyEqual(headers["content-length"][0],
 837                                        str(len(self.BAR_CONTENTS)))
 838             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
 839         d.addCallback(_got)
 840         return d
 841
 842     def test_GET_FILEURL_named(self):
 843         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 844         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 845         d = self.GET(base + "/@@name=/blah.txt")
 846         d.addCallback(self.failUnlessIsBarDotTxt)
 847         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 848         d.addCallback(self.failUnlessIsBarDotTxt)
 849         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 850         d.addCallback(self.failUnlessIsBarDotTxt)
 851         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 852         d.addCallback(self.failUnlessIsBarDotTxt)
 853         save_url = base + "?save=true&filename=blah.txt"
 854         d.addCallback(lambda res: self.GET(save_url))
 855         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 856         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 857         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 858         u_url = base + "?save=true&filename=" + u_fn_e
 859         d.addCallback(lambda res: self.GET(u_url))
 860         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 861         return d
 862
 863     def test_PUT_FILEURL_named_bad(self):
 864         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 865         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 866                              "400 Bad Request",
 867                              "/file can only be used with GET or HEAD",
 868                              self.PUT, base + "/@@name=/blah.txt", "")
 869         return d
 870
 871
 872     def test_GET_DIRURL_named_bad(self):
 873         base = "/file/%s" % urllib.quote(self._foo_uri)
 874         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 875                              "400 Bad Request",
 876                              "is not a file-cap",
 877                              self.GET, base + "/@@name=/blah.txt")
 878         return d
 879
 880     def test_GET_slash_file_bad(self):
 881         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 882                              "404 Not Found",
 883                              "/file must be followed by a file-cap and a name",
 884                              self.GET, "/file")
 885         return d
 886
 887     def test_GET_unhandled_URI_named(self):
 888         contents, n, newuri = self.makefile(12)
 889         verifier_cap = n.get_verify_cap().to_string()
 890         base = "/file/%s" % urllib.quote(verifier_cap)
 891         # client.create_node_from_uri() can't handle verify-caps
 892         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 893                              "400 Bad Request", "is not a file-cap",
 894                              self.GET, base)
 895         return d
 896
 897     def test_GET_unhandled_URI(self):
 898         contents, n, newuri = self.makefile(12)
 899         verifier_cap = n.get_verify_cap().to_string()
 900         base = "/uri/%s" % urllib.quote(verifier_cap)
 901         # client.create_node_from_uri() can't handle verify-caps
 902         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 903                              "400 Bad Request",
 904                              "GET unknown URI type: can only do t=info",
 905                              self.GET, base)
 906         return d
 907
 908     def test_GET_FILE_URI(self):
 909         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 910         d = self.GET(base)
 911         d.addCallback(self.failUnlessIsBarDotTxt)
 912         return d
 913
 914     def test_GET_FILE_URI_mdmf(self):
 915         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 916         d = self.GET(base)
 917         d.addCallback(self.failUnlessIsQuuxDotTxt)
 918         return d
 919
 920     def test_GET_FILE_URI_mdmf_extensions(self):
 921         base = "/uri/%s" % urllib.quote("%s:RANDOMSTUFF" % self._quux_txt_uri)
 922         d = self.GET(base)
 923         d.addCallback(self.failUnlessIsQuuxDotTxt)
 924         return d
 925
 926     def test_GET_FILE_URI_mdmf_readonly(self):
 927         base = "/uri/%s" % urllib.quote(self._quux_txt_readonly_uri)
 928         d = self.GET(base)
 929         d.addCallback(self.failUnlessIsQuuxDotTxt)
 930         return d
 931
 932     def test_GET_FILE_URI_badchild(self):
 933         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 934         errmsg = "Files have no children, certainly not named 'boguschild'"
 935         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 936                              "400 Bad Request", errmsg,
 937                              self.GET, base)
 938         return d
 939
 940     def test_PUT_FILE_URI_badchild(self):
 941         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 942         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 943         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 944                              "400 Bad Request", errmsg,
 945                              self.PUT, base, "")
 946         return d
 947
 948     def test_PUT_FILE_URI_mdmf(self):
 949         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 950         self._quux_new_contents = "new_contents"
 951         d = self.GET(base)
 952         d.addCallback(lambda res:
 953             self.failUnlessIsQuuxDotTxt(res))
 954         d.addCallback(lambda ignored:
 955             self.PUT(base, self._quux_new_contents))
 956         d.addCallback(lambda ignored:
 957             self.GET(base))
 958         d.addCallback(lambda res:
 959             self.failUnlessReallyEqual(res, self._quux_new_contents))
 960         return d
 961
 962     def test_PUT_FILE_URI_mdmf_extensions(self):
 963         base = "/uri/%s" % urllib.quote("%s:EXTENSIONSTUFF" % self._quux_txt_uri)
 964         self._quux_new_contents = "new_contents"
 965         d = self.GET(base)
 966         d.addCallback(lambda res: self.failUnlessIsQuuxDotTxt(res))
 967         d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
 968         d.addCallback(lambda ignored: self.GET(base))
 969         d.addCallback(lambda res: self.failUnlessEqual(self._quux_new_contents,
 970                                                        res))
 971         return d
 972
 973     def test_PUT_FILE_URI_mdmf_readonly(self):
 974         # We're not allowed to PUT things to a readonly cap.
 975         base = "/uri/%s" % self._quux_txt_readonly_uri
 976         d = self.GET(base)
 977         d.addCallback(lambda res:
 978             self.failUnlessIsQuuxDotTxt(res))
 979         # What should we get here? We get a 500 error now; that's not right.
 980         d.addCallback(lambda ignored:
 981             self.shouldFail2(error.Error, "test_PUT_FILE_URI_mdmf_readonly",
 982                              "400 Bad Request", "read-only cap",
 983                              self.PUT, base, "new data"))
 984         return d
 985
 986     def test_PUT_FILE_URI_sdmf_readonly(self):
 987         # We're not allowed to put things to a readonly cap.
 988         base = "/uri/%s" % self._baz_txt_readonly_uri
 989         d = self.GET(base)
 990         d.addCallback(lambda res:
 991             self.failUnlessIsBazDotTxt(res))
 992         d.addCallback(lambda ignored:
 993             self.shouldFail2(error.Error, "test_PUT_FILE_URI_sdmf_readonly",
 994                              "400 Bad Request", "read-only cap",
 995                              self.PUT, base, "new_data"))
 996         return d
 997
 998     def test_GET_etags(self):
 999
1000         def _check_etags(uri):
1001             d1 = _get_etag(uri)
1002             d2 = _get_etag(uri, 'json')
1003             d = defer.DeferredList([d1, d2], consumeErrors=True)
1004             def _check(results):
1005                 # All deferred must succeed
1006                 self.failUnless(all([r[0] for r in results]))
1007                 # the etag for the t=json form should be just like the etag
1008                 # fo the default t='' form, but with a 'json' suffix
1009                 self.failUnlessEqual(results[0][1] + 'json', results[1][1])
1010             d.addCallback(_check)
1011             return d
1012
1013         def _get_etag(uri, t=''):
1014             targetbase = "/uri/%s?t=%s" % (urllib.quote(uri.strip()), t)
1015             d = self.GET(targetbase, return_response=True, followRedirect=True)
1016             def _just_the_etag(result):
1017                 data, response, headers = result
1018                 etag = headers['etag'][0]
1019                 if uri.startswith('URI:DIR'):
1020                     self.failUnless(etag.startswith('DIR:'), etag)
1021                 return etag
1022             return d.addCallback(_just_the_etag)
1023
1024         # Check that etags work with immutable directories
1025         (newkids, caps) = self._create_immutable_children()
1026         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1027                       simplejson.dumps(newkids))
1028         def _stash_immdir_uri(uri):
1029             self._immdir_uri = uri
1030             return uri
1031         d.addCallback(_stash_immdir_uri)
1032         d.addCallback(_check_etags)
1033
1034         # Check that etags work with immutable files
1035         d.addCallback(lambda _: _check_etags(self._bar_txt_uri))
1036
1037         # use the ETag on GET
1038         def _check_match(ign):
1039             uri = "/uri/%s" % self._bar_txt_uri
1040             d = self.GET(uri, return_response=True)
1041             # extract the ETag
1042             d.addCallback(lambda (data, code, headers):
1043                           headers['etag'][0])
1044             # do a GET that's supposed to match the ETag
1045             d.addCallback(lambda etag:
1046                           self.GET(uri, return_response=True,
1047                                    headers={"If-None-Match": etag}))
1048             # make sure it short-circuited (304 instead of 200)
1049             d.addCallback(lambda (data, code, headers):
1050                           self.failUnlessEqual(int(code), http.NOT_MODIFIED))
1051             return d
1052         d.addCallback(_check_match)
1053
1054         def _no_etag(uri, t):
1055             target = "/uri/%s?t=%s" % (uri, t)
1056             d = self.GET(target, return_response=True, followRedirect=True)
1057             d.addCallback(lambda (data, code, headers):
1058                           self.failIf("etag" in headers, target))
1059             return d
1060         def _yes_etag(uri, t):
1061             target = "/uri/%s?t=%s" % (uri, t)
1062             d = self.GET(target, return_response=True, followRedirect=True)
1063             d.addCallback(lambda (data, code, headers):
1064                           self.failUnless("etag" in headers, target))
1065             return d
1066
1067         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, ""))
1068         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "json"))
1069         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "uri"))
1070         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "readonly-uri"))
1071         d.addCallback(lambda ign: _no_etag(self._bar_txt_uri, "info"))
1072
1073         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, ""))
1074         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "json"))
1075         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "uri"))
1076         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "readonly-uri"))
1077         d.addCallback(lambda ign: _no_etag(self._immdir_uri, "info"))
1078         d.addCallback(lambda ign: _no_etag(self._immdir_uri, "rename-form"))
1079
1080         return d
1081
1082     # TODO: version of this with a Unicode filename
1083     def test_GET_FILEURL_save(self):
1084         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
1085                      return_response=True)
1086         def _got((res, statuscode, headers)):
1087             content_disposition = headers["content-disposition"][0]
1088             self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
1089             self.failUnlessIsBarDotTxt(res)
1090         d.addCallback(_got)
1091         return d
1092
1093     def test_GET_FILEURL_missing(self):
1094         d = self.GET(self.public_url + "/foo/missing")
1095         d.addBoth(self.should404, "test_GET_FILEURL_missing")
1096         return d
1097
1098     def test_GET_FILEURL_info_mdmf(self):
1099         d = self.GET("/uri/%s?t=info" % self._quux_txt_uri)
1100         def _got(res):
1101             self.failUnlessIn("mutable file (mdmf)", res)
1102             self.failUnlessIn(self._quux_txt_uri, res)
1103             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1104         d.addCallback(_got)
1105         return d
1106
1107     def test_GET_FILEURL_info_mdmf_readonly(self):
1108         d = self.GET("/uri/%s?t=info" % self._quux_txt_readonly_uri)
1109         def _got(res):
1110             self.failUnlessIn("mutable file (mdmf)", res)
1111             self.failIfIn(self._quux_txt_uri, res)
1112             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1113         d.addCallback(_got)
1114         return d
1115
1116     def test_GET_FILEURL_info_sdmf(self):
1117         d = self.GET("/uri/%s?t=info" % self._baz_txt_uri)
1118         def _got(res):
1119             self.failUnlessIn("mutable file (sdmf)", res)
1120             self.failUnlessIn(self._baz_txt_uri, res)
1121         d.addCallback(_got)
1122         return d
1123
1124     def test_GET_FILEURL_info_mdmf_extensions(self):
1125         d = self.GET("/uri/%s:STUFF?t=info" % self._quux_txt_uri)
1126         def _got(res):
1127             self.failUnlessIn("mutable file (mdmf)", res)
1128             self.failUnlessIn(self._quux_txt_uri, res)
1129             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1130         d.addCallback(_got)
1131         return d
1132
1133     def test_PUT_overwrite_only_files(self):
1134         # create a directory, put a file in that directory.
1135         contents, n, filecap = self.makefile(8)
1136         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
1137         d.addCallback(lambda res:
1138             self.PUT(self.public_url + "/foo/dir/file1.txt",
1139                      self.NEWFILE_CONTENTS))
1140         # try to overwrite the file with replace=only-files
1141         # (this should work)
1142         d.addCallback(lambda res:
1143             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
1144                      filecap))
1145         d.addCallback(lambda res:
1146             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
1147                  "There was already a child by that name, and you asked me "
1148                  "to not replace it",
1149                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
1150                  filecap))
1151         return d
1152
1153     def test_PUT_NEWFILEURL(self):
1154         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
1155         # TODO: we lose the response code, so we can't check this
1156         #self.failUnlessReallyEqual(responsecode, 201)
1157         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1158         d.addCallback(lambda res:
1159                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1160                                                       self.NEWFILE_CONTENTS))
1161         return d
1162
1163     def test_PUT_NEWFILEURL_not_mutable(self):
1164         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
1165                      self.NEWFILE_CONTENTS)
1166         # TODO: we lose the response code, so we can't check this
1167         #self.failUnlessReallyEqual(responsecode, 201)
1168         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1169         d.addCallback(lambda res:
1170                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1171                                                       self.NEWFILE_CONTENTS))
1172         return d
1173
1174     def test_PUT_NEWFILEURL_unlinked_mdmf(self):
1175         # this should get us a few segments of an MDMF mutable file,
1176         # which we can then test for.
1177         contents = self.NEWFILE_CONTENTS * 300000
1178         d = self.PUT("/uri?format=mdmf",
1179                      contents)
1180         def _got_filecap(filecap):
1181             self.failUnless(filecap.startswith("URI:MDMF"))
1182             return filecap
1183         d.addCallback(_got_filecap)
1184         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1185         d.addCallback(lambda json: self.failUnlessIn("MDMF", json))
1186         return d
1187
1188     def test_PUT_NEWFILEURL_unlinked_sdmf(self):
1189         contents = self.NEWFILE_CONTENTS * 300000
1190         d = self.PUT("/uri?format=sdmf",
1191                      contents)
1192         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1193         d.addCallback(lambda json: self.failUnlessIn("SDMF", json))
1194         return d
1195
1196     def test_PUT_NEWFILEURL_unlinked_bad_format(self):
1197         contents = self.NEWFILE_CONTENTS * 300000
1198         return self.shouldHTTPError("PUT_NEWFILEURL_unlinked_bad_format",
1199                                     400, "Bad Request", "Unknown format: foo",
1200                                     self.PUT, "/uri?format=foo",
1201                                     contents)
1202
1203     def test_PUT_NEWFILEURL_range_bad(self):
1204         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
1205         target = self.public_url + "/foo/new.txt"
1206         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
1207                              "501 Not Implemented",
1208                              "Content-Range in PUT not yet supported",
1209                              # (and certainly not for immutable files)
1210                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
1211                              headers=headers)
1212         d.addCallback(lambda res:
1213                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
1214         return d
1215
1216     def test_PUT_NEWFILEURL_mutable(self):
1217         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1218                      self.NEWFILE_CONTENTS)
1219         # TODO: we lose the response code, so we can't check this
1220         #self.failUnlessReallyEqual(responsecode, 201)
1221         def _check_uri(res):
1222             u = uri.from_string_mutable_filenode(res)
1223             self.failUnless(u.is_mutable())
1224             self.failIf(u.is_readonly())
1225             return res
1226         d.addCallback(_check_uri)
1227         d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
1228         d.addCallback(lambda res:
1229                       self.failUnlessMutableChildContentsAre(self._foo_node,
1230                                                              u"new.txt",
1231                                                              self.NEWFILE_CONTENTS))
1232         return d
1233
1234     def test_PUT_NEWFILEURL_mutable_toobig(self):
1235         # It is okay to upload large mutable files, so we should be able
1236         # to do that.
1237         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1238                      "b" * (self.s.MUTABLE_SIZELIMIT + 1))
1239         return d
1240
1241     def test_PUT_NEWFILEURL_replace(self):
1242         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
1243         # TODO: we lose the response code, so we can't check this
1244         #self.failUnlessReallyEqual(responsecode, 200)
1245         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
1246         d.addCallback(lambda res:
1247                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1248                                                       self.NEWFILE_CONTENTS))
1249         return d
1250
1251     def test_PUT_NEWFILEURL_bad_t(self):
1252         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
1253                              "PUT to a file: bad t=bogus",
1254                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
1255                              "contents")
1256         return d
1257
1258     def test_PUT_NEWFILEURL_no_replace(self):
1259         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
1260                      self.NEWFILE_CONTENTS)
1261         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
1262                   "409 Conflict",
1263                   "There was already a child by that name, and you asked me "
1264                   "to not replace it")
1265         return d
1266
1267     def test_PUT_NEWFILEURL_mkdirs(self):
1268         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
1269         fn = self._foo_node
1270         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
1271         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
1272         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
1273         d.addCallback(lambda res:
1274                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
1275                                                       self.NEWFILE_CONTENTS))
1276         return d
1277
1278     def test_PUT_NEWFILEURL_blocked(self):
1279         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
1280                      self.NEWFILE_CONTENTS)
1281         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
1282                   "409 Conflict",
1283                   "Unable to create directory 'blockingfile': a file was in the way")
1284         return d
1285
1286     def test_PUT_NEWFILEURL_emptyname(self):
1287         # an empty pathname component (i.e. a double-slash) is disallowed
1288         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
1289                              "400 Bad Request",
1290                              "The webapi does not allow empty pathname components",
1291                              self.PUT, self.public_url + "/foo//new.txt", "")
1292         return d
1293
1294     def test_DELETE_FILEURL(self):
1295         d = self.DELETE(self.public_url + "/foo/bar.txt")
1296         d.addCallback(lambda res:
1297                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1298         return d
1299
1300     def test_DELETE_FILEURL_missing(self):
1301         d = self.DELETE(self.public_url + "/foo/missing")
1302         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
1303         return d
1304
1305     def test_DELETE_FILEURL_missing2(self):
1306         d = self.DELETE(self.public_url + "/missing/missing")
1307         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
1308         return d
1309
1310     def failUnlessHasBarDotTxtMetadata(self, res):
1311         data = simplejson.loads(res)
1312         self.failUnless(isinstance(data, list))
1313         self.failUnlessIn("metadata", data[1])
1314         self.failUnlessIn("tahoe", data[1]["metadata"])
1315         self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
1316         self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
1317         self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
1318                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
1319
1320     def test_GET_FILEURL_json(self):
1321         # twisted.web.http.parse_qs ignores any query args without an '=', so
1322         # I can't do "GET /path?json", I have to do "GET /path/t=json"
1323         # instead. This may make it tricky to emulate the S3 interface
1324         # completely.
1325         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
1326         def _check1(data):
1327             self.failUnlessIsBarJSON(data)
1328             self.failUnlessHasBarDotTxtMetadata(data)
1329             return
1330         d.addCallback(_check1)
1331         return d
1332
1333     def test_GET_FILEURL_json_mutable_type(self):
1334         # The JSON should include format, which says whether the
1335         # file is SDMF or MDMF
1336         d = self.PUT("/uri?format=mdmf",
1337                      self.NEWFILE_CONTENTS * 300000)
1338         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1339         def _got_json(json, version):
1340             data = simplejson.loads(json)
1341             assert "filenode" == data[0]
1342             data = data[1]
1343             assert isinstance(data, dict)
1344
1345             self.failUnlessIn("format", data)
1346             self.failUnlessEqual(data["format"], version)
1347
1348         d.addCallback(_got_json, "MDMF")
1349         # Now make an SDMF file and check that it is reported correctly.
1350         d.addCallback(lambda ignored:
1351             self.PUT("/uri?format=sdmf",
1352                       self.NEWFILE_CONTENTS * 300000))
1353         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1354         d.addCallback(_got_json, "SDMF")
1355         return d
1356
1357     def test_GET_FILEURL_json_mdmf(self):
1358         d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
1359         d.addCallback(self.failUnlessIsQuuxJSON)
1360         return d
1361
1362     def test_GET_FILEURL_json_missing(self):
1363         d = self.GET(self.public_url + "/foo/missing?json")
1364         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
1365         return d
1366
1367     def test_GET_FILEURL_uri(self):
1368         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
1369         def _check(res):
1370             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1371         d.addCallback(_check)
1372         d.addCallback(lambda res:
1373                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
1374         def _check2(res):
1375             # for now, for files, uris and readonly-uris are the same
1376             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1377         d.addCallback(_check2)
1378         return d
1379
1380     def test_GET_FILEURL_badtype(self):
1381         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
1382                                  "bad t=bogus",
1383                                  self.GET,
1384                                  self.public_url + "/foo/bar.txt?t=bogus")
1385         return d
1386
1387     def test_CSS_FILE(self):
1388         d = self.GET("/tahoe.css", followRedirect=True)
1389         def _check(res):
1390             CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
1391             self.failUnless(CSS_STYLE.search(res), res)
1392         d.addCallback(_check)
1393         return d
1394
1395     def test_GET_FILEURL_uri_missing(self):
1396         d = self.GET(self.public_url + "/foo/missing?t=uri")
1397         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
1398         return d
1399
1400     def _check_upload_and_mkdir_forms(self, html):
1401         # We should have a form to create a file, with radio buttons that allow
1402         # the user to toggle whether it is a CHK/LIT (default), SDMF, or MDMF file.
1403         self.failUnlessIn('name="t" value="upload"', html)
1404         self.failUnlessIn('input checked="checked" type="radio" id="upload-chk" value="chk" name="format"', html)
1405         self.failUnlessIn('input type="radio" id="upload-sdmf" value="sdmf" name="format"', html)
1406         self.failUnlessIn('input type="radio" id="upload-mdmf" value="mdmf" name="format"', html)
1407
1408         # We should also have the ability to create a mutable directory, with
1409         # radio buttons that allow the user to toggle whether it is an SDMF (default)
1410         # or MDMF directory.
1411         self.failUnlessIn('name="t" value="mkdir"', html)
1412         self.failUnlessIn('input checked="checked" type="radio" id="mkdir-sdmf" value="sdmf" name="format"', html)
1413         self.failUnlessIn('input type="radio" id="mkdir-mdmf" value="mdmf" name="format"', html)
1414
1415         self.failUnlessIn(FAVICON_MARKUP, html)
1416
1417     def test_GET_DIRECTORY_html(self):
1418         d = self.GET(self.public_url + "/foo", followRedirect=True)
1419         def _check(html):
1420             self.failUnlessIn('<div class="toolbar-item"><a href="../../..">Return to Welcome page</a></div>', html)
1421             self._check_upload_and_mkdir_forms(html)
1422             self.failUnlessIn("quux", html)
1423         d.addCallback(_check)
1424         return d
1425
1426     def test_GET_DIRECTORY_html_filenode_encoding(self):
1427         d = self.GET(self.public_url + "/foo", followRedirect=True)
1428         def _check(html):
1429             # Check if encoded entries are there
1430             self.failUnlessIn('@@named=/' + self._htmlname_urlencoded + '">'
1431                               + self._htmlname_escaped + '</a>', html)
1432             self.failUnlessIn('value="' + self._htmlname_escaped_attr + '"', html)
1433             self.failIfIn(self._htmlname_escaped_double, html)
1434             # Make sure that Nevow escaping actually works by checking for unsafe characters
1435             # and that '&' is escaped.
1436             for entity in '<>':
1437                 self.failUnlessIn(entity, self._htmlname_raw)
1438                 self.failIfIn(entity, self._htmlname_escaped)
1439             self.failUnlessIn('&', re.sub(r'&(amp|lt|gt|quot|apos);', '', self._htmlname_raw))
1440             self.failIfIn('&', re.sub(r'&(amp|lt|gt|quot|apos);', '', self._htmlname_escaped))
1441         d.addCallback(_check)
1442         return d
1443
1444     def test_GET_root_html(self):
1445         d = self.GET("/")
1446         d.addCallback(self._check_upload_and_mkdir_forms)
1447         return d
1448
1449     def test_GET_DIRURL(self):
1450         # the addSlash means we get a redirect here
1451         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1452         ROOT = "../../.."
1453         d = self.GET(self.public_url + "/foo", followRedirect=True)
1454         def _check(res):
1455             self.failUnlessIn('<a href="%s">Return to Welcome page' % ROOT, res)
1456
1457             # the FILE reference points to a URI, but it should end in bar.txt
1458             bar_url = ("%s/file/%s/@@named=/bar.txt" %
1459                        (ROOT, urllib.quote(self._bar_txt_uri)))
1460             get_bar = "".join([r'<td>FILE</td>',
1461                                r'\s+<td>',
1462                                r'<a href="%s">bar.txt</a>' % bar_url,
1463                                r'</td>',
1464                                r'\s+<td align="right">%d</td>' % len(self.BAR_CONTENTS),
1465                                ])
1466             self.failUnless(re.search(get_bar, res), res)
1467             for label in ['unlink', 'rename/move']:
1468                 for line in res.split("\n"):
1469                     # find the line that contains the relevant button for bar.txt
1470                     if ("form action" in line and
1471                         ('value="%s"' % (label,)) in line and
1472                         'value="bar.txt"' in line):
1473                         # the form target should use a relative URL
1474                         foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1475                         self.failUnlessIn('action="%s"' % foo_url, line)
1476                         # and the when_done= should too
1477                         #done_url = urllib.quote(???)
1478                         #self.failUnlessIn('name="when_done" value="%s"' % done_url, line)
1479
1480                         # 'unlink' needs to use POST because it directly has a side effect
1481                         if label == 'unlink':
1482                             self.failUnlessIn('method="post"', line)
1483                         break
1484                 else:
1485                     self.fail("unable to find '%s bar.txt' line" % (label,))
1486
1487             # the DIR reference just points to a URI
1488             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1489             get_sub = ((r'<td>DIR</td>')
1490                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1491             self.failUnless(re.search(get_sub, res), res)
1492         d.addCallback(_check)
1493
1494         # look at a readonly directory
1495         d.addCallback(lambda res:
1496                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
1497         def _check2(res):
1498             self.failUnlessIn("(read-only)", res)
1499             self.failIfIn("Upload a file", res)
1500         d.addCallback(_check2)
1501
1502         # and at a directory that contains a readonly directory
1503         d.addCallback(lambda res:
1504                       self.GET(self.public_url, followRedirect=True))
1505         def _check3(res):
1506             self.failUnless(re.search('<td>DIR-RO</td>'
1507                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1508         d.addCallback(_check3)
1509
1510         # and an empty directory
1511         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1512         def _check4(res):
1513             self.failUnlessIn("directory is empty", res)
1514             MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
1515             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1516         d.addCallback(_check4)
1517
1518         # and at a literal directory
1519         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1520         d.addCallback(lambda res:
1521                       self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1522         def _check5(res):
1523             self.failUnlessIn('(immutable)', res)
1524             self.failUnless(re.search('<td>FILE</td>'
1525                                       r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1526         d.addCallback(_check5)
1527         return d
1528
1529     def test_GET_DIRURL_badtype(self):
1530         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1531                                  400, "Bad Request",
1532                                  "bad t=bogus",
1533                                  self.GET,
1534                                  self.public_url + "/foo?t=bogus")
1535         return d
1536
1537     def test_GET_DIRURL_json(self):
1538         d = self.GET(self.public_url + "/foo?t=json")
1539         d.addCallback(self.failUnlessIsFooJSON)
1540         return d
1541
1542     def test_GET_DIRURL_json_format(self):
1543         d = self.PUT(self.public_url + \
1544                      "/foo/sdmf.txt?format=sdmf",
1545                      self.NEWFILE_CONTENTS * 300000)
1546         d.addCallback(lambda ignored:
1547             self.PUT(self.public_url + \
1548                      "/foo/mdmf.txt?format=mdmf",
1549                      self.NEWFILE_CONTENTS * 300000))
1550         # Now we have an MDMF and SDMF file in the directory. If we GET
1551         # its JSON, we should see their encodings.
1552         d.addCallback(lambda ignored:
1553             self.GET(self.public_url + "/foo?t=json"))
1554         def _got_json(json):
1555             data = simplejson.loads(json)
1556             assert data[0] == "dirnode"
1557
1558             data = data[1]
1559             kids = data['children']
1560
1561             mdmf_data = kids['mdmf.txt'][1]
1562             self.failUnlessIn("format", mdmf_data)
1563             self.failUnlessEqual(mdmf_data["format"], "MDMF")
1564
1565             sdmf_data = kids['sdmf.txt'][1]
1566             self.failUnlessIn("format", sdmf_data)
1567             self.failUnlessEqual(sdmf_data["format"], "SDMF")
1568         d.addCallback(_got_json)
1569         return d
1570
1571
1572     def test_POST_DIRURL_manifest_no_ophandle(self):
1573         d = self.shouldFail2(error.Error,
1574                              "test_POST_DIRURL_manifest_no_ophandle",
1575                              "400 Bad Request",
1576                              "slow operation requires ophandle=",
1577                              self.POST, self.public_url, t="start-manifest")
1578         return d
1579
1580     def test_POST_DIRURL_manifest(self):
1581         d = defer.succeed(None)
1582         def getman(ignored, output):
1583             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1584                           followRedirect=True)
1585             d.addCallback(self.wait_for_operation, "125")
1586             d.addCallback(self.get_operation_results, "125", output)
1587             return d
1588         d.addCallback(getman, None)
1589         def _got_html(manifest):
1590             self.failUnlessIn("Manifest of SI=", manifest)
1591             self.failUnlessIn("<td>sub</td>", manifest)
1592             self.failUnlessIn(self._sub_uri, manifest)
1593             self.failUnlessIn("<td>sub/baz.txt</td>", manifest)
1594             self.failUnlessIn(FAVICON_MARKUP, manifest)
1595         d.addCallback(_got_html)
1596
1597         # both t=status and unadorned GET should be identical
1598         d.addCallback(lambda res: self.GET("/operations/125"))
1599         d.addCallback(_got_html)
1600
1601         d.addCallback(getman, "html")
1602         d.addCallback(_got_html)
1603         d.addCallback(getman, "text")
1604         def _got_text(manifest):
1605             self.failUnlessIn("\nsub " + self._sub_uri + "\n", manifest)
1606             self.failUnlessIn("\nsub/baz.txt URI:CHK:", manifest)
1607         d.addCallback(_got_text)
1608         d.addCallback(getman, "JSON")
1609         def _got_json(res):
1610             data = res["manifest"]
1611             got = {}
1612             for (path_list, cap) in data:
1613                 got[tuple(path_list)] = cap
1614             self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
1615             self.failUnlessIn((u"sub", u"baz.txt"), got)
1616             self.failUnlessIn("finished", res)
1617             self.failUnlessIn("origin", res)
1618             self.failUnlessIn("storage-index", res)
1619             self.failUnlessIn("verifycaps", res)
1620             self.failUnlessIn("stats", res)
1621         d.addCallback(_got_json)
1622         return d
1623
1624     def test_POST_DIRURL_deepsize_no_ophandle(self):
1625         d = self.shouldFail2(error.Error,
1626                              "test_POST_DIRURL_deepsize_no_ophandle",
1627                              "400 Bad Request",
1628                              "slow operation requires ophandle=",
1629                              self.POST, self.public_url, t="start-deep-size")
1630         return d
1631
1632     def test_POST_DIRURL_deepsize(self):
1633         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1634                       followRedirect=True)
1635         d.addCallback(self.wait_for_operation, "126")
1636         d.addCallback(self.get_operation_results, "126", "json")
1637         def _got_json(data):
1638             self.failUnlessReallyEqual(data["finished"], True)
1639             size = data["size"]
1640             self.failUnless(size > 1000)
1641         d.addCallback(_got_json)
1642         d.addCallback(self.get_operation_results, "126", "text")
1643         def _got_text(res):
1644             mo = re.search(r'^size: (\d+)$', res, re.M)
1645             self.failUnless(mo, res)
1646             size = int(mo.group(1))
1647             # with directories, the size varies.
1648             self.failUnless(size > 1000)
1649         d.addCallback(_got_text)
1650         return d
1651
1652     def test_POST_DIRURL_deepstats_no_ophandle(self):
1653         d = self.shouldFail2(error.Error,
1654                              "test_POST_DIRURL_deepstats_no_ophandle",
1655                              "400 Bad Request",
1656                              "slow operation requires ophandle=",
1657                              self.POST, self.public_url, t="start-deep-stats")
1658         return d
1659
1660     def test_POST_DIRURL_deepstats(self):
1661         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1662                       followRedirect=True)
1663         d.addCallback(self.wait_for_operation, "127")
1664         d.addCallback(self.get_operation_results, "127", "json")
1665         def _got_json(stats):
1666             expected = {"count-immutable-files": 4,
1667                         "count-mutable-files": 2,
1668                         "count-literal-files": 0,
1669                         "count-files": 6,
1670                         "count-directories": 3,
1671                         "size-immutable-files": 76,
1672                         "size-literal-files": 0,
1673                         #"size-directories": 1912, # varies
1674                         #"largest-directory": 1590,
1675                         "largest-directory-children": 8,
1676                         "largest-immutable-file": 19,
1677                         }
1678             for k,v in expected.iteritems():
1679                 self.failUnlessReallyEqual(stats[k], v,
1680                                            "stats[%s] was %s, not %s" %
1681                                            (k, stats[k], v))
1682             self.failUnlessReallyEqual(stats["size-files-histogram"],
1683                                        [ [11, 31, 4] ])
1684         d.addCallback(_got_json)
1685         return d
1686
1687     def test_POST_DIRURL_stream_manifest(self):
1688         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1689         def _check(res):
1690             self.failUnless(res.endswith("\n"))
1691             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1692             self.failUnlessReallyEqual(len(units), 10)
1693             self.failUnlessEqual(units[-1]["type"], "stats")
1694             first = units[0]
1695             self.failUnlessEqual(first["path"], [])
1696             self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri)
1697             self.failUnlessEqual(first["type"], "directory")
1698             baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0]
1699             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1700             self.failIfEqual(baz["storage-index"], None)
1701             self.failIfEqual(baz["verifycap"], None)
1702             self.failIfEqual(baz["repaircap"], None)
1703             # XXX: Add quux and baz to this test.
1704             return
1705         d.addCallback(_check)
1706         return d
1707
1708     def test_GET_DIRURL_uri(self):
1709         d = self.GET(self.public_url + "/foo?t=uri")
1710         def _check(res):
1711             self.failUnlessReallyEqual(to_str(res), self._foo_uri)
1712         d.addCallback(_check)
1713         return d
1714
1715     def test_GET_DIRURL_readonly_uri(self):
1716         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1717         def _check(res):
1718             self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri)
1719         d.addCallback(_check)
1720         return d
1721
1722     def test_PUT_NEWDIRURL(self):
1723         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1724         d.addCallback(lambda res:
1725                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1726         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1727         d.addCallback(self.failUnlessNodeKeysAre, [])
1728         return d
1729
1730     def test_PUT_NEWDIRURL_mdmf(self):
1731         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1732         d.addCallback(lambda res:
1733                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1734         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1735         d.addCallback(lambda node:
1736             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1737         return d
1738
1739     def test_PUT_NEWDIRURL_sdmf(self):
1740         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=sdmf",
1741                      "")
1742         d.addCallback(lambda res:
1743                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1744         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1745         d.addCallback(lambda node:
1746             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1747         return d
1748
1749     def test_PUT_NEWDIRURL_bad_format(self):
1750         return self.shouldHTTPError("PUT_NEWDIRURL_bad_format",
1751                                     400, "Bad Request", "Unknown format: foo",
1752                                     self.PUT, self.public_url +
1753                                     "/foo/newdir=?t=mkdir&format=foo", "")
1754
1755     def test_POST_NEWDIRURL(self):
1756         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1757         d.addCallback(lambda res:
1758                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1759         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1760         d.addCallback(self.failUnlessNodeKeysAre, [])
1761         return d
1762
1763     def test_POST_NEWDIRURL_mdmf(self):
1764         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1765         d.addCallback(lambda res:
1766                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1767         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1768         d.addCallback(lambda node:
1769             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1770         return d
1771
1772     def test_POST_NEWDIRURL_sdmf(self):
1773         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=sdmf", "")
1774         d.addCallback(lambda res:
1775             self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1776         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1777         d.addCallback(lambda node:
1778             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1779         return d
1780
1781     def test_POST_NEWDIRURL_bad_format(self):
1782         return self.shouldHTTPError("POST_NEWDIRURL_bad_format",
1783                                     400, "Bad Request", "Unknown format: foo",
1784                                     self.POST2, self.public_url + \
1785                                     "/foo/newdir?t=mkdir&format=foo", "")
1786
1787     def test_POST_NEWDIRURL_emptyname(self):
1788         # an empty pathname component (i.e. a double-slash) is disallowed
1789         d = self.shouldFail2(error.Error, "POST_NEWDIRURL_emptyname",
1790                              "400 Bad Request",
1791                              "The webapi does not allow empty pathname components, i.e. a double slash",
1792                              self.POST, self.public_url + "//?t=mkdir")
1793         return d
1794
1795     def _do_POST_NEWDIRURL_initial_children_test(self, version=None):
1796         (newkids, caps) = self._create_initial_children()
1797         query = "/foo/newdir?t=mkdir-with-children"
1798         if version == MDMF_VERSION:
1799             query += "&format=mdmf"
1800         elif version == SDMF_VERSION:
1801             query += "&format=sdmf"
1802         else:
1803             version = SDMF_VERSION # for later
1804         d = self.POST2(self.public_url + query,
1805                        simplejson.dumps(newkids))
1806         def _check(uri):
1807             n = self.s.create_node_from_uri(uri.strip())
1808             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1809             self.failUnlessEqual(n._node.get_version(), version)
1810             d2.addCallback(lambda ign:
1811                            self.failUnlessROChildURIIs(n, u"child-imm",
1812                                                        caps['filecap1']))
1813             d2.addCallback(lambda ign:
1814                            self.failUnlessRWChildURIIs(n, u"child-mutable",
1815                                                        caps['filecap2']))
1816             d2.addCallback(lambda ign:
1817                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1818                                                        caps['filecap3']))
1819             d2.addCallback(lambda ign:
1820                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1821                                                        caps['unknown_rocap']))
1822             d2.addCallback(lambda ign:
1823                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1824                                                        caps['unknown_rwcap']))
1825             d2.addCallback(lambda ign:
1826                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1827                                                        caps['unknown_immcap']))
1828             d2.addCallback(lambda ign:
1829                            self.failUnlessRWChildURIIs(n, u"dirchild",
1830                                                        caps['dircap']))
1831             d2.addCallback(lambda ign:
1832                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1833                                                        caps['litdircap']))
1834             d2.addCallback(lambda ign:
1835                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1836                                                        caps['emptydircap']))
1837             return d2
1838         d.addCallback(_check)
1839         d.addCallback(lambda res:
1840                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1841         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1842         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1843         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1844         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1845         return d
1846
1847     def test_POST_NEWDIRURL_initial_children(self):
1848         return self._do_POST_NEWDIRURL_initial_children_test()
1849
1850     def test_POST_NEWDIRURL_initial_children_mdmf(self):
1851         return self._do_POST_NEWDIRURL_initial_children_test(MDMF_VERSION)
1852
1853     def test_POST_NEWDIRURL_initial_children_sdmf(self):
1854         return self._do_POST_NEWDIRURL_initial_children_test(SDMF_VERSION)
1855
1856     def test_POST_NEWDIRURL_initial_children_bad_format(self):
1857         (newkids, caps) = self._create_initial_children()
1858         return self.shouldHTTPError("POST_NEWDIRURL_initial_children_bad_format",
1859                                     400, "Bad Request", "Unknown format: foo",
1860                                     self.POST2, self.public_url + \
1861                                     "/foo/newdir?t=mkdir-with-children&format=foo",
1862                                     simplejson.dumps(newkids))
1863
1864     def test_POST_NEWDIRURL_immutable(self):
1865         (newkids, caps) = self._create_immutable_children()
1866         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1867                        simplejson.dumps(newkids))
1868         def _check(uri):
1869             n = self.s.create_node_from_uri(uri.strip())
1870             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1871             d2.addCallback(lambda ign:
1872                            self.failUnlessROChildURIIs(n, u"child-imm",
1873                                                        caps['filecap1']))
1874             d2.addCallback(lambda ign:
1875                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1876                                                        caps['unknown_immcap']))
1877             d2.addCallback(lambda ign:
1878                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
1879                                                        caps['immdircap']))
1880             d2.addCallback(lambda ign:
1881                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1882                                                        caps['litdircap']))
1883             d2.addCallback(lambda ign:
1884                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1885                                                        caps['emptydircap']))
1886             return d2
1887         d.addCallback(_check)
1888         d.addCallback(lambda res:
1889                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1890         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1891         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1892         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1893         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1894         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1895         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1896         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1897         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
1898         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1899         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
1900         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1901         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
1902         d.addErrback(self.explain_web_error)
1903         return d
1904
1905     def test_POST_NEWDIRURL_immutable_bad(self):
1906         (newkids, caps) = self._create_initial_children()
1907         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
1908                              "400 Bad Request",
1909                              "needed to be immutable but was not",
1910                              self.POST2,
1911                              self.public_url + "/foo/newdir?t=mkdir-immutable",
1912                              simplejson.dumps(newkids))
1913         return d
1914
1915     def test_PUT_NEWDIRURL_exists(self):
1916         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1917         d.addCallback(lambda res:
1918                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1919         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1920         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1921         return d
1922
1923     def test_PUT_NEWDIRURL_blocked(self):
1924         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1925                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1926                              self.PUT,
1927                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1928         d.addCallback(lambda res:
1929                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1930         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1931         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1932         return d
1933
1934     def test_PUT_NEWDIRURL_mkdirs(self):
1935         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1936         d.addCallback(lambda res:
1937                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1938         d.addCallback(lambda res:
1939                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1940         d.addCallback(lambda res:
1941                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1942         d.addCallback(self.failUnlessNodeKeysAre, [])
1943         return d
1944
1945     def test_PUT_NEWDIRURL_mkdirs_mdmf(self):
1946         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=mdmf", "")
1947         d.addCallback(lambda ignored:
1948             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1949         d.addCallback(lambda ignored:
1950             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1951         d.addCallback(lambda ignored:
1952             self._foo_node.get_child_at_path(u"subdir"))
1953         def _got_subdir(subdir):
1954             # XXX: What we want?
1955             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1956             self.failUnlessNodeHasChild(subdir, u"newdir")
1957             return subdir.get_child_at_path(u"newdir")
1958         d.addCallback(_got_subdir)
1959         d.addCallback(lambda newdir:
1960             self.failUnlessEqual(newdir._node.get_version(), MDMF_VERSION))
1961         return d
1962
1963     def test_PUT_NEWDIRURL_mkdirs_sdmf(self):
1964         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=sdmf", "")
1965         d.addCallback(lambda ignored:
1966             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1967         d.addCallback(lambda ignored:
1968             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1969         d.addCallback(lambda ignored:
1970             self._foo_node.get_child_at_path(u"subdir"))
1971         def _got_subdir(subdir):
1972             # XXX: What we want?
1973             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1974             self.failUnlessNodeHasChild(subdir, u"newdir")
1975             return subdir.get_child_at_path(u"newdir")
1976         d.addCallback(_got_subdir)
1977         d.addCallback(lambda newdir:
1978             self.failUnlessEqual(newdir._node.get_version(), SDMF_VERSION))
1979         return d
1980
1981     def test_PUT_NEWDIRURL_mkdirs_bad_format(self):
1982         return self.shouldHTTPError("PUT_NEWDIRURL_mkdirs_bad_format",
1983                                     400, "Bad Request", "Unknown format: foo",
1984                                     self.PUT, self.public_url + \
1985                                     "/foo/subdir/newdir?t=mkdir&format=foo",
1986                                     "")
1987
1988     def test_DELETE_DIRURL(self):
1989         d = self.DELETE(self.public_url + "/foo")
1990         d.addCallback(lambda res:
1991                       self.failIfNodeHasChild(self.public_root, u"foo"))
1992         return d
1993
1994     def test_DELETE_DIRURL_missing(self):
1995         d = self.DELETE(self.public_url + "/foo/missing")
1996         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1997         d.addCallback(lambda res:
1998                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1999         return d
2000
2001     def test_DELETE_DIRURL_missing2(self):
2002         d = self.DELETE(self.public_url + "/missing")
2003         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
2004         return d
2005
2006     def dump_root(self):
2007         print "NODEWALK"
2008         w = webish.DirnodeWalkerMixin()
2009         def visitor(childpath, childnode, metadata):
2010             print childpath
2011         d = w.walk(self.public_root, visitor)
2012         return d
2013
2014     def failUnlessNodeKeysAre(self, node, expected_keys):
2015         for k in expected_keys:
2016             assert isinstance(k, unicode)
2017         d = node.list()
2018         def _check(children):
2019             self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
2020         d.addCallback(_check)
2021         return d
2022     def failUnlessNodeHasChild(self, node, name):
2023         assert isinstance(name, unicode)
2024         d = node.list()
2025         def _check(children):
2026             self.failUnlessIn(name, children)
2027         d.addCallback(_check)
2028         return d
2029     def failIfNodeHasChild(self, node, name):
2030         assert isinstance(name, unicode)
2031         d = node.list()
2032         def _check(children):
2033             self.failIfIn(name, children)
2034         d.addCallback(_check)
2035         return d
2036
2037     def failUnlessChildContentsAre(self, node, name, expected_contents):
2038         assert isinstance(name, unicode)
2039         d = node.get_child_at_path(name)
2040         d.addCallback(lambda node: download_to_data(node))
2041         def _check(contents):
2042             self.failUnlessReallyEqual(contents, expected_contents)
2043         d.addCallback(_check)
2044         return d
2045
2046     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
2047         assert isinstance(name, unicode)
2048         d = node.get_child_at_path(name)
2049         d.addCallback(lambda node: node.download_best_version())
2050         def _check(contents):
2051             self.failUnlessReallyEqual(contents, expected_contents)
2052         d.addCallback(_check)
2053         return d
2054
2055     def failUnlessRWChildURIIs(self, node, name, expected_uri):
2056         assert isinstance(name, unicode)
2057         d = node.get_child_at_path(name)
2058         def _check(child):
2059             self.failUnless(child.is_unknown() or not child.is_readonly())
2060             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
2061             self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
2062             expected_ro_uri = self._make_readonly(expected_uri)
2063             if expected_ro_uri:
2064                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
2065         d.addCallback(_check)
2066         return d
2067
2068     def failUnlessROChildURIIs(self, node, name, expected_uri):
2069         assert isinstance(name, unicode)
2070         d = node.get_child_at_path(name)
2071         def _check(child):
2072             self.failUnless(child.is_unknown() or child.is_readonly())
2073             self.failUnlessReallyEqual(child.get_write_uri(), None)
2074             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
2075             self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
2076         d.addCallback(_check)
2077         return d
2078
2079     def failUnlessURIMatchesRWChild(self, got_uri, node, name):
2080         assert isinstance(name, unicode)
2081         d = node.get_child_at_path(name)
2082         def _check(child):
2083             self.failUnless(child.is_unknown() or not child.is_readonly())
2084             self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
2085             self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
2086             expected_ro_uri = self._make_readonly(got_uri)
2087             if expected_ro_uri:
2088                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
2089         d.addCallback(_check)
2090         return d
2091
2092     def failUnlessURIMatchesROChild(self, got_uri, node, name):
2093         assert isinstance(name, unicode)
2094         d = node.get_child_at_path(name)
2095         def _check(child):
2096             self.failUnless(child.is_unknown() or child.is_readonly())
2097             self.failUnlessReallyEqual(child.get_write_uri(), None)
2098             self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
2099             self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
2100         d.addCallback(_check)
2101         return d
2102
2103     def failUnlessCHKURIHasContents(self, got_uri, contents):
2104         self.failUnless(self.get_all_contents()[got_uri] == contents)
2105
2106     def test_POST_upload(self):
2107         d = self.POST(self.public_url + "/foo", t="upload",
2108                       file=("new.txt", self.NEWFILE_CONTENTS))
2109         fn = self._foo_node
2110         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2111         d.addCallback(lambda res:
2112                       self.failUnlessChildContentsAre(fn, u"new.txt",
2113                                                       self.NEWFILE_CONTENTS))
2114         return d
2115
2116     def test_POST_upload_unicode(self):
2117         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2118         d = self.POST(self.public_url + "/foo", t="upload",
2119                       file=(filename, self.NEWFILE_CONTENTS))
2120         fn = self._foo_node
2121         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2122         d.addCallback(lambda res:
2123                       self.failUnlessChildContentsAre(fn, filename,
2124                                                       self.NEWFILE_CONTENTS))
2125         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2126         d.addCallback(lambda res: self.GET(target_url))
2127         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2128                                                                   self.NEWFILE_CONTENTS,
2129                                                                   contents))
2130         return d
2131
2132     def test_POST_upload_unicode_named(self):
2133         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2134         d = self.POST(self.public_url + "/foo", t="upload",
2135                       name=filename,
2136                       file=("overridden", self.NEWFILE_CONTENTS))
2137         fn = self._foo_node
2138         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2139         d.addCallback(lambda res:
2140                       self.failUnlessChildContentsAre(fn, filename,
2141                                                       self.NEWFILE_CONTENTS))
2142         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2143         d.addCallback(lambda res: self.GET(target_url))
2144         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2145                                                                   self.NEWFILE_CONTENTS,
2146                                                                   contents))
2147         return d
2148
2149     def test_POST_upload_no_link(self):
2150         d = self.POST("/uri", t="upload",
2151                       file=("new.txt", self.NEWFILE_CONTENTS))
2152         def _check_upload_results(page):
2153             # this should be a page which describes the results of the upload
2154             # that just finished.
2155             self.failUnlessIn("Upload Results:", page)
2156             self.failUnlessIn("URI:", page)
2157             uri_re = re.compile("URI: <tt><span>(.*)</span>")
2158             mo = uri_re.search(page)
2159             self.failUnless(mo, page)
2160             new_uri = mo.group(1)
2161             return new_uri
2162         d.addCallback(_check_upload_results)
2163         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
2164         return d
2165
2166     def test_POST_upload_no_link_whendone(self):
2167         d = self.POST("/uri", t="upload", when_done="/",
2168                       file=("new.txt", self.NEWFILE_CONTENTS))
2169         d.addBoth(self.shouldRedirect, "/")
2170         return d
2171
2172     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
2173         d = defer.maybeDeferred(callable, *args, **kwargs)
2174         def done(res):
2175             if isinstance(res, failure.Failure):
2176                 res.trap(error.PageRedirect)
2177                 statuscode = res.value.status
2178                 target = res.value.location
2179                 return checker(statuscode, target)
2180             self.fail("%s: callable was supposed to redirect, not return '%s'"
2181                       % (which, res))
2182         d.addBoth(done)
2183         return d
2184
2185     def test_POST_upload_no_link_whendone_results(self):
2186         def check(statuscode, target):
2187             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2188             self.failUnless(target.startswith(self.webish_url), target)
2189             return client.getPage(target, method="GET")
2190         # We encode "uri" as "%75ri" to exercise a case affected by ticket #1860.
2191         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
2192                                  check,
2193                                  self.POST, "/uri", t="upload",
2194                                  when_done="/%75ri/%(uri)s",
2195                                  file=("new.txt", self.NEWFILE_CONTENTS))
2196         d.addCallback(lambda res:
2197                       self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
2198         return d
2199
2200     def test_POST_upload_no_link_mutable(self):
2201         d = self.POST("/uri", t="upload", mutable="true",
2202                       file=("new.txt", self.NEWFILE_CONTENTS))
2203         def _check(filecap):
2204             filecap = filecap.strip()
2205             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2206             self.filecap = filecap
2207             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2208             self.failUnlessIn(u.get_storage_index(), self.get_all_contents())
2209             n = self.s.create_node_from_uri(filecap)
2210             return n.download_best_version()
2211         d.addCallback(_check)
2212         def _check2(data):
2213             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2214             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2215         d.addCallback(_check2)
2216         def _check3(data):
2217             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2218             return self.GET("/file/%s" % urllib.quote(self.filecap))
2219         d.addCallback(_check3)
2220         def _check4(data):
2221             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2222         d.addCallback(_check4)
2223         return d
2224
2225     def test_POST_upload_no_link_mutable_toobig(self):
2226         # The SDMF size limit is no longer in place, so we should be
2227         # able to upload mutable files that are as large as we want them
2228         # to be.
2229         d = self.POST("/uri", t="upload", mutable="true",
2230                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2231         return d
2232
2233
2234     def test_POST_upload_format_unlinked(self):
2235         def _check_upload_unlinked(ign, format, uri_prefix):
2236             filename = format + ".txt"
2237             d = self.POST("/uri?t=upload&format=" + format,
2238                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2239             def _got_results(results):
2240                 if format.upper() in ("SDMF", "MDMF"):
2241                     # webapi.rst says this returns a filecap
2242                     filecap = results
2243                 else:
2244                     # for immutable, it returns an "upload results page", and
2245                     # the filecap is buried inside
2246                     line = [l for l in results.split("\n") if "URI: " in l][0]
2247                     mo = re.search(r'<span>([^<]+)</span>', line)
2248                     filecap = mo.group(1)
2249                 self.failUnless(filecap.startswith(uri_prefix),
2250                                 (uri_prefix, filecap))
2251                 return self.GET("/uri/%s?t=json" % filecap)
2252             d.addCallback(_got_results)
2253             def _got_json(json):
2254                 data = simplejson.loads(json)
2255                 data = data[1]
2256                 self.failUnlessIn("format", data)
2257                 self.failUnlessEqual(data["format"], format.upper())
2258             d.addCallback(_got_json)
2259             return d
2260         d = defer.succeed(None)
2261         d.addCallback(_check_upload_unlinked, "chk", "URI:CHK")
2262         d.addCallback(_check_upload_unlinked, "CHK", "URI:CHK")
2263         d.addCallback(_check_upload_unlinked, "sdmf", "URI:SSK")
2264         d.addCallback(_check_upload_unlinked, "mdmf", "URI:MDMF")
2265         return d
2266
2267     def test_POST_upload_bad_format_unlinked(self):
2268         return self.shouldHTTPError("POST_upload_bad_format_unlinked",
2269                                     400, "Bad Request", "Unknown format: foo",
2270                                     self.POST,
2271                                     "/uri?t=upload&format=foo",
2272                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2273
2274     def test_POST_upload_format(self):
2275         def _check_upload(ign, format, uri_prefix, fn=None):
2276             filename = format + ".txt"
2277             d = self.POST(self.public_url +
2278                           "/foo?t=upload&format=" + format,
2279                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2280             def _got_filecap(filecap):
2281                 if fn is not None:
2282                     filenameu = unicode(filename)
2283                     self.failUnlessURIMatchesRWChild(filecap, fn, filenameu)
2284                 self.failUnless(filecap.startswith(uri_prefix))
2285                 return self.GET(self.public_url + "/foo/%s?t=json" % filename)
2286             d.addCallback(_got_filecap)
2287             def _got_json(json):
2288                 data = simplejson.loads(json)
2289                 data = data[1]
2290                 self.failUnlessIn("format", data)
2291                 self.failUnlessEqual(data["format"], format.upper())
2292             d.addCallback(_got_json)
2293             return d
2294
2295         d = defer.succeed(None)
2296         d.addCallback(_check_upload, "chk", "URI:CHK")
2297         d.addCallback(_check_upload, "sdmf", "URI:SSK", self._foo_node)
2298         d.addCallback(_check_upload, "mdmf", "URI:MDMF")
2299         d.addCallback(_check_upload, "MDMF", "URI:MDMF")
2300         return d
2301
2302     def test_POST_upload_bad_format(self):
2303         return self.shouldHTTPError("POST_upload_bad_format",
2304                                     400, "Bad Request", "Unknown format: foo",
2305                                     self.POST, self.public_url + \
2306                                     "/foo?t=upload&format=foo",
2307                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2308
2309     def test_POST_upload_mutable(self):
2310         # this creates a mutable file
2311         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
2312                       file=("new.txt", self.NEWFILE_CONTENTS))
2313         fn = self._foo_node
2314         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2315         d.addCallback(lambda res:
2316                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2317                                                              self.NEWFILE_CONTENTS))
2318         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2319         def _got(newnode):
2320             self.failUnless(IMutableFileNode.providedBy(newnode))
2321             self.failUnless(newnode.is_mutable())
2322             self.failIf(newnode.is_readonly())
2323             self._mutable_node = newnode
2324             self._mutable_uri = newnode.get_uri()
2325         d.addCallback(_got)
2326
2327         # now upload it again and make sure that the URI doesn't change
2328         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
2329         d.addCallback(lambda res:
2330                       self.POST(self.public_url + "/foo", t="upload",
2331                                 mutable="true",
2332                                 file=("new.txt", NEWER_CONTENTS)))
2333         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2334         d.addCallback(lambda res:
2335                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2336                                                              NEWER_CONTENTS))
2337         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2338         def _got2(newnode):
2339             self.failUnless(IMutableFileNode.providedBy(newnode))
2340             self.failUnless(newnode.is_mutable())
2341             self.failIf(newnode.is_readonly())
2342             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2343         d.addCallback(_got2)
2344
2345         # upload a second time, using PUT instead of POST
2346         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
2347         d.addCallback(lambda res:
2348                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
2349         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2350         d.addCallback(lambda res:
2351                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2352                                                              NEW2_CONTENTS))
2353
2354         # finally list the directory, since mutable files are displayed
2355         # slightly differently
2356
2357         d.addCallback(lambda res:
2358                       self.GET(self.public_url + "/foo/",
2359                                followRedirect=True))
2360         def _check_page(res):
2361             # TODO: assert more about the contents
2362             self.failUnlessIn("SSK", res)
2363             return res
2364         d.addCallback(_check_page)
2365
2366         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2367         def _got3(newnode):
2368             self.failUnless(IMutableFileNode.providedBy(newnode))
2369             self.failUnless(newnode.is_mutable())
2370             self.failIf(newnode.is_readonly())
2371             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2372         d.addCallback(_got3)
2373
2374         # look at the JSON form of the enclosing directory
2375         d.addCallback(lambda res:
2376                       self.GET(self.public_url + "/foo/?t=json",
2377                                followRedirect=True))
2378         def _check_page_json(res):
2379             parsed = simplejson.loads(res)
2380             self.failUnlessEqual(parsed[0], "dirnode")
2381             children = dict( [(unicode(name),value)
2382                               for (name,value)
2383                               in parsed[1]["children"].iteritems()] )
2384             self.failUnlessIn(u"new.txt", children)
2385             new_json = children[u"new.txt"]
2386             self.failUnlessEqual(new_json[0], "filenode")
2387             self.failUnless(new_json[1]["mutable"])
2388             self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri)
2389             ro_uri = self._mutable_node.get_readonly().to_string()
2390             self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri)
2391         d.addCallback(_check_page_json)
2392
2393         # and the JSON form of the file
2394         d.addCallback(lambda res:
2395                       self.GET(self.public_url + "/foo/new.txt?t=json"))
2396         def _check_file_json(res):
2397             parsed = simplejson.loads(res)
2398             self.failUnlessEqual(parsed[0], "filenode")
2399             self.failUnless(parsed[1]["mutable"])
2400             self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri)
2401             ro_uri = self._mutable_node.get_readonly().to_string()
2402             self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri)
2403         d.addCallback(_check_file_json)
2404
2405         # and look at t=uri and t=readonly-uri
2406         d.addCallback(lambda res:
2407                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
2408         d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
2409         d.addCallback(lambda res:
2410                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
2411         def _check_ro_uri(res):
2412             ro_uri = self._mutable_node.get_readonly().to_string()
2413             self.failUnlessReallyEqual(res, ro_uri)
2414         d.addCallback(_check_ro_uri)
2415
2416         # make sure we can get to it from /uri/URI
2417         d.addCallback(lambda res:
2418                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
2419         d.addCallback(lambda res:
2420                       self.failUnlessReallyEqual(res, NEW2_CONTENTS))
2421
2422         # and that HEAD computes the size correctly
2423         d.addCallback(lambda res:
2424                       self.HEAD(self.public_url + "/foo/new.txt",
2425                                 return_response=True))
2426         def _got_headers((res, status, headers)):
2427             self.failUnlessReallyEqual(res, "")
2428             self.failUnlessReallyEqual(headers["content-length"][0],
2429                                        str(len(NEW2_CONTENTS)))
2430             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
2431         d.addCallback(_got_headers)
2432
2433         # make sure that outdated size limits aren't enforced anymore.
2434         d.addCallback(lambda ignored:
2435             self.POST(self.public_url + "/foo", t="upload",
2436                       mutable="true",
2437                       file=("new.txt",
2438                             "b" * (self.s.MUTABLE_SIZELIMIT+1))))
2439         d.addErrback(self.dump_error)
2440         return d
2441
2442     def test_POST_upload_mutable_toobig(self):
2443         # SDMF had a size limti that was removed a while ago. MDMF has
2444         # never had a size limit. Test to make sure that we do not
2445         # encounter errors when trying to upload large mutable files,
2446         # since there should be no coded prohibitions regarding large
2447         # mutable files.
2448         d = self.POST(self.public_url + "/foo",
2449                       t="upload", mutable="true",
2450                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2451         return d
2452
2453     def dump_error(self, f):
2454         # if the web server returns an error code (like 400 Bad Request),
2455         # web.client.getPage puts the HTTP response body into the .response
2456         # attribute of the exception object that it gives back. It does not
2457         # appear in the Failure's repr(), so the ERROR that trial displays
2458         # will be rather terse and unhelpful. addErrback this method to the
2459         # end of your chain to get more information out of these errors.
2460         if f.check(error.Error):
2461             print "web.error.Error:"
2462             print f
2463             print f.value.response
2464         return f
2465
2466     def test_POST_upload_replace(self):
2467         d = self.POST(self.public_url + "/foo", t="upload",
2468                       file=("bar.txt", self.NEWFILE_CONTENTS))
2469         fn = self._foo_node
2470         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
2471         d.addCallback(lambda res:
2472                       self.failUnlessChildContentsAre(fn, u"bar.txt",
2473                                                       self.NEWFILE_CONTENTS))
2474         return d
2475
2476     def test_POST_upload_no_replace_ok(self):
2477         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2478                       file=("new.txt", self.NEWFILE_CONTENTS))
2479         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
2480         d.addCallback(lambda res: self.failUnlessReallyEqual(res,
2481                                                              self.NEWFILE_CONTENTS))
2482         return d
2483
2484     def test_POST_upload_no_replace_queryarg(self):
2485         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2486                       file=("bar.txt", self.NEWFILE_CONTENTS))
2487         d.addBoth(self.shouldFail, error.Error,
2488                   "POST_upload_no_replace_queryarg",
2489                   "409 Conflict",
2490                   "There was already a child by that name, and you asked me "
2491                   "to not replace it")
2492         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2493         d.addCallback(self.failUnlessIsBarDotTxt)
2494         return d
2495
2496     def test_POST_upload_no_replace_field(self):
2497         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
2498                       file=("bar.txt", self.NEWFILE_CONTENTS))
2499         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
2500                   "409 Conflict",
2501                   "There was already a child by that name, and you asked me "
2502                   "to not replace it")
2503         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2504         d.addCallback(self.failUnlessIsBarDotTxt)
2505         return d
2506
2507     def test_POST_upload_whendone(self):
2508         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
2509                       file=("new.txt", self.NEWFILE_CONTENTS))
2510         d.addBoth(self.shouldRedirect, "/THERE")
2511         fn = self._foo_node
2512         d.addCallback(lambda res:
2513                       self.failUnlessChildContentsAre(fn, u"new.txt",
2514                                                       self.NEWFILE_CONTENTS))
2515         return d
2516
2517     def test_POST_upload_named(self):
2518         fn = self._foo_node
2519         d = self.POST(self.public_url + "/foo", t="upload",
2520                       name="new.txt", file=self.NEWFILE_CONTENTS)
2521         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2522         d.addCallback(lambda res:
2523                       self.failUnlessChildContentsAre(fn, u"new.txt",
2524                                                       self.NEWFILE_CONTENTS))
2525         return d
2526
2527     def test_POST_upload_named_badfilename(self):
2528         d = self.POST(self.public_url + "/foo", t="upload",
2529                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
2530         d.addBoth(self.shouldFail, error.Error,
2531                   "test_POST_upload_named_badfilename",
2532                   "400 Bad Request",
2533                   "name= may not contain a slash",
2534                   )
2535         # make sure that nothing was added
2536         d.addCallback(lambda res:
2537                       self.failUnlessNodeKeysAre(self._foo_node,
2538                                                  [self._htmlname_unicode,
2539                                                   u"bar.txt", u"baz.txt", u"blockingfile",
2540                                                   u"empty", u"n\u00fc.txt", u"quux.txt",
2541                                                   u"sub"]))
2542         return d
2543
2544     def test_POST_FILEURL_check(self):
2545         bar_url = self.public_url + "/foo/bar.txt"
2546         d = self.POST(bar_url, t="check")
2547         def _check(res):
2548             self.failUnlessIn("Healthy :", res)
2549         d.addCallback(_check)
2550         redir_url = "http://allmydata.org/TARGET"
2551         def _check2(statuscode, target):
2552             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2553             self.failUnlessReallyEqual(target, redir_url)
2554         d.addCallback(lambda res:
2555                       self.shouldRedirect2("test_POST_FILEURL_check",
2556                                            _check2,
2557                                            self.POST, bar_url,
2558                                            t="check",
2559                                            when_done=redir_url))
2560         d.addCallback(lambda res:
2561                       self.POST(bar_url, t="check", return_to=redir_url))
2562         def _check3(res):
2563             self.failUnlessIn("Healthy :", res)
2564             self.failUnlessIn("Return to file", res)
2565             self.failUnlessIn(redir_url, res)
2566         d.addCallback(_check3)
2567
2568         d.addCallback(lambda res:
2569                       self.POST(bar_url, t="check", output="JSON"))
2570         def _check_json(res):
2571             data = simplejson.loads(res)
2572             self.failUnlessIn("storage-index", data)
2573             self.failUnless(data["results"]["healthy"])
2574         d.addCallback(_check_json)
2575
2576         return d
2577
2578     def test_POST_FILEURL_check_and_repair(self):
2579         bar_url = self.public_url + "/foo/bar.txt"
2580         d = self.POST(bar_url, t="check", repair="true")
2581         def _check(res):
2582             self.failUnlessIn("Healthy :", res)
2583         d.addCallback(_check)
2584         redir_url = "http://allmydata.org/TARGET"
2585         def _check2(statuscode, target):
2586             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2587             self.failUnlessReallyEqual(target, redir_url)
2588         d.addCallback(lambda res:
2589                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
2590                                            _check2,
2591                                            self.POST, bar_url,
2592                                            t="check", repair="true",
2593                                            when_done=redir_url))
2594         d.addCallback(lambda res:
2595                       self.POST(bar_url, t="check", return_to=redir_url))
2596         def _check3(res):
2597             self.failUnlessIn("Healthy :", res)
2598             self.failUnlessIn("Return to file", res)
2599             self.failUnlessIn(redir_url, res)
2600         d.addCallback(_check3)
2601         return d
2602
2603     def test_POST_DIRURL_check(self):
2604         foo_url = self.public_url + "/foo/"
2605         d = self.POST(foo_url, t="check")
2606         def _check(res):
2607             self.failUnlessIn("Healthy :", res)
2608         d.addCallback(_check)
2609         redir_url = "http://allmydata.org/TARGET"
2610         def _check2(statuscode, target):
2611             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2612             self.failUnlessReallyEqual(target, redir_url)
2613         d.addCallback(lambda res:
2614                       self.shouldRedirect2("test_POST_DIRURL_check",
2615                                            _check2,
2616                                            self.POST, foo_url,
2617                                            t="check",
2618                                            when_done=redir_url))
2619         d.addCallback(lambda res:
2620                       self.POST(foo_url, t="check", return_to=redir_url))
2621         def _check3(res):
2622             self.failUnlessIn("Healthy :", res)
2623             self.failUnlessIn("Return to file/directory", res)
2624             self.failUnlessIn(redir_url, res)
2625         d.addCallback(_check3)
2626
2627         d.addCallback(lambda res:
2628                       self.POST(foo_url, t="check", output="JSON"))
2629         def _check_json(res):
2630             data = simplejson.loads(res)
2631             self.failUnlessIn("storage-index", data)
2632             self.failUnless(data["results"]["healthy"])
2633         d.addCallback(_check_json)
2634
2635         return d
2636
2637     def test_POST_DIRURL_check_and_repair(self):
2638         foo_url = self.public_url + "/foo/"
2639         d = self.POST(foo_url, t="check", repair="true")
2640         def _check(res):
2641             self.failUnlessIn("Healthy :", res)
2642         d.addCallback(_check)
2643         redir_url = "http://allmydata.org/TARGET"
2644         def _check2(statuscode, target):
2645             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2646             self.failUnlessReallyEqual(target, redir_url)
2647         d.addCallback(lambda res:
2648                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2649                                            _check2,
2650                                            self.POST, foo_url,
2651                                            t="check", repair="true",
2652                                            when_done=redir_url))
2653         d.addCallback(lambda res:
2654                       self.POST(foo_url, t="check", return_to=redir_url))
2655         def _check3(res):
2656             self.failUnlessIn("Healthy :", res)
2657             self.failUnlessIn("Return to file/directory", res)
2658             self.failUnlessIn(redir_url, res)
2659         d.addCallback(_check3)
2660         return d
2661
2662     def test_POST_FILEURL_mdmf_check(self):
2663         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2664         d = self.POST(quux_url, t="check")
2665         def _check(res):
2666             self.failUnlessIn("Healthy", res)
2667         d.addCallback(_check)
2668         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2669         d.addCallback(lambda ignored:
2670                       self.POST(quux_extension_url, t="check"))
2671         d.addCallback(_check)
2672         return d
2673
2674     def test_POST_FILEURL_mdmf_check_and_repair(self):
2675         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2676         d = self.POST(quux_url, t="check", repair="true")
2677         def _check(res):
2678             self.failUnlessIn("Healthy", res)
2679         d.addCallback(_check)
2680         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2681         d.addCallback(lambda ignored:
2682                       self.POST(quux_extension_url, t="check", repair="true"))
2683         d.addCallback(_check)
2684         return d
2685
2686     def wait_for_operation(self, ignored, ophandle):
2687         url = "/operations/" + ophandle
2688         url += "?t=status&output=JSON"
2689         d = self.GET(url)
2690         def _got(res):
2691             data = simplejson.loads(res)
2692             if not data["finished"]:
2693                 d = self.stall(delay=1.0)
2694                 d.addCallback(self.wait_for_operation, ophandle)
2695                 return d
2696             return data
2697         d.addCallback(_got)
2698         return d
2699
2700     def get_operation_results(self, ignored, ophandle, output=None):
2701         url = "/operations/" + ophandle
2702         url += "?t=status"
2703         if output:
2704             url += "&output=" + output
2705         d = self.GET(url)
2706         def _got(res):
2707             if output and output.lower() == "json":
2708                 return simplejson.loads(res)
2709             return res
2710         d.addCallback(_got)
2711         return d
2712
2713     def test_POST_DIRURL_deepcheck_no_ophandle(self):
2714         d = self.shouldFail2(error.Error,
2715                              "test_POST_DIRURL_deepcheck_no_ophandle",
2716                              "400 Bad Request",
2717                              "slow operation requires ophandle=",
2718                              self.POST, self.public_url, t="start-deep-check")
2719         return d
2720
2721     def test_POST_DIRURL_deepcheck(self):
2722         def _check_redirect(statuscode, target):
2723             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2724             self.failUnless(target.endswith("/operations/123"))
2725         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2726                                  self.POST, self.public_url,
2727                                  t="start-deep-check", ophandle="123")
2728         d.addCallback(self.wait_for_operation, "123")
2729         def _check_json(data):
2730             self.failUnlessReallyEqual(data["finished"], True)
2731             self.failUnlessReallyEqual(data["count-objects-checked"], 11)
2732             self.failUnlessReallyEqual(data["count-objects-healthy"], 11)
2733         d.addCallback(_check_json)
2734         d.addCallback(self.get_operation_results, "123", "html")
2735         def _check_html(res):
2736             self.failUnlessIn("Objects Checked: <span>11</span>", res)
2737             self.failUnlessIn("Objects Healthy: <span>11</span>", res)
2738             self.failUnlessIn(FAVICON_MARKUP, res)
2739         d.addCallback(_check_html)
2740
2741         d.addCallback(lambda res:
2742                       self.GET("/operations/123/"))
2743         d.addCallback(_check_html) # should be the same as without the slash
2744
2745         d.addCallback(lambda res:
2746                       self.shouldFail2(error.Error, "one", "404 Not Found",
2747                                        "No detailed results for SI bogus",
2748                                        self.GET, "/operations/123/bogus"))
2749
2750         foo_si = self._foo_node.get_storage_index()
2751         foo_si_s = base32.b2a(foo_si)
2752         d.addCallback(lambda res:
2753                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2754         def _check_foo_json(res):
2755             data = simplejson.loads(res)
2756             self.failUnlessEqual(data["storage-index"], foo_si_s)
2757             self.failUnless(data["results"]["healthy"])
2758         d.addCallback(_check_foo_json)
2759         return d
2760
2761     def test_POST_DIRURL_deepcheck_and_repair(self):
2762         d = self.POST(self.public_url, t="start-deep-check", repair="true",
2763                       ophandle="124", output="json", followRedirect=True)
2764         d.addCallback(self.wait_for_operation, "124")
2765         def _check_json(data):
2766             self.failUnlessReallyEqual(data["finished"], True)
2767             self.failUnlessReallyEqual(data["count-objects-checked"], 11)
2768             self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 11)
2769             self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2770             self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2771             self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2772             self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2773             self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2774             self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 11)
2775             self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2776             self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2777         d.addCallback(_check_json)
2778         d.addCallback(self.get_operation_results, "124", "html")
2779         def _check_html(res):
2780             self.failUnlessIn("Objects Checked: <span>11</span>", res)
2781
2782             self.failUnlessIn("Objects Healthy (before repair): <span>11</span>", res)
2783             self.failUnlessIn("Objects Unhealthy (before repair): <span>0</span>", res)
2784             self.failUnlessIn("Corrupt Shares (before repair): <span>0</span>", res)
2785
2786             self.failUnlessIn("Repairs Attempted: <span>0</span>", res)
2787             self.failUnlessIn("Repairs Successful: <span>0</span>", res)
2788             self.failUnlessIn("Repairs Unsuccessful: <span>0</span>", res)
2789
2790             self.failUnlessIn("Objects Healthy (after repair): <span>11</span>", res)
2791             self.failUnlessIn("Objects Unhealthy (after repair): <span>0</span>", res)
2792             self.failUnlessIn("Corrupt Shares (after repair): <span>0</span>", res)
2793
2794             self.failUnlessIn(FAVICON_MARKUP, res)
2795         d.addCallback(_check_html)
2796         return d
2797
2798     def test_POST_FILEURL_bad_t(self):
2799         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2800                              "POST to file: bad t=bogus",
2801                              self.POST, self.public_url + "/foo/bar.txt",
2802                              t="bogus")
2803         return d
2804
2805     def test_POST_mkdir(self): # return value?
2806         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2807         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2808         d.addCallback(self.failUnlessNodeKeysAre, [])
2809         return d
2810
2811     def test_POST_mkdir_mdmf(self):
2812         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=mdmf")
2813         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2814         d.addCallback(lambda node:
2815             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2816         return d
2817
2818     def test_POST_mkdir_sdmf(self):
2819         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=sdmf")
2820         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2821         d.addCallback(lambda node:
2822             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2823         return d
2824
2825     def test_POST_mkdir_bad_format(self):
2826         return self.shouldHTTPError("POST_mkdir_bad_format",
2827                                     400, "Bad Request", "Unknown format: foo",
2828                                     self.POST, self.public_url +
2829                                     "/foo?t=mkdir&name=newdir&format=foo")
2830
2831     def test_POST_mkdir_initial_children(self):
2832         (newkids, caps) = self._create_initial_children()
2833         d = self.POST2(self.public_url +
2834                        "/foo?t=mkdir-with-children&name=newdir",
2835                        simplejson.dumps(newkids))
2836         d.addCallback(lambda res:
2837                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2838         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2839         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2840         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2841         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2842         return d
2843
2844     def test_POST_mkdir_initial_children_mdmf(self):
2845         (newkids, caps) = self._create_initial_children()
2846         d = self.POST2(self.public_url +
2847                        "/foo?t=mkdir-with-children&name=newdir&format=mdmf",
2848                        simplejson.dumps(newkids))
2849         d.addCallback(lambda res:
2850                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2851         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2852         d.addCallback(lambda node:
2853             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2854         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2855         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2856                        caps['filecap1'])
2857         return d
2858
2859     # XXX: Duplication.
2860     def test_POST_mkdir_initial_children_sdmf(self):
2861         (newkids, caps) = self._create_initial_children()
2862         d = self.POST2(self.public_url +
2863                        "/foo?t=mkdir-with-children&name=newdir&format=sdmf",
2864                        simplejson.dumps(newkids))
2865         d.addCallback(lambda res:
2866                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2867         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2868         d.addCallback(lambda node:
2869             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2870         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2871         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2872                        caps['filecap1'])
2873         return d
2874
2875     def test_POST_mkdir_initial_children_bad_format(self):
2876         (newkids, caps) = self._create_initial_children()
2877         return self.shouldHTTPError("POST_mkdir_initial_children_bad_format",
2878                                     400, "Bad Request", "Unknown format: foo",
2879                                     self.POST, self.public_url + \
2880                                     "/foo?t=mkdir-with-children&name=newdir&format=foo",
2881                                     simplejson.dumps(newkids))
2882
2883     def test_POST_mkdir_immutable(self):
2884         (newkids, caps) = self._create_immutable_children()
2885         d = self.POST2(self.public_url +
2886                        "/foo?t=mkdir-immutable&name=newdir",
2887                        simplejson.dumps(newkids))
2888         d.addCallback(lambda res:
2889                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2890         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2891         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2892         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2893         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2894         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2895         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2896         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2897         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2898         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2899         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2900         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2901         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2902         return d
2903
2904     def test_POST_mkdir_immutable_bad(self):
2905         (newkids, caps) = self._create_initial_children()
2906         d = self.shouldFail2(error.Error, "POST_mkdir_immutable_bad",
2907                              "400 Bad Request",
2908                              "needed to be immutable but was not",
2909                              self.POST2,
2910                              self.public_url +
2911                              "/foo?t=mkdir-immutable&name=newdir",
2912                              simplejson.dumps(newkids))
2913         return d
2914
2915     def test_POST_mkdir_2(self):
2916         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
2917         d.addCallback(lambda res:
2918                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2919         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2920         d.addCallback(self.failUnlessNodeKeysAre, [])
2921         return d
2922
2923     def test_POST_mkdirs_2(self):
2924         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
2925         d.addCallback(lambda res:
2926                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
2927         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
2928         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
2929         d.addCallback(self.failUnlessNodeKeysAre, [])
2930         return d
2931
2932     def test_POST_mkdir_no_parentdir_noredirect(self):
2933         d = self.POST("/uri?t=mkdir")
2934         def _after_mkdir(res):
2935             uri.DirectoryURI.init_from_string(res)
2936         d.addCallback(_after_mkdir)
2937         return d
2938
2939     def test_POST_mkdir_no_parentdir_noredirect_mdmf(self):
2940         d = self.POST("/uri?t=mkdir&format=mdmf")
2941         def _after_mkdir(res):
2942             u = uri.from_string(res)
2943             # Check that this is an MDMF writecap
2944             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
2945         d.addCallback(_after_mkdir)
2946         return d
2947
2948     def test_POST_mkdir_no_parentdir_noredirect_sdmf(self):
2949         d = self.POST("/uri?t=mkdir&format=sdmf")
2950         def _after_mkdir(res):
2951             u = uri.from_string(res)
2952             self.failUnlessIsInstance(u, uri.DirectoryURI)
2953         d.addCallback(_after_mkdir)
2954         return d
2955
2956     def test_POST_mkdir_no_parentdir_noredirect_bad_format(self):
2957         return self.shouldHTTPError("POST_mkdir_no_parentdir_noredirect_bad_format",
2958                                     400, "Bad Request", "Unknown format: foo",
2959                                     self.POST, self.public_url +
2960                                     "/uri?t=mkdir&format=foo")
2961
2962     def test_POST_mkdir_no_parentdir_noredirect2(self):
2963         # make sure form-based arguments (as on the welcome page) still work
2964         d = self.POST("/uri", t="mkdir")
2965         def _after_mkdir(res):
2966             uri.DirectoryURI.init_from_string(res)
2967         d.addCallback(_after_mkdir)
2968         d.addErrback(self.explain_web_error)
2969         return d
2970
2971     def test_POST_mkdir_no_parentdir_redirect(self):
2972         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
2973         d.addBoth(self.shouldRedirect, None, statuscode='303')
2974         def _check_target(target):
2975             target = urllib.unquote(target)
2976             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2977         d.addCallback(_check_target)
2978         return d
2979
2980     def test_POST_mkdir_no_parentdir_redirect2(self):
2981         d = self.POST("/uri", t="mkdir", redirect_to_result="true")
2982         d.addBoth(self.shouldRedirect, None, statuscode='303')
2983         def _check_target(target):
2984             target = urllib.unquote(target)
2985             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2986         d.addCallback(_check_target)
2987         d.addErrback(self.explain_web_error)
2988         return d
2989
2990     def _make_readonly(self, u):
2991         ro_uri = uri.from_string(u).get_readonly()
2992         if ro_uri is None:
2993             return None
2994         return ro_uri.to_string()
2995
2996     def _create_initial_children(self):
2997         contents, n, filecap1 = self.makefile(12)
2998         md1 = {"metakey1": "metavalue1"}
2999         filecap2 = make_mutable_file_uri()
3000         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
3001         filecap3 = node3.get_readonly_uri()
3002         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
3003         dircap = DirectoryNode(node4, None, None).get_uri()
3004         mdmfcap = make_mutable_file_uri(mdmf=True)
3005         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
3006         emptydircap = "URI:DIR2-LIT:"
3007         newkids = {u"child-imm":        ["filenode", {"rw_uri": filecap1,
3008                                                       "ro_uri": self._make_readonly(filecap1),
3009                                                       "metadata": md1, }],
3010                    u"child-mutable":    ["filenode", {"rw_uri": filecap2,
3011                                                       "ro_uri": self._make_readonly(filecap2)}],
3012                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
3013                    u"unknownchild-rw":  ["unknown",  {"rw_uri": unknown_rwcap,
3014                                                       "ro_uri": unknown_rocap}],
3015                    u"unknownchild-ro":  ["unknown",  {"ro_uri": unknown_rocap}],
3016                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
3017                    u"dirchild":         ["dirnode",  {"rw_uri": dircap,
3018                                                       "ro_uri": self._make_readonly(dircap)}],
3019                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
3020                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
3021                    u"child-mutable-mdmf": ["filenode", {"rw_uri": mdmfcap,
3022                                                         "ro_uri": self._make_readonly(mdmfcap)}],
3023                    }
3024         return newkids, {'filecap1': filecap1,
3025                          'filecap2': filecap2,
3026                          'filecap3': filecap3,
3027                          'unknown_rwcap': unknown_rwcap,
3028                          'unknown_rocap': unknown_rocap,
3029                          'unknown_immcap': unknown_immcap,
3030                          'dircap': dircap,
3031                          'litdircap': litdircap,
3032                          'emptydircap': emptydircap,
3033                          'mdmfcap': mdmfcap}
3034
3035     def _create_immutable_children(self):
3036         contents, n, filecap1 = self.makefile(12)
3037         md1 = {"metakey1": "metavalue1"}
3038         tnode = create_chk_filenode("immutable directory contents\n"*10,
3039                                     self.get_all_contents())
3040         dnode = DirectoryNode(tnode, None, None)
3041         assert not dnode.is_mutable()
3042         immdircap = dnode.get_uri()
3043         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
3044         emptydircap = "URI:DIR2-LIT:"
3045         newkids = {u"child-imm":        ["filenode", {"ro_uri": filecap1,
3046                                                       "metadata": md1, }],
3047                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
3048                    u"dirchild-imm":     ["dirnode",  {"ro_uri": immdircap}],
3049                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
3050                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
3051                    }
3052         return newkids, {'filecap1': filecap1,
3053                          'unknown_immcap': unknown_immcap,
3054                          'immdircap': immdircap,
3055                          'litdircap': litdircap,
3056                          'emptydircap': emptydircap}
3057
3058     def test_POST_mkdir_no_parentdir_initial_children(self):
3059         (newkids, caps) = self._create_initial_children()
3060         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
3061         def _after_mkdir(res):
3062             self.failUnless(res.startswith("URI:DIR"), res)
3063             n = self.s.create_node_from_uri(res)
3064             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3065             d2.addCallback(lambda ign:
3066                            self.failUnlessROChildURIIs(n, u"child-imm",
3067                                                        caps['filecap1']))
3068             d2.addCallback(lambda ign:
3069                            self.failUnlessRWChildURIIs(n, u"child-mutable",
3070                                                        caps['filecap2']))
3071             d2.addCallback(lambda ign:
3072                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
3073                                                        caps['filecap3']))
3074             d2.addCallback(lambda ign:
3075                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
3076                                                        caps['unknown_rwcap']))
3077             d2.addCallback(lambda ign:
3078                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
3079                                                        caps['unknown_rocap']))
3080             d2.addCallback(lambda ign:
3081                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3082                                                        caps['unknown_immcap']))
3083             d2.addCallback(lambda ign:
3084                            self.failUnlessRWChildURIIs(n, u"dirchild",
3085                                                        caps['dircap']))
3086             return d2
3087         d.addCallback(_after_mkdir)
3088         return d
3089
3090     def test_POST_mkdir_no_parentdir_unexpected_children(self):
3091         # the regular /uri?t=mkdir operation is specified to ignore its body.
3092         # Only t=mkdir-with-children pays attention to it.
3093         (newkids, caps) = self._create_initial_children()
3094         d = self.shouldHTTPError("POST_mkdir_no_parentdir_unexpected_children",
3095                                  400, "Bad Request",
3096                                  "t=mkdir does not accept children=, "
3097                                  "try t=mkdir-with-children instead",
3098                                  self.POST2, "/uri?t=mkdir", # without children
3099                                  simplejson.dumps(newkids))
3100         return d
3101
3102     def test_POST_noparent_bad(self):
3103         d = self.shouldHTTPError("POST_noparent_bad",
3104                                  400, "Bad Request",
3105                                  "/uri accepts only PUT, PUT?t=mkdir, "
3106                                  "POST?t=upload, and POST?t=mkdir",
3107                                  self.POST, "/uri?t=bogus")
3108         return d
3109
3110     def test_POST_mkdir_no_parentdir_immutable(self):
3111         (newkids, caps) = self._create_immutable_children()
3112         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
3113         def _after_mkdir(res):
3114             self.failUnless(res.startswith("URI:DIR"), res)
3115             n = self.s.create_node_from_uri(res)
3116             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3117             d2.addCallback(lambda ign:
3118                            self.failUnlessROChildURIIs(n, u"child-imm",
3119                                                           caps['filecap1']))
3120             d2.addCallback(lambda ign:
3121                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3122                                                           caps['unknown_immcap']))
3123             d2.addCallback(lambda ign:
3124                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
3125                                                           caps['immdircap']))
3126             d2.addCallback(lambda ign:
3127                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
3128                                                           caps['litdircap']))
3129             d2.addCallback(lambda ign:
3130                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
3131                                                           caps['emptydircap']))
3132             return d2
3133         d.addCallback(_after_mkdir)
3134         return d
3135
3136     def test_POST_mkdir_no_parentdir_immutable_bad(self):
3137         (newkids, caps) = self._create_initial_children()
3138         d = self.shouldFail2(error.Error,
3139                              "test_POST_mkdir_no_parentdir_immutable_bad",
3140                              "400 Bad Request",
3141                              "needed to be immutable but was not",
3142                              self.POST2,
3143                              "/uri?t=mkdir-immutable",
3144                              simplejson.dumps(newkids))
3145         return d
3146
3147     def test_welcome_page_mkdir_button(self):
3148         # Fetch the welcome page.
3149         d = self.GET("/")
3150         def _after_get_welcome_page(res):
3151             MKDIR_BUTTON_RE = re.compile(
3152                 '<form action="([^"]*)" method="post".*?'
3153                 '<input type="hidden" name="t" value="([^"]*)" />'
3154                 '<input type="hidden" name="([^"]*)" value="([^"]*)" />'
3155                 '<input type="submit" value="Create a directory" />',
3156                 re.I)
3157             mo = MKDIR_BUTTON_RE.search(res)
3158             formaction = mo.group(1)
3159             formt = mo.group(2)
3160             formaname = mo.group(3)
3161             formavalue = mo.group(4)
3162             return (formaction, formt, formaname, formavalue)
3163         d.addCallback(_after_get_welcome_page)
3164         def _after_parse_form(res):
3165             (formaction, formt, formaname, formavalue) = res
3166             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
3167         d.addCallback(_after_parse_form)
3168         d.addBoth(self.shouldRedirect, None, statuscode='303')
3169         return d
3170
3171     def test_POST_mkdir_replace(self): # return value?
3172         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
3173         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3174         d.addCallback(self.failUnlessNodeKeysAre, [])
3175         return d
3176
3177     def test_POST_mkdir_no_replace_queryarg(self): # return value?
3178         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
3179         d.addBoth(self.shouldFail, error.Error,
3180                   "POST_mkdir_no_replace_queryarg",
3181                   "409 Conflict",
3182                   "There was already a child by that name, and you asked me "
3183                   "to not replace it")
3184         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3185         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3186         return d
3187
3188     def test_POST_mkdir_no_replace_field(self): # return value?
3189         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
3190                       replace="false")
3191         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
3192                   "409 Conflict",
3193                   "There was already a child by that name, and you asked me "
3194                   "to not replace it")
3195         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3196         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3197         return d
3198
3199     def test_POST_mkdir_whendone_field(self):
3200         d = self.POST(self.public_url + "/foo",
3201                       t="mkdir", name="newdir", when_done="/THERE")
3202         d.addBoth(self.shouldRedirect, "/THERE")
3203         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3204         d.addCallback(self.failUnlessNodeKeysAre, [])
3205         return d
3206
3207     def test_POST_mkdir_whendone_queryarg(self):
3208         d = self.POST(self.public_url + "/foo?when_done=/THERE",
3209                       t="mkdir", name="newdir")
3210         d.addBoth(self.shouldRedirect, "/THERE")
3211         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3212         d.addCallback(self.failUnlessNodeKeysAre, [])
3213         return d
3214
3215     def test_POST_bad_t(self):
3216         d = self.shouldFail2(error.Error, "POST_bad_t",
3217                              "400 Bad Request",
3218                              "POST to a directory with bad t=BOGUS",
3219                              self.POST, self.public_url + "/foo", t="BOGUS")
3220         return d
3221
3222     def test_POST_set_children(self, command_name="set_children"):
3223         contents9, n9, newuri9 = self.makefile(9)
3224         contents10, n10, newuri10 = self.makefile(10)
3225         contents11, n11, newuri11 = self.makefile(11)
3226
3227         reqbody = """{
3228                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
3229                                                 "size": 0,
3230                                                 "metadata": {
3231                                                   "ctime": 1002777696.7564139,
3232                                                   "mtime": 1002777696.7564139
3233                                                  }
3234                                                } ],
3235                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
3236                                                 "size": 1,
3237                                                 "metadata": {
3238                                                   "ctime": 1002777696.7564139,
3239                                                   "mtime": 1002777696.7564139
3240                                                  }
3241                                                } ],
3242                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
3243                                                 "size": 2,
3244                                                 "metadata": {
3245                                                   "ctime": 1002777696.7564139,
3246                                                   "mtime": 1002777696.7564139
3247                                                  }
3248                                                } ]
3249                     }""" % (newuri9, newuri10, newuri11)
3250
3251         url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
3252
3253         d = client.getPage(url, method="POST", postdata=reqbody)
3254         def _then(res):
3255             self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
3256             self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
3257             self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
3258
3259         d.addCallback(_then)
3260         d.addErrback(self.dump_error)
3261         return d
3262
3263     def test_POST_set_children_with_hyphen(self):
3264         return self.test_POST_set_children(command_name="set-children")
3265
3266     def test_POST_link_uri(self):
3267         contents, n, newuri = self.makefile(8)
3268         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
3269         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
3270         d.addCallback(lambda res:
3271                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3272                                                       contents))
3273         return d
3274
3275     def test_POST_link_uri_replace(self):
3276         contents, n, newuri = self.makefile(8)
3277         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
3278         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
3279         d.addCallback(lambda res:
3280                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3281                                                       contents))
3282         return d
3283
3284     def test_POST_link_uri_unknown_bad(self):
3285         d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
3286         d.addBoth(self.shouldFail, error.Error,
3287                   "POST_link_uri_unknown_bad",
3288                   "400 Bad Request",
3289                   "unknown cap in a write slot")
3290         return d
3291
3292     def test_POST_link_uri_unknown_ro_good(self):
3293         d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
3294         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
3295         return d
3296
3297     def test_POST_link_uri_unknown_imm_good(self):
3298         d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
3299         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
3300         return d
3301
3302     def test_POST_link_uri_no_replace_queryarg(self):
3303         contents, n, newuri = self.makefile(8)
3304         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
3305                       name="bar.txt", uri=newuri)
3306         d.addBoth(self.shouldFail, error.Error,
3307                   "POST_link_uri_no_replace_queryarg",
3308                   "409 Conflict",
3309                   "There was already a child by that name, and you asked me "
3310                   "to not replace it")
3311         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3312         d.addCallback(self.failUnlessIsBarDotTxt)
3313         return d
3314
3315     def test_POST_link_uri_no_replace_field(self):
3316         contents, n, newuri = self.makefile(8)
3317         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
3318                       name="bar.txt", uri=newuri)
3319         d.addBoth(self.shouldFail, error.Error,
3320                   "POST_link_uri_no_replace_field",
3321                   "409 Conflict",
3322                   "There was already a child by that name, and you asked me "
3323                   "to not replace it")
3324         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3325         d.addCallback(self.failUnlessIsBarDotTxt)
3326         return d
3327
3328     def test_POST_delete(self, command_name='delete'):
3329         d = self._foo_node.list()
3330         def _check_before(children):
3331             self.failUnlessIn(u"bar.txt", children)
3332         d.addCallback(_check_before)
3333         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t=command_name, name="bar.txt"))
3334         d.addCallback(lambda res: self._foo_node.list())
3335         def _check_after(children):
3336             self.failIfIn(u"bar.txt", children)
3337         d.addCallback(_check_after)
3338         return d
3339
3340     def test_POST_unlink(self):
3341         return self.test_POST_delete(command_name='unlink')
3342
3343     def test_POST_rename_file(self):
3344         d = self.POST(self.public_url + "/foo", t="rename",
3345                       from_name="bar.txt", to_name='wibble.txt')
3346         d.addCallback(lambda res:
3347                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3348         d.addCallback(lambda res:
3349                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
3350         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
3351         d.addCallback(self.failUnlessIsBarDotTxt)
3352         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
3353         d.addCallback(self.failUnlessIsBarJSON)
3354         return d
3355
3356     def test_POST_rename_file_redundant(self):
3357         d = self.POST(self.public_url + "/foo", t="rename",
3358                       from_name="bar.txt", to_name='bar.txt')
3359         d.addCallback(lambda res:
3360                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3361         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3362         d.addCallback(self.failUnlessIsBarDotTxt)
3363         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3364         d.addCallback(self.failUnlessIsBarJSON)
3365         return d
3366
3367     def test_POST_rename_file_replace(self):
3368         # rename a file and replace a directory with it
3369         d = self.POST(self.public_url + "/foo", t="rename",
3370                       from_name="bar.txt", to_name='empty')
3371         d.addCallback(lambda res:
3372                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3373         d.addCallback(lambda res:
3374                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
3375         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
3376         d.addCallback(self.failUnlessIsBarDotTxt)
3377         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3378         d.addCallback(self.failUnlessIsBarJSON)
3379         return d
3380
3381     def test_POST_rename_file_no_replace_queryarg(self):
3382         # rename a file and replace a directory with it
3383         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
3384                       from_name="bar.txt", to_name='empty')
3385         d.addBoth(self.shouldFail, error.Error,
3386                   "POST_rename_file_no_replace_queryarg",
3387                   "409 Conflict",
3388                   "There was already a child by that name, and you asked me "
3389                   "to not replace it")
3390         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3391         d.addCallback(self.failUnlessIsEmptyJSON)
3392         return d
3393
3394     def test_POST_rename_file_no_replace_field(self):
3395         # rename a file and replace a directory with it
3396         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
3397                       from_name="bar.txt", to_name='empty')
3398         d.addBoth(self.shouldFail, error.Error,
3399                   "POST_rename_file_no_replace_field",
3400                   "409 Conflict",
3401                   "There was already a child by that name, and you asked me "
3402                   "to not replace it")
3403         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3404         d.addCallback(self.failUnlessIsEmptyJSON)
3405         return d
3406
3407     def failUnlessIsEmptyJSON(self, res):
3408         data = simplejson.loads(res)
3409         self.failUnlessEqual(data[0], "dirnode", data)
3410         self.failUnlessReallyEqual(len(data[1]["children"]), 0)
3411
3412     def test_POST_rename_file_slash_fail(self):
3413         d = self.POST(self.public_url + "/foo", t="rename",
3414                       from_name="bar.txt", to_name='kirk/spock.txt')
3415         d.addBoth(self.shouldFail, error.Error,
3416                   "test_POST_rename_file_slash_fail",
3417                   "400 Bad Request",
3418                   "to_name= may not contain a slash",
3419                   )
3420         d.addCallback(lambda res:
3421                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3422         return d
3423
3424     def test_POST_rename_dir(self):
3425         d = self.POST(self.public_url, t="rename",
3426                       from_name="foo", to_name='plunk')
3427         d.addCallback(lambda res:
3428                       self.failIfNodeHasChild(self.public_root, u"foo"))
3429         d.addCallback(lambda res:
3430                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
3431         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
3432         d.addCallback(self.failUnlessIsFooJSON)
3433         return d
3434
3435     def test_POST_move_file(self):
3436         d = self.POST(self.public_url + "/foo", t="move",
3437                       from_name="bar.txt", to_dir="sub")
3438         d.addCallback(lambda res:
3439                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3440         d.addCallback(lambda res:
3441                       self.failUnlessNodeHasChild(self._sub_node, u"bar.txt"))
3442         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3443         d.addCallback(self.failUnlessIsBarDotTxt)
3444         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3445         d.addCallback(self.failUnlessIsBarJSON)
3446         return d
3447
3448     def test_POST_move_file_new_name(self):
3449         d = self.POST(self.public_url + "/foo", t="move",
3450                       from_name="bar.txt", to_name="wibble.txt", to_dir="sub")
3451         d.addCallback(lambda res:
3452                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3453         d.addCallback(lambda res:
3454                       self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3455         d.addCallback(lambda res:
3456                       self.failUnlessNodeHasChild(self._sub_node, u"wibble.txt"))
3457         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt"))
3458         d.addCallback(self.failUnlessIsBarDotTxt)
3459         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt?t=json"))
3460         d.addCallback(self.failUnlessIsBarJSON)
3461         return d
3462
3463     def test_POST_move_file_replace(self):
3464         d = self.POST(self.public_url + "/foo", t="move",
3465                       from_name="bar.txt", to_name="baz.txt", to_dir="sub")
3466         d.addCallback(lambda res:
3467                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3468         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3469         d.addCallback(self.failUnlessIsBarDotTxt)
3470         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt?t=json"))
3471         d.addCallback(self.failUnlessIsBarJSON)
3472         return d
3473
3474     def test_POST_move_file_no_replace(self):
3475         d = self.shouldFail2(error.Error, "POST_move_file_no_replace",
3476                              "409 Conflict",
3477                              "There was already a child by that name, and you asked me to not replace it",
3478                              self.POST, self.public_url + "/foo", t="move",
3479                              replace="false", from_name="bar.txt",
3480                              to_name="baz.txt", to_dir="sub")
3481         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3482         d.addCallback(self.failUnlessIsBarDotTxt)
3483         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3484         d.addCallback(self.failUnlessIsBarJSON)
3485         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3486         d.addCallback(self.failUnlessIsSubBazDotTxt)
3487         return d
3488
3489     def test_POST_move_file_slash_fail(self):
3490         d = self.shouldFail2(error.Error, "test_POST_rename_file_slash_fail",
3491                              "400 Bad Request",
3492                              "to_name= may not contain a slash",
3493                              self.POST, self.public_url + "/foo", t="move",
3494                              from_name="bar.txt",
3495                              to_name="slash/fail.txt", to_dir="sub")
3496         d.addCallback(lambda res:
3497                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3498         d.addCallback(lambda res:
3499                       self.failIfNodeHasChild(self._sub_node, u"slash/fail.txt"))
3500         d.addCallback(lambda ign:
3501                       self.shouldFail2(error.Error,
3502                                        "test_POST_rename_file_slash_fail2",
3503                                        "400 Bad Request",
3504                                        "from_name= may not contain a slash",
3505                                        self.POST, self.public_url + "/foo",
3506                                        t="move",
3507                                        from_name="nope/bar.txt",
3508                                        to_name="fail.txt", to_dir="sub"))
3509         return d
3510
3511     def test_POST_move_file_no_target(self):
3512         d = self.shouldFail2(error.Error, "POST_move_file_no_target",
3513                              "400 Bad Request",
3514                              "move requires from_name and to_dir",
3515                              self.POST, self.public_url + "/foo", t="move",
3516                              from_name="bar.txt", to_name="baz.txt")
3517         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3518         d.addCallback(self.failUnlessIsBarDotTxt)
3519         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3520         d.addCallback(self.failUnlessIsBarJSON)
3521         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3522         d.addCallback(self.failUnlessIsBazDotTxt)
3523         return d
3524
3525     def test_POST_move_file_bad_target_type(self):
3526         d = self.shouldFail2(error.Error, "test_POST_move_file_bad_target_type",
3527                              "400 Bad Request", "invalid target_type parameter",
3528                              self.POST,
3529                              self.public_url + "/foo", t="move",
3530                              target_type="*D", from_name="bar.txt",
3531                              to_dir="sub")
3532         return d
3533
3534     def test_POST_move_file_multi_level(self):
3535         d = self.POST(self.public_url + "/foo/sub/level2?t=mkdir", "")
3536         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t="move",
3537                       from_name="bar.txt", to_dir="sub/level2"))
3538         d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3539         d.addCallback(lambda res: self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3540         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt"))
3541         d.addCallback(self.failUnlessIsBarDotTxt)
3542         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt?t=json"))
3543         d.addCallback(self.failUnlessIsBarJSON)
3544         return d
3545
3546     def test_POST_move_file_to_uri(self):
3547         d = self.POST(self.public_url + "/foo", t="move", target_type="uri",
3548                       from_name="bar.txt", to_dir=self._sub_uri)
3549         d.addCallback(lambda res:
3550                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3551         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3552         d.addCallback(self.failUnlessIsBarDotTxt)
3553         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3554         d.addCallback(self.failUnlessIsBarJSON)
3555         return d
3556
3557     def test_POST_move_file_to_nonexist_dir(self):
3558         d = self.shouldFail2(error.Error, "POST_move_file_to_nonexist_dir",
3559                             "404 Not Found", "No such child: nopechucktesta",
3560                             self.POST, self.public_url + "/foo", t="move",
3561                             from_name="bar.txt", to_dir="nopechucktesta")
3562         return d
3563
3564     def test_POST_move_file_into_file(self):
3565         d = self.shouldFail2(error.Error, "POST_move_file_into_file",
3566                              "400 Bad Request", "to_dir is not a directory",
3567                              self.POST, self.public_url + "/foo", t="move",
3568                              from_name="bar.txt", to_dir="baz.txt")
3569         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3570         d.addCallback(self.failUnlessIsBazDotTxt)
3571         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3572         d.addCallback(self.failUnlessIsBarDotTxt)
3573         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3574         d.addCallback(self.failUnlessIsBarJSON)
3575         return d
3576
3577     def test_POST_move_file_to_bad_uri(self):
3578         d =  self.shouldFail2(error.Error, "POST_move_file_to_bad_uri",
3579                               "400 Bad Request", "to_dir is not a directory",
3580                               self.POST, self.public_url + "/foo", t="move",
3581                               from_name="bar.txt", target_type="uri",
3582                               to_dir="URI:DIR2:mn5jlyjnrjeuydyswlzyui72i:rmneifcj6k6sycjljjhj3f6majsq2zqffydnnul5hfa4j577arma")
3583         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3584         d.addCallback(self.failUnlessIsBarDotTxt)
3585         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3586         d.addCallback(self.failUnlessIsBarJSON)
3587         return d
3588
3589     def test_POST_move_dir(self):
3590         d = self.POST(self.public_url + "/foo", t="move",
3591                       from_name="bar.txt", to_dir="empty")
3592         d.addCallback(lambda res: self.POST(self.public_url + "/foo",
3593                       t="move", from_name="empty", to_dir="sub"))
3594         d.addCallback(lambda res:
3595                       self.failIfNodeHasChild(self._foo_node, u"empty"))
3596         d.addCallback(lambda res:
3597                       self.failUnlessNodeHasChild(self._sub_node, u"empty"))
3598         d.addCallback(lambda res:
3599                       self._sub_node.get_child_at_path(u"empty"))
3600         d.addCallback(lambda node:
3601                       self.failUnlessNodeHasChild(node, u"bar.txt"))
3602         d.addCallback(lambda res:
3603                       self.GET(self.public_url + "/foo/sub/empty/bar.txt"))
3604         d.addCallback(self.failUnlessIsBarDotTxt)
3605         return d
3606
3607     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
3608         """ If target is not None then the redirection has to go to target.  If
3609         statuscode is not None then the redirection has to be accomplished with
3610         that HTTP status code."""
3611         if not isinstance(res, failure.Failure):
3612             to_where = (target is None) and "somewhere" or ("to " + target)
3613             self.fail("%s: we were expecting to get redirected %s, not get an"
3614                       " actual page: %s" % (which, to_where, res))
3615         res.trap(error.PageRedirect)
3616         if statuscode is not None:
3617             self.failUnlessReallyEqual(res.value.status, statuscode,
3618                                        "%s: not a redirect" % which)
3619         if target is not None:
3620             # the PageRedirect does not seem to capture the uri= query arg
3621             # properly, so we can't check for it.
3622             realtarget = self.webish_url + target
3623             self.failUnlessReallyEqual(res.value.location, realtarget,
3624                                        "%s: wrong target" % which)
3625         return res.value.location
3626
3627     def test_GET_URI_form(self):
3628         base = "/uri?uri=%s" % self._bar_txt_uri
3629         # this is supposed to give us a redirect to /uri/$URI, plus arguments
3630         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
3631         d = self.GET(base)
3632         d.addBoth(self.shouldRedirect, targetbase)
3633         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
3634         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
3635         d.addCallback(lambda res: self.GET(base+"&t=json"))
3636         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
3637         d.addCallback(self.log, "about to get file by uri")
3638         d.addCallback(lambda res: self.GET(base, followRedirect=True))
3639         d.addCallback(self.failUnlessIsBarDotTxt)
3640         d.addCallback(self.log, "got file by uri, about to get dir by uri")
3641         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
3642                                            followRedirect=True))
3643         d.addCallback(self.failUnlessIsFooJSON)
3644         d.addCallback(self.log, "got dir by uri")
3645
3646         return d
3647
3648     def test_GET_URI_form_bad(self):
3649         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
3650                              "400 Bad Request", "GET /uri requires uri=",
3651                              self.GET, "/uri")
3652         return d
3653
3654     def test_GET_rename_form(self):
3655         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
3656                      followRedirect=True)
3657         def _check(res):
3658             self.failUnlessIn('name="when_done" value="."', res)
3659             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
3660             self.failUnlessIn(FAVICON_MARKUP, res)
3661         d.addCallback(_check)
3662         return d
3663
3664     def log(self, res, msg):
3665         #print "MSG: %s  RES: %s" % (msg, res)
3666         log.msg(msg)
3667         return res
3668
3669     def test_GET_URI_URL(self):
3670         base = "/uri/%s" % self._bar_txt_uri
3671         d = self.GET(base)
3672         d.addCallback(self.failUnlessIsBarDotTxt)
3673         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
3674         d.addCallback(self.failUnlessIsBarDotTxt)
3675         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
3676         d.addCallback(self.failUnlessIsBarDotTxt)
3677         return d
3678
3679     def test_GET_URI_URL_dir(self):
3680         base = "/uri/%s?t=json" % self._foo_uri
3681         d = self.GET(base)
3682         d.addCallback(self.failUnlessIsFooJSON)
3683         return d
3684
3685     def test_GET_URI_URL_missing(self):
3686         base = "/uri/%s" % self._bad_file_uri
3687         d = self.shouldHTTPError("test_GET_URI_URL_missing",
3688                                  http.GONE, None, "NotEnoughSharesError",
3689                                  self.GET, base)
3690         # TODO: how can we exercise both sides of WebDownloadTarget.fail
3691         # here? we must arrange for a download to fail after target.open()
3692         # has been called, and then inspect the response to see that it is
3693         # shorter than we expected.
3694         return d
3695
3696     def test_PUT_DIRURL_uri(self):
3697         d = self.s.create_dirnode()
3698         def _made_dir(dn):
3699             new_uri = dn.get_uri()
3700             # replace /foo with a new (empty) directory
3701             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
3702             d.addCallback(lambda res:
3703                           self.failUnlessReallyEqual(res.strip(), new_uri))
3704             d.addCallback(lambda res:
3705                           self.failUnlessRWChildURIIs(self.public_root,
3706                                                       u"foo",
3707                                                       new_uri))
3708             return d
3709         d.addCallback(_made_dir)
3710         return d
3711
3712     def test_PUT_DIRURL_uri_noreplace(self):
3713         d = self.s.create_dirnode()
3714         def _made_dir(dn):
3715             new_uri = dn.get_uri()
3716             # replace /foo with a new (empty) directory, but ask that
3717             # replace=false, so it should fail
3718             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
3719                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
3720                                  self.PUT,
3721                                  self.public_url + "/foo?t=uri&replace=false",
3722                                  new_uri)
3723             d.addCallback(lambda res:
3724                           self.failUnlessRWChildURIIs(self.public_root,
3725                                                       u"foo",
3726                                                       self._foo_uri))
3727             return d
3728         d.addCallback(_made_dir)
3729         return d
3730
3731     def test_PUT_DIRURL_bad_t(self):
3732         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
3733                              "400 Bad Request", "PUT to a directory",
3734                              self.PUT, self.public_url + "/foo?t=BOGUS", "")
3735         d.addCallback(lambda res:
3736                       self.failUnlessRWChildURIIs(self.public_root,
3737                                                   u"foo",
3738                                                   self._foo_uri))
3739         return d
3740
3741     def test_PUT_NEWFILEURL_uri(self):
3742         contents, n, new_uri = self.makefile(8)
3743         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
3744         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3745         d.addCallback(lambda res:
3746                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3747                                                       contents))
3748         return d
3749
3750     def test_PUT_NEWFILEURL_mdmf(self):
3751         new_contents = self.NEWFILE_CONTENTS * 300000
3752         d = self.PUT(self.public_url + \
3753                      "/foo/mdmf.txt?format=mdmf",
3754                      new_contents)
3755         d.addCallback(lambda ignored:
3756             self.GET(self.public_url + "/foo/mdmf.txt?t=json"))
3757         def _got_json(json):
3758             data = simplejson.loads(json)
3759             data = data[1]
3760             self.failUnlessIn("format", data)
3761             self.failUnlessEqual(data["format"], "MDMF")
3762             self.failUnless(data['rw_uri'].startswith("URI:MDMF"))
3763             self.failUnless(data['ro_uri'].startswith("URI:MDMF"))
3764         d.addCallback(_got_json)
3765         return d
3766
3767     def test_PUT_NEWFILEURL_sdmf(self):
3768         new_contents = self.NEWFILE_CONTENTS * 300000
3769         d = self.PUT(self.public_url + \
3770                      "/foo/sdmf.txt?format=sdmf",
3771                      new_contents)
3772         d.addCallback(lambda ignored:
3773             self.GET(self.public_url + "/foo/sdmf.txt?t=json"))
3774         def _got_json(json):
3775             data = simplejson.loads(json)
3776             data = data[1]
3777             self.failUnlessIn("format", data)
3778             self.failUnlessEqual(data["format"], "SDMF")
3779         d.addCallback(_got_json)
3780         return d
3781
3782     def test_PUT_NEWFILEURL_bad_format(self):
3783         new_contents = self.NEWFILE_CONTENTS * 300000
3784         return self.shouldHTTPError("PUT_NEWFILEURL_bad_format",
3785                                     400, "Bad Request", "Unknown format: foo",
3786                                     self.PUT, self.public_url + \
3787                                     "/foo/foo.txt?format=foo",
3788                                     new_contents)
3789
3790     def test_PUT_NEWFILEURL_uri_replace(self):
3791         contents, n, new_uri = self.makefile(8)
3792         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
3793         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3794         d.addCallback(lambda res:
3795                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3796                                                       contents))
3797         return d
3798
3799     def test_PUT_NEWFILEURL_uri_no_replace(self):
3800         contents, n, new_uri = self.makefile(8)
3801         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
3802         d.addBoth(self.shouldFail, error.Error,
3803                   "PUT_NEWFILEURL_uri_no_replace",
3804                   "409 Conflict",
3805                   "There was already a child by that name, and you asked me "
3806                   "to not replace it")
3807         return d
3808
3809     def test_PUT_NEWFILEURL_uri_unknown_bad(self):
3810         d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
3811         d.addBoth(self.shouldFail, error.Error,
3812                   "POST_put_uri_unknown_bad",
3813                   "400 Bad Request",
3814                   "unknown cap in a write slot")
3815         return d
3816
3817     def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
3818         d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
3819         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3820                       u"put-future-ro.txt")
3821         return d
3822
3823     def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
3824         d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
3825         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3826                       u"put-future-imm.txt")
3827         return d
3828
3829     def test_PUT_NEWFILE_URI(self):
3830         file_contents = "New file contents here\n"
3831         d = self.PUT("/uri", file_contents)
3832         def _check(uri):
3833             assert isinstance(uri, str), uri
3834             self.failUnlessIn(uri, self.get_all_contents())
3835             self.failUnlessReallyEqual(self.get_all_contents()[uri],
3836                                        file_contents)
3837             return self.GET("/uri/%s" % uri)
3838         d.addCallback(_check)
3839         def _check2(res):
3840             self.failUnlessReallyEqual(res, file_contents)
3841         d.addCallback(_check2)
3842         return d
3843
3844     def test_PUT_NEWFILE_URI_not_mutable(self):
3845         file_contents = "New file contents here\n"
3846         d = self.PUT("/uri?mutable=false", file_contents)
3847         def _check(uri):
3848             assert isinstance(uri, str), uri
3849             self.failUnlessIn(uri, self.get_all_contents())
3850             self.failUnlessReallyEqual(self.get_all_contents()[uri],
3851                                        file_contents)
3852             return self.GET("/uri/%s" % uri)
3853         d.addCallback(_check)
3854         def _check2(res):
3855             self.failUnlessReallyEqual(res, file_contents)
3856         d.addCallback(_check2)
3857         return d
3858
3859     def test_PUT_NEWFILE_URI_only_PUT(self):
3860         d = self.PUT("/uri?t=bogus", "")
3861         d.addBoth(self.shouldFail, error.Error,
3862                   "PUT_NEWFILE_URI_only_PUT",
3863                   "400 Bad Request",
3864                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
3865         return d
3866
3867     def test_PUT_NEWFILE_URI_mutable(self):
3868         file_contents = "New file contents here\n"
3869         d = self.PUT("/uri?mutable=true", file_contents)
3870         def _check1(filecap):
3871             filecap = filecap.strip()
3872             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
3873             self.filecap = filecap
3874             u = uri.WriteableSSKFileURI.init_from_string(filecap)
3875             self.failUnlessIn(u.get_storage_index(), self.get_all_contents())
3876             n = self.s.create_node_from_uri(filecap)
3877             return n.download_best_version()
3878         d.addCallback(_check1)
3879         def _check2(data):
3880             self.failUnlessReallyEqual(data, file_contents)
3881             return self.GET("/uri/%s" % urllib.quote(self.filecap))
3882         d.addCallback(_check2)
3883         def _check3(res):
3884             self.failUnlessReallyEqual(res, file_contents)
3885         d.addCallback(_check3)
3886         return d
3887
3888     def test_PUT_mkdir(self):
3889         d = self.PUT("/uri?t=mkdir", "")
3890         def _check(uri):
3891             n = self.s.create_node_from_uri(uri.strip())
3892             d2 = self.failUnlessNodeKeysAre(n, [])
3893             d2.addCallback(lambda res:
3894                            self.GET("/uri/%s?t=json" % uri))
3895             return d2
3896         d.addCallback(_check)
3897         d.addCallback(self.failUnlessIsEmptyJSON)
3898         return d
3899
3900     def test_PUT_mkdir_mdmf(self):
3901         d = self.PUT("/uri?t=mkdir&format=mdmf", "")
3902         def _got(res):
3903             u = uri.from_string(res)
3904             # Check that this is an MDMF writecap
3905             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
3906         d.addCallback(_got)
3907         return d
3908
3909     def test_PUT_mkdir_sdmf(self):
3910         d = self.PUT("/uri?t=mkdir&format=sdmf", "")
3911         def _got(res):
3912             u = uri.from_string(res)
3913             self.failUnlessIsInstance(u, uri.DirectoryURI)
3914         d.addCallback(_got)
3915         return d
3916
3917     def test_PUT_mkdir_bad_format(self):
3918         return self.shouldHTTPError("PUT_mkdir_bad_format",
3919                                     400, "Bad Request", "Unknown format: foo",
3920                                     self.PUT, "/uri?t=mkdir&format=foo",
3921                                     "")
3922
3923     def test_POST_check(self):
3924         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
3925         def _done(res):
3926             # this returns a string form of the results, which are probably
3927             # None since we're using fake filenodes.
3928             # TODO: verify that the check actually happened, by changing
3929             # FakeCHKFileNode to count how many times .check() has been
3930             # called.
3931             pass
3932         d.addCallback(_done)
3933         return d
3934
3935
3936     def test_PUT_update_at_offset(self):
3937         file_contents = "test file" * 100000 # about 900 KiB
3938         d = self.PUT("/uri?mutable=true", file_contents)
3939         def _then(filecap):
3940             self.filecap = filecap
3941             new_data = file_contents[:100]
3942             new = "replaced and so on"
3943             new_data += new
3944             new_data += file_contents[len(new_data):]
3945             assert len(new_data) == len(file_contents)
3946             self.new_data = new_data
3947         d.addCallback(_then)
3948         d.addCallback(lambda ignored:
3949             self.PUT("/uri/%s?replace=True&offset=100" % self.filecap,
3950                      "replaced and so on"))
3951         def _get_data(filecap):
3952             n = self.s.create_node_from_uri(filecap)
3953             return n.download_best_version()
3954         d.addCallback(_get_data)
3955         d.addCallback(lambda results:
3956             self.failUnlessEqual(results, self.new_data))
3957         # Now try appending things to the file
3958         d.addCallback(lambda ignored:
3959             self.PUT("/uri/%s?offset=%d" % (self.filecap, len(self.new_data)),
3960                      "puppies" * 100))
3961         d.addCallback(_get_data)
3962         d.addCallback(lambda results:
3963             self.failUnlessEqual(results, self.new_data + ("puppies" * 100)))
3964         # and try replacing the beginning of the file
3965         d.addCallback(lambda ignored:
3966             self.PUT("/uri/%s?offset=0" % self.filecap, "begin"))
3967         d.addCallback(_get_data)
3968         d.addCallback(lambda results:
3969             self.failUnlessEqual(results, "begin"+self.new_data[len("begin"):]+("puppies"*100)))
3970         return d
3971
3972     def test_PUT_update_at_invalid_offset(self):
3973         file_contents = "test file" * 100000 # about 900 KiB
3974         d = self.PUT("/uri?mutable=true", file_contents)
3975         def _then(filecap):
3976             self.filecap = filecap
3977         d.addCallback(_then)
3978         # Negative offsets should cause an error.
3979         d.addCallback(lambda ignored:
3980             self.shouldHTTPError("PUT_update_at_invalid_offset",
3981                                  400, "Bad Request",
3982                                  "Invalid offset",
3983                                  self.PUT,
3984                                  "/uri/%s?offset=-1" % self.filecap,
3985                                  "foo"))
3986         return d
3987
3988     def test_PUT_update_at_offset_immutable(self):
3989         file_contents = "Test file" * 100000
3990         d = self.PUT("/uri", file_contents)
3991         def _then(filecap):
3992             self.filecap = filecap
3993         d.addCallback(_then)
3994         d.addCallback(lambda ignored:
3995             self.shouldHTTPError("PUT_update_at_offset_immutable",
3996                                  400, "Bad Request",
3997                                  "immutable",
3998                                  self.PUT,
3999                                  "/uri/%s?offset=50" % self.filecap,
4000                                  "foo"))
4001         return d
4002
4003
4004     def test_bad_method(self):
4005         url = self.webish_url + self.public_url + "/foo/bar.txt"
4006         d = self.shouldHTTPError("bad_method",
4007                                  501, "Not Implemented",
4008                                  "I don't know how to treat a BOGUS request.",
4009                                  client.getPage, url, method="BOGUS")
4010         return d
4011
4012     def test_short_url(self):
4013         url = self.webish_url + "/uri"
4014         d = self.shouldHTTPError("short_url", 501, "Not Implemented",
4015                                  "I don't know how to treat a DELETE request.",
4016                                  client.getPage, url, method="DELETE")
4017         return d
4018
4019     def test_ophandle_bad(self):
4020         url = self.webish_url + "/operations/bogus?t=status"
4021         d = self.shouldHTTPError("ophandle_bad", 404, "404 Not Found",
4022                                  "unknown/expired handle 'bogus'",
4023                                  client.getPage, url)
4024         return d
4025
4026     def test_ophandle_cancel(self):
4027         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
4028                       followRedirect=True)
4029         d.addCallback(lambda ignored:
4030                       self.GET("/operations/128?t=status&output=JSON"))
4031         def _check1(res):
4032             data = simplejson.loads(res)
4033             self.failUnless("finished" in data, res)
4034             monitor = self.ws.root.child_operations.handles["128"][0]
4035             d = self.POST("/operations/128?t=cancel&output=JSON")
4036             def _check2(res):
4037                 data = simplejson.loads(res)
4038                 self.failUnless("finished" in data, res)
4039                 # t=cancel causes the handle to be forgotten
4040                 self.failUnless(monitor.is_cancelled())
4041             d.addCallback(_check2)
4042             return d
4043         d.addCallback(_check1)
4044         d.addCallback(lambda ignored:
4045                       self.shouldHTTPError("ophandle_cancel",
4046                                            404, "404 Not Found",
4047                                            "unknown/expired handle '128'",
4048                                            self.GET,
4049                                            "/operations/128?t=status&output=JSON"))
4050         return d
4051
4052     def test_ophandle_retainfor(self):
4053         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
4054                       followRedirect=True)
4055         d.addCallback(lambda ignored:
4056                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
4057         def _check1(res):
4058             data = simplejson.loads(res)
4059             self.failUnless("finished" in data, res)
4060         d.addCallback(_check1)
4061         # the retain-for=0 will cause the handle to be expired very soon
4062         d.addCallback(lambda ign:
4063             self.clock.advance(2.0))
4064         d.addCallback(lambda ignored:
4065                       self.shouldHTTPError("ophandle_retainfor",
4066                                            404, "404 Not Found",
4067                                            "unknown/expired handle '129'",
4068                                            self.GET,
4069                                            "/operations/129?t=status&output=JSON"))
4070         return d
4071
4072     def test_ophandle_release_after_complete(self):
4073         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
4074                       followRedirect=True)
4075         d.addCallback(self.wait_for_operation, "130")
4076         d.addCallback(lambda ignored:
4077                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
4078         # the release-after-complete=true will cause the handle to be expired
4079         d.addCallback(lambda ignored:
4080                       self.shouldHTTPError("ophandle_release_after_complete",
4081                                            404, "404 Not Found",
4082                                            "unknown/expired handle '130'",
4083                                            self.GET,
4084                                            "/operations/130?t=status&output=JSON"))
4085         return d
4086
4087     def test_uncollected_ophandle_expiration(self):
4088         # uncollected ophandles should expire after 4 days
4089         def _make_uncollected_ophandle(ophandle):
4090             d = self.POST(self.public_url +
4091                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
4092                           followRedirect=False)
4093             # When we start the operation, the webapi server will want
4094             # to redirect us to the page for the ophandle, so we get
4095             # confirmation that the operation has started. If the
4096             # manifest operation has finished by the time we get there,
4097             # following that redirect (by setting followRedirect=True
4098             # above) has the side effect of collecting the ophandle that
4099             # we've just created, which means that we can't use the
4100             # ophandle to test the uncollected timeout anymore. So,
4101             # instead, catch the 302 here and don't follow it.
4102             d.addBoth(self.should302, "uncollected_ophandle_creation")
4103             return d
4104         # Create an ophandle, don't collect it, then advance the clock by
4105         # 4 days - 1 second and make sure that the ophandle is still there.
4106         d = _make_uncollected_ophandle(131)
4107         d.addCallback(lambda ign:
4108             self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
4109         d.addCallback(lambda ign:
4110             self.GET("/operations/131?t=status&output=JSON"))
4111         def _check1(res):
4112             data = simplejson.loads(res)
4113             self.failUnless("finished" in data, res)
4114         d.addCallback(_check1)
4115         # Create an ophandle, don't collect it, then try to collect it
4116         # after 4 days. It should be gone.
4117         d.addCallback(lambda ign:
4118             _make_uncollected_ophandle(132))
4119         d.addCallback(lambda ign:
4120             self.clock.advance(96*60*60))
4121         d.addCallback(lambda ign:
4122             self.shouldHTTPError("uncollected_ophandle_expired_after_100_hours",
4123                                  404, "404 Not Found",
4124                                  "unknown/expired handle '132'",
4125                                  self.GET,
4126                                  "/operations/132?t=status&output=JSON"))
4127         return d
4128
4129     def test_collected_ophandle_expiration(self):
4130         # collected ophandles should expire after 1 day
4131         def _make_collected_ophandle(ophandle):
4132             d = self.POST(self.public_url +
4133                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
4134                           followRedirect=True)
4135             # By following the initial redirect, we collect the ophandle
4136             # we've just created.
4137             return d
4138         # Create a collected ophandle, then collect it after 23 hours
4139         # and 59 seconds to make sure that it is still there.
4140         d = _make_collected_ophandle(133)
4141         d.addCallback(lambda ign:
4142             self.clock.advance((24*60*60) - 1))
4143         d.addCallback(lambda ign:
4144             self.GET("/operations/133?t=status&output=JSON"))
4145         def _check1(res):
4146             data = simplejson.loads(res)
4147             self.failUnless("finished" in data, res)
4148         d.addCallback(_check1)
4149         # Create another uncollected ophandle, then try to collect it
4150         # after 24 hours to make sure that it is gone.
4151         d.addCallback(lambda ign:
4152             _make_collected_ophandle(134))
4153         d.addCallback(lambda ign:
4154             self.clock.advance(24*60*60))
4155         d.addCallback(lambda ign:
4156             self.shouldHTTPError("collected_ophandle_expired_after_1_day",
4157                                  404, "404 Not Found",
4158                                  "unknown/expired handle '134'",
4159                                  self.GET,
4160                                  "/operations/134?t=status&output=JSON"))
4161         return d
4162
4163     def test_incident(self):
4164         d = self.POST("/report_incident", details="eek")
4165         def _done(res):
4166             self.failIfIn("<html>", res)
4167             self.failUnlessIn("Thank you for your report!", res)
4168         d.addCallback(_done)
4169         return d
4170
4171     def test_static(self):
4172         webdir = os.path.join(self.staticdir, "subdir")
4173         fileutil.make_dirs(webdir)
4174         f = open(os.path.join(webdir, "hello.txt"), "wb")
4175         f.write("hello")
4176         f.close()
4177
4178         d = self.GET("/static/subdir/hello.txt")
4179         def _check(res):
4180             self.failUnlessReallyEqual(res, "hello")
4181         d.addCallback(_check)
4182         return d
4183
4184
4185 class IntroducerWeb(unittest.TestCase):
4186     def setUp(self):
4187         self.node = None
4188
4189     def tearDown(self):
4190         d = defer.succeed(None)
4191         if self.node:
4192             d.addCallback(lambda ign: self.node.stopService())
4193         d.addCallback(flushEventualQueue)
4194         return d
4195
4196     def test_welcome(self):
4197         basedir = "web.IntroducerWeb.test_welcome"
4198         os.mkdir(basedir)
4199         fileutil.write(os.path.join(basedir, "tahoe.cfg"), "[node]\nweb.port = tcp:0\n")
4200         self.node = IntroducerNode(basedir)
4201         self.ws = self.node.getServiceNamed("webish")
4202
4203         d = fireEventually(None)
4204         d.addCallback(lambda ign: self.node.startService())
4205         d.addCallback(lambda ign: self.node.when_tub_ready())
4206
4207         d.addCallback(lambda ign: self.GET("/"))
4208         def _check(res):
4209             self.failUnlessIn('Welcome to the Tahoe-LAFS Introducer', res)
4210             self.failUnlessIn(FAVICON_MARKUP, res)
4211         d.addCallback(_check)
4212         return d
4213
4214     def GET(self, urlpath, followRedirect=False, return_response=False,
4215             **kwargs):
4216         # if return_response=True, this fires with (data, statuscode,
4217         # respheaders) instead of just data.
4218         assert not isinstance(urlpath, unicode)
4219         url = self.ws.getURL().rstrip('/') + urlpath
4220         factory = HTTPClientGETFactory(url, method="GET",
4221                                        followRedirect=followRedirect, **kwargs)
4222         reactor.connectTCP("localhost", self.ws.getPortnum(), factory)
4223         d = factory.deferred
4224         def _got_data(data):
4225             return (data, factory.status, factory.response_headers)
4226         if return_response:
4227             d.addCallback(_got_data)
4228         return factory.deferred
4229
4230
4231 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4232     def test_load_file(self):
4233         # This will raise an exception unless a well-formed XML file is found under that name.
4234         common.getxmlfile('directory.xhtml').load()
4235
4236     def test_parse_replace_arg(self):
4237         self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
4238         self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
4239         self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
4240                                    "only-files")
4241         self.shouldFail(AssertionError, "test_parse_replace_arg", "",
4242                         common.parse_replace_arg, "only_fles")
4243
4244     def test_abbreviate_time(self):
4245         self.failUnlessReallyEqual(common.abbreviate_time(None), "")
4246         self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
4247         self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
4248         self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
4249         self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
4250         self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
4251
4252     def test_compute_rate(self):
4253         self.failUnlessReallyEqual(common.compute_rate(None, None), None)
4254         self.failUnlessReallyEqual(common.compute_rate(None, 1), None)
4255         self.failUnlessReallyEqual(common.compute_rate(250000, None), None)
4256         self.failUnlessReallyEqual(common.compute_rate(250000, 0), None)
4257         self.failUnlessReallyEqual(common.compute_rate(250000, 10), 25000.0)
4258         self.failUnlessReallyEqual(common.compute_rate(0, 10), 0.0)
4259         self.shouldFail(AssertionError, "test_compute_rate", "",
4260                         common.compute_rate, -100, 10)
4261         self.shouldFail(AssertionError, "test_compute_rate", "",
4262                         common.compute_rate, 100, -10)
4263
4264         # Sanity check
4265         rate = common.compute_rate(10*1000*1000, 1)
4266         self.failUnlessReallyEqual(common.abbreviate_rate(rate), "10.00MBps")
4267
4268     def test_abbreviate_rate(self):
4269         self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
4270         self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
4271         self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
4272         self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
4273
4274     def test_abbreviate_size(self):
4275         self.failUnlessReallyEqual(common.abbreviate_size(None), "")
4276         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
4277         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
4278         self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
4279         self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
4280
4281     def test_plural(self):
4282         def convert(s):
4283             return "%d second%s" % (s, status.plural(s))
4284         self.failUnlessReallyEqual(convert(0), "0 seconds")
4285         self.failUnlessReallyEqual(convert(1), "1 second")
4286         self.failUnlessReallyEqual(convert(2), "2 seconds")
4287         def convert2(s):
4288             return "has share%s: %s" % (status.plural(s), ",".join(s))
4289         self.failUnlessReallyEqual(convert2([]), "has shares: ")
4290         self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
4291         self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
4292
4293
4294 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4295
4296     def CHECK(self, ign, which, args, clientnum=0):
4297         fileurl = self.fileurls[which]
4298         url = fileurl + "?" + args
4299         return self.GET(url, method="POST", clientnum=clientnum)
4300
4301     def test_filecheck(self):
4302         self.basedir = "web/Grid/filecheck"
4303         self.set_up_grid()
4304         c0 = self.g.clients[0]
4305         self.uris = {}
4306         DATA = "data" * 100
4307         d = c0.upload(upload.Data(DATA, convergence=""))
4308         def _stash_uri(ur, which):
4309             self.uris[which] = ur.get_uri()
4310         d.addCallback(_stash_uri, "good")
4311         d.addCallback(lambda ign:
4312                       c0.upload(upload.Data(DATA+"1", convergence="")))
4313         d.addCallback(_stash_uri, "sick")
4314         d.addCallback(lambda ign:
4315                       c0.upload(upload.Data(DATA+"2", convergence="")))
4316         d.addCallback(_stash_uri, "dead")
4317         def _stash_mutable_uri(n, which):
4318             self.uris[which] = n.get_uri()
4319             assert isinstance(self.uris[which], str)
4320         d.addCallback(lambda ign:
4321             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4322         d.addCallback(_stash_mutable_uri, "corrupt")
4323         d.addCallback(lambda ign:
4324                       c0.upload(upload.Data("literal", convergence="")))
4325         d.addCallback(_stash_uri, "small")
4326         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
4327         d.addCallback(_stash_mutable_uri, "smalldir")
4328
4329         def _compute_fileurls(ignored):
4330             self.fileurls = {}
4331             for which in self.uris:
4332                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4333         d.addCallback(_compute_fileurls)
4334
4335         def _clobber_shares(ignored):
4336             good_shares = self.find_uri_shares(self.uris["good"])
4337             self.failUnlessReallyEqual(len(good_shares), 10)
4338             sick_shares = self.find_uri_shares(self.uris["sick"])
4339             os.unlink(sick_shares[0][2])
4340             dead_shares = self.find_uri_shares(self.uris["dead"])
4341             for i in range(1, 10):
4342                 os.unlink(dead_shares[i][2])
4343             c_shares = self.find_uri_shares(self.uris["corrupt"])
4344             cso = CorruptShareOptions()
4345             cso.stdout = StringIO()
4346             cso.parseOptions([c_shares[0][2]])
4347             corrupt_share(cso)
4348         d.addCallback(_clobber_shares)
4349
4350         d.addCallback(self.CHECK, "good", "t=check")
4351         def _got_html_good(res):
4352             self.failUnlessIn("Healthy", res)
4353             self.failIfIn("Not Healthy", res)
4354             self.failUnlessIn(FAVICON_MARKUP, res)
4355         d.addCallback(_got_html_good)
4356         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
4357         def _got_html_good_return_to(res):
4358             self.failUnlessIn("Healthy", res)
4359             self.failIfIn("Not Healthy", res)
4360             self.failUnlessIn('<a href="somewhere">Return to file', res)
4361         d.addCallback(_got_html_good_return_to)
4362         d.addCallback(self.CHECK, "good", "t=check&output=json")
4363         def _got_json_good(res):
4364             r = simplejson.loads(res)
4365             self.failUnlessEqual(r["summary"], "Healthy")
4366             self.failUnless(r["results"]["healthy"])
4367             self.failIf(r["results"]["needs-rebalancing"])
4368             self.failUnless(r["results"]["recoverable"])
4369         d.addCallback(_got_json_good)
4370
4371         d.addCallback(self.CHECK, "small", "t=check")
4372         def _got_html_small(res):
4373             self.failUnlessIn("Literal files are always healthy", res)
4374             self.failIfIn("Not Healthy", res)
4375         d.addCallback(_got_html_small)
4376         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
4377         def _got_html_small_return_to(res):
4378             self.failUnlessIn("Literal files are always healthy", res)
4379             self.failIfIn("Not Healthy", res)
4380             self.failUnlessIn('<a href="somewhere">Return to file', res)
4381         d.addCallback(_got_html_small_return_to)
4382         d.addCallback(self.CHECK, "small", "t=check&output=json")
4383         def _got_json_small(res):
4384             r = simplejson.loads(res)
4385             self.failUnlessEqual(r["storage-index"], "")
4386             self.failUnless(r["results"]["healthy"])
4387         d.addCallback(_got_json_small)
4388
4389         d.addCallback(self.CHECK, "smalldir", "t=check")
4390         def _got_html_smalldir(res):
4391             self.failUnlessIn("Literal files are always healthy", res)
4392             self.failIfIn("Not Healthy", res)
4393         d.addCallback(_got_html_smalldir)
4394         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
4395         def _got_json_smalldir(res):
4396             r = simplejson.loads(res)
4397             self.failUnlessEqual(r["storage-index"], "")
4398             self.failUnless(r["results"]["healthy"])
4399         d.addCallback(_got_json_smalldir)
4400
4401         d.addCallback(self.CHECK, "sick", "t=check")
4402         def _got_html_sick(res):
4403             self.failUnlessIn("Not Healthy", res)
4404         d.addCallback(_got_html_sick)
4405         d.addCallback(self.CHECK, "sick", "t=check&output=json")
4406         def _got_json_sick(res):
4407             r = simplejson.loads(res)
4408             self.failUnlessEqual(r["summary"],
4409                                  "Not Healthy: 9 shares (enc 3-of-10)")
4410             self.failIf(r["results"]["healthy"])
4411             self.failIf(r["results"]["needs-rebalancing"])
4412             self.failUnless(r["results"]["recoverable"])
4413         d.addCallback(_got_json_sick)
4414
4415         d.addCallback(self.CHECK, "dead", "t=check")
4416         def _got_html_dead(res):
4417             self.failUnlessIn("Not Healthy", res)
4418         d.addCallback(_got_html_dead)
4419         d.addCallback(self.CHECK, "dead", "t=check&output=json")
4420         def _got_json_dead(res):
4421             r = simplejson.loads(res)
4422             self.failUnlessEqual(r["summary"],
4423                                  "Not Healthy: 1 shares (enc 3-of-10)")
4424             self.failIf(r["results"]["healthy"])
4425             self.failIf(r["results"]["needs-rebalancing"])
4426             self.failIf(r["results"]["recoverable"])
4427         d.addCallback(_got_json_dead)
4428
4429         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
4430         def _got_html_corrupt(res):
4431             self.failUnlessIn("Not Healthy! : Unhealthy", res)
4432         d.addCallback(_got_html_corrupt)
4433         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
4434         def _got_json_corrupt(res):
4435             r = simplejson.loads(res)
4436             self.failUnlessIn("Unhealthy: 9 shares (enc 3-of-10)", r["summary"])
4437             self.failIf(r["results"]["healthy"])
4438             self.failUnless(r["results"]["recoverable"])
4439             self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
4440             self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
4441         d.addCallback(_got_json_corrupt)
4442
4443         d.addErrback(self.explain_web_error)
4444         return d
4445
4446     def test_repair_html(self):
4447         self.basedir = "web/Grid/repair_html"
4448         self.set_up_grid()
4449         c0 = self.g.clients[0]
4450         self.uris = {}
4451         DATA = "data" * 100
4452         d = c0.upload(upload.Data(DATA, convergence=""))
4453         def _stash_uri(ur, which):
4454             self.uris[which] = ur.get_uri()
4455         d.addCallback(_stash_uri, "good")
4456         d.addCallback(lambda ign:
4457                       c0.upload(upload.Data(DATA+"1", convergence="")))
4458         d.addCallback(_stash_uri, "sick")
4459         d.addCallback(lambda ign:
4460                       c0.upload(upload.Data(DATA+"2", convergence="")))
4461         d.addCallback(_stash_uri, "dead")
4462         def _stash_mutable_uri(n, which):
4463             self.uris[which] = n.get_uri()
4464             assert isinstance(self.uris[which], str)
4465         d.addCallback(lambda ign:
4466             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4467         d.addCallback(_stash_mutable_uri, "corrupt")
4468
4469         def _compute_fileurls(ignored):
4470             self.fileurls = {}
4471             for which in self.uris:
4472                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4473         d.addCallback(_compute_fileurls)
4474
4475         def _clobber_shares(ignored):
4476             good_shares = self.find_uri_shares(self.uris["good"])
4477             self.failUnlessReallyEqual(len(good_shares), 10)
4478             sick_shares = self.find_uri_shares(self.uris["sick"])
4479             os.unlink(sick_shares[0][2])
4480             dead_shares = self.find_uri_shares(self.uris["dead"])
4481             for i in range(1, 10):
4482                 os.unlink(dead_shares[i][2])
4483             c_shares = self.find_uri_shares(self.uris["corrupt"])
4484             cso = CorruptShareOptions()
4485             cso.stdout = StringIO()
4486             cso.parseOptions([c_shares[0][2]])
4487             corrupt_share(cso)
4488         d.addCallback(_clobber_shares)
4489
4490         d.addCallback(self.CHECK, "good", "t=check&repair=true")
4491         def _got_html_good(res):
4492             self.failUnlessIn("Healthy", res)
4493             self.failIfIn("Not Healthy", res)
4494             self.failUnlessIn("No repair necessary", res)
4495             self.failUnlessIn(FAVICON_MARKUP, res)
4496         d.addCallback(_got_html_good)
4497
4498         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
4499         def _got_html_sick(res):
4500             self.failUnlessIn("Healthy : healthy", res)
4501             self.failIfIn("Not Healthy", res)
4502             self.failUnlessIn("Repair successful", res)
4503         d.addCallback(_got_html_sick)
4504
4505         # repair of a dead file will fail, of course, but it isn't yet
4506         # clear how this should be reported. Right now it shows up as
4507         # a "410 Gone".
4508         #
4509         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
4510         #def _got_html_dead(res):
4511         #    print res
4512         #    self.failUnlessIn("Healthy : healthy", res)
4513         #    self.failIfIn("Not Healthy", res)
4514         #    self.failUnlessIn("No repair necessary", res)
4515         #d.addCallback(_got_html_dead)
4516
4517         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
4518         def _got_html_corrupt(res):
4519             self.failUnlessIn("Healthy : Healthy", res)
4520             self.failIfIn("Not Healthy", res)
4521             self.failUnlessIn("Repair successful", res)
4522         d.addCallback(_got_html_corrupt)
4523
4524         d.addErrback(self.explain_web_error)
4525         return d
4526
4527     def test_repair_json(self):
4528         self.basedir = "web/Grid/repair_json"
4529         self.set_up_grid()
4530         c0 = self.g.clients[0]
4531         self.uris = {}
4532         DATA = "data" * 100
4533         d = c0.upload(upload.Data(DATA+"1", convergence=""))
4534         def _stash_uri(ur, which):
4535             self.uris[which] = ur.get_uri()
4536         d.addCallback(_stash_uri, "sick")
4537
4538         def _compute_fileurls(ignored):
4539             self.fileurls = {}
4540             for which in self.uris:
4541                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4542         d.addCallback(_compute_fileurls)
4543
4544         def _clobber_shares(ignored):
4545             sick_shares = self.find_uri_shares(self.uris["sick"])
4546             os.unlink(sick_shares[0][2])
4547         d.addCallback(_clobber_shares)
4548
4549         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
4550         def _got_json_sick(res):
4551             r = simplejson.loads(res)
4552             self.failUnlessReallyEqual(r["repair-attempted"], True)
4553             self.failUnlessReallyEqual(r["repair-successful"], True)
4554             self.failUnlessEqual(r["pre-repair-results"]["summary"],
4555                                  "Not Healthy: 9 shares (enc 3-of-10)")
4556             self.failIf(r["pre-repair-results"]["results"]["healthy"])
4557             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
4558             self.failUnless(r["post-repair-results"]["results"]["healthy"])
4559         d.addCallback(_got_json_sick)
4560
4561         d.addErrback(self.explain_web_error)
4562         return d
4563
4564     def test_unknown(self, immutable=False):
4565         self.basedir = "web/Grid/unknown"
4566         if immutable:
4567             self.basedir = "web/Grid/unknown-immutable"
4568
4569         self.set_up_grid()
4570         c0 = self.g.clients[0]
4571         self.uris = {}
4572         self.fileurls = {}
4573
4574         # the future cap format may contain slashes, which must be tolerated
4575         expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
4576                                                            safe="")
4577
4578         if immutable:
4579             name = u"future-imm"
4580             future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
4581             d = c0.create_immutable_dirnode({name: (future_node, {})})
4582         else:
4583             name = u"future"
4584             future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4585             d = c0.create_dirnode()
4586
4587         def _stash_root_and_create_file(n):
4588             self.rootnode = n
4589             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
4590             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
4591             if not immutable:
4592                 return self.rootnode.set_node(name, future_node)
4593         d.addCallback(_stash_root_and_create_file)
4594
4595         # make sure directory listing tolerates unknown nodes
4596         d.addCallback(lambda ign: self.GET(self.rooturl))
4597         def _check_directory_html(res, expected_type_suffix):
4598             pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
4599                                   '<td>%s</td>' % (expected_type_suffix, str(name)),
4600                                  re.DOTALL)
4601             self.failUnless(re.search(pattern, res), res)
4602             # find the More Info link for name, should be relative
4603             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4604             info_url = mo.group(1)
4605             self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
4606         if immutable:
4607             d.addCallback(_check_directory_html, "-IMM")
4608         else:
4609             d.addCallback(_check_directory_html, "")
4610
4611         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4612         def _check_directory_json(res, expect_rw_uri):
4613             data = simplejson.loads(res)
4614             self.failUnlessEqual(data[0], "dirnode")
4615             f = data[1]["children"][name]
4616             self.failUnlessEqual(f[0], "unknown")
4617             if expect_rw_uri:
4618                 self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data)
4619             else:
4620                 self.failIfIn("rw_uri", f[1])
4621             if immutable:
4622                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
4623             else:
4624                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
4625             self.failUnlessIn("metadata", f[1])
4626         d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
4627
4628         def _check_info(res, expect_rw_uri, expect_ro_uri):
4629             self.failUnlessIn("Object Type: <span>unknown</span>", res)
4630             if expect_rw_uri:
4631                 self.failUnlessIn(unknown_rwcap, res)
4632             if expect_ro_uri:
4633                 if immutable:
4634                     self.failUnlessIn(unknown_immcap, res)
4635                 else:
4636                     self.failUnlessIn(unknown_rocap, res)
4637             else:
4638                 self.failIfIn(unknown_rocap, res)
4639             self.failIfIn("Raw data as", res)
4640             self.failIfIn("Directory writecap", res)
4641             self.failIfIn("Checker Operations", res)
4642             self.failIfIn("Mutable File Operations", res)
4643             self.failIfIn("Directory Operations", res)
4644
4645         # FIXME: these should have expect_rw_uri=not immutable; I don't know
4646         # why they fail. Possibly related to ticket #922.
4647
4648         d.addCallback(lambda ign: self.GET(expected_info_url))
4649         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
4650         d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
4651         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
4652
4653         def _check_json(res, expect_rw_uri):
4654             data = simplejson.loads(res)
4655             self.failUnlessEqual(data[0], "unknown")
4656             if expect_rw_uri:
4657                 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data)
4658             else:
4659                 self.failIfIn("rw_uri", data[1])
4660
4661             if immutable:
4662                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data)
4663                 self.failUnlessReallyEqual(data[1]["mutable"], False)
4664             elif expect_rw_uri:
4665                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4666                 self.failUnlessReallyEqual(data[1]["mutable"], True)
4667             else:
4668                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4669                 self.failIfIn("mutable", data[1])
4670
4671             # TODO: check metadata contents
4672             self.failUnlessIn("metadata", data[1])
4673
4674         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
4675         d.addCallback(_check_json, expect_rw_uri=not immutable)
4676
4677         # and make sure that a read-only version of the directory can be
4678         # rendered too. This version will not have unknown_rwcap, whether
4679         # or not future_node was immutable.
4680         d.addCallback(lambda ign: self.GET(self.rourl))
4681         if immutable:
4682             d.addCallback(_check_directory_html, "-IMM")
4683         else:
4684             d.addCallback(_check_directory_html, "-RO")
4685
4686         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
4687         d.addCallback(_check_directory_json, expect_rw_uri=False)
4688
4689         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
4690         d.addCallback(_check_json, expect_rw_uri=False)
4691
4692         # TODO: check that getting t=info from the Info link in the ro directory
4693         # works, and does not include the writecap URI.
4694         return d
4695
4696     def test_immutable_unknown(self):
4697         return self.test_unknown(immutable=True)
4698
4699     def test_mutant_dirnodes_are_omitted(self):
4700         self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
4701
4702         self.set_up_grid()
4703         c = self.g.clients[0]
4704         nm = c.nodemaker
4705         self.uris = {}
4706         self.fileurls = {}
4707
4708         lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
4709         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
4710         mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
4711
4712         # This method tests mainly dirnode, but we'd have to duplicate code in order to
4713         # test the dirnode and web layers separately.
4714
4715         # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
4716         # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
4717         # When the directory is read, the mutants should be silently disposed of, leaving
4718         # their lonely sibling.
4719         # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
4720         # because immutable directories don't have a writecap and therefore that field
4721         # isn't (and can't be) decrypted.
4722         # TODO: The field still exists in the netstring. Technically we should check what
4723         # happens if something is put there (_unpack_contents should raise ValueError),
4724         # but that can wait.
4725
4726         lonely_child = nm.create_from_cap(lonely_uri)
4727         mutant_ro_child = nm.create_from_cap(mut_read_uri)
4728         mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
4729
4730         def _by_hook_or_by_crook():
4731             return True
4732         for n in [mutant_ro_child, mutant_write_in_ro_child]:
4733             n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
4734
4735         mutant_write_in_ro_child.get_write_uri    = lambda: None
4736         mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
4737
4738         kids = {u"lonely":      (lonely_child, {}),
4739                 u"ro":          (mutant_ro_child, {}),
4740                 u"write-in-ro": (mutant_write_in_ro_child, {}),
4741                 }
4742         d = c.create_immutable_dirnode(kids)
4743
4744         def _created(dn):
4745             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
4746             self.failIf(dn.is_mutable())
4747             self.failUnless(dn.is_readonly())
4748             # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
4749             self.failIf(hasattr(dn._node, 'get_writekey'))
4750             rep = str(dn)
4751             self.failUnlessIn("RO-IMM", rep)
4752             cap = dn.get_cap()
4753             self.failUnlessIn("CHK", cap.to_string())
4754             self.cap = cap
4755             self.rootnode = dn
4756             self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
4757             return download_to_data(dn._node)
4758         d.addCallback(_created)
4759
4760         def _check_data(data):
4761             # Decode the netstring representation of the directory to check that all children
4762             # are present. This is a bit of an abstraction violation, but there's not really
4763             # any other way to do it given that the real DirectoryNode._unpack_contents would
4764             # strip the mutant children out (which is what we're trying to test, later).
4765             position = 0
4766             numkids = 0
4767             while position < len(data):
4768                 entries, position = split_netstring(data, 1, position)
4769                 entry = entries[0]
4770                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
4771                 name = name_utf8.decode("utf-8")
4772                 self.failUnlessEqual(rwcapdata, "")
4773                 self.failUnlessIn(name, kids)
4774                 (expected_child, ign) = kids[name]
4775                 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
4776                 numkids += 1
4777
4778             self.failUnlessReallyEqual(numkids, 3)
4779             return self.rootnode.list()
4780         d.addCallback(_check_data)
4781
4782         # Now when we use the real directory listing code, the mutants should be absent.
4783         def _check_kids(children):
4784             self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
4785             lonely_node, lonely_metadata = children[u"lonely"]
4786
4787             self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
4788             self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
4789         d.addCallback(_check_kids)
4790
4791         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
4792         d.addCallback(lambda n: n.list())
4793         d.addCallback(_check_kids)  # again with dirnode recreated from cap
4794
4795         # Make sure the lonely child can be listed in HTML...
4796         d.addCallback(lambda ign: self.GET(self.rooturl))
4797         def _check_html(res):
4798             self.failIfIn("URI:SSK", res)
4799             get_lonely = "".join([r'<td>FILE</td>',
4800                                   r'\s+<td>',
4801                                   r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
4802                                   r'</td>',
4803                                   r'\s+<td align="right">%d</td>' % len("one"),
4804                                   ])
4805             self.failUnless(re.search(get_lonely, res), res)
4806
4807             # find the More Info link for name, should be relative
4808             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4809             info_url = mo.group(1)
4810             self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
4811         d.addCallback(_check_html)
4812
4813         # ... and in JSON.
4814         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4815         def _check_json(res):
4816             data = simplejson.loads(res)
4817             self.failUnlessEqual(data[0], "dirnode")
4818             listed_children = data[1]["children"]
4819             self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
4820             ll_type, ll_data = listed_children[u"lonely"]
4821             self.failUnlessEqual(ll_type, "filenode")
4822             self.failIfIn("rw_uri", ll_data)
4823             self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
4824         d.addCallback(_check_json)
4825         return d
4826
4827     def test_deep_check(self):
4828         self.basedir = "web/Grid/deep_check"
4829         self.set_up_grid()
4830         c0 = self.g.clients[0]
4831         self.uris = {}
4832         self.fileurls = {}
4833         DATA = "data" * 100
4834         d = c0.create_dirnode()
4835         def _stash_root_and_create_file(n):
4836             self.rootnode = n
4837             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4838             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4839         d.addCallback(_stash_root_and_create_file)
4840         def _stash_uri(fn, which):
4841             self.uris[which] = fn.get_uri()
4842             return fn
4843         d.addCallback(_stash_uri, "good")
4844         d.addCallback(lambda ign:
4845                       self.rootnode.add_file(u"small",
4846                                              upload.Data("literal",
4847                                                         convergence="")))
4848         d.addCallback(_stash_uri, "small")
4849         d.addCallback(lambda ign:
4850                       self.rootnode.add_file(u"sick",
4851                                              upload.Data(DATA+"1",
4852                                                         convergence="")))
4853         d.addCallback(_stash_uri, "sick")
4854
4855         # this tests that deep-check and stream-manifest will ignore
4856         # UnknownNode instances. Hopefully this will also cover deep-stats.
4857         future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4858         d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
4859
4860         def _clobber_shares(ignored):
4861             self.delete_shares_numbered(self.uris["sick"], [0,1])
4862         d.addCallback(_clobber_shares)
4863
4864         # root
4865         # root/good
4866         # root/small
4867         # root/sick
4868         # root/future
4869
4870         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4871         def _done(res):
4872             try:
4873                 units = [simplejson.loads(line)
4874                          for line in res.splitlines()
4875                          if line]
4876             except ValueError:
4877                 print "response is:", res
4878                 print "undecodeable line was '%s'" % line
4879                 raise
4880             self.failUnlessReallyEqual(len(units), 5+1)
4881             # should be parent-first
4882             u0 = units[0]
4883             self.failUnlessEqual(u0["path"], [])
4884             self.failUnlessEqual(u0["type"], "directory")
4885             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4886             u0cr = u0["check-results"]
4887             self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
4888
4889             ugood = [u for u in units
4890                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4891             self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
4892             ugoodcr = ugood["check-results"]
4893             self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
4894
4895             stats = units[-1]
4896             self.failUnlessEqual(stats["type"], "stats")
4897             s = stats["stats"]
4898             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4899             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4900             self.failUnlessReallyEqual(s["count-directories"], 1)
4901             self.failUnlessReallyEqual(s["count-unknown"], 1)
4902         d.addCallback(_done)
4903
4904         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4905         def _check_manifest(res):
4906             self.failUnless(res.endswith("\n"))
4907             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
4908             self.failUnlessReallyEqual(len(units), 5+1)
4909             self.failUnlessEqual(units[-1]["type"], "stats")
4910             first = units[0]
4911             self.failUnlessEqual(first["path"], [])
4912             self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri())
4913             self.failUnlessEqual(first["type"], "directory")
4914             stats = units[-1]["stats"]
4915             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4916             self.failUnlessReallyEqual(stats["count-literal-files"], 1)
4917             self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
4918             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4919             self.failUnlessReallyEqual(stats["count-unknown"], 1)
4920         d.addCallback(_check_manifest)
4921
4922         # now add root/subdir and root/subdir/grandchild, then make subdir
4923         # unrecoverable, then see what happens
4924
4925         d.addCallback(lambda ign:
4926                       self.rootnode.create_subdirectory(u"subdir"))
4927         d.addCallback(_stash_uri, "subdir")
4928         d.addCallback(lambda subdir_node:
4929                       subdir_node.add_file(u"grandchild",
4930                                            upload.Data(DATA+"2",
4931                                                        convergence="")))
4932         d.addCallback(_stash_uri, "grandchild")
4933
4934         d.addCallback(lambda ign:
4935                       self.delete_shares_numbered(self.uris["subdir"],
4936                                                   range(1, 10)))
4937
4938         # root
4939         # root/good
4940         # root/small
4941         # root/sick
4942         # root/future
4943         # root/subdir [unrecoverable]
4944         # root/subdir/grandchild
4945
4946         # how should a streaming-JSON API indicate fatal error?
4947         # answer: emit ERROR: instead of a JSON string
4948
4949         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4950         def _check_broken_manifest(res):
4951             lines = res.splitlines()
4952             error_lines = [i
4953                            for (i,line) in enumerate(lines)
4954                            if line.startswith("ERROR:")]
4955             if not error_lines:
4956                 self.fail("no ERROR: in output: %s" % (res,))
4957             first_error = error_lines[0]
4958             error_line = lines[first_error]
4959             error_msg = lines[first_error+1:]
4960             error_msg_s = "\n".join(error_msg) + "\n"
4961             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4962                               error_line)
4963             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4964             units = [simplejson.loads(line) for line in lines[:first_error]]
4965             self.failUnlessReallyEqual(len(units), 6) # includes subdir
4966             last_unit = units[-1]
4967             self.failUnlessEqual(last_unit["path"], ["subdir"])
4968         d.addCallback(_check_broken_manifest)
4969
4970         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4971         def _check_broken_deepcheck(res):
4972             lines = res.splitlines()
4973             error_lines = [i
4974                            for (i,line) in enumerate(lines)
4975                            if line.startswith("ERROR:")]
4976             if not error_lines:
4977                 self.fail("no ERROR: in output: %s" % (res,))
4978             first_error = error_lines[0]
4979             error_line = lines[first_error]
4980             error_msg = lines[first_error+1:]
4981             error_msg_s = "\n".join(error_msg) + "\n"
4982             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4983                               error_line)
4984             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4985             units = [simplejson.loads(line) for line in lines[:first_error]]
4986             self.failUnlessReallyEqual(len(units), 6) # includes subdir
4987             last_unit = units[-1]
4988             self.failUnlessEqual(last_unit["path"], ["subdir"])
4989             r = last_unit["check-results"]["results"]
4990             self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
4991             self.failUnlessReallyEqual(r["count-shares-good"], 1)
4992             self.failUnlessReallyEqual(r["recoverable"], False)
4993         d.addCallback(_check_broken_deepcheck)
4994
4995         d.addErrback(self.explain_web_error)
4996         return d
4997
4998     def test_deep_check_and_repair(self):
4999         self.basedir = "web/Grid/deep_check_and_repair"
5000         self.set_up_grid()
5001         c0 = self.g.clients[0]
5002         self.uris = {}
5003         self.fileurls = {}
5004         DATA = "data" * 100
5005         d = c0.create_dirnode()
5006         def _stash_root_and_create_file(n):
5007             self.rootnode = n
5008             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5009             return n.add_file(u"good", upload.Data(DATA, convergence=""))
5010         d.addCallback(_stash_root_and_create_file)
5011         def _stash_uri(fn, which):
5012             self.uris[which] = fn.get_uri()
5013         d.addCallback(_stash_uri, "good")
5014         d.addCallback(lambda ign:
5015                       self.rootnode.add_file(u"small",
5016                                              upload.Data("literal",
5017                                                         convergence="")))
5018         d.addCallback(_stash_uri, "small")
5019         d.addCallback(lambda ign:
5020                       self.rootnode.add_file(u"sick",
5021                                              upload.Data(DATA+"1",
5022                                                         convergence="")))
5023         d.addCallback(_stash_uri, "sick")
5024         #d.addCallback(lambda ign:
5025         #              self.rootnode.add_file(u"dead",
5026         #                                     upload.Data(DATA+"2",
5027         #                                                convergence="")))
5028         #d.addCallback(_stash_uri, "dead")
5029
5030         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
5031         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
5032         #d.addCallback(_stash_uri, "corrupt")
5033
5034         def _clobber_shares(ignored):
5035             good_shares = self.find_uri_shares(self.uris["good"])
5036             self.failUnlessReallyEqual(len(good_shares), 10)
5037             sick_shares = self.find_uri_shares(self.uris["sick"])
5038             os.unlink(sick_shares[0][2])
5039             #dead_shares = self.find_uri_shares(self.uris["dead"])
5040             #for i in range(1, 10):
5041             #    os.unlink(dead_shares[i][2])
5042
5043             #c_shares = self.find_uri_shares(self.uris["corrupt"])
5044             #cso = CorruptShareOptions()
5045             #cso.stdout = StringIO()
5046             #cso.parseOptions([c_shares[0][2]])
5047             #corrupt_share(cso)
5048         d.addCallback(_clobber_shares)
5049
5050         # root
5051         # root/good   CHK, 10 shares
5052         # root/small  LIT
5053         # root/sick   CHK, 9 shares
5054
5055         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
5056         def _done(res):
5057             units = [simplejson.loads(line)
5058                      for line in res.splitlines()
5059                      if line]
5060             self.failUnlessReallyEqual(len(units), 4+1)
5061             # should be parent-first
5062             u0 = units[0]
5063             self.failUnlessEqual(u0["path"], [])
5064             self.failUnlessEqual(u0["type"], "directory")
5065             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
5066             u0crr = u0["check-and-repair-results"]
5067             self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
5068             self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
5069
5070             ugood = [u for u in units
5071                      if u["type"] == "file" and u["path"] == [u"good"]][0]
5072             self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
5073             ugoodcrr = ugood["check-and-repair-results"]
5074             self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
5075             self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
5076
5077             usick = [u for u in units
5078                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
5079             self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"])
5080             usickcrr = usick["check-and-repair-results"]
5081             self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
5082             self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
5083             self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
5084             self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
5085
5086             stats = units[-1]
5087             self.failUnlessEqual(stats["type"], "stats")
5088             s = stats["stats"]
5089             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
5090             self.failUnlessReallyEqual(s["count-literal-files"], 1)
5091             self.failUnlessReallyEqual(s["count-directories"], 1)
5092         d.addCallback(_done)
5093
5094         d.addErrback(self.explain_web_error)
5095         return d
5096
5097     def _count_leases(self, ignored, which):
5098         u = self.uris[which]
5099         shares = self.find_uri_shares(u)
5100         lease_counts = []
5101         for shnum, serverid, fn in shares:
5102             sf = get_share_file(fn)
5103             num_leases = len(list(sf.get_leases()))
5104             lease_counts.append( (fn, num_leases) )
5105         return lease_counts
5106
5107     def _assert_leasecount(self, lease_counts, expected):
5108         for (fn, num_leases) in lease_counts:
5109             if num_leases != expected:
5110                 self.fail("expected %d leases, have %d, on %s" %
5111                           (expected, num_leases, fn))
5112
5113     def test_add_lease(self):
5114         self.basedir = "web/Grid/add_lease"
5115         self.set_up_grid(num_clients=2)
5116         c0 = self.g.clients[0]
5117         self.uris = {}
5118         DATA = "data" * 100
5119         d = c0.upload(upload.Data(DATA, convergence=""))
5120         def _stash_uri(ur, which):
5121             self.uris[which] = ur.get_uri()
5122         d.addCallback(_stash_uri, "one")
5123         d.addCallback(lambda ign:
5124                       c0.upload(upload.Data(DATA+"1", convergence="")))
5125         d.addCallback(_stash_uri, "two")
5126         def _stash_mutable_uri(n, which):
5127             self.uris[which] = n.get_uri()
5128             assert isinstance(self.uris[which], str)
5129         d.addCallback(lambda ign:
5130             c0.create_mutable_file(publish.MutableData(DATA+"2")))
5131         d.addCallback(_stash_mutable_uri, "mutable")
5132
5133         def _compute_fileurls(ignored):
5134             self.fileurls = {}
5135             for which in self.uris:
5136                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
5137         d.addCallback(_compute_fileurls)
5138
5139         d.addCallback(self._count_leases, "one")
5140         d.addCallback(self._assert_leasecount, 1)
5141         d.addCallback(self._count_leases, "two")
5142         d.addCallback(self._assert_leasecount, 1)
5143         d.addCallback(self._count_leases, "mutable")
5144         d.addCallback(self._assert_leasecount, 1)
5145
5146         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
5147         def _got_html_good(res):
5148             self.failUnlessIn("Healthy", res)
5149             self.failIfIn("Not Healthy", res)
5150         d.addCallback(_got_html_good)
5151
5152         d.addCallback(self._count_leases, "one")
5153         d.addCallback(self._assert_leasecount, 1)
5154         d.addCallback(self._count_leases, "two")
5155         d.addCallback(self._assert_leasecount, 1)
5156         d.addCallback(self._count_leases, "mutable")
5157         d.addCallback(self._assert_leasecount, 1)
5158
5159         # this CHECK uses the original client, which uses the same
5160         # lease-secrets, so it will just renew the original lease
5161         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
5162         d.addCallback(_got_html_good)
5163
5164         d.addCallback(self._count_leases, "one")
5165         d.addCallback(self._assert_leasecount, 1)
5166         d.addCallback(self._count_leases, "two")
5167         d.addCallback(self._assert_leasecount, 1)
5168         d.addCallback(self._count_leases, "mutable")
5169         d.addCallback(self._assert_leasecount, 1)
5170
5171         # this CHECK uses an alternate client, which adds a second lease
5172         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
5173         d.addCallback(_got_html_good)
5174
5175         d.addCallback(self._count_leases, "one")
5176         d.addCallback(self._assert_leasecount, 2)
5177         d.addCallback(self._count_leases, "two")
5178         d.addCallback(self._assert_leasecount, 1)
5179         d.addCallback(self._count_leases, "mutable")
5180         d.addCallback(self._assert_leasecount, 1)
5181
5182         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
5183         d.addCallback(_got_html_good)
5184
5185         d.addCallback(self._count_leases, "one")
5186         d.addCallback(self._assert_leasecount, 2)
5187         d.addCallback(self._count_leases, "two")
5188         d.addCallback(self._assert_leasecount, 1)
5189         d.addCallback(self._count_leases, "mutable")
5190         d.addCallback(self._assert_leasecount, 1)
5191
5192         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
5193                       clientnum=1)
5194         d.addCallback(_got_html_good)
5195
5196         d.addCallback(self._count_leases, "one")
5197         d.addCallback(self._assert_leasecount, 2)
5198         d.addCallback(self._count_leases, "two")
5199         d.addCallback(self._assert_leasecount, 1)
5200         d.addCallback(self._count_leases, "mutable")
5201         d.addCallback(self._assert_leasecount, 2)
5202
5203         d.addErrback(self.explain_web_error)
5204         return d
5205
5206     def test_deep_add_lease(self):
5207         self.basedir = "web/Grid/deep_add_lease"
5208         self.set_up_grid(num_clients=2)
5209         c0 = self.g.clients[0]
5210         self.uris = {}
5211         self.fileurls = {}
5212         DATA = "data" * 100
5213         d = c0.create_dirnode()
5214         def _stash_root_and_create_file(n):
5215             self.rootnode = n
5216             self.uris["root"] = n.get_uri()
5217             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5218             return n.add_file(u"one", upload.Data(DATA, convergence=""))
5219         d.addCallback(_stash_root_and_create_file)
5220         def _stash_uri(fn, which):
5221             self.uris[which] = fn.get_uri()
5222         d.addCallback(_stash_uri, "one")
5223         d.addCallback(lambda ign:
5224                       self.rootnode.add_file(u"small",
5225                                              upload.Data("literal",
5226                                                         convergence="")))
5227         d.addCallback(_stash_uri, "small")
5228
5229         d.addCallback(lambda ign:
5230             c0.create_mutable_file(publish.MutableData("mutable")))
5231         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
5232         d.addCallback(_stash_uri, "mutable")
5233
5234         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
5235         def _done(res):
5236             units = [simplejson.loads(line)
5237                      for line in res.splitlines()
5238                      if line]
5239             # root, one, small, mutable,   stats
5240             self.failUnlessReallyEqual(len(units), 4+1)
5241         d.addCallback(_done)
5242
5243         d.addCallback(self._count_leases, "root")
5244         d.addCallback(self._assert_leasecount, 1)
5245         d.addCallback(self._count_leases, "one")
5246         d.addCallback(self._assert_leasecount, 1)
5247         d.addCallback(self._count_leases, "mutable")
5248         d.addCallback(self._assert_leasecount, 1)
5249
5250         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
5251         d.addCallback(_done)
5252
5253         d.addCallback(self._count_leases, "root")
5254         d.addCallback(self._assert_leasecount, 1)
5255         d.addCallback(self._count_leases, "one")
5256         d.addCallback(self._assert_leasecount, 1)
5257         d.addCallback(self._count_leases, "mutable")
5258         d.addCallback(self._assert_leasecount, 1)
5259
5260         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
5261                       clientnum=1)
5262         d.addCallback(_done)
5263
5264         d.addCallback(self._count_leases, "root")
5265         d.addCallback(self._assert_leasecount, 2)
5266         d.addCallback(self._count_leases, "one")
5267         d.addCallback(self._assert_leasecount, 2)
5268         d.addCallback(self._count_leases, "mutable")
5269         d.addCallback(self._assert_leasecount, 2)
5270
5271         d.addErrback(self.explain_web_error)
5272         return d
5273
5274
5275     def test_exceptions(self):
5276         self.basedir = "web/Grid/exceptions"
5277         self.set_up_grid(num_clients=1, num_servers=2)
5278         c0 = self.g.clients[0]
5279         c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 2
5280         self.fileurls = {}
5281         DATA = "data" * 100
5282         d = c0.create_dirnode()
5283         def _stash_root(n):
5284             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5285             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
5286             return n
5287         d.addCallback(_stash_root)
5288         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
5289         def _stash_bad(ur):
5290             self.fileurls["1share"] = "uri/" + urllib.quote(ur.get_uri())
5291             self.delete_shares_numbered(ur.get_uri(), range(1,10))
5292
5293             u = uri.from_string(ur.get_uri())
5294             u.key = testutil.flip_bit(u.key, 0)
5295             baduri = u.to_string()
5296             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
5297         d.addCallback(_stash_bad)
5298         d.addCallback(lambda ign: c0.create_dirnode())
5299         def _mangle_dirnode_1share(n):
5300             u = n.get_uri()
5301             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
5302             self.fileurls["dir-1share-json"] = url + "?t=json"
5303             self.delete_shares_numbered(u, range(1,10))
5304         d.addCallback(_mangle_dirnode_1share)
5305         d.addCallback(lambda ign: c0.create_dirnode())
5306         def _mangle_dirnode_0share(n):
5307             u = n.get_uri()
5308             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
5309             self.fileurls["dir-0share-json"] = url + "?t=json"
5310             self.delete_shares_numbered(u, range(0,10))
5311         d.addCallback(_mangle_dirnode_0share)
5312
5313         # NotEnoughSharesError should be reported sensibly, with a
5314         # text/plain explanation of the problem, and perhaps some
5315         # information on which shares *could* be found.
5316
5317         d.addCallback(lambda ignored:
5318                       self.shouldHTTPError("GET unrecoverable",
5319                                            410, "Gone", "NoSharesError",
5320                                            self.GET, self.fileurls["0shares"]))
5321         def _check_zero_shares(body):
5322             self.failIfIn("<html>", body)
5323             body = " ".join(body.strip().split())
5324             exp = ("NoSharesError: no shares could be found. "
5325                    "Zero shares usually indicates a corrupt URI, or that "
5326                    "no servers were connected, but it might also indicate "
5327                    "severe corruption. You should perform a filecheck on "
5328                    "this object to learn more. The full error message is: "
5329                    "no shares (need 3). Last failure: None")
5330             self.failUnlessReallyEqual(exp, body)
5331         d.addCallback(_check_zero_shares)
5332
5333
5334         d.addCallback(lambda ignored:
5335                       self.shouldHTTPError("GET 1share",
5336                                            410, "Gone", "NotEnoughSharesError",
5337                                            self.GET, self.fileurls["1share"]))
5338         def _check_one_share(body):
5339             self.failIfIn("<html>", body)
5340             body = " ".join(body.strip().split())
5341             msgbase = ("NotEnoughSharesError: This indicates that some "
5342                        "servers were unavailable, or that shares have been "
5343                        "lost to server departure, hard drive failure, or disk "
5344                        "corruption. You should perform a filecheck on "
5345                        "this object to learn more. The full error message is:"
5346                        )
5347             msg1 = msgbase + (" ran out of shares:"
5348                               " complete=sh0"
5349                               " pending="
5350                               " overdue= unused= need 3. Last failure: None")
5351             msg2 = msgbase + (" ran out of shares:"
5352                               " complete="
5353                               " pending=Share(sh0-on-xgru5)"
5354                               " overdue= unused= need 3. Last failure: None")
5355             self.failUnless(body == msg1 or body == msg2, body)
5356         d.addCallback(_check_one_share)
5357
5358         d.addCallback(lambda ignored:
5359                       self.shouldHTTPError("GET imaginary",
5360                                            404, "Not Found", None,
5361                                            self.GET, self.fileurls["imaginary"]))
5362         def _missing_child(body):
5363             self.failUnlessIn("No such child: imaginary", body)
5364         d.addCallback(_missing_child)
5365
5366         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
5367         def _check_0shares_dir_html(body):
5368             self.failUnlessIn("<html>", body)
5369             # we should see the regular page, but without the child table or
5370             # the dirops forms
5371             body = " ".join(body.strip().split())
5372             self.failUnlessIn('href="?t=info">More info on this directory',
5373                               body)
5374             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5375                    "could not be retrieved, because there were insufficient "
5376                    "good shares. This might indicate that no servers were "
5377                    "connected, insufficient servers were connected, the URI "
5378                    "was corrupt, or that shares have been lost due to server "
5379                    "departure, hard drive failure, or disk corruption. You "
5380                    "should perform a filecheck on this object to learn more.")
5381             self.failUnlessIn(exp, body)
5382             self.failUnlessIn("No upload forms: directory is unreadable", body)
5383         d.addCallback(_check_0shares_dir_html)
5384
5385         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
5386         def _check_1shares_dir_html(body):
5387             # at some point, we'll split UnrecoverableFileError into 0-shares
5388             # and some-shares like we did for immutable files (since there
5389             # are different sorts of advice to offer in each case). For now,
5390             # they present the same way.
5391             self.failUnlessIn("<html>", body)
5392             body = " ".join(body.strip().split())
5393             self.failUnlessIn('href="?t=info">More info on this directory',
5394                               body)
5395             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5396                    "could not be retrieved, because there were insufficient "
5397                    "good shares. This might indicate that no servers were "
5398                    "connected, insufficient servers were connected, the URI "
5399                    "was corrupt, or that shares have been lost due to server "
5400                    "departure, hard drive failure, or disk corruption. You "
5401                    "should perform a filecheck on this object to learn more.")
5402             self.failUnlessIn(exp, body)
5403             self.failUnlessIn("No upload forms: directory is unreadable", body)
5404         d.addCallback(_check_1shares_dir_html)
5405
5406         d.addCallback(lambda ignored:
5407                       self.shouldHTTPError("GET dir-0share-json",
5408                                            410, "Gone", "UnrecoverableFileError",
5409                                            self.GET,
5410                                            self.fileurls["dir-0share-json"]))
5411         def _check_unrecoverable_file(body):
5412             self.failIfIn("<html>", body)
5413             body = " ".join(body.strip().split())
5414             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5415                    "could not be retrieved, because there were insufficient "
5416                    "good shares. This might indicate that no servers were "
5417                    "connected, insufficient servers were connected, the URI "
5418                    "was corrupt, or that shares have been lost due to server "
5419                    "departure, hard drive failure, or disk corruption. You "
5420                    "should perform a filecheck on this object to learn more.")
5421             self.failUnlessReallyEqual(exp, body)
5422         d.addCallback(_check_unrecoverable_file)
5423
5424         d.addCallback(lambda ignored:
5425                       self.shouldHTTPError("GET dir-1share-json",
5426                                            410, "Gone", "UnrecoverableFileError",
5427                                            self.GET,
5428                                            self.fileurls["dir-1share-json"]))
5429         d.addCallback(_check_unrecoverable_file)
5430
5431         d.addCallback(lambda ignored:
5432                       self.shouldHTTPError("GET imaginary",
5433                                            404, "Not Found", None,
5434                                            self.GET, self.fileurls["imaginary"]))
5435
5436         # attach a webapi child that throws a random error, to test how it
5437         # gets rendered.
5438         w = c0.getServiceNamed("webish")
5439         w.root.putChild("ERRORBOOM", ErrorBoom())
5440
5441         # "Accept: */*" :        should get a text/html stack trace
5442         # "Accept: text/plain" : should get a text/plain stack trace
5443         # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
5444         # no Accept header:      should get a text/html stack trace
5445
5446         d.addCallback(lambda ignored:
5447                       self.shouldHTTPError("GET errorboom_html",
5448                                            500, "Internal Server Error", None,
5449                                            self.GET, "ERRORBOOM",
5450                                            headers={"accept": "*/*"}))
5451         def _internal_error_html1(body):
5452             self.failUnlessIn("<html>", "expected HTML, not '%s'" % body)
5453         d.addCallback(_internal_error_html1)
5454
5455         d.addCallback(lambda ignored:
5456                       self.shouldHTTPError("GET errorboom_text",
5457                                            500, "Internal Server Error", None,
5458                                            self.GET, "ERRORBOOM",
5459                                            headers={"accept": "text/plain"}))
5460         def _internal_error_text2(body):
5461             self.failIfIn("<html>", body)
5462             self.failUnless(body.startswith("Traceback "), body)
5463         d.addCallback(_internal_error_text2)
5464
5465         CLI_accepts = "text/plain, application/octet-stream"
5466         d.addCallback(lambda ignored:
5467                       self.shouldHTTPError("GET errorboom_text",
5468                                            500, "Internal Server Error", None,
5469                                            self.GET, "ERRORBOOM",
5470                                            headers={"accept": CLI_accepts}))
5471         def _internal_error_text3(body):
5472             self.failIfIn("<html>", body)
5473             self.failUnless(body.startswith("Traceback "), body)
5474         d.addCallback(_internal_error_text3)
5475
5476         d.addCallback(lambda ignored:
5477                       self.shouldHTTPError("GET errorboom_text",
5478                                            500, "Internal Server Error", None,
5479                                            self.GET, "ERRORBOOM"))
5480         def _internal_error_html4(body):
5481             self.failUnlessIn("<html>", body)
5482         d.addCallback(_internal_error_html4)
5483
5484         def _flush_errors(res):
5485             # Trial: please ignore the CompletelyUnhandledError in the logs
5486             self.flushLoggedErrors(CompletelyUnhandledError)
5487             return res
5488         d.addBoth(_flush_errors)
5489
5490         return d
5491
5492     def test_blacklist(self):
5493         # download from a blacklisted URI, get an error
5494         self.basedir = "web/Grid/blacklist"
5495         self.set_up_grid()
5496         c0 = self.g.clients[0]
5497         c0_basedir = c0.basedir
5498         fn = os.path.join(c0_basedir, "access.blacklist")
5499         self.uris = {}
5500         DATA = "off-limits " * 50
5501
5502         d = c0.upload(upload.Data(DATA, convergence=""))
5503         def _stash_uri_and_create_dir(ur):
5504             self.uri = ur.get_uri()
5505             self.url = "uri/"+self.uri
5506             u = uri.from_string_filenode(self.uri)
5507             self.si = u.get_storage_index()
5508             childnode = c0.create_node_from_uri(self.uri, None)
5509             return c0.create_dirnode({u"blacklisted.txt": (childnode,{}) })
5510         d.addCallback(_stash_uri_and_create_dir)
5511         def _stash_dir(node):
5512             self.dir_node = node
5513             self.dir_uri = node.get_uri()
5514             self.dir_url = "uri/"+self.dir_uri
5515         d.addCallback(_stash_dir)
5516         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5517         def _check_dir_html(body):
5518             self.failUnlessIn("<html>", body)
5519             self.failUnlessIn("blacklisted.txt</a>", body)
5520         d.addCallback(_check_dir_html)
5521         d.addCallback(lambda ign: self.GET(self.url))
5522         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5523
5524         def _blacklist(ign):
5525             f = open(fn, "w")
5526             f.write(" # this is a comment\n")
5527             f.write(" \n")
5528             f.write("\n") # also exercise blank lines
5529             f.write("%s %s\n" % (base32.b2a(self.si), "off-limits to you"))
5530             f.close()
5531             # clients should be checking the blacklist each time, so we don't
5532             # need to restart the client
5533         d.addCallback(_blacklist)
5534         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_uri",
5535                                                        403, "Forbidden",
5536                                                        "Access Prohibited: off-limits",
5537                                                        self.GET, self.url))
5538
5539         # We should still be able to list the parent directory, in HTML...
5540         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5541         def _check_dir_html2(body):
5542             self.failUnlessIn("<html>", body)
5543             self.failUnlessIn("blacklisted.txt</strike>", body)
5544         d.addCallback(_check_dir_html2)
5545
5546         # ... and in JSON (used by CLI).
5547         d.addCallback(lambda ign: self.GET(self.dir_url+"?t=json", followRedirect=True))
5548         def _check_dir_json(res):
5549             data = simplejson.loads(res)
5550             self.failUnless(isinstance(data, list), data)
5551             self.failUnlessEqual(data[0], "dirnode")
5552             self.failUnless(isinstance(data[1], dict), data)
5553             self.failUnlessIn("children", data[1])
5554             self.failUnlessIn("blacklisted.txt", data[1]["children"])
5555             childdata = data[1]["children"]["blacklisted.txt"]
5556             self.failUnless(isinstance(childdata, list), data)
5557             self.failUnlessEqual(childdata[0], "filenode")
5558             self.failUnless(isinstance(childdata[1], dict), data)
5559         d.addCallback(_check_dir_json)
5560
5561         def _unblacklist(ign):
5562             open(fn, "w").close()
5563             # the Blacklist object watches mtime to tell when the file has
5564             # changed, but on windows this test will run faster than the
5565             # filesystem's mtime resolution. So we edit Blacklist.last_mtime
5566             # to force a reload.
5567             self.g.clients[0].blacklist.last_mtime -= 2.0
5568         d.addCallback(_unblacklist)
5569
5570         # now a read should work
5571         d.addCallback(lambda ign: self.GET(self.url))
5572         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5573
5574         # read again to exercise the blacklist-is-unchanged logic
5575         d.addCallback(lambda ign: self.GET(self.url))
5576         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5577
5578         # now add a blacklisted directory, and make sure files under it are
5579         # refused too
5580         def _add_dir(ign):
5581             childnode = c0.create_node_from_uri(self.uri, None)
5582             return c0.create_dirnode({u"child": (childnode,{}) })
5583         d.addCallback(_add_dir)
5584         def _get_dircap(dn):
5585             self.dir_si_b32 = base32.b2a(dn.get_storage_index())
5586             self.dir_url_base = "uri/"+dn.get_write_uri()
5587             self.dir_url_json1 = "uri/"+dn.get_write_uri()+"?t=json"
5588             self.dir_url_json2 = "uri/"+dn.get_write_uri()+"/?t=json"
5589             self.dir_url_json_ro = "uri/"+dn.get_readonly_uri()+"/?t=json"
5590             self.child_url = "uri/"+dn.get_readonly_uri()+"/child"
5591         d.addCallback(_get_dircap)
5592         d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True))
5593         d.addCallback(lambda body: self.failUnlessIn("<html>", body))
5594         d.addCallback(lambda ign: self.GET(self.dir_url_json1))
5595         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5596         d.addCallback(lambda ign: self.GET(self.dir_url_json2))
5597         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5598         d.addCallback(lambda ign: self.GET(self.dir_url_json_ro))
5599         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5600         d.addCallback(lambda ign: self.GET(self.child_url))
5601         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5602
5603         def _block_dir(ign):
5604             f = open(fn, "w")
5605             f.write("%s %s\n" % (self.dir_si_b32, "dir-off-limits to you"))
5606             f.close()
5607             self.g.clients[0].blacklist.last_mtime -= 2.0
5608         d.addCallback(_block_dir)
5609         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir base",
5610                                                        403, "Forbidden",
5611                                                        "Access Prohibited: dir-off-limits",
5612                                                        self.GET, self.dir_url_base))
5613         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json1",
5614                                                        403, "Forbidden",
5615                                                        "Access Prohibited: dir-off-limits",
5616                                                        self.GET, self.dir_url_json1))
5617         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json2",
5618                                                        403, "Forbidden",
5619                                                        "Access Prohibited: dir-off-limits",
5620                                                        self.GET, self.dir_url_json2))
5621         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json_ro",
5622                                                        403, "Forbidden",
5623                                                        "Access Prohibited: dir-off-limits",
5624                                                        self.GET, self.dir_url_json_ro))
5625         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir child",
5626                                                        403, "Forbidden",
5627                                                        "Access Prohibited: dir-off-limits",
5628                                                        self.GET, self.child_url))
5629         return d
5630
5631
5632 class CompletelyUnhandledError(Exception):
5633     pass
5634 class ErrorBoom(rend.Page):
5635     def beforeRender(self, ctx):
5636         raise CompletelyUnhandledError("whoops")