src/allmydata/test/test_web.py

   1 import os.path, re, urllib, time
   2 import simplejson
   3 from StringIO import StringIO
   4
   5 from twisted.application import service
   6 from twisted.trial import unittest
   7 from twisted.internet import defer, reactor
   8 from twisted.internet.task import Clock
   9 from twisted.web import client, error, http, html
  10 from twisted.python import failure, log
  11
  12 from foolscap.api import fireEventually, flushEventualQueue
  13
  14 from nevow.util import escapeToXML
  15 from nevow import rend
  16
  17 from allmydata import interfaces, uri, webish, dirnode
  18 from allmydata.storage.shares import get_share_file
  19 from allmydata.storage_client import StorageFarmBroker, StubServer
  20 from allmydata.immutable import upload
  21 from allmydata.immutable.downloader.status import DownloadStatus
  22 from allmydata.dirnode import DirectoryNode
  23 from allmydata.nodemaker import NodeMaker
  24 from allmydata.unknown import UnknownNode
  25 from allmydata.web import status, common
  26 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  27 from allmydata.util import fileutil, base32, hashutil
  28 from allmydata.util.consumer import download_to_data
  29 from allmydata.util.netstring import split_netstring
  30 from allmydata.util.encodingutil import to_str
  31 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  32      create_chk_filenode, WebErrorMixin, ShouldFailMixin, \
  33      make_mutable_file_uri, create_mutable_filenode
  34 from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
  35 from allmydata.mutable import servermap, publish, retrieve
  36 import allmydata.test.common_util as testutil
  37 from allmydata.test.no_network import GridTestMixin
  38 from allmydata.test.common_web import HTTPClientGETFactory, \
  39      HTTPClientHEADFactory
  40 from allmydata.client import Client, SecretHolder
  41 from allmydata.introducer import IntroducerNode
  42
  43 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  44 # create a webserver that works against them
  45
  46 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  47
  48 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
  49 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
  50 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
  51
  52 FAVICON_MARKUP = '<link href="/icon.png" rel="shortcut icon" />'
  53
  54
  55 class FakeStatsProvider:
  56     def get_stats(self):
  57         stats = {'stats': {}, 'counters': {}}
  58         return stats
  59
  60 class FakeNodeMaker(NodeMaker):
  61     encoding_params = {
  62         'k': 3,
  63         'n': 10,
  64         'happy': 7,
  65         'max_segment_size':128*1024 # 1024=KiB
  66     }
  67     def _create_lit(self, cap):
  68         return FakeCHKFileNode(cap, self.all_contents)
  69     def _create_immutable(self, cap):
  70         return FakeCHKFileNode(cap, self.all_contents)
  71     def _create_mutable(self, cap):
  72         return FakeMutableFileNode(None, None,
  73                                    self.encoding_params, None,
  74                                    self.all_contents).init_from_cap(cap)
  75     def create_mutable_file(self, contents="", keysize=None,
  76                             version=SDMF_VERSION):
  77         n = FakeMutableFileNode(None, None, self.encoding_params, None,
  78                                 self.all_contents)
  79         return n.create(contents, version=version)
  80
  81 class FakeUploader(service.Service):
  82     name = "uploader"
  83     helper_furl = None
  84     helper_connected = False
  85
  86     def upload(self, uploadable):
  87         d = uploadable.get_size()
  88         d.addCallback(lambda size: uploadable.read(size))
  89         def _got_data(datav):
  90             data = "".join(datav)
  91             n = create_chk_filenode(data, self.all_contents)
  92             ur = upload.UploadResults(file_size=len(data),
  93                                       ciphertext_fetched=0,
  94                                       preexisting_shares=0,
  95                                       pushed_shares=10,
  96                                       sharemap={},
  97                                       servermap={},
  98                                       timings={},
  99                                       uri_extension_data={},
 100                                       uri_extension_hash="fake",
 101                                       verifycapstr="fakevcap")
 102             ur.set_uri(n.get_uri())
 103             return ur
 104         d.addCallback(_got_data)
 105         return d
 106
 107     def get_helper_info(self):
 108         return (self.helper_furl, self.helper_connected)
 109
 110
 111 def build_one_ds():
 112     ds = DownloadStatus("storage_index", 1234)
 113     now = time.time()
 114
 115     serverA = StubServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
 116     serverB = StubServer(hashutil.tagged_hash("foo", "serverid_b")[:20])
 117     storage_index = hashutil.storage_index_hash("SI")
 118     e0 = ds.add_segment_request(0, now)
 119     e0.activate(now+0.5)
 120     e0.deliver(now+1, 0, 100, 0.5) # when, start,len, decodetime
 121     e1 = ds.add_segment_request(1, now+2)
 122     e1.error(now+3)
 123     # two outstanding requests
 124     e2 = ds.add_segment_request(2, now+4)
 125     e3 = ds.add_segment_request(3, now+5)
 126     del e2,e3 # hush pyflakes
 127
 128     # simulate a segment which gets delivered faster than a system clock tick (ticket #1166)
 129     e = ds.add_segment_request(4, now)
 130     e.activate(now)
 131     e.deliver(now, 0, 140, 0.5)
 132
 133     e = ds.add_dyhb_request(serverA, now)
 134     e.finished([1,2], now+1)
 135     e = ds.add_dyhb_request(serverB, now+2) # left unfinished
 136
 137     e = ds.add_read_event(0, 120, now)
 138     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 139     e.finished(now+1)
 140     e = ds.add_read_event(120, 30, now+2) # left unfinished
 141
 142     e = ds.add_block_request(serverA, 1, 100, 20, now)
 143     e.finished(20, now+1)
 144     e = ds.add_block_request(serverB, 1, 120, 30, now+1) # left unfinished
 145
 146     # make sure that add_read_event() can come first too
 147     ds1 = DownloadStatus(storage_index, 1234)
 148     e = ds1.add_read_event(0, 120, now)
 149     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 150     e.finished(now+1)
 151
 152     return ds
 153
 154 class FakeHistory:
 155     _all_upload_status = [upload.UploadStatus()]
 156     _all_download_status = [build_one_ds()]
 157     _all_mapupdate_statuses = [servermap.UpdateStatus()]
 158     _all_publish_statuses = [publish.PublishStatus()]
 159     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
 160
 161     def list_all_upload_statuses(self):
 162         return self._all_upload_status
 163     def list_all_download_statuses(self):
 164         return self._all_download_status
 165     def list_all_mapupdate_statuses(self):
 166         return self._all_mapupdate_statuses
 167     def list_all_publish_statuses(self):
 168         return self._all_publish_statuses
 169     def list_all_retrieve_statuses(self):
 170         return self._all_retrieve_statuses
 171     def list_all_helper_statuses(self):
 172         return []
 173
 174 class FakeDisplayableServer(StubServer):
 175     def __init__(self, serverid, nickname):
 176         StubServer.__init__(self, serverid)
 177         self.announcement = {"my-version": "allmydata-tahoe-fake",
 178                              "service-name": "storage",
 179                              "nickname": nickname}
 180     def is_connected(self):
 181         return True
 182     def get_permutation_seed(self):
 183         return ""
 184     def get_remote_host(self):
 185         return ""
 186     def get_last_loss_time(self):
 187         return None
 188     def get_announcement_time(self):
 189         return None
 190     def get_announcement(self):
 191         return self.announcement
 192     def get_nickname(self):
 193         return self.announcement["nickname"]
 194
 195 class FakeBucketCounter(object):
 196     def get_state(self):
 197         return {"last-complete-bucket-count": 0}
 198     def get_progress(self):
 199         return {"estimated-time-per-cycle": 0,
 200                 "cycle-in-progress": False,
 201                 "remaining-wait-time": 0}
 202
 203 class FakeLeaseChecker(object):
 204     def __init__(self):
 205         self.expiration_enabled = False
 206         self.mode = "age"
 207         self.override_lease_duration = None
 208         self.sharetypes_to_expire = {}
 209     def get_state(self):
 210         return {"history": None}
 211     def get_progress(self):
 212         return {"estimated-time-per-cycle": 0,
 213                 "cycle-in-progress": False,
 214                 "remaining-wait-time": 0}
 215
 216 class FakeStorageServer(service.MultiService):
 217     name = 'storage'
 218     def __init__(self, nodeid, nickname):
 219         service.MultiService.__init__(self)
 220         self.my_nodeid = nodeid
 221         self.nickname = nickname
 222         self.bucket_counter = FakeBucketCounter()
 223         self.lease_checker = FakeLeaseChecker()
 224     def get_stats(self):
 225         return {"storage_server.accepting_immutable_shares": False}
 226
 227 class FakeClient(Client):
 228     def __init__(self):
 229         # don't upcall to Client.__init__, since we only want to initialize a
 230         # minimal subset
 231         service.MultiService.__init__(self)
 232         self.all_contents = {}
 233         self.nodeid = "fake_nodeid"
 234         self.nickname = u"fake_nickname \u263A"
 235         self.introducer_furl = "None"
 236         self.stats_provider = FakeStatsProvider()
 237         self._secret_holder = SecretHolder("lease secret", "convergence secret")
 238         self.helper = None
 239         self.convergence = "some random string"
 240         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 241         # fake knowledge of another server
 242         self.storage_broker.test_add_server("other_nodeid",
 243                                             FakeDisplayableServer("other_nodeid", u"other_nickname \u263B"))
 244         self.introducer_client = None
 245         self.history = FakeHistory()
 246         self.uploader = FakeUploader()
 247         self.uploader.all_contents = self.all_contents
 248         self.uploader.setServiceParent(self)
 249         self.blacklist = None
 250         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 251                                        self.uploader, None,
 252                                        None, None, None)
 253         self.nodemaker.all_contents = self.all_contents
 254         self.mutable_file_default = SDMF_VERSION
 255         self.addService(FakeStorageServer(self.nodeid, self.nickname))
 256
 257     def startService(self):
 258         return service.MultiService.startService(self)
 259     def stopService(self):
 260         return service.MultiService.stopService(self)
 261
 262     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 263
 264 class WebMixin(object):
 265     def setUp(self):
 266         self.s = FakeClient()
 267         self.s.startService()
 268         self.staticdir = self.mktemp()
 269         self.clock = Clock()
 270         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
 271                                       clock=self.clock)
 272         self.ws.setServiceParent(self.s)
 273         self.webish_port = self.ws.getPortnum()
 274         self.webish_url = self.ws.getURL()
 275         assert self.webish_url.endswith("/")
 276         self.webish_url = self.webish_url[:-1] # these tests add their own /
 277
 278         l = [ self.s.create_dirnode() for x in range(6) ]
 279         d = defer.DeferredList(l)
 280         def _then(res):
 281             self.public_root = res[0][1]
 282             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 283             self.public_url = "/uri/" + self.public_root.get_uri()
 284             self.private_root = res[1][1]
 285
 286             foo = res[2][1]
 287             self._foo_node = foo
 288             self._foo_uri = foo.get_uri()
 289             self._foo_readonly_uri = foo.get_readonly_uri()
 290             self._foo_verifycap = foo.get_verify_cap().to_string()
 291             # NOTE: we ignore the deferred on all set_uri() calls, because we
 292             # know the fake nodes do these synchronously
 293             self.public_root.set_uri(u"foo", foo.get_uri(),
 294                                      foo.get_readonly_uri())
 295
 296             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 297             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 298             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 299
 300             # sdmf
 301             # XXX: Do we ever use this?
 302             self.BAZ_CONTENTS, n, self._baz_txt_uri, self._baz_txt_readonly_uri = self.makefile_mutable(0)
 303
 304             foo.set_uri(u"baz.txt", self._baz_txt_uri, self._baz_txt_readonly_uri)
 305
 306             # mdmf
 307             self.QUUX_CONTENTS, n, self._quux_txt_uri, self._quux_txt_readonly_uri = self.makefile_mutable(0, mdmf=True)
 308             assert self._quux_txt_uri.startswith("URI:MDMF")
 309             foo.set_uri(u"quux.txt", self._quux_txt_uri, self._quux_txt_readonly_uri)
 310
 311             foo.set_uri(u"empty", res[3][1].get_uri(),
 312                         res[3][1].get_readonly_uri())
 313             sub_uri = res[4][1].get_uri()
 314             self._sub_uri = sub_uri
 315             foo.set_uri(u"sub", sub_uri, sub_uri)
 316             sub = self.s.create_node_from_uri(sub_uri)
 317             self._sub_node = sub
 318
 319             _ign, n, blocking_uri = self.makefile(1)
 320             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 321
 322             # filenode to test for html encoding issues
 323             self._htmlname_unicode = u"<&weirdly'named\"file>>>_<iframe />.txt"
 324             self._htmlname_raw = self._htmlname_unicode.encode('utf-8')
 325             self._htmlname_urlencoded = urllib.quote(self._htmlname_raw, '')
 326             self._htmlname_escaped = escapeToXML(self._htmlname_raw)
 327             self._htmlname_escaped_attr = html.escape(self._htmlname_raw)
 328             self._htmlname_escaped_double = escapeToXML(html.escape(self._htmlname_raw))
 329             self.HTMLNAME_CONTENTS, n, self._htmlname_txt_uri = self.makefile(0)
 330             foo.set_uri(self._htmlname_unicode, self._htmlname_txt_uri, self._htmlname_txt_uri)
 331
 332             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 333             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 334             # still think of it as an umlaut
 335             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 336
 337             self.SUBBAZ_CONTENTS, n, baz_file = self.makefile(2)
 338             self._baz_file_uri = baz_file
 339             sub.set_uri(u"baz.txt", baz_file, baz_file)
 340
 341             _ign, n, self._bad_file_uri = self.makefile(3)
 342             # this uri should not be downloadable
 343             del self.s.all_contents[self._bad_file_uri]
 344
 345             rodir = res[5][1]
 346             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 347                                      rodir.get_readonly_uri())
 348             rodir.set_uri(u"nor", baz_file, baz_file)
 349
 350             # public/
 351             # public/foo/
 352             # public/foo/bar.txt
 353             # public/foo/baz.txt
 354             # public/foo/quux.txt
 355             # public/foo/blockingfile
 356             # public/foo/<&weirdly'named\"file>>>_<iframe />.txt
 357             # public/foo/empty/
 358             # public/foo/sub/
 359             # public/foo/sub/baz.txt
 360             # public/reedownlee/
 361             # public/reedownlee/nor
 362             self.NEWFILE_CONTENTS = "newfile contents\n"
 363
 364             return foo.get_metadata_for(u"bar.txt")
 365         d.addCallback(_then)
 366         def _got_metadata(metadata):
 367             self._bar_txt_metadata = metadata
 368         d.addCallback(_got_metadata)
 369         return d
 370
 371     def get_all_contents(self):
 372         return self.s.all_contents
 373
 374     def makefile(self, number):
 375         contents = "contents of file %s\n" % number
 376         n = create_chk_filenode(contents, self.get_all_contents())
 377         return contents, n, n.get_uri()
 378
 379     def makefile_mutable(self, number, mdmf=False):
 380         contents = "contents of mutable file %s\n" % number
 381         n = create_mutable_filenode(contents, mdmf, self.s.all_contents)
 382         return contents, n, n.get_uri(), n.get_readonly_uri()
 383
 384     def tearDown(self):
 385         return self.s.stopService()
 386
 387     def failUnlessIsBarDotTxt(self, res):
 388         self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
 389
 390     def failUnlessIsQuuxDotTxt(self, res):
 391         self.failUnlessReallyEqual(res, self.QUUX_CONTENTS, res)
 392
 393     def failUnlessIsBazDotTxt(self, res):
 394         self.failUnlessReallyEqual(res, self.BAZ_CONTENTS, res)
 395
 396     def failUnlessIsSubBazDotTxt(self, res):
 397         self.failUnlessReallyEqual(res, self.SUBBAZ_CONTENTS, res)
 398
 399     def failUnlessIsBarJSON(self, res):
 400         data = simplejson.loads(res)
 401         self.failUnless(isinstance(data, list))
 402         self.failUnlessEqual(data[0], "filenode")
 403         self.failUnless(isinstance(data[1], dict))
 404         self.failIf(data[1]["mutable"])
 405         self.failIfIn("rw_uri", data[1]) # immutable
 406         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
 407         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
 408         self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
 409
 410     def failUnlessIsQuuxJSON(self, res, readonly=False):
 411         data = simplejson.loads(res)
 412         self.failUnless(isinstance(data, list))
 413         self.failUnlessEqual(data[0], "filenode")
 414         self.failUnless(isinstance(data[1], dict))
 415         metadata = data[1]
 416         return self.failUnlessIsQuuxDotTxtMetadata(metadata, readonly)
 417
 418     def failUnlessIsQuuxDotTxtMetadata(self, metadata, readonly):
 419         self.failUnless(metadata['mutable'])
 420         if readonly:
 421             self.failIfIn("rw_uri", metadata)
 422         else:
 423             self.failUnlessIn("rw_uri", metadata)
 424             self.failUnlessEqual(metadata['rw_uri'], self._quux_txt_uri)
 425         self.failUnlessIn("ro_uri", metadata)
 426         self.failUnlessEqual(metadata['ro_uri'], self._quux_txt_readonly_uri)
 427         self.failUnlessReallyEqual(metadata['size'], len(self.QUUX_CONTENTS))
 428
 429     def failUnlessIsFooJSON(self, res):
 430         data = simplejson.loads(res)
 431         self.failUnless(isinstance(data, list))
 432         self.failUnlessEqual(data[0], "dirnode", res)
 433         self.failUnless(isinstance(data[1], dict))
 434         self.failUnless(data[1]["mutable"])
 435         self.failUnlessIn("rw_uri", data[1]) # mutable
 436         self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
 437         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
 438         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
 439
 440         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 441         self.failUnlessEqual(kidnames,
 442                              [self._htmlname_unicode, u"bar.txt", u"baz.txt",
 443                               u"blockingfile", u"empty", u"n\u00fc.txt", u"quux.txt", u"sub"])
 444         kids = dict( [(unicode(name),value)
 445                       for (name,value)
 446                       in data[1]["children"].iteritems()] )
 447         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 448         self.failUnlessIn("metadata", kids[u"sub"][1])
 449         self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
 450         tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
 451         self.failUnlessIn("linkcrtime", tahoe_md)
 452         self.failUnlessIn("linkmotime", tahoe_md)
 453         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 454         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 455         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri)
 456         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]),
 457                                    self._bar_txt_verifycap)
 458         self.failUnlessIn("metadata", kids[u"bar.txt"][1])
 459         self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
 460         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
 461                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
 462         self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
 463                                    self._bar_txt_uri)
 464         self.failUnlessIn("quux.txt", kids)
 465         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["rw_uri"]),
 466                                    self._quux_txt_uri)
 467         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["ro_uri"]),
 468                                    self._quux_txt_readonly_uri)
 469
 470     def GET(self, urlpath, followRedirect=False, return_response=False,
 471             **kwargs):
 472         # if return_response=True, this fires with (data, statuscode,
 473         # respheaders) instead of just data.
 474         assert not isinstance(urlpath, unicode)
 475         url = self.webish_url + urlpath
 476         factory = HTTPClientGETFactory(url, method="GET",
 477                                        followRedirect=followRedirect, **kwargs)
 478         reactor.connectTCP("localhost", self.webish_port, factory)
 479         d = factory.deferred
 480         def _got_data(data):
 481             return (data, factory.status, factory.response_headers)
 482         if return_response:
 483             d.addCallback(_got_data)
 484         return factory.deferred
 485
 486     def HEAD(self, urlpath, return_response=False, **kwargs):
 487         # this requires some surgery, because twisted.web.client doesn't want
 488         # to give us back the response headers.
 489         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 490         reactor.connectTCP("localhost", self.webish_port, factory)
 491         d = factory.deferred
 492         def _got_data(data):
 493             return (data, factory.status, factory.response_headers)
 494         if return_response:
 495             d.addCallback(_got_data)
 496         return factory.deferred
 497
 498     def PUT(self, urlpath, data, **kwargs):
 499         url = self.webish_url + urlpath
 500         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 501
 502     def DELETE(self, urlpath):
 503         url = self.webish_url + urlpath
 504         return client.getPage(url, method="DELETE")
 505
 506     def POST(self, urlpath, followRedirect=False, **fields):
 507         sepbase = "boogabooga"
 508         sep = "--" + sepbase
 509         form = []
 510         form.append(sep)
 511         form.append('Content-Disposition: form-data; name="_charset"')
 512         form.append('')
 513         form.append('UTF-8')
 514         form.append(sep)
 515         for name, value in fields.iteritems():
 516             if isinstance(value, tuple):
 517                 filename, value = value
 518                 form.append('Content-Disposition: form-data; name="%s"; '
 519                             'filename="%s"' % (name, filename.encode("utf-8")))
 520             else:
 521                 form.append('Content-Disposition: form-data; name="%s"' % name)
 522             form.append('')
 523             if isinstance(value, unicode):
 524                 value = value.encode("utf-8")
 525             else:
 526                 value = str(value)
 527             assert isinstance(value, str)
 528             form.append(value)
 529             form.append(sep)
 530         form[-1] += "--"
 531         body = ""
 532         headers = {}
 533         if fields:
 534             body = "\r\n".join(form) + "\r\n"
 535             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 536         return self.POST2(urlpath, body, headers, followRedirect)
 537
 538     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 539         url = self.webish_url + urlpath
 540         return client.getPage(url, method="POST", postdata=body,
 541                               headers=headers, followRedirect=followRedirect)
 542
 543     def shouldFail(self, res, expected_failure, which,
 544                    substring=None, response_substring=None):
 545         if isinstance(res, failure.Failure):
 546             res.trap(expected_failure)
 547             if substring:
 548                 self.failUnlessIn(substring, str(res), which)
 549             if response_substring:
 550                 self.failUnlessIn(response_substring, res.value.response, which)
 551         else:
 552             self.fail("%s was supposed to raise %s, not get '%s'" %
 553                       (which, expected_failure, res))
 554
 555     def shouldFail2(self, expected_failure, which, substring,
 556                     response_substring,
 557                     callable, *args, **kwargs):
 558         assert substring is None or isinstance(substring, str)
 559         assert response_substring is None or isinstance(response_substring, str)
 560         d = defer.maybeDeferred(callable, *args, **kwargs)
 561         def done(res):
 562             if isinstance(res, failure.Failure):
 563                 res.trap(expected_failure)
 564                 if substring:
 565                     self.failUnlessIn(substring, str(res),
 566                                       "'%s' not in '%s' for test '%s'" % \
 567                                       (substring, str(res), which))
 568                 if response_substring:
 569                     self.failUnlessIn(response_substring, res.value.response,
 570                                       "'%s' not in '%s' for test '%s'" % \
 571                                       (response_substring, res.value.response,
 572                                        which))
 573             else:
 574                 self.fail("%s was supposed to raise %s, not get '%s'" %
 575                           (which, expected_failure, res))
 576         d.addBoth(done)
 577         return d
 578
 579     def should404(self, res, which):
 580         if isinstance(res, failure.Failure):
 581             res.trap(error.Error)
 582             self.failUnlessReallyEqual(res.value.status, "404")
 583         else:
 584             self.fail("%s was supposed to Error(404), not get '%s'" %
 585                       (which, res))
 586
 587     def should302(self, res, which):
 588         if isinstance(res, failure.Failure):
 589             res.trap(error.Error)
 590             self.failUnlessReallyEqual(res.value.status, "302")
 591         else:
 592             self.fail("%s was supposed to Error(302), not get '%s'" %
 593                       (which, res))
 594
 595
 596 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
 597     def test_create(self):
 598         pass
 599
 600     def test_welcome(self):
 601         d = self.GET("/")
 602         def _check(res):
 603             self.failUnlessIn('Welcome to Tahoe-LAFS', res)
 604             self.failUnlessIn(FAVICON_MARKUP, res)
 605             self.failUnlessIn('href="https://tahoe-lafs.org/"', res)
 606             res_u = res.decode('utf-8')
 607             self.failUnlessIn(u'<th>My nickname:</th> <td class="nickname mine">fake_nickname \u263A</td></tr>', res_u)
 608             self.failUnlessIn(u'<div class="nickname">other_nickname \u263B</div>', res_u)
 609
 610             self.s.basedir = 'web/test_welcome'
 611             fileutil.make_dirs("web/test_welcome")
 612             fileutil.make_dirs("web/test_welcome/private")
 613             return self.GET("/")
 614         d.addCallback(_check)
 615         return d
 616
 617     def test_helper_status(self):
 618         d = defer.succeed(None)
 619
 620         # set helper furl to None
 621         def _set_no_helper(ign):
 622             self.s.uploader.helper_furl = None
 623             return self.GET("/")
 624         d.addCallback(_set_no_helper)
 625         d.addCallback(lambda res:
 626                       self.failUnlessIn('Connected to helper?: <span>not configured</span>', res))
 627
 628         # enable helper, not connected
 629         def _set_helper_not_connected(ign):
 630             self.s.uploader.helper_furl = "pb://someHelper"
 631             self.s.uploader.helper_connected = False
 632             return self.GET("/")
 633         d.addCallback(_set_helper_not_connected)
 634         d.addCallback(lambda res:
 635                       self.failUnlessIn('Connected to helper?: <span>no</span>', res))
 636
 637         # enable helper, connected
 638         def _set_helper_connected(ign):
 639             self.s.uploader.helper_furl = "pb://someHelper"
 640             self.s.uploader.helper_connected = True
 641             return self.GET("/")
 642         d.addCallback(_set_helper_connected)
 643         d.addCallback(lambda res:
 644                       self.failUnlessIn('Connected to helper?: <span>yes</span>', res))
 645         return d
 646
 647     def test_storage(self):
 648         d = self.GET("/storage")
 649         def _check(res):
 650             self.failUnlessIn('Storage Server Status', res)
 651             self.failUnlessIn(FAVICON_MARKUP, res)
 652             res_u = res.decode('utf-8')
 653             self.failUnlessIn(u'<li>Server Nickname: <span class="nickname mine">fake_nickname \u263A</span></li>', res_u)
 654         d.addCallback(_check)
 655         return d
 656
 657     def test_status(self):
 658         h = self.s.get_history()
 659         dl_num = h.list_all_download_statuses()[0].get_counter()
 660         ul_num = h.list_all_upload_statuses()[0].get_counter()
 661         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 662         pub_num = h.list_all_publish_statuses()[0].get_counter()
 663         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 664         d = self.GET("/status", followRedirect=True)
 665         def _check(res):
 666             self.failUnlessIn('Upload and Download Status', res)
 667             self.failUnlessIn('"down-%d"' % dl_num, res)
 668             self.failUnlessIn('"up-%d"' % ul_num, res)
 669             self.failUnlessIn('"mapupdate-%d"' % mu_num, res)
 670             self.failUnlessIn('"publish-%d"' % pub_num, res)
 671             self.failUnlessIn('"retrieve-%d"' % ret_num, res)
 672         d.addCallback(_check)
 673         d.addCallback(lambda res: self.GET("/status/?t=json"))
 674         def _check_json(res):
 675             data = simplejson.loads(res)
 676             self.failUnless(isinstance(data, dict))
 677             #active = data["active"]
 678             # TODO: test more. We need a way to fake an active operation
 679             # here.
 680         d.addCallback(_check_json)
 681
 682         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 683         def _check_dl(res):
 684             self.failUnlessIn("File Download Status", res)
 685         d.addCallback(_check_dl)
 686         d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
 687         def _check_dl_json(res):
 688             data = simplejson.loads(res)
 689             self.failUnless(isinstance(data, dict))
 690             self.failUnlessIn("read", data)
 691             self.failUnlessEqual(data["read"][0]["length"], 120)
 692             self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
 693             self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
 694             self.failUnlessEqual(data["segment"][2]["finish_time"], None)
 695             phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
 696             cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
 697             # serverids[] keys are strings, since that's what JSON does, but
 698             # we'd really like them to be ints
 699             self.failUnlessEqual(data["serverids"]["0"], "phwrsjte")
 700             self.failUnless(data["serverids"].has_key("1"),
 701                             str(data["serverids"]))
 702             self.failUnlessEqual(data["serverids"]["1"], "cmpuvkjm",
 703                                  str(data["serverids"]))
 704             self.failUnlessEqual(data["server_info"][phwr_id]["short"],
 705                                  "phwrsjte")
 706             self.failUnlessEqual(data["server_info"][cmpu_id]["short"],
 707                                  "cmpuvkjm")
 708             self.failUnlessIn("dyhb", data)
 709             self.failUnlessIn("misc", data)
 710         d.addCallback(_check_dl_json)
 711         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 712         def _check_ul(res):
 713             self.failUnlessIn("File Upload Status", res)
 714         d.addCallback(_check_ul)
 715         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 716         def _check_mapupdate(res):
 717             self.failUnlessIn("Mutable File Servermap Update Status", res)
 718         d.addCallback(_check_mapupdate)
 719         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 720         def _check_publish(res):
 721             self.failUnlessIn("Mutable File Publish Status", res)
 722         d.addCallback(_check_publish)
 723         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 724         def _check_retrieve(res):
 725             self.failUnlessIn("Mutable File Retrieve Status", res)
 726         d.addCallback(_check_retrieve)
 727
 728         return d
 729
 730     def test_status_numbers(self):
 731         drrm = status.DownloadResultsRendererMixin()
 732         self.failUnlessReallyEqual(drrm.render_time(None, None), "")
 733         self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
 734         self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
 735         self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
 736         self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
 737         self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
 738         self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 739         self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
 740         self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
 741
 742         urrm = status.UploadResultsRendererMixin()
 743         self.failUnlessReallyEqual(urrm.render_time(None, None), "")
 744         self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
 745         self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
 746         self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
 747         self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
 748         self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
 749         self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 750         self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
 751         self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
 752
 753     def test_GET_FILEURL(self):
 754         d = self.GET(self.public_url + "/foo/bar.txt")
 755         d.addCallback(self.failUnlessIsBarDotTxt)
 756         return d
 757
 758     def test_GET_FILEURL_range(self):
 759         headers = {"range": "bytes=1-10"}
 760         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 761                      return_response=True)
 762         def _got((res, status, headers)):
 763             self.failUnlessReallyEqual(int(status), 206)
 764             self.failUnless(headers.has_key("content-range"))
 765             self.failUnlessReallyEqual(headers["content-range"][0],
 766                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 767             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
 768         d.addCallback(_got)
 769         return d
 770
 771     def test_GET_FILEURL_partial_range(self):
 772         headers = {"range": "bytes=5-"}
 773         length  = len(self.BAR_CONTENTS)
 774         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 775                      return_response=True)
 776         def _got((res, status, headers)):
 777             self.failUnlessReallyEqual(int(status), 206)
 778             self.failUnless(headers.has_key("content-range"))
 779             self.failUnlessReallyEqual(headers["content-range"][0],
 780                                        "bytes 5-%d/%d" % (length-1, length))
 781             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
 782         d.addCallback(_got)
 783         return d
 784
 785     def test_GET_FILEURL_partial_end_range(self):
 786         headers = {"range": "bytes=-5"}
 787         length  = len(self.BAR_CONTENTS)
 788         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 789                      return_response=True)
 790         def _got((res, status, headers)):
 791             self.failUnlessReallyEqual(int(status), 206)
 792             self.failUnless(headers.has_key("content-range"))
 793             self.failUnlessReallyEqual(headers["content-range"][0],
 794                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 795             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
 796         d.addCallback(_got)
 797         return d
 798
 799     def test_GET_FILEURL_partial_range_overrun(self):
 800         headers = {"range": "bytes=100-200"}
 801         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
 802                              "416 Requested Range not satisfiable",
 803                              "First beyond end of file",
 804                              self.GET, self.public_url + "/foo/bar.txt",
 805                              headers=headers)
 806         return d
 807
 808     def test_HEAD_FILEURL_range(self):
 809         headers = {"range": "bytes=1-10"}
 810         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 811                      return_response=True)
 812         def _got((res, status, headers)):
 813             self.failUnlessReallyEqual(res, "")
 814             self.failUnlessReallyEqual(int(status), 206)
 815             self.failUnless(headers.has_key("content-range"))
 816             self.failUnlessReallyEqual(headers["content-range"][0],
 817                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 818         d.addCallback(_got)
 819         return d
 820
 821     def test_HEAD_FILEURL_partial_range(self):
 822         headers = {"range": "bytes=5-"}
 823         length  = len(self.BAR_CONTENTS)
 824         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 825                      return_response=True)
 826         def _got((res, status, headers)):
 827             self.failUnlessReallyEqual(int(status), 206)
 828             self.failUnless(headers.has_key("content-range"))
 829             self.failUnlessReallyEqual(headers["content-range"][0],
 830                                        "bytes 5-%d/%d" % (length-1, length))
 831         d.addCallback(_got)
 832         return d
 833
 834     def test_HEAD_FILEURL_partial_end_range(self):
 835         headers = {"range": "bytes=-5"}
 836         length  = len(self.BAR_CONTENTS)
 837         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 838                      return_response=True)
 839         def _got((res, status, headers)):
 840             self.failUnlessReallyEqual(int(status), 206)
 841             self.failUnless(headers.has_key("content-range"))
 842             self.failUnlessReallyEqual(headers["content-range"][0],
 843                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 844         d.addCallback(_got)
 845         return d
 846
 847     def test_HEAD_FILEURL_partial_range_overrun(self):
 848         headers = {"range": "bytes=100-200"}
 849         d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
 850                              "416 Requested Range not satisfiable",
 851                              "",
 852                              self.HEAD, self.public_url + "/foo/bar.txt",
 853                              headers=headers)
 854         return d
 855
 856     def test_GET_FILEURL_range_bad(self):
 857         headers = {"range": "BOGUS=fizbop-quarnak"}
 858         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 859                      return_response=True)
 860         def _got((res, status, headers)):
 861             self.failUnlessReallyEqual(int(status), 200)
 862             self.failUnless(not headers.has_key("content-range"))
 863             self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
 864         d.addCallback(_got)
 865         return d
 866
 867     def test_HEAD_FILEURL(self):
 868         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 869         def _got((res, status, headers)):
 870             self.failUnlessReallyEqual(res, "")
 871             self.failUnlessReallyEqual(headers["content-length"][0],
 872                                        str(len(self.BAR_CONTENTS)))
 873             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
 874         d.addCallback(_got)
 875         return d
 876
 877     def test_GET_FILEURL_named(self):
 878         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 879         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 880         d = self.GET(base + "/@@name=/blah.txt")
 881         d.addCallback(self.failUnlessIsBarDotTxt)
 882         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 883         d.addCallback(self.failUnlessIsBarDotTxt)
 884         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 885         d.addCallback(self.failUnlessIsBarDotTxt)
 886         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 887         d.addCallback(self.failUnlessIsBarDotTxt)
 888         save_url = base + "?save=true&filename=blah.txt"
 889         d.addCallback(lambda res: self.GET(save_url))
 890         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 891         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 892         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 893         u_url = base + "?save=true&filename=" + u_fn_e
 894         d.addCallback(lambda res: self.GET(u_url))
 895         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 896         return d
 897
 898     def test_PUT_FILEURL_named_bad(self):
 899         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 900         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 901                              "400 Bad Request",
 902                              "/file can only be used with GET or HEAD",
 903                              self.PUT, base + "/@@name=/blah.txt", "")
 904         return d
 905
 906
 907     def test_GET_DIRURL_named_bad(self):
 908         base = "/file/%s" % urllib.quote(self._foo_uri)
 909         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 910                              "400 Bad Request",
 911                              "is not a file-cap",
 912                              self.GET, base + "/@@name=/blah.txt")
 913         return d
 914
 915     def test_GET_slash_file_bad(self):
 916         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 917                              "404 Not Found",
 918                              "/file must be followed by a file-cap and a name",
 919                              self.GET, "/file")
 920         return d
 921
 922     def test_GET_unhandled_URI_named(self):
 923         contents, n, newuri = self.makefile(12)
 924         verifier_cap = n.get_verify_cap().to_string()
 925         base = "/file/%s" % urllib.quote(verifier_cap)
 926         # client.create_node_from_uri() can't handle verify-caps
 927         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 928                              "400 Bad Request", "is not a file-cap",
 929                              self.GET, base)
 930         return d
 931
 932     def test_GET_unhandled_URI(self):
 933         contents, n, newuri = self.makefile(12)
 934         verifier_cap = n.get_verify_cap().to_string()
 935         base = "/uri/%s" % urllib.quote(verifier_cap)
 936         # client.create_node_from_uri() can't handle verify-caps
 937         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 938                              "400 Bad Request",
 939                              "GET unknown URI type: can only do t=info",
 940                              self.GET, base)
 941         return d
 942
 943     def test_GET_FILE_URI(self):
 944         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 945         d = self.GET(base)
 946         d.addCallback(self.failUnlessIsBarDotTxt)
 947         return d
 948
 949     def test_GET_FILE_URI_mdmf(self):
 950         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 951         d = self.GET(base)
 952         d.addCallback(self.failUnlessIsQuuxDotTxt)
 953         return d
 954
 955     def test_GET_FILE_URI_mdmf_extensions(self):
 956         base = "/uri/%s" % urllib.quote("%s:RANDOMSTUFF" % self._quux_txt_uri)
 957         d = self.GET(base)
 958         d.addCallback(self.failUnlessIsQuuxDotTxt)
 959         return d
 960
 961     def test_GET_FILE_URI_mdmf_readonly(self):
 962         base = "/uri/%s" % urllib.quote(self._quux_txt_readonly_uri)
 963         d = self.GET(base)
 964         d.addCallback(self.failUnlessIsQuuxDotTxt)
 965         return d
 966
 967     def test_GET_FILE_URI_badchild(self):
 968         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 969         errmsg = "Files have no children, certainly not named 'boguschild'"
 970         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 971                              "400 Bad Request", errmsg,
 972                              self.GET, base)
 973         return d
 974
 975     def test_PUT_FILE_URI_badchild(self):
 976         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 977         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 978         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 979                              "400 Bad Request", errmsg,
 980                              self.PUT, base, "")
 981         return d
 982
 983     def test_PUT_FILE_URI_mdmf(self):
 984         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 985         self._quux_new_contents = "new_contents"
 986         d = self.GET(base)
 987         d.addCallback(lambda res:
 988             self.failUnlessIsQuuxDotTxt(res))
 989         d.addCallback(lambda ignored:
 990             self.PUT(base, self._quux_new_contents))
 991         d.addCallback(lambda ignored:
 992             self.GET(base))
 993         d.addCallback(lambda res:
 994             self.failUnlessReallyEqual(res, self._quux_new_contents))
 995         return d
 996
 997     def test_PUT_FILE_URI_mdmf_extensions(self):
 998         base = "/uri/%s" % urllib.quote("%s:EXTENSIONSTUFF" % self._quux_txt_uri)
 999         self._quux_new_contents = "new_contents"
1000         d = self.GET(base)
1001         d.addCallback(lambda res: self.failUnlessIsQuuxDotTxt(res))
1002         d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
1003         d.addCallback(lambda ignored: self.GET(base))
1004         d.addCallback(lambda res: self.failUnlessEqual(self._quux_new_contents,
1005                                                        res))
1006         return d
1007
1008     def test_PUT_FILE_URI_mdmf_readonly(self):
1009         # We're not allowed to PUT things to a readonly cap.
1010         base = "/uri/%s" % self._quux_txt_readonly_uri
1011         d = self.GET(base)
1012         d.addCallback(lambda res:
1013             self.failUnlessIsQuuxDotTxt(res))
1014         # What should we get here? We get a 500 error now; that's not right.
1015         d.addCallback(lambda ignored:
1016             self.shouldFail2(error.Error, "test_PUT_FILE_URI_mdmf_readonly",
1017                              "400 Bad Request", "read-only cap",
1018                              self.PUT, base, "new data"))
1019         return d
1020
1021     def test_PUT_FILE_URI_sdmf_readonly(self):
1022         # We're not allowed to put things to a readonly cap.
1023         base = "/uri/%s" % self._baz_txt_readonly_uri
1024         d = self.GET(base)
1025         d.addCallback(lambda res:
1026             self.failUnlessIsBazDotTxt(res))
1027         d.addCallback(lambda ignored:
1028             self.shouldFail2(error.Error, "test_PUT_FILE_URI_sdmf_readonly",
1029                              "400 Bad Request", "read-only cap",
1030                              self.PUT, base, "new_data"))
1031         return d
1032
1033     def test_GET_etags(self):
1034
1035         def _check_etags(uri):
1036             d1 = _get_etag(uri)
1037             d2 = _get_etag(uri, 'json')
1038             d = defer.DeferredList([d1, d2], consumeErrors=True)
1039             def _check(results):
1040                 # All deferred must succeed
1041                 self.failUnless(all([r[0] for r in results]))
1042                 # the etag for the t=json form should be just like the etag
1043                 # fo the default t='' form, but with a 'json' suffix
1044                 self.failUnlessEqual(results[0][1] + 'json', results[1][1])
1045             d.addCallback(_check)
1046             return d
1047
1048         def _get_etag(uri, t=''):
1049             targetbase = "/uri/%s?t=%s" % (urllib.quote(uri.strip()), t)
1050             d = self.GET(targetbase, return_response=True, followRedirect=True)
1051             def _just_the_etag(result):
1052                 data, response, headers = result
1053                 etag = headers['etag'][0]
1054                 if uri.startswith('URI:DIR'):
1055                     self.failUnless(etag.startswith('DIR:'), etag)
1056                 return etag
1057             return d.addCallback(_just_the_etag)
1058
1059         # Check that etags work with immutable directories
1060         (newkids, caps) = self._create_immutable_children()
1061         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1062                       simplejson.dumps(newkids))
1063         def _stash_immdir_uri(uri):
1064             self._immdir_uri = uri
1065             return uri
1066         d.addCallback(_stash_immdir_uri)
1067         d.addCallback(_check_etags)
1068
1069         # Check that etags work with immutable files
1070         d.addCallback(lambda _: _check_etags(self._bar_txt_uri))
1071
1072         # use the ETag on GET
1073         def _check_match(ign):
1074             uri = "/uri/%s" % self._bar_txt_uri
1075             d = self.GET(uri, return_response=True)
1076             # extract the ETag
1077             d.addCallback(lambda (data, code, headers):
1078                           headers['etag'][0])
1079             # do a GET that's supposed to match the ETag
1080             d.addCallback(lambda etag:
1081                           self.GET(uri, return_response=True,
1082                                    headers={"If-None-Match": etag}))
1083             # make sure it short-circuited (304 instead of 200)
1084             d.addCallback(lambda (data, code, headers):
1085                           self.failUnlessEqual(int(code), http.NOT_MODIFIED))
1086             return d
1087         d.addCallback(_check_match)
1088
1089         def _no_etag(uri, t):
1090             target = "/uri/%s?t=%s" % (uri, t)
1091             d = self.GET(target, return_response=True, followRedirect=True)
1092             d.addCallback(lambda (data, code, headers):
1093                           self.failIf("etag" in headers, target))
1094             return d
1095         def _yes_etag(uri, t):
1096             target = "/uri/%s?t=%s" % (uri, t)
1097             d = self.GET(target, return_response=True, followRedirect=True)
1098             d.addCallback(lambda (data, code, headers):
1099                           self.failUnless("etag" in headers, target))
1100             return d
1101
1102         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, ""))
1103         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "json"))
1104         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "uri"))
1105         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "readonly-uri"))
1106         d.addCallback(lambda ign: _no_etag(self._bar_txt_uri, "info"))
1107
1108         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, ""))
1109         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "json"))
1110         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "uri"))
1111         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "readonly-uri"))
1112         d.addCallback(lambda ign: _no_etag(self._immdir_uri, "info"))
1113         d.addCallback(lambda ign: _no_etag(self._immdir_uri, "rename-form"))
1114
1115         return d
1116
1117     # TODO: version of this with a Unicode filename
1118     def test_GET_FILEURL_save(self):
1119         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
1120                      return_response=True)
1121         def _got((res, statuscode, headers)):
1122             content_disposition = headers["content-disposition"][0]
1123             self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
1124             self.failUnlessIsBarDotTxt(res)
1125         d.addCallback(_got)
1126         return d
1127
1128     def test_GET_FILEURL_missing(self):
1129         d = self.GET(self.public_url + "/foo/missing")
1130         d.addBoth(self.should404, "test_GET_FILEURL_missing")
1131         return d
1132
1133     def test_GET_FILEURL_info_mdmf(self):
1134         d = self.GET("/uri/%s?t=info" % self._quux_txt_uri)
1135         def _got(res):
1136             self.failUnlessIn("mutable file (mdmf)", res)
1137             self.failUnlessIn(self._quux_txt_uri, res)
1138             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1139         d.addCallback(_got)
1140         return d
1141
1142     def test_GET_FILEURL_info_mdmf_readonly(self):
1143         d = self.GET("/uri/%s?t=info" % self._quux_txt_readonly_uri)
1144         def _got(res):
1145             self.failUnlessIn("mutable file (mdmf)", res)
1146             self.failIfIn(self._quux_txt_uri, res)
1147             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1148         d.addCallback(_got)
1149         return d
1150
1151     def test_GET_FILEURL_info_sdmf(self):
1152         d = self.GET("/uri/%s?t=info" % self._baz_txt_uri)
1153         def _got(res):
1154             self.failUnlessIn("mutable file (sdmf)", res)
1155             self.failUnlessIn(self._baz_txt_uri, res)
1156         d.addCallback(_got)
1157         return d
1158
1159     def test_GET_FILEURL_info_mdmf_extensions(self):
1160         d = self.GET("/uri/%s:STUFF?t=info" % self._quux_txt_uri)
1161         def _got(res):
1162             self.failUnlessIn("mutable file (mdmf)", res)
1163             self.failUnlessIn(self._quux_txt_uri, res)
1164             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1165         d.addCallback(_got)
1166         return d
1167
1168     def test_PUT_overwrite_only_files(self):
1169         # create a directory, put a file in that directory.
1170         contents, n, filecap = self.makefile(8)
1171         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
1172         d.addCallback(lambda res:
1173             self.PUT(self.public_url + "/foo/dir/file1.txt",
1174                      self.NEWFILE_CONTENTS))
1175         # try to overwrite the file with replace=only-files
1176         # (this should work)
1177         d.addCallback(lambda res:
1178             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
1179                      filecap))
1180         d.addCallback(lambda res:
1181             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
1182                  "There was already a child by that name, and you asked me "
1183                  "to not replace it",
1184                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
1185                  filecap))
1186         return d
1187
1188     def test_PUT_NEWFILEURL(self):
1189         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
1190         # TODO: we lose the response code, so we can't check this
1191         #self.failUnlessReallyEqual(responsecode, 201)
1192         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1193         d.addCallback(lambda res:
1194                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1195                                                       self.NEWFILE_CONTENTS))
1196         return d
1197
1198     def test_PUT_NEWFILEURL_not_mutable(self):
1199         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
1200                      self.NEWFILE_CONTENTS)
1201         # TODO: we lose the response code, so we can't check this
1202         #self.failUnlessReallyEqual(responsecode, 201)
1203         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1204         d.addCallback(lambda res:
1205                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1206                                                       self.NEWFILE_CONTENTS))
1207         return d
1208
1209     def test_PUT_NEWFILEURL_unlinked_mdmf(self):
1210         # this should get us a few segments of an MDMF mutable file,
1211         # which we can then test for.
1212         contents = self.NEWFILE_CONTENTS * 300000
1213         d = self.PUT("/uri?format=mdmf",
1214                      contents)
1215         def _got_filecap(filecap):
1216             self.failUnless(filecap.startswith("URI:MDMF"))
1217             return filecap
1218         d.addCallback(_got_filecap)
1219         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1220         d.addCallback(lambda json: self.failUnlessIn("MDMF", json))
1221         return d
1222
1223     def test_PUT_NEWFILEURL_unlinked_sdmf(self):
1224         contents = self.NEWFILE_CONTENTS * 300000
1225         d = self.PUT("/uri?format=sdmf",
1226                      contents)
1227         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1228         d.addCallback(lambda json: self.failUnlessIn("SDMF", json))
1229         return d
1230
1231     def test_PUT_NEWFILEURL_unlinked_bad_format(self):
1232         contents = self.NEWFILE_CONTENTS * 300000
1233         return self.shouldHTTPError("PUT_NEWFILEURL_unlinked_bad_format",
1234                                     400, "Bad Request", "Unknown format: foo",
1235                                     self.PUT, "/uri?format=foo",
1236                                     contents)
1237
1238     def test_PUT_NEWFILEURL_range_bad(self):
1239         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
1240         target = self.public_url + "/foo/new.txt"
1241         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
1242                              "501 Not Implemented",
1243                              "Content-Range in PUT not yet supported",
1244                              # (and certainly not for immutable files)
1245                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
1246                              headers=headers)
1247         d.addCallback(lambda res:
1248                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
1249         return d
1250
1251     def test_PUT_NEWFILEURL_mutable(self):
1252         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1253                      self.NEWFILE_CONTENTS)
1254         # TODO: we lose the response code, so we can't check this
1255         #self.failUnlessReallyEqual(responsecode, 201)
1256         def _check_uri(res):
1257             u = uri.from_string_mutable_filenode(res)
1258             self.failUnless(u.is_mutable())
1259             self.failIf(u.is_readonly())
1260             return res
1261         d.addCallback(_check_uri)
1262         d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
1263         d.addCallback(lambda res:
1264                       self.failUnlessMutableChildContentsAre(self._foo_node,
1265                                                              u"new.txt",
1266                                                              self.NEWFILE_CONTENTS))
1267         return d
1268
1269     def test_PUT_NEWFILEURL_mutable_toobig(self):
1270         # It is okay to upload large mutable files, so we should be able
1271         # to do that.
1272         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1273                      "b" * (self.s.MUTABLE_SIZELIMIT + 1))
1274         return d
1275
1276     def test_PUT_NEWFILEURL_replace(self):
1277         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
1278         # TODO: we lose the response code, so we can't check this
1279         #self.failUnlessReallyEqual(responsecode, 200)
1280         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
1281         d.addCallback(lambda res:
1282                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1283                                                       self.NEWFILE_CONTENTS))
1284         return d
1285
1286     def test_PUT_NEWFILEURL_bad_t(self):
1287         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
1288                              "PUT to a file: bad t=bogus",
1289                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
1290                              "contents")
1291         return d
1292
1293     def test_PUT_NEWFILEURL_no_replace(self):
1294         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
1295                      self.NEWFILE_CONTENTS)
1296         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
1297                   "409 Conflict",
1298                   "There was already a child by that name, and you asked me "
1299                   "to not replace it")
1300         return d
1301
1302     def test_PUT_NEWFILEURL_mkdirs(self):
1303         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
1304         fn = self._foo_node
1305         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
1306         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
1307         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
1308         d.addCallback(lambda res:
1309                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
1310                                                       self.NEWFILE_CONTENTS))
1311         return d
1312
1313     def test_PUT_NEWFILEURL_blocked(self):
1314         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
1315                      self.NEWFILE_CONTENTS)
1316         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
1317                   "409 Conflict",
1318                   "Unable to create directory 'blockingfile': a file was in the way")
1319         return d
1320
1321     def test_PUT_NEWFILEURL_emptyname(self):
1322         # an empty pathname component (i.e. a double-slash) is disallowed
1323         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
1324                              "400 Bad Request",
1325                              "The webapi does not allow empty pathname components",
1326                              self.PUT, self.public_url + "/foo//new.txt", "")
1327         return d
1328
1329     def test_DELETE_FILEURL(self):
1330         d = self.DELETE(self.public_url + "/foo/bar.txt")
1331         d.addCallback(lambda res:
1332                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1333         return d
1334
1335     def test_DELETE_FILEURL_missing(self):
1336         d = self.DELETE(self.public_url + "/foo/missing")
1337         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
1338         return d
1339
1340     def test_DELETE_FILEURL_missing2(self):
1341         d = self.DELETE(self.public_url + "/missing/missing")
1342         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
1343         return d
1344
1345     def failUnlessHasBarDotTxtMetadata(self, res):
1346         data = simplejson.loads(res)
1347         self.failUnless(isinstance(data, list))
1348         self.failUnlessIn("metadata", data[1])
1349         self.failUnlessIn("tahoe", data[1]["metadata"])
1350         self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
1351         self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
1352         self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
1353                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
1354
1355     def test_GET_FILEURL_json(self):
1356         # twisted.web.http.parse_qs ignores any query args without an '=', so
1357         # I can't do "GET /path?json", I have to do "GET /path/t=json"
1358         # instead. This may make it tricky to emulate the S3 interface
1359         # completely.
1360         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
1361         def _check1(data):
1362             self.failUnlessIsBarJSON(data)
1363             self.failUnlessHasBarDotTxtMetadata(data)
1364             return
1365         d.addCallback(_check1)
1366         return d
1367
1368     def test_GET_FILEURL_json_mutable_type(self):
1369         # The JSON should include format, which says whether the
1370         # file is SDMF or MDMF
1371         d = self.PUT("/uri?format=mdmf",
1372                      self.NEWFILE_CONTENTS * 300000)
1373         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1374         def _got_json(json, version):
1375             data = simplejson.loads(json)
1376             assert "filenode" == data[0]
1377             data = data[1]
1378             assert isinstance(data, dict)
1379
1380             self.failUnlessIn("format", data)
1381             self.failUnlessEqual(data["format"], version)
1382
1383         d.addCallback(_got_json, "MDMF")
1384         # Now make an SDMF file and check that it is reported correctly.
1385         d.addCallback(lambda ignored:
1386             self.PUT("/uri?format=sdmf",
1387                       self.NEWFILE_CONTENTS * 300000))
1388         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1389         d.addCallback(_got_json, "SDMF")
1390         return d
1391
1392     def test_GET_FILEURL_json_mdmf(self):
1393         d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
1394         d.addCallback(self.failUnlessIsQuuxJSON)
1395         return d
1396
1397     def test_GET_FILEURL_json_missing(self):
1398         d = self.GET(self.public_url + "/foo/missing?json")
1399         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
1400         return d
1401
1402     def test_GET_FILEURL_uri(self):
1403         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
1404         def _check(res):
1405             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1406         d.addCallback(_check)
1407         d.addCallback(lambda res:
1408                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
1409         def _check2(res):
1410             # for now, for files, uris and readonly-uris are the same
1411             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1412         d.addCallback(_check2)
1413         return d
1414
1415     def test_GET_FILEURL_badtype(self):
1416         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
1417                                  "bad t=bogus",
1418                                  self.GET,
1419                                  self.public_url + "/foo/bar.txt?t=bogus")
1420         return d
1421
1422     def test_CSS_FILE(self):
1423         d = self.GET("/tahoe.css", followRedirect=True)
1424         def _check(res):
1425             CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
1426             self.failUnless(CSS_STYLE.search(res), res)
1427         d.addCallback(_check)
1428         return d
1429
1430     def test_GET_FILEURL_uri_missing(self):
1431         d = self.GET(self.public_url + "/foo/missing?t=uri")
1432         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
1433         return d
1434
1435     def _check_upload_and_mkdir_forms(self, html):
1436         # We should have a form to create a file, with radio buttons that allow
1437         # the user to toggle whether it is a CHK/LIT (default), SDMF, or MDMF file.
1438         self.failUnlessIn('name="t" value="upload"', html)
1439         self.failUnlessIn('input checked="checked" type="radio" id="upload-chk" value="chk" name="format"', html)
1440         self.failUnlessIn('input type="radio" id="upload-sdmf" value="sdmf" name="format"', html)
1441         self.failUnlessIn('input type="radio" id="upload-mdmf" value="mdmf" name="format"', html)
1442
1443         # We should also have the ability to create a mutable directory, with
1444         # radio buttons that allow the user to toggle whether it is an SDMF (default)
1445         # or MDMF directory.
1446         self.failUnlessIn('name="t" value="mkdir"', html)
1447         self.failUnlessIn('input checked="checked" type="radio" id="mkdir-sdmf" value="sdmf" name="format"', html)
1448         self.failUnlessIn('input type="radio" id="mkdir-mdmf" value="mdmf" name="format"', html)
1449
1450         self.failUnlessIn(FAVICON_MARKUP, html)
1451
1452     def test_GET_DIRECTORY_html(self):
1453         d = self.GET(self.public_url + "/foo", followRedirect=True)
1454         def _check(html):
1455             self.failUnlessIn('<div class="toolbar-item"><a href="../../..">Return to Welcome page</a></div>', html)
1456             self._check_upload_and_mkdir_forms(html)
1457             self.failUnlessIn("quux", html)
1458         d.addCallback(_check)
1459         return d
1460
1461     def test_GET_DIRECTORY_html_filenode_encoding(self):
1462         d = self.GET(self.public_url + "/foo", followRedirect=True)
1463         def _check(html):
1464             # Check if encoded entries are there
1465             self.failUnlessIn('@@named=/' + self._htmlname_urlencoded + '">'
1466                               + self._htmlname_escaped + '</a>', html)
1467             self.failUnlessIn('value="' + self._htmlname_escaped_attr + '"', html)
1468             self.failIfIn(self._htmlname_escaped_double, html)
1469             # Make sure that Nevow escaping actually works by checking for unsafe characters
1470             # and that '&' is escaped.
1471             for entity in '<>':
1472                 self.failUnlessIn(entity, self._htmlname_raw)
1473                 self.failIfIn(entity, self._htmlname_escaped)
1474             self.failUnlessIn('&', re.sub(r'&(amp|lt|gt|quot|apos);', '', self._htmlname_raw))
1475             self.failIfIn('&', re.sub(r'&(amp|lt|gt|quot|apos);', '', self._htmlname_escaped))
1476         d.addCallback(_check)
1477         return d
1478
1479     def test_GET_root_html(self):
1480         d = self.GET("/")
1481         d.addCallback(self._check_upload_and_mkdir_forms)
1482         return d
1483
1484     def test_GET_DIRURL(self):
1485         # the addSlash means we get a redirect here
1486         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1487         ROOT = "../../.."
1488         d = self.GET(self.public_url + "/foo", followRedirect=True)
1489         def _check(res):
1490             self.failUnlessIn('<a href="%s">Return to Welcome page' % ROOT, res)
1491
1492             # the FILE reference points to a URI, but it should end in bar.txt
1493             bar_url = ("%s/file/%s/@@named=/bar.txt" %
1494                        (ROOT, urllib.quote(self._bar_txt_uri)))
1495             get_bar = "".join([r'<td>FILE</td>',
1496                                r'\s+<td>',
1497                                r'<a href="%s">bar.txt</a>' % bar_url,
1498                                r'</td>',
1499                                r'\s+<td align="right">%d</td>' % len(self.BAR_CONTENTS),
1500                                ])
1501             self.failUnless(re.search(get_bar, res), res)
1502             for label in ['unlink', 'rename/move']:
1503                 for line in res.split("\n"):
1504                     # find the line that contains the relevant button for bar.txt
1505                     if ("form action" in line and
1506                         ('value="%s"' % (label,)) in line and
1507                         'value="bar.txt"' in line):
1508                         # the form target should use a relative URL
1509                         foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1510                         self.failUnlessIn('action="%s"' % foo_url, line)
1511                         # and the when_done= should too
1512                         #done_url = urllib.quote(???)
1513                         #self.failUnlessIn('name="when_done" value="%s"' % done_url, line)
1514
1515                         # 'unlink' needs to use POST because it directly has a side effect
1516                         if label == 'unlink':
1517                             self.failUnlessIn('method="post"', line)
1518                         break
1519                 else:
1520                     self.fail("unable to find '%s bar.txt' line" % (label,))
1521
1522             # the DIR reference just points to a URI
1523             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1524             get_sub = ((r'<td>DIR</td>')
1525                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1526             self.failUnless(re.search(get_sub, res), res)
1527         d.addCallback(_check)
1528
1529         # look at a readonly directory
1530         d.addCallback(lambda res:
1531                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
1532         def _check2(res):
1533             self.failUnlessIn("(read-only)", res)
1534             self.failIfIn("Upload a file", res)
1535         d.addCallback(_check2)
1536
1537         # and at a directory that contains a readonly directory
1538         d.addCallback(lambda res:
1539                       self.GET(self.public_url, followRedirect=True))
1540         def _check3(res):
1541             self.failUnless(re.search('<td>DIR-RO</td>'
1542                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1543         d.addCallback(_check3)
1544
1545         # and an empty directory
1546         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1547         def _check4(res):
1548             self.failUnlessIn("directory is empty", res)
1549             MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
1550             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1551         d.addCallback(_check4)
1552
1553         # and at a literal directory
1554         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1555         d.addCallback(lambda res:
1556                       self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1557         def _check5(res):
1558             self.failUnlessIn('(immutable)', res)
1559             self.failUnless(re.search('<td>FILE</td>'
1560                                       r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1561         d.addCallback(_check5)
1562         return d
1563
1564     def test_GET_DIRURL_badtype(self):
1565         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1566                                  400, "Bad Request",
1567                                  "bad t=bogus",
1568                                  self.GET,
1569                                  self.public_url + "/foo?t=bogus")
1570         return d
1571
1572     def test_GET_DIRURL_json(self):
1573         d = self.GET(self.public_url + "/foo?t=json")
1574         d.addCallback(self.failUnlessIsFooJSON)
1575         return d
1576
1577     def test_GET_DIRURL_json_format(self):
1578         d = self.PUT(self.public_url + \
1579                      "/foo/sdmf.txt?format=sdmf",
1580                      self.NEWFILE_CONTENTS * 300000)
1581         d.addCallback(lambda ignored:
1582             self.PUT(self.public_url + \
1583                      "/foo/mdmf.txt?format=mdmf",
1584                      self.NEWFILE_CONTENTS * 300000))
1585         # Now we have an MDMF and SDMF file in the directory. If we GET
1586         # its JSON, we should see their encodings.
1587         d.addCallback(lambda ignored:
1588             self.GET(self.public_url + "/foo?t=json"))
1589         def _got_json(json):
1590             data = simplejson.loads(json)
1591             assert data[0] == "dirnode"
1592
1593             data = data[1]
1594             kids = data['children']
1595
1596             mdmf_data = kids['mdmf.txt'][1]
1597             self.failUnlessIn("format", mdmf_data)
1598             self.failUnlessEqual(mdmf_data["format"], "MDMF")
1599
1600             sdmf_data = kids['sdmf.txt'][1]
1601             self.failUnlessIn("format", sdmf_data)
1602             self.failUnlessEqual(sdmf_data["format"], "SDMF")
1603         d.addCallback(_got_json)
1604         return d
1605
1606
1607     def test_POST_DIRURL_manifest_no_ophandle(self):
1608         d = self.shouldFail2(error.Error,
1609                              "test_POST_DIRURL_manifest_no_ophandle",
1610                              "400 Bad Request",
1611                              "slow operation requires ophandle=",
1612                              self.POST, self.public_url, t="start-manifest")
1613         return d
1614
1615     def test_POST_DIRURL_manifest(self):
1616         d = defer.succeed(None)
1617         def getman(ignored, output):
1618             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1619                           followRedirect=True)
1620             d.addCallback(self.wait_for_operation, "125")
1621             d.addCallback(self.get_operation_results, "125", output)
1622             return d
1623         d.addCallback(getman, None)
1624         def _got_html(manifest):
1625             self.failUnlessIn("Manifest of SI=", manifest)
1626             self.failUnlessIn("<td>sub</td>", manifest)
1627             self.failUnlessIn(self._sub_uri, manifest)
1628             self.failUnlessIn("<td>sub/baz.txt</td>", manifest)
1629             self.failUnlessIn(FAVICON_MARKUP, manifest)
1630         d.addCallback(_got_html)
1631
1632         # both t=status and unadorned GET should be identical
1633         d.addCallback(lambda res: self.GET("/operations/125"))
1634         d.addCallback(_got_html)
1635
1636         d.addCallback(getman, "html")
1637         d.addCallback(_got_html)
1638         d.addCallback(getman, "text")
1639         def _got_text(manifest):
1640             self.failUnlessIn("\nsub " + self._sub_uri + "\n", manifest)
1641             self.failUnlessIn("\nsub/baz.txt URI:CHK:", manifest)
1642         d.addCallback(_got_text)
1643         d.addCallback(getman, "JSON")
1644         def _got_json(res):
1645             data = res["manifest"]
1646             got = {}
1647             for (path_list, cap) in data:
1648                 got[tuple(path_list)] = cap
1649             self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
1650             self.failUnlessIn((u"sub", u"baz.txt"), got)
1651             self.failUnlessIn("finished", res)
1652             self.failUnlessIn("origin", res)
1653             self.failUnlessIn("storage-index", res)
1654             self.failUnlessIn("verifycaps", res)
1655             self.failUnlessIn("stats", res)
1656         d.addCallback(_got_json)
1657         return d
1658
1659     def test_POST_DIRURL_deepsize_no_ophandle(self):
1660         d = self.shouldFail2(error.Error,
1661                              "test_POST_DIRURL_deepsize_no_ophandle",
1662                              "400 Bad Request",
1663                              "slow operation requires ophandle=",
1664                              self.POST, self.public_url, t="start-deep-size")
1665         return d
1666
1667     def test_POST_DIRURL_deepsize(self):
1668         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1669                       followRedirect=True)
1670         d.addCallback(self.wait_for_operation, "126")
1671         d.addCallback(self.get_operation_results, "126", "json")
1672         def _got_json(data):
1673             self.failUnlessReallyEqual(data["finished"], True)
1674             size = data["size"]
1675             self.failUnless(size > 1000)
1676         d.addCallback(_got_json)
1677         d.addCallback(self.get_operation_results, "126", "text")
1678         def _got_text(res):
1679             mo = re.search(r'^size: (\d+)$', res, re.M)
1680             self.failUnless(mo, res)
1681             size = int(mo.group(1))
1682             # with directories, the size varies.
1683             self.failUnless(size > 1000)
1684         d.addCallback(_got_text)
1685         return d
1686
1687     def test_POST_DIRURL_deepstats_no_ophandle(self):
1688         d = self.shouldFail2(error.Error,
1689                              "test_POST_DIRURL_deepstats_no_ophandle",
1690                              "400 Bad Request",
1691                              "slow operation requires ophandle=",
1692                              self.POST, self.public_url, t="start-deep-stats")
1693         return d
1694
1695     def test_POST_DIRURL_deepstats(self):
1696         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1697                       followRedirect=True)
1698         d.addCallback(self.wait_for_operation, "127")
1699         d.addCallback(self.get_operation_results, "127", "json")
1700         def _got_json(stats):
1701             expected = {"count-immutable-files": 4,
1702                         "count-mutable-files": 2,
1703                         "count-literal-files": 0,
1704                         "count-files": 6,
1705                         "count-directories": 3,
1706                         "size-immutable-files": 76,
1707                         "size-literal-files": 0,
1708                         #"size-directories": 1912, # varies
1709                         #"largest-directory": 1590,
1710                         "largest-directory-children": 8,
1711                         "largest-immutable-file": 19,
1712                         }
1713             for k,v in expected.iteritems():
1714                 self.failUnlessReallyEqual(stats[k], v,
1715                                            "stats[%s] was %s, not %s" %
1716                                            (k, stats[k], v))
1717             self.failUnlessReallyEqual(stats["size-files-histogram"],
1718                                        [ [11, 31, 4] ])
1719         d.addCallback(_got_json)
1720         return d
1721
1722     def test_POST_DIRURL_stream_manifest(self):
1723         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1724         def _check(res):
1725             self.failUnless(res.endswith("\n"))
1726             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1727             self.failUnlessReallyEqual(len(units), 10)
1728             self.failUnlessEqual(units[-1]["type"], "stats")
1729             first = units[0]
1730             self.failUnlessEqual(first["path"], [])
1731             self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri)
1732             self.failUnlessEqual(first["type"], "directory")
1733             baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0]
1734             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1735             self.failIfEqual(baz["storage-index"], None)
1736             self.failIfEqual(baz["verifycap"], None)
1737             self.failIfEqual(baz["repaircap"], None)
1738             # XXX: Add quux and baz to this test.
1739             return
1740         d.addCallback(_check)
1741         return d
1742
1743     def test_GET_DIRURL_uri(self):
1744         d = self.GET(self.public_url + "/foo?t=uri")
1745         def _check(res):
1746             self.failUnlessReallyEqual(to_str(res), self._foo_uri)
1747         d.addCallback(_check)
1748         return d
1749
1750     def test_GET_DIRURL_readonly_uri(self):
1751         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1752         def _check(res):
1753             self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri)
1754         d.addCallback(_check)
1755         return d
1756
1757     def test_PUT_NEWDIRURL(self):
1758         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1759         d.addCallback(lambda res:
1760                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1761         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1762         d.addCallback(self.failUnlessNodeKeysAre, [])
1763         return d
1764
1765     def test_PUT_NEWDIRURL_mdmf(self):
1766         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1767         d.addCallback(lambda res:
1768                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1769         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1770         d.addCallback(lambda node:
1771             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1772         return d
1773
1774     def test_PUT_NEWDIRURL_sdmf(self):
1775         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=sdmf",
1776                      "")
1777         d.addCallback(lambda res:
1778                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1779         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1780         d.addCallback(lambda node:
1781             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1782         return d
1783
1784     def test_PUT_NEWDIRURL_bad_format(self):
1785         return self.shouldHTTPError("PUT_NEWDIRURL_bad_format",
1786                                     400, "Bad Request", "Unknown format: foo",
1787                                     self.PUT, self.public_url +
1788                                     "/foo/newdir=?t=mkdir&format=foo", "")
1789
1790     def test_POST_NEWDIRURL(self):
1791         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1792         d.addCallback(lambda res:
1793                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1794         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1795         d.addCallback(self.failUnlessNodeKeysAre, [])
1796         return d
1797
1798     def test_POST_NEWDIRURL_mdmf(self):
1799         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1800         d.addCallback(lambda res:
1801                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1802         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1803         d.addCallback(lambda node:
1804             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1805         return d
1806
1807     def test_POST_NEWDIRURL_sdmf(self):
1808         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=sdmf", "")
1809         d.addCallback(lambda res:
1810             self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1811         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1812         d.addCallback(lambda node:
1813             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1814         return d
1815
1816     def test_POST_NEWDIRURL_bad_format(self):
1817         return self.shouldHTTPError("POST_NEWDIRURL_bad_format",
1818                                     400, "Bad Request", "Unknown format: foo",
1819                                     self.POST2, self.public_url + \
1820                                     "/foo/newdir?t=mkdir&format=foo", "")
1821
1822     def test_POST_NEWDIRURL_emptyname(self):
1823         # an empty pathname component (i.e. a double-slash) is disallowed
1824         d = self.shouldFail2(error.Error, "POST_NEWDIRURL_emptyname",
1825                              "400 Bad Request",
1826                              "The webapi does not allow empty pathname components, i.e. a double slash",
1827                              self.POST, self.public_url + "//?t=mkdir")
1828         return d
1829
1830     def _do_POST_NEWDIRURL_initial_children_test(self, version=None):
1831         (newkids, caps) = self._create_initial_children()
1832         query = "/foo/newdir?t=mkdir-with-children"
1833         if version == MDMF_VERSION:
1834             query += "&format=mdmf"
1835         elif version == SDMF_VERSION:
1836             query += "&format=sdmf"
1837         else:
1838             version = SDMF_VERSION # for later
1839         d = self.POST2(self.public_url + query,
1840                        simplejson.dumps(newkids))
1841         def _check(uri):
1842             n = self.s.create_node_from_uri(uri.strip())
1843             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1844             self.failUnlessEqual(n._node.get_version(), version)
1845             d2.addCallback(lambda ign:
1846                            self.failUnlessROChildURIIs(n, u"child-imm",
1847                                                        caps['filecap1']))
1848             d2.addCallback(lambda ign:
1849                            self.failUnlessRWChildURIIs(n, u"child-mutable",
1850                                                        caps['filecap2']))
1851             d2.addCallback(lambda ign:
1852                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1853                                                        caps['filecap3']))
1854             d2.addCallback(lambda ign:
1855                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1856                                                        caps['unknown_rocap']))
1857             d2.addCallback(lambda ign:
1858                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1859                                                        caps['unknown_rwcap']))
1860             d2.addCallback(lambda ign:
1861                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1862                                                        caps['unknown_immcap']))
1863             d2.addCallback(lambda ign:
1864                            self.failUnlessRWChildURIIs(n, u"dirchild",
1865                                                        caps['dircap']))
1866             d2.addCallback(lambda ign:
1867                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1868                                                        caps['litdircap']))
1869             d2.addCallback(lambda ign:
1870                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1871                                                        caps['emptydircap']))
1872             return d2
1873         d.addCallback(_check)
1874         d.addCallback(lambda res:
1875                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1876         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1877         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1878         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1879         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1880         return d
1881
1882     def test_POST_NEWDIRURL_initial_children(self):
1883         return self._do_POST_NEWDIRURL_initial_children_test()
1884
1885     def test_POST_NEWDIRURL_initial_children_mdmf(self):
1886         return self._do_POST_NEWDIRURL_initial_children_test(MDMF_VERSION)
1887
1888     def test_POST_NEWDIRURL_initial_children_sdmf(self):
1889         return self._do_POST_NEWDIRURL_initial_children_test(SDMF_VERSION)
1890
1891     def test_POST_NEWDIRURL_initial_children_bad_format(self):
1892         (newkids, caps) = self._create_initial_children()
1893         return self.shouldHTTPError("POST_NEWDIRURL_initial_children_bad_format",
1894                                     400, "Bad Request", "Unknown format: foo",
1895                                     self.POST2, self.public_url + \
1896                                     "/foo/newdir?t=mkdir-with-children&format=foo",
1897                                     simplejson.dumps(newkids))
1898
1899     def test_POST_NEWDIRURL_immutable(self):
1900         (newkids, caps) = self._create_immutable_children()
1901         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1902                        simplejson.dumps(newkids))
1903         def _check(uri):
1904             n = self.s.create_node_from_uri(uri.strip())
1905             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1906             d2.addCallback(lambda ign:
1907                            self.failUnlessROChildURIIs(n, u"child-imm",
1908                                                        caps['filecap1']))
1909             d2.addCallback(lambda ign:
1910                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1911                                                        caps['unknown_immcap']))
1912             d2.addCallback(lambda ign:
1913                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
1914                                                        caps['immdircap']))
1915             d2.addCallback(lambda ign:
1916                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1917                                                        caps['litdircap']))
1918             d2.addCallback(lambda ign:
1919                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1920                                                        caps['emptydircap']))
1921             return d2
1922         d.addCallback(_check)
1923         d.addCallback(lambda res:
1924                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1925         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1926         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1927         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1928         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1929         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1930         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1931         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1932         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
1933         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1934         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
1935         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1936         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
1937         d.addErrback(self.explain_web_error)
1938         return d
1939
1940     def test_POST_NEWDIRURL_immutable_bad(self):
1941         (newkids, caps) = self._create_initial_children()
1942         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
1943                              "400 Bad Request",
1944                              "needed to be immutable but was not",
1945                              self.POST2,
1946                              self.public_url + "/foo/newdir?t=mkdir-immutable",
1947                              simplejson.dumps(newkids))
1948         return d
1949
1950     def test_PUT_NEWDIRURL_exists(self):
1951         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1952         d.addCallback(lambda res:
1953                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1954         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1955         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1956         return d
1957
1958     def test_PUT_NEWDIRURL_blocked(self):
1959         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1960                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1961                              self.PUT,
1962                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1963         d.addCallback(lambda res:
1964                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1965         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1966         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1967         return d
1968
1969     def test_PUT_NEWDIRURL_mkdirs(self):
1970         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1971         d.addCallback(lambda res:
1972                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1973         d.addCallback(lambda res:
1974                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1975         d.addCallback(lambda res:
1976                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1977         d.addCallback(self.failUnlessNodeKeysAre, [])
1978         return d
1979
1980     def test_PUT_NEWDIRURL_mkdirs_mdmf(self):
1981         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=mdmf", "")
1982         d.addCallback(lambda ignored:
1983             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1984         d.addCallback(lambda ignored:
1985             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1986         d.addCallback(lambda ignored:
1987             self._foo_node.get_child_at_path(u"subdir"))
1988         def _got_subdir(subdir):
1989             # XXX: What we want?
1990             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1991             self.failUnlessNodeHasChild(subdir, u"newdir")
1992             return subdir.get_child_at_path(u"newdir")
1993         d.addCallback(_got_subdir)
1994         d.addCallback(lambda newdir:
1995             self.failUnlessEqual(newdir._node.get_version(), MDMF_VERSION))
1996         return d
1997
1998     def test_PUT_NEWDIRURL_mkdirs_sdmf(self):
1999         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=sdmf", "")
2000         d.addCallback(lambda ignored:
2001             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
2002         d.addCallback(lambda ignored:
2003             self.failIfNodeHasChild(self._foo_node, u"newdir"))
2004         d.addCallback(lambda ignored:
2005             self._foo_node.get_child_at_path(u"subdir"))
2006         def _got_subdir(subdir):
2007             # XXX: What we want?
2008             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
2009             self.failUnlessNodeHasChild(subdir, u"newdir")
2010             return subdir.get_child_at_path(u"newdir")
2011         d.addCallback(_got_subdir)
2012         d.addCallback(lambda newdir:
2013             self.failUnlessEqual(newdir._node.get_version(), SDMF_VERSION))
2014         return d
2015
2016     def test_PUT_NEWDIRURL_mkdirs_bad_format(self):
2017         return self.shouldHTTPError("PUT_NEWDIRURL_mkdirs_bad_format",
2018                                     400, "Bad Request", "Unknown format: foo",
2019                                     self.PUT, self.public_url + \
2020                                     "/foo/subdir/newdir?t=mkdir&format=foo",
2021                                     "")
2022
2023     def test_DELETE_DIRURL(self):
2024         d = self.DELETE(self.public_url + "/foo")
2025         d.addCallback(lambda res:
2026                       self.failIfNodeHasChild(self.public_root, u"foo"))
2027         return d
2028
2029     def test_DELETE_DIRURL_missing(self):
2030         d = self.DELETE(self.public_url + "/foo/missing")
2031         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
2032         d.addCallback(lambda res:
2033                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
2034         return d
2035
2036     def test_DELETE_DIRURL_missing2(self):
2037         d = self.DELETE(self.public_url + "/missing")
2038         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
2039         return d
2040
2041     def dump_root(self):
2042         print "NODEWALK"
2043         w = webish.DirnodeWalkerMixin()
2044         def visitor(childpath, childnode, metadata):
2045             print childpath
2046         d = w.walk(self.public_root, visitor)
2047         return d
2048
2049     def failUnlessNodeKeysAre(self, node, expected_keys):
2050         for k in expected_keys:
2051             assert isinstance(k, unicode)
2052         d = node.list()
2053         def _check(children):
2054             self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
2055         d.addCallback(_check)
2056         return d
2057     def failUnlessNodeHasChild(self, node, name):
2058         assert isinstance(name, unicode)
2059         d = node.list()
2060         def _check(children):
2061             self.failUnlessIn(name, children)
2062         d.addCallback(_check)
2063         return d
2064     def failIfNodeHasChild(self, node, name):
2065         assert isinstance(name, unicode)
2066         d = node.list()
2067         def _check(children):
2068             self.failIfIn(name, children)
2069         d.addCallback(_check)
2070         return d
2071
2072     def failUnlessChildContentsAre(self, node, name, expected_contents):
2073         assert isinstance(name, unicode)
2074         d = node.get_child_at_path(name)
2075         d.addCallback(lambda node: download_to_data(node))
2076         def _check(contents):
2077             self.failUnlessReallyEqual(contents, expected_contents)
2078         d.addCallback(_check)
2079         return d
2080
2081     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
2082         assert isinstance(name, unicode)
2083         d = node.get_child_at_path(name)
2084         d.addCallback(lambda node: node.download_best_version())
2085         def _check(contents):
2086             self.failUnlessReallyEqual(contents, expected_contents)
2087         d.addCallback(_check)
2088         return d
2089
2090     def failUnlessRWChildURIIs(self, node, name, expected_uri):
2091         assert isinstance(name, unicode)
2092         d = node.get_child_at_path(name)
2093         def _check(child):
2094             self.failUnless(child.is_unknown() or not child.is_readonly())
2095             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
2096             self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
2097             expected_ro_uri = self._make_readonly(expected_uri)
2098             if expected_ro_uri:
2099                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
2100         d.addCallback(_check)
2101         return d
2102
2103     def failUnlessROChildURIIs(self, node, name, expected_uri):
2104         assert isinstance(name, unicode)
2105         d = node.get_child_at_path(name)
2106         def _check(child):
2107             self.failUnless(child.is_unknown() or child.is_readonly())
2108             self.failUnlessReallyEqual(child.get_write_uri(), None)
2109             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
2110             self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
2111         d.addCallback(_check)
2112         return d
2113
2114     def failUnlessURIMatchesRWChild(self, got_uri, node, name):
2115         assert isinstance(name, unicode)
2116         d = node.get_child_at_path(name)
2117         def _check(child):
2118             self.failUnless(child.is_unknown() or not child.is_readonly())
2119             self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
2120             self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
2121             expected_ro_uri = self._make_readonly(got_uri)
2122             if expected_ro_uri:
2123                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
2124         d.addCallback(_check)
2125         return d
2126
2127     def failUnlessURIMatchesROChild(self, got_uri, node, name):
2128         assert isinstance(name, unicode)
2129         d = node.get_child_at_path(name)
2130         def _check(child):
2131             self.failUnless(child.is_unknown() or child.is_readonly())
2132             self.failUnlessReallyEqual(child.get_write_uri(), None)
2133             self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
2134             self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
2135         d.addCallback(_check)
2136         return d
2137
2138     def failUnlessCHKURIHasContents(self, got_uri, contents):
2139         self.failUnless(self.get_all_contents()[got_uri] == contents)
2140
2141     def test_POST_upload(self):
2142         d = self.POST(self.public_url + "/foo", t="upload",
2143                       file=("new.txt", self.NEWFILE_CONTENTS))
2144         fn = self._foo_node
2145         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2146         d.addCallback(lambda res:
2147                       self.failUnlessChildContentsAre(fn, u"new.txt",
2148                                                       self.NEWFILE_CONTENTS))
2149         return d
2150
2151     def test_POST_upload_unicode(self):
2152         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2153         d = self.POST(self.public_url + "/foo", t="upload",
2154                       file=(filename, self.NEWFILE_CONTENTS))
2155         fn = self._foo_node
2156         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2157         d.addCallback(lambda res:
2158                       self.failUnlessChildContentsAre(fn, filename,
2159                                                       self.NEWFILE_CONTENTS))
2160         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2161         d.addCallback(lambda res: self.GET(target_url))
2162         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2163                                                                   self.NEWFILE_CONTENTS,
2164                                                                   contents))
2165         return d
2166
2167     def test_POST_upload_unicode_named(self):
2168         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2169         d = self.POST(self.public_url + "/foo", t="upload",
2170                       name=filename,
2171                       file=("overridden", self.NEWFILE_CONTENTS))
2172         fn = self._foo_node
2173         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2174         d.addCallback(lambda res:
2175                       self.failUnlessChildContentsAre(fn, filename,
2176                                                       self.NEWFILE_CONTENTS))
2177         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2178         d.addCallback(lambda res: self.GET(target_url))
2179         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2180                                                                   self.NEWFILE_CONTENTS,
2181                                                                   contents))
2182         return d
2183
2184     def test_POST_upload_no_link(self):
2185         d = self.POST("/uri", t="upload",
2186                       file=("new.txt", self.NEWFILE_CONTENTS))
2187         def _check_upload_results(page):
2188             # this should be a page which describes the results of the upload
2189             # that just finished.
2190             self.failUnlessIn("Upload Results:", page)
2191             self.failUnlessIn("URI:", page)
2192             uri_re = re.compile("URI: <tt><span>(.*)</span>")
2193             mo = uri_re.search(page)
2194             self.failUnless(mo, page)
2195             new_uri = mo.group(1)
2196             return new_uri
2197         d.addCallback(_check_upload_results)
2198         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
2199         return d
2200
2201     def test_POST_upload_no_link_whendone(self):
2202         d = self.POST("/uri", t="upload", when_done="/",
2203                       file=("new.txt", self.NEWFILE_CONTENTS))
2204         d.addBoth(self.shouldRedirect, "/")
2205         return d
2206
2207     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
2208         d = defer.maybeDeferred(callable, *args, **kwargs)
2209         def done(res):
2210             if isinstance(res, failure.Failure):
2211                 res.trap(error.PageRedirect)
2212                 statuscode = res.value.status
2213                 target = res.value.location
2214                 return checker(statuscode, target)
2215             self.fail("%s: callable was supposed to redirect, not return '%s'"
2216                       % (which, res))
2217         d.addBoth(done)
2218         return d
2219
2220     def test_POST_upload_no_link_whendone_results(self):
2221         def check(statuscode, target):
2222             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2223             self.failUnless(target.startswith(self.webish_url), target)
2224             return client.getPage(target, method="GET")
2225         # We encode "uri" as "%75ri" to exercise a case affected by ticket #1860.
2226         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
2227                                  check,
2228                                  self.POST, "/uri", t="upload",
2229                                  when_done="/%75ri/%(uri)s",
2230                                  file=("new.txt", self.NEWFILE_CONTENTS))
2231         d.addCallback(lambda res:
2232                       self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
2233         return d
2234
2235     def test_POST_upload_no_link_mutable(self):
2236         d = self.POST("/uri", t="upload", mutable="true",
2237                       file=("new.txt", self.NEWFILE_CONTENTS))
2238         def _check(filecap):
2239             filecap = filecap.strip()
2240             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2241             self.filecap = filecap
2242             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2243             self.failUnlessIn(u.get_storage_index(), self.get_all_contents())
2244             n = self.s.create_node_from_uri(filecap)
2245             return n.download_best_version()
2246         d.addCallback(_check)
2247         def _check2(data):
2248             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2249             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2250         d.addCallback(_check2)
2251         def _check3(data):
2252             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2253             return self.GET("/file/%s" % urllib.quote(self.filecap))
2254         d.addCallback(_check3)
2255         def _check4(data):
2256             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2257         d.addCallback(_check4)
2258         return d
2259
2260     def test_POST_upload_no_link_mutable_toobig(self):
2261         # The SDMF size limit is no longer in place, so we should be
2262         # able to upload mutable files that are as large as we want them
2263         # to be.
2264         d = self.POST("/uri", t="upload", mutable="true",
2265                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2266         return d
2267
2268
2269     def test_POST_upload_format_unlinked(self):
2270         def _check_upload_unlinked(ign, format, uri_prefix):
2271             filename = format + ".txt"
2272             d = self.POST("/uri?t=upload&format=" + format,
2273                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2274             def _got_results(results):
2275                 if format.upper() in ("SDMF", "MDMF"):
2276                     # webapi.rst says this returns a filecap
2277                     filecap = results
2278                 else:
2279                     # for immutable, it returns an "upload results page", and
2280                     # the filecap is buried inside
2281                     line = [l for l in results.split("\n") if "URI: " in l][0]
2282                     mo = re.search(r'<span>([^<]+)</span>', line)
2283                     filecap = mo.group(1)
2284                 self.failUnless(filecap.startswith(uri_prefix),
2285                                 (uri_prefix, filecap))
2286                 return self.GET("/uri/%s?t=json" % filecap)
2287             d.addCallback(_got_results)
2288             def _got_json(json):
2289                 data = simplejson.loads(json)
2290                 data = data[1]
2291                 self.failUnlessIn("format", data)
2292                 self.failUnlessEqual(data["format"], format.upper())
2293             d.addCallback(_got_json)
2294             return d
2295         d = defer.succeed(None)
2296         d.addCallback(_check_upload_unlinked, "chk", "URI:CHK")
2297         d.addCallback(_check_upload_unlinked, "CHK", "URI:CHK")
2298         d.addCallback(_check_upload_unlinked, "sdmf", "URI:SSK")
2299         d.addCallback(_check_upload_unlinked, "mdmf", "URI:MDMF")
2300         return d
2301
2302     def test_POST_upload_bad_format_unlinked(self):
2303         return self.shouldHTTPError("POST_upload_bad_format_unlinked",
2304                                     400, "Bad Request", "Unknown format: foo",
2305                                     self.POST,
2306                                     "/uri?t=upload&format=foo",
2307                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2308
2309     def test_POST_upload_format(self):
2310         def _check_upload(ign, format, uri_prefix, fn=None):
2311             filename = format + ".txt"
2312             d = self.POST(self.public_url +
2313                           "/foo?t=upload&format=" + format,
2314                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2315             def _got_filecap(filecap):
2316                 if fn is not None:
2317                     filenameu = unicode(filename)
2318                     self.failUnlessURIMatchesRWChild(filecap, fn, filenameu)
2319                 self.failUnless(filecap.startswith(uri_prefix))
2320                 return self.GET(self.public_url + "/foo/%s?t=json" % filename)
2321             d.addCallback(_got_filecap)
2322             def _got_json(json):
2323                 data = simplejson.loads(json)
2324                 data = data[1]
2325                 self.failUnlessIn("format", data)
2326                 self.failUnlessEqual(data["format"], format.upper())
2327             d.addCallback(_got_json)
2328             return d
2329
2330         d = defer.succeed(None)
2331         d.addCallback(_check_upload, "chk", "URI:CHK")
2332         d.addCallback(_check_upload, "sdmf", "URI:SSK", self._foo_node)
2333         d.addCallback(_check_upload, "mdmf", "URI:MDMF")
2334         d.addCallback(_check_upload, "MDMF", "URI:MDMF")
2335         return d
2336
2337     def test_POST_upload_bad_format(self):
2338         return self.shouldHTTPError("POST_upload_bad_format",
2339                                     400, "Bad Request", "Unknown format: foo",
2340                                     self.POST, self.public_url + \
2341                                     "/foo?t=upload&format=foo",
2342                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2343
2344     def test_POST_upload_mutable(self):
2345         # this creates a mutable file
2346         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
2347                       file=("new.txt", self.NEWFILE_CONTENTS))
2348         fn = self._foo_node
2349         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2350         d.addCallback(lambda res:
2351                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2352                                                              self.NEWFILE_CONTENTS))
2353         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2354         def _got(newnode):
2355             self.failUnless(IMutableFileNode.providedBy(newnode))
2356             self.failUnless(newnode.is_mutable())
2357             self.failIf(newnode.is_readonly())
2358             self._mutable_node = newnode
2359             self._mutable_uri = newnode.get_uri()
2360         d.addCallback(_got)
2361
2362         # now upload it again and make sure that the URI doesn't change
2363         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
2364         d.addCallback(lambda res:
2365                       self.POST(self.public_url + "/foo", t="upload",
2366                                 mutable="true",
2367                                 file=("new.txt", NEWER_CONTENTS)))
2368         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2369         d.addCallback(lambda res:
2370                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2371                                                              NEWER_CONTENTS))
2372         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2373         def _got2(newnode):
2374             self.failUnless(IMutableFileNode.providedBy(newnode))
2375             self.failUnless(newnode.is_mutable())
2376             self.failIf(newnode.is_readonly())
2377             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2378         d.addCallback(_got2)
2379
2380         # upload a second time, using PUT instead of POST
2381         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
2382         d.addCallback(lambda res:
2383                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
2384         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2385         d.addCallback(lambda res:
2386                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2387                                                              NEW2_CONTENTS))
2388
2389         # finally list the directory, since mutable files are displayed
2390         # slightly differently
2391
2392         d.addCallback(lambda res:
2393                       self.GET(self.public_url + "/foo/",
2394                                followRedirect=True))
2395         def _check_page(res):
2396             # TODO: assert more about the contents
2397             self.failUnlessIn("SSK", res)
2398             return res
2399         d.addCallback(_check_page)
2400
2401         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2402         def _got3(newnode):
2403             self.failUnless(IMutableFileNode.providedBy(newnode))
2404             self.failUnless(newnode.is_mutable())
2405             self.failIf(newnode.is_readonly())
2406             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2407         d.addCallback(_got3)
2408
2409         # look at the JSON form of the enclosing directory
2410         d.addCallback(lambda res:
2411                       self.GET(self.public_url + "/foo/?t=json",
2412                                followRedirect=True))
2413         def _check_page_json(res):
2414             parsed = simplejson.loads(res)
2415             self.failUnlessEqual(parsed[0], "dirnode")
2416             children = dict( [(unicode(name),value)
2417                               for (name,value)
2418                               in parsed[1]["children"].iteritems()] )
2419             self.failUnlessIn(u"new.txt", children)
2420             new_json = children[u"new.txt"]
2421             self.failUnlessEqual(new_json[0], "filenode")
2422             self.failUnless(new_json[1]["mutable"])
2423             self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri)
2424             ro_uri = self._mutable_node.get_readonly().to_string()
2425             self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri)
2426         d.addCallback(_check_page_json)
2427
2428         # and the JSON form of the file
2429         d.addCallback(lambda res:
2430                       self.GET(self.public_url + "/foo/new.txt?t=json"))
2431         def _check_file_json(res):
2432             parsed = simplejson.loads(res)
2433             self.failUnlessEqual(parsed[0], "filenode")
2434             self.failUnless(parsed[1]["mutable"])
2435             self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri)
2436             ro_uri = self._mutable_node.get_readonly().to_string()
2437             self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri)
2438         d.addCallback(_check_file_json)
2439
2440         # and look at t=uri and t=readonly-uri
2441         d.addCallback(lambda res:
2442                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
2443         d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
2444         d.addCallback(lambda res:
2445                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
2446         def _check_ro_uri(res):
2447             ro_uri = self._mutable_node.get_readonly().to_string()
2448             self.failUnlessReallyEqual(res, ro_uri)
2449         d.addCallback(_check_ro_uri)
2450
2451         # make sure we can get to it from /uri/URI
2452         d.addCallback(lambda res:
2453                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
2454         d.addCallback(lambda res:
2455                       self.failUnlessReallyEqual(res, NEW2_CONTENTS))
2456
2457         # and that HEAD computes the size correctly
2458         d.addCallback(lambda res:
2459                       self.HEAD(self.public_url + "/foo/new.txt",
2460                                 return_response=True))
2461         def _got_headers((res, status, headers)):
2462             self.failUnlessReallyEqual(res, "")
2463             self.failUnlessReallyEqual(headers["content-length"][0],
2464                                        str(len(NEW2_CONTENTS)))
2465             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
2466         d.addCallback(_got_headers)
2467
2468         # make sure that outdated size limits aren't enforced anymore.
2469         d.addCallback(lambda ignored:
2470             self.POST(self.public_url + "/foo", t="upload",
2471                       mutable="true",
2472                       file=("new.txt",
2473                             "b" * (self.s.MUTABLE_SIZELIMIT+1))))
2474         d.addErrback(self.dump_error)
2475         return d
2476
2477     def test_POST_upload_mutable_toobig(self):
2478         # SDMF had a size limti that was removed a while ago. MDMF has
2479         # never had a size limit. Test to make sure that we do not
2480         # encounter errors when trying to upload large mutable files,
2481         # since there should be no coded prohibitions regarding large
2482         # mutable files.
2483         d = self.POST(self.public_url + "/foo",
2484                       t="upload", mutable="true",
2485                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2486         return d
2487
2488     def dump_error(self, f):
2489         # if the web server returns an error code (like 400 Bad Request),
2490         # web.client.getPage puts the HTTP response body into the .response
2491         # attribute of the exception object that it gives back. It does not
2492         # appear in the Failure's repr(), so the ERROR that trial displays
2493         # will be rather terse and unhelpful. addErrback this method to the
2494         # end of your chain to get more information out of these errors.
2495         if f.check(error.Error):
2496             print "web.error.Error:"
2497             print f
2498             print f.value.response
2499         return f
2500
2501     def test_POST_upload_replace(self):
2502         d = self.POST(self.public_url + "/foo", t="upload",
2503                       file=("bar.txt", self.NEWFILE_CONTENTS))
2504         fn = self._foo_node
2505         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
2506         d.addCallback(lambda res:
2507                       self.failUnlessChildContentsAre(fn, u"bar.txt",
2508                                                       self.NEWFILE_CONTENTS))
2509         return d
2510
2511     def test_POST_upload_no_replace_ok(self):
2512         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2513                       file=("new.txt", self.NEWFILE_CONTENTS))
2514         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
2515         d.addCallback(lambda res: self.failUnlessReallyEqual(res,
2516                                                              self.NEWFILE_CONTENTS))
2517         return d
2518
2519     def test_POST_upload_no_replace_queryarg(self):
2520         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2521                       file=("bar.txt", self.NEWFILE_CONTENTS))
2522         d.addBoth(self.shouldFail, error.Error,
2523                   "POST_upload_no_replace_queryarg",
2524                   "409 Conflict",
2525                   "There was already a child by that name, and you asked me "
2526                   "to not replace it")
2527         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2528         d.addCallback(self.failUnlessIsBarDotTxt)
2529         return d
2530
2531     def test_POST_upload_no_replace_field(self):
2532         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
2533                       file=("bar.txt", self.NEWFILE_CONTENTS))
2534         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
2535                   "409 Conflict",
2536                   "There was already a child by that name, and you asked me "
2537                   "to not replace it")
2538         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2539         d.addCallback(self.failUnlessIsBarDotTxt)
2540         return d
2541
2542     def test_POST_upload_whendone(self):
2543         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
2544                       file=("new.txt", self.NEWFILE_CONTENTS))
2545         d.addBoth(self.shouldRedirect, "/THERE")
2546         fn = self._foo_node
2547         d.addCallback(lambda res:
2548                       self.failUnlessChildContentsAre(fn, u"new.txt",
2549                                                       self.NEWFILE_CONTENTS))
2550         return d
2551
2552     def test_POST_upload_named(self):
2553         fn = self._foo_node
2554         d = self.POST(self.public_url + "/foo", t="upload",
2555                       name="new.txt", file=self.NEWFILE_CONTENTS)
2556         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2557         d.addCallback(lambda res:
2558                       self.failUnlessChildContentsAre(fn, u"new.txt",
2559                                                       self.NEWFILE_CONTENTS))
2560         return d
2561
2562     def test_POST_upload_named_badfilename(self):
2563         d = self.POST(self.public_url + "/foo", t="upload",
2564                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
2565         d.addBoth(self.shouldFail, error.Error,
2566                   "test_POST_upload_named_badfilename",
2567                   "400 Bad Request",
2568                   "name= may not contain a slash",
2569                   )
2570         # make sure that nothing was added
2571         d.addCallback(lambda res:
2572                       self.failUnlessNodeKeysAre(self._foo_node,
2573                                                  [self._htmlname_unicode,
2574                                                   u"bar.txt", u"baz.txt", u"blockingfile",
2575                                                   u"empty", u"n\u00fc.txt", u"quux.txt",
2576                                                   u"sub"]))
2577         return d
2578
2579     def test_POST_FILEURL_check(self):
2580         bar_url = self.public_url + "/foo/bar.txt"
2581         d = self.POST(bar_url, t="check")
2582         def _check(res):
2583             self.failUnlessIn("Healthy :", res)
2584         d.addCallback(_check)
2585         redir_url = "http://allmydata.org/TARGET"
2586         def _check2(statuscode, target):
2587             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2588             self.failUnlessReallyEqual(target, redir_url)
2589         d.addCallback(lambda res:
2590                       self.shouldRedirect2("test_POST_FILEURL_check",
2591                                            _check2,
2592                                            self.POST, bar_url,
2593                                            t="check",
2594                                            when_done=redir_url))
2595         d.addCallback(lambda res:
2596                       self.POST(bar_url, t="check", return_to=redir_url))
2597         def _check3(res):
2598             self.failUnlessIn("Healthy :", res)
2599             self.failUnlessIn("Return to file", res)
2600             self.failUnlessIn(redir_url, res)
2601         d.addCallback(_check3)
2602
2603         d.addCallback(lambda res:
2604                       self.POST(bar_url, t="check", output="JSON"))
2605         def _check_json(res):
2606             data = simplejson.loads(res)
2607             self.failUnlessIn("storage-index", data)
2608             self.failUnless(data["results"]["healthy"])
2609         d.addCallback(_check_json)
2610
2611         return d
2612
2613     def test_POST_FILEURL_check_and_repair(self):
2614         bar_url = self.public_url + "/foo/bar.txt"
2615         d = self.POST(bar_url, t="check", repair="true")
2616         def _check(res):
2617             self.failUnlessIn("Healthy :", res)
2618         d.addCallback(_check)
2619         redir_url = "http://allmydata.org/TARGET"
2620         def _check2(statuscode, target):
2621             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2622             self.failUnlessReallyEqual(target, redir_url)
2623         d.addCallback(lambda res:
2624                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
2625                                            _check2,
2626                                            self.POST, bar_url,
2627                                            t="check", repair="true",
2628                                            when_done=redir_url))
2629         d.addCallback(lambda res:
2630                       self.POST(bar_url, t="check", return_to=redir_url))
2631         def _check3(res):
2632             self.failUnlessIn("Healthy :", res)
2633             self.failUnlessIn("Return to file", res)
2634             self.failUnlessIn(redir_url, res)
2635         d.addCallback(_check3)
2636         return d
2637
2638     def test_POST_DIRURL_check(self):
2639         foo_url = self.public_url + "/foo/"
2640         d = self.POST(foo_url, t="check")
2641         def _check(res):
2642             self.failUnlessIn("Healthy :", res)
2643         d.addCallback(_check)
2644         redir_url = "http://allmydata.org/TARGET"
2645         def _check2(statuscode, target):
2646             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2647             self.failUnlessReallyEqual(target, redir_url)
2648         d.addCallback(lambda res:
2649                       self.shouldRedirect2("test_POST_DIRURL_check",
2650                                            _check2,
2651                                            self.POST, foo_url,
2652                                            t="check",
2653                                            when_done=redir_url))
2654         d.addCallback(lambda res:
2655                       self.POST(foo_url, t="check", return_to=redir_url))
2656         def _check3(res):
2657             self.failUnlessIn("Healthy :", res)
2658             self.failUnlessIn("Return to file/directory", res)
2659             self.failUnlessIn(redir_url, res)
2660         d.addCallback(_check3)
2661
2662         d.addCallback(lambda res:
2663                       self.POST(foo_url, t="check", output="JSON"))
2664         def _check_json(res):
2665             data = simplejson.loads(res)
2666             self.failUnlessIn("storage-index", data)
2667             self.failUnless(data["results"]["healthy"])
2668         d.addCallback(_check_json)
2669
2670         return d
2671
2672     def test_POST_DIRURL_check_and_repair(self):
2673         foo_url = self.public_url + "/foo/"
2674         d = self.POST(foo_url, t="check", repair="true")
2675         def _check(res):
2676             self.failUnlessIn("Healthy :", res)
2677         d.addCallback(_check)
2678         redir_url = "http://allmydata.org/TARGET"
2679         def _check2(statuscode, target):
2680             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2681             self.failUnlessReallyEqual(target, redir_url)
2682         d.addCallback(lambda res:
2683                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2684                                            _check2,
2685                                            self.POST, foo_url,
2686                                            t="check", repair="true",
2687                                            when_done=redir_url))
2688         d.addCallback(lambda res:
2689                       self.POST(foo_url, t="check", return_to=redir_url))
2690         def _check3(res):
2691             self.failUnlessIn("Healthy :", res)
2692             self.failUnlessIn("Return to file/directory", res)
2693             self.failUnlessIn(redir_url, res)
2694         d.addCallback(_check3)
2695         return d
2696
2697     def test_POST_FILEURL_mdmf_check(self):
2698         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2699         d = self.POST(quux_url, t="check")
2700         def _check(res):
2701             self.failUnlessIn("Healthy", res)
2702         d.addCallback(_check)
2703         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2704         d.addCallback(lambda ignored:
2705                       self.POST(quux_extension_url, t="check"))
2706         d.addCallback(_check)
2707         return d
2708
2709     def test_POST_FILEURL_mdmf_check_and_repair(self):
2710         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2711         d = self.POST(quux_url, t="check", repair="true")
2712         def _check(res):
2713             self.failUnlessIn("Healthy", res)
2714         d.addCallback(_check)
2715         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2716         d.addCallback(lambda ignored:
2717                       self.POST(quux_extension_url, t="check", repair="true"))
2718         d.addCallback(_check)
2719         return d
2720
2721     def wait_for_operation(self, ignored, ophandle):
2722         url = "/operations/" + ophandle
2723         url += "?t=status&output=JSON"
2724         d = self.GET(url)
2725         def _got(res):
2726             data = simplejson.loads(res)
2727             if not data["finished"]:
2728                 d = self.stall(delay=1.0)
2729                 d.addCallback(self.wait_for_operation, ophandle)
2730                 return d
2731             return data
2732         d.addCallback(_got)
2733         return d
2734
2735     def get_operation_results(self, ignored, ophandle, output=None):
2736         url = "/operations/" + ophandle
2737         url += "?t=status"
2738         if output:
2739             url += "&output=" + output
2740         d = self.GET(url)
2741         def _got(res):
2742             if output and output.lower() == "json":
2743                 return simplejson.loads(res)
2744             return res
2745         d.addCallback(_got)
2746         return d
2747
2748     def test_POST_DIRURL_deepcheck_no_ophandle(self):
2749         d = self.shouldFail2(error.Error,
2750                              "test_POST_DIRURL_deepcheck_no_ophandle",
2751                              "400 Bad Request",
2752                              "slow operation requires ophandle=",
2753                              self.POST, self.public_url, t="start-deep-check")
2754         return d
2755
2756     def test_POST_DIRURL_deepcheck(self):
2757         def _check_redirect(statuscode, target):
2758             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2759             self.failUnless(target.endswith("/operations/123"))
2760         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2761                                  self.POST, self.public_url,
2762                                  t="start-deep-check", ophandle="123")
2763         d.addCallback(self.wait_for_operation, "123")
2764         def _check_json(data):
2765             self.failUnlessReallyEqual(data["finished"], True)
2766             self.failUnlessReallyEqual(data["count-objects-checked"], 11)
2767             self.failUnlessReallyEqual(data["count-objects-healthy"], 11)
2768         d.addCallback(_check_json)
2769         d.addCallback(self.get_operation_results, "123", "html")
2770         def _check_html(res):
2771             self.failUnlessIn("Objects Checked: <span>11</span>", res)
2772             self.failUnlessIn("Objects Healthy: <span>11</span>", res)
2773             self.failUnlessIn(FAVICON_MARKUP, res)
2774         d.addCallback(_check_html)
2775
2776         d.addCallback(lambda res:
2777                       self.GET("/operations/123/"))
2778         d.addCallback(_check_html) # should be the same as without the slash
2779
2780         d.addCallback(lambda res:
2781                       self.shouldFail2(error.Error, "one", "404 Not Found",
2782                                        "No detailed results for SI bogus",
2783                                        self.GET, "/operations/123/bogus"))
2784
2785         foo_si = self._foo_node.get_storage_index()
2786         foo_si_s = base32.b2a(foo_si)
2787         d.addCallback(lambda res:
2788                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2789         def _check_foo_json(res):
2790             data = simplejson.loads(res)
2791             self.failUnlessEqual(data["storage-index"], foo_si_s)
2792             self.failUnless(data["results"]["healthy"])
2793         d.addCallback(_check_foo_json)
2794         return d
2795
2796     def test_POST_DIRURL_deepcheck_and_repair(self):
2797         d = self.POST(self.public_url, t="start-deep-check", repair="true",
2798                       ophandle="124", output="json", followRedirect=True)
2799         d.addCallback(self.wait_for_operation, "124")
2800         def _check_json(data):
2801             self.failUnlessReallyEqual(data["finished"], True)
2802             self.failUnlessReallyEqual(data["count-objects-checked"], 11)
2803             self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 11)
2804             self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2805             self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2806             self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2807             self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2808             self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2809             self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 11)
2810             self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2811             self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2812         d.addCallback(_check_json)
2813         d.addCallback(self.get_operation_results, "124", "html")
2814         def _check_html(res):
2815             self.failUnlessIn("Objects Checked: <span>11</span>", res)
2816
2817             self.failUnlessIn("Objects Healthy (before repair): <span>11</span>", res)
2818             self.failUnlessIn("Objects Unhealthy (before repair): <span>0</span>", res)
2819             self.failUnlessIn("Corrupt Shares (before repair): <span>0</span>", res)
2820
2821             self.failUnlessIn("Repairs Attempted: <span>0</span>", res)
2822             self.failUnlessIn("Repairs Successful: <span>0</span>", res)
2823             self.failUnlessIn("Repairs Unsuccessful: <span>0</span>", res)
2824
2825             self.failUnlessIn("Objects Healthy (after repair): <span>11</span>", res)
2826             self.failUnlessIn("Objects Unhealthy (after repair): <span>0</span>", res)
2827             self.failUnlessIn("Corrupt Shares (after repair): <span>0</span>", res)
2828
2829             self.failUnlessIn(FAVICON_MARKUP, res)
2830         d.addCallback(_check_html)
2831         return d
2832
2833     def test_POST_FILEURL_bad_t(self):
2834         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2835                              "POST to file: bad t=bogus",
2836                              self.POST, self.public_url + "/foo/bar.txt",
2837                              t="bogus")
2838         return d
2839
2840     def test_POST_mkdir(self): # return value?
2841         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2842         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2843         d.addCallback(self.failUnlessNodeKeysAre, [])
2844         return d
2845
2846     def test_POST_mkdir_mdmf(self):
2847         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=mdmf")
2848         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2849         d.addCallback(lambda node:
2850             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2851         return d
2852
2853     def test_POST_mkdir_sdmf(self):
2854         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=sdmf")
2855         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2856         d.addCallback(lambda node:
2857             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2858         return d
2859
2860     def test_POST_mkdir_bad_format(self):
2861         return self.shouldHTTPError("POST_mkdir_bad_format",
2862                                     400, "Bad Request", "Unknown format: foo",
2863                                     self.POST, self.public_url +
2864                                     "/foo?t=mkdir&name=newdir&format=foo")
2865
2866     def test_POST_mkdir_initial_children(self):
2867         (newkids, caps) = self._create_initial_children()
2868         d = self.POST2(self.public_url +
2869                        "/foo?t=mkdir-with-children&name=newdir",
2870                        simplejson.dumps(newkids))
2871         d.addCallback(lambda res:
2872                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2873         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2874         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2875         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2876         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2877         return d
2878
2879     def test_POST_mkdir_initial_children_mdmf(self):
2880         (newkids, caps) = self._create_initial_children()
2881         d = self.POST2(self.public_url +
2882                        "/foo?t=mkdir-with-children&name=newdir&format=mdmf",
2883                        simplejson.dumps(newkids))
2884         d.addCallback(lambda res:
2885                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2886         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2887         d.addCallback(lambda node:
2888             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2889         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2890         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2891                        caps['filecap1'])
2892         return d
2893
2894     # XXX: Duplication.
2895     def test_POST_mkdir_initial_children_sdmf(self):
2896         (newkids, caps) = self._create_initial_children()
2897         d = self.POST2(self.public_url +
2898                        "/foo?t=mkdir-with-children&name=newdir&format=sdmf",
2899                        simplejson.dumps(newkids))
2900         d.addCallback(lambda res:
2901                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2902         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2903         d.addCallback(lambda node:
2904             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2905         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2906         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2907                        caps['filecap1'])
2908         return d
2909
2910     def test_POST_mkdir_initial_children_bad_format(self):
2911         (newkids, caps) = self._create_initial_children()
2912         return self.shouldHTTPError("POST_mkdir_initial_children_bad_format",
2913                                     400, "Bad Request", "Unknown format: foo",
2914                                     self.POST, self.public_url + \
2915                                     "/foo?t=mkdir-with-children&name=newdir&format=foo",
2916                                     simplejson.dumps(newkids))
2917
2918     def test_POST_mkdir_immutable(self):
2919         (newkids, caps) = self._create_immutable_children()
2920         d = self.POST2(self.public_url +
2921                        "/foo?t=mkdir-immutable&name=newdir",
2922                        simplejson.dumps(newkids))
2923         d.addCallback(lambda res:
2924                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2925         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2926         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2927         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2928         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2929         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2930         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2931         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2932         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2933         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2934         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2935         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2936         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2937         return d
2938
2939     def test_POST_mkdir_immutable_bad(self):
2940         (newkids, caps) = self._create_initial_children()
2941         d = self.shouldFail2(error.Error, "POST_mkdir_immutable_bad",
2942                              "400 Bad Request",
2943                              "needed to be immutable but was not",
2944                              self.POST2,
2945                              self.public_url +
2946                              "/foo?t=mkdir-immutable&name=newdir",
2947                              simplejson.dumps(newkids))
2948         return d
2949
2950     def test_POST_mkdir_2(self):
2951         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
2952         d.addCallback(lambda res:
2953                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2954         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2955         d.addCallback(self.failUnlessNodeKeysAre, [])
2956         return d
2957
2958     def test_POST_mkdirs_2(self):
2959         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
2960         d.addCallback(lambda res:
2961                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
2962         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
2963         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
2964         d.addCallback(self.failUnlessNodeKeysAre, [])
2965         return d
2966
2967     def test_POST_mkdir_no_parentdir_noredirect(self):
2968         d = self.POST("/uri?t=mkdir")
2969         def _after_mkdir(res):
2970             uri.DirectoryURI.init_from_string(res)
2971         d.addCallback(_after_mkdir)
2972         return d
2973
2974     def test_POST_mkdir_no_parentdir_noredirect_mdmf(self):
2975         d = self.POST("/uri?t=mkdir&format=mdmf")
2976         def _after_mkdir(res):
2977             u = uri.from_string(res)
2978             # Check that this is an MDMF writecap
2979             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
2980         d.addCallback(_after_mkdir)
2981         return d
2982
2983     def test_POST_mkdir_no_parentdir_noredirect_sdmf(self):
2984         d = self.POST("/uri?t=mkdir&format=sdmf")
2985         def _after_mkdir(res):
2986             u = uri.from_string(res)
2987             self.failUnlessIsInstance(u, uri.DirectoryURI)
2988         d.addCallback(_after_mkdir)
2989         return d
2990
2991     def test_POST_mkdir_no_parentdir_noredirect_bad_format(self):
2992         return self.shouldHTTPError("POST_mkdir_no_parentdir_noredirect_bad_format",
2993                                     400, "Bad Request", "Unknown format: foo",
2994                                     self.POST, self.public_url +
2995                                     "/uri?t=mkdir&format=foo")
2996
2997     def test_POST_mkdir_no_parentdir_noredirect2(self):
2998         # make sure form-based arguments (as on the welcome page) still work
2999         d = self.POST("/uri", t="mkdir")
3000         def _after_mkdir(res):
3001             uri.DirectoryURI.init_from_string(res)
3002         d.addCallback(_after_mkdir)
3003         d.addErrback(self.explain_web_error)
3004         return d
3005
3006     def test_POST_mkdir_no_parentdir_redirect(self):
3007         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
3008         d.addBoth(self.shouldRedirect, None, statuscode='303')
3009         def _check_target(target):
3010             target = urllib.unquote(target)
3011             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
3012         d.addCallback(_check_target)
3013         return d
3014
3015     def test_POST_mkdir_no_parentdir_redirect2(self):
3016         d = self.POST("/uri", t="mkdir", redirect_to_result="true")
3017         d.addBoth(self.shouldRedirect, None, statuscode='303')
3018         def _check_target(target):
3019             target = urllib.unquote(target)
3020             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
3021         d.addCallback(_check_target)
3022         d.addErrback(self.explain_web_error)
3023         return d
3024
3025     def _make_readonly(self, u):
3026         ro_uri = uri.from_string(u).get_readonly()
3027         if ro_uri is None:
3028             return None
3029         return ro_uri.to_string()
3030
3031     def _create_initial_children(self):
3032         contents, n, filecap1 = self.makefile(12)
3033         md1 = {"metakey1": "metavalue1"}
3034         filecap2 = make_mutable_file_uri()
3035         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
3036         filecap3 = node3.get_readonly_uri()
3037         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
3038         dircap = DirectoryNode(node4, None, None).get_uri()
3039         mdmfcap = make_mutable_file_uri(mdmf=True)
3040         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
3041         emptydircap = "URI:DIR2-LIT:"
3042         newkids = {u"child-imm":        ["filenode", {"rw_uri": filecap1,
3043                                                       "ro_uri": self._make_readonly(filecap1),
3044                                                       "metadata": md1, }],
3045                    u"child-mutable":    ["filenode", {"rw_uri": filecap2,
3046                                                       "ro_uri": self._make_readonly(filecap2)}],
3047                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
3048                    u"unknownchild-rw":  ["unknown",  {"rw_uri": unknown_rwcap,
3049                                                       "ro_uri": unknown_rocap}],
3050                    u"unknownchild-ro":  ["unknown",  {"ro_uri": unknown_rocap}],
3051                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
3052                    u"dirchild":         ["dirnode",  {"rw_uri": dircap,
3053                                                       "ro_uri": self._make_readonly(dircap)}],
3054                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
3055                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
3056                    u"child-mutable-mdmf": ["filenode", {"rw_uri": mdmfcap,
3057                                                         "ro_uri": self._make_readonly(mdmfcap)}],
3058                    }
3059         return newkids, {'filecap1': filecap1,
3060                          'filecap2': filecap2,
3061                          'filecap3': filecap3,
3062                          'unknown_rwcap': unknown_rwcap,
3063                          'unknown_rocap': unknown_rocap,
3064                          'unknown_immcap': unknown_immcap,
3065                          'dircap': dircap,
3066                          'litdircap': litdircap,
3067                          'emptydircap': emptydircap,
3068                          'mdmfcap': mdmfcap}
3069
3070     def _create_immutable_children(self):
3071         contents, n, filecap1 = self.makefile(12)
3072         md1 = {"metakey1": "metavalue1"}
3073         tnode = create_chk_filenode("immutable directory contents\n"*10,
3074                                     self.get_all_contents())
3075         dnode = DirectoryNode(tnode, None, None)
3076         assert not dnode.is_mutable()
3077         immdircap = dnode.get_uri()
3078         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
3079         emptydircap = "URI:DIR2-LIT:"
3080         newkids = {u"child-imm":        ["filenode", {"ro_uri": filecap1,
3081                                                       "metadata": md1, }],
3082                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
3083                    u"dirchild-imm":     ["dirnode",  {"ro_uri": immdircap}],
3084                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
3085                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
3086                    }
3087         return newkids, {'filecap1': filecap1,
3088                          'unknown_immcap': unknown_immcap,
3089                          'immdircap': immdircap,
3090                          'litdircap': litdircap,
3091                          'emptydircap': emptydircap}
3092
3093     def test_POST_mkdir_no_parentdir_initial_children(self):
3094         (newkids, caps) = self._create_initial_children()
3095         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
3096         def _after_mkdir(res):
3097             self.failUnless(res.startswith("URI:DIR"), res)
3098             n = self.s.create_node_from_uri(res)
3099             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3100             d2.addCallback(lambda ign:
3101                            self.failUnlessROChildURIIs(n, u"child-imm",
3102                                                        caps['filecap1']))
3103             d2.addCallback(lambda ign:
3104                            self.failUnlessRWChildURIIs(n, u"child-mutable",
3105                                                        caps['filecap2']))
3106             d2.addCallback(lambda ign:
3107                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
3108                                                        caps['filecap3']))
3109             d2.addCallback(lambda ign:
3110                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
3111                                                        caps['unknown_rwcap']))
3112             d2.addCallback(lambda ign:
3113                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
3114                                                        caps['unknown_rocap']))
3115             d2.addCallback(lambda ign:
3116                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3117                                                        caps['unknown_immcap']))
3118             d2.addCallback(lambda ign:
3119                            self.failUnlessRWChildURIIs(n, u"dirchild",
3120                                                        caps['dircap']))
3121             return d2
3122         d.addCallback(_after_mkdir)
3123         return d
3124
3125     def test_POST_mkdir_no_parentdir_unexpected_children(self):
3126         # the regular /uri?t=mkdir operation is specified to ignore its body.
3127         # Only t=mkdir-with-children pays attention to it.
3128         (newkids, caps) = self._create_initial_children()
3129         d = self.shouldHTTPError("POST_mkdir_no_parentdir_unexpected_children",
3130                                  400, "Bad Request",
3131                                  "t=mkdir does not accept children=, "
3132                                  "try t=mkdir-with-children instead",
3133                                  self.POST2, "/uri?t=mkdir", # without children
3134                                  simplejson.dumps(newkids))
3135         return d
3136
3137     def test_POST_noparent_bad(self):
3138         d = self.shouldHTTPError("POST_noparent_bad",
3139                                  400, "Bad Request",
3140                                  "/uri accepts only PUT, PUT?t=mkdir, "
3141                                  "POST?t=upload, and POST?t=mkdir",
3142                                  self.POST, "/uri?t=bogus")
3143         return d
3144
3145     def test_POST_mkdir_no_parentdir_immutable(self):
3146         (newkids, caps) = self._create_immutable_children()
3147         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
3148         def _after_mkdir(res):
3149             self.failUnless(res.startswith("URI:DIR"), res)
3150             n = self.s.create_node_from_uri(res)
3151             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3152             d2.addCallback(lambda ign:
3153                            self.failUnlessROChildURIIs(n, u"child-imm",
3154                                                           caps['filecap1']))
3155             d2.addCallback(lambda ign:
3156                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3157                                                           caps['unknown_immcap']))
3158             d2.addCallback(lambda ign:
3159                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
3160                                                           caps['immdircap']))
3161             d2.addCallback(lambda ign:
3162                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
3163                                                           caps['litdircap']))
3164             d2.addCallback(lambda ign:
3165                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
3166                                                           caps['emptydircap']))
3167             return d2
3168         d.addCallback(_after_mkdir)
3169         return d
3170
3171     def test_POST_mkdir_no_parentdir_immutable_bad(self):
3172         (newkids, caps) = self._create_initial_children()
3173         d = self.shouldFail2(error.Error,
3174                              "test_POST_mkdir_no_parentdir_immutable_bad",
3175                              "400 Bad Request",
3176                              "needed to be immutable but was not",
3177                              self.POST2,
3178                              "/uri?t=mkdir-immutable",
3179                              simplejson.dumps(newkids))
3180         return d
3181
3182     def test_welcome_page_mkdir_button(self):
3183         # Fetch the welcome page.
3184         d = self.GET("/")
3185         def _after_get_welcome_page(res):
3186             MKDIR_BUTTON_RE = re.compile(
3187                 '<form action="([^"]*)" method="post".*?'
3188                 '<input type="hidden" name="t" value="([^"]*)" />'
3189                 '<input type="hidden" name="([^"]*)" value="([^"]*)" />'
3190                 '<input type="submit" value="Create a directory" />',
3191                 re.I)
3192             mo = MKDIR_BUTTON_RE.search(res)
3193             formaction = mo.group(1)
3194             formt = mo.group(2)
3195             formaname = mo.group(3)
3196             formavalue = mo.group(4)
3197             return (formaction, formt, formaname, formavalue)
3198         d.addCallback(_after_get_welcome_page)
3199         def _after_parse_form(res):
3200             (formaction, formt, formaname, formavalue) = res
3201             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
3202         d.addCallback(_after_parse_form)
3203         d.addBoth(self.shouldRedirect, None, statuscode='303')
3204         return d
3205
3206     def test_POST_mkdir_replace(self): # return value?
3207         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
3208         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3209         d.addCallback(self.failUnlessNodeKeysAre, [])
3210         return d
3211
3212     def test_POST_mkdir_no_replace_queryarg(self): # return value?
3213         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
3214         d.addBoth(self.shouldFail, error.Error,
3215                   "POST_mkdir_no_replace_queryarg",
3216                   "409 Conflict",
3217                   "There was already a child by that name, and you asked me "
3218                   "to not replace it")
3219         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3220         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3221         return d
3222
3223     def test_POST_mkdir_no_replace_field(self): # return value?
3224         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
3225                       replace="false")
3226         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
3227                   "409 Conflict",
3228                   "There was already a child by that name, and you asked me "
3229                   "to not replace it")
3230         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3231         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3232         return d
3233
3234     def test_POST_mkdir_whendone_field(self):
3235         d = self.POST(self.public_url + "/foo",
3236                       t="mkdir", name="newdir", when_done="/THERE")
3237         d.addBoth(self.shouldRedirect, "/THERE")
3238         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3239         d.addCallback(self.failUnlessNodeKeysAre, [])
3240         return d
3241
3242     def test_POST_mkdir_whendone_queryarg(self):
3243         d = self.POST(self.public_url + "/foo?when_done=/THERE",
3244                       t="mkdir", name="newdir")
3245         d.addBoth(self.shouldRedirect, "/THERE")
3246         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3247         d.addCallback(self.failUnlessNodeKeysAre, [])
3248         return d
3249
3250     def test_POST_bad_t(self):
3251         d = self.shouldFail2(error.Error, "POST_bad_t",
3252                              "400 Bad Request",
3253                              "POST to a directory with bad t=BOGUS",
3254                              self.POST, self.public_url + "/foo", t="BOGUS")
3255         return d
3256
3257     def test_POST_set_children(self, command_name="set_children"):
3258         contents9, n9, newuri9 = self.makefile(9)
3259         contents10, n10, newuri10 = self.makefile(10)
3260         contents11, n11, newuri11 = self.makefile(11)
3261
3262         reqbody = """{
3263                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
3264                                                 "size": 0,
3265                                                 "metadata": {
3266                                                   "ctime": 1002777696.7564139,
3267                                                   "mtime": 1002777696.7564139
3268                                                  }
3269                                                } ],
3270                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
3271                                                 "size": 1,
3272                                                 "metadata": {
3273                                                   "ctime": 1002777696.7564139,
3274                                                   "mtime": 1002777696.7564139
3275                                                  }
3276                                                } ],
3277                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
3278                                                 "size": 2,
3279                                                 "metadata": {
3280                                                   "ctime": 1002777696.7564139,
3281                                                   "mtime": 1002777696.7564139
3282                                                  }
3283                                                } ]
3284                     }""" % (newuri9, newuri10, newuri11)
3285
3286         url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
3287
3288         d = client.getPage(url, method="POST", postdata=reqbody)
3289         def _then(res):
3290             self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
3291             self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
3292             self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
3293
3294         d.addCallback(_then)
3295         d.addErrback(self.dump_error)
3296         return d
3297
3298     def test_POST_set_children_with_hyphen(self):
3299         return self.test_POST_set_children(command_name="set-children")
3300
3301     def test_POST_link_uri(self):
3302         contents, n, newuri = self.makefile(8)
3303         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
3304         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
3305         d.addCallback(lambda res:
3306                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3307                                                       contents))
3308         return d
3309
3310     def test_POST_link_uri_replace(self):
3311         contents, n, newuri = self.makefile(8)
3312         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
3313         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
3314         d.addCallback(lambda res:
3315                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3316                                                       contents))
3317         return d
3318
3319     def test_POST_link_uri_unknown_bad(self):
3320         d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
3321         d.addBoth(self.shouldFail, error.Error,
3322                   "POST_link_uri_unknown_bad",
3323                   "400 Bad Request",
3324                   "unknown cap in a write slot")
3325         return d
3326
3327     def test_POST_link_uri_unknown_ro_good(self):
3328         d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
3329         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
3330         return d
3331
3332     def test_POST_link_uri_unknown_imm_good(self):
3333         d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
3334         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
3335         return d
3336
3337     def test_POST_link_uri_no_replace_queryarg(self):
3338         contents, n, newuri = self.makefile(8)
3339         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
3340                       name="bar.txt", uri=newuri)
3341         d.addBoth(self.shouldFail, error.Error,
3342                   "POST_link_uri_no_replace_queryarg",
3343                   "409 Conflict",
3344                   "There was already a child by that name, and you asked me "
3345                   "to not replace it")
3346         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3347         d.addCallback(self.failUnlessIsBarDotTxt)
3348         return d
3349
3350     def test_POST_link_uri_no_replace_field(self):
3351         contents, n, newuri = self.makefile(8)
3352         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
3353                       name="bar.txt", uri=newuri)
3354         d.addBoth(self.shouldFail, error.Error,
3355                   "POST_link_uri_no_replace_field",
3356                   "409 Conflict",
3357                   "There was already a child by that name, and you asked me "
3358                   "to not replace it")
3359         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3360         d.addCallback(self.failUnlessIsBarDotTxt)
3361         return d
3362
3363     def test_POST_delete(self, command_name='delete'):
3364         d = self._foo_node.list()
3365         def _check_before(children):
3366             self.failUnlessIn(u"bar.txt", children)
3367         d.addCallback(_check_before)
3368         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t=command_name, name="bar.txt"))
3369         d.addCallback(lambda res: self._foo_node.list())
3370         def _check_after(children):
3371             self.failIfIn(u"bar.txt", children)
3372         d.addCallback(_check_after)
3373         return d
3374
3375     def test_POST_unlink(self):
3376         return self.test_POST_delete(command_name='unlink')
3377
3378     def test_POST_rename_file(self):
3379         d = self.POST(self.public_url + "/foo", t="rename",
3380                       from_name="bar.txt", to_name='wibble.txt')
3381         d.addCallback(lambda res:
3382                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3383         d.addCallback(lambda res:
3384                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
3385         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
3386         d.addCallback(self.failUnlessIsBarDotTxt)
3387         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
3388         d.addCallback(self.failUnlessIsBarJSON)
3389         return d
3390
3391     def test_POST_rename_file_redundant(self):
3392         d = self.POST(self.public_url + "/foo", t="rename",
3393                       from_name="bar.txt", to_name='bar.txt')
3394         d.addCallback(lambda res:
3395                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3396         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3397         d.addCallback(self.failUnlessIsBarDotTxt)
3398         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3399         d.addCallback(self.failUnlessIsBarJSON)
3400         return d
3401
3402     def test_POST_rename_file_replace(self):
3403         # rename a file and replace a directory with it
3404         d = self.POST(self.public_url + "/foo", t="rename",
3405                       from_name="bar.txt", to_name='empty')
3406         d.addCallback(lambda res:
3407                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3408         d.addCallback(lambda res:
3409                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
3410         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
3411         d.addCallback(self.failUnlessIsBarDotTxt)
3412         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3413         d.addCallback(self.failUnlessIsBarJSON)
3414         return d
3415
3416     def test_POST_rename_file_no_replace_queryarg(self):
3417         # rename a file and replace a directory with it
3418         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
3419                       from_name="bar.txt", to_name='empty')
3420         d.addBoth(self.shouldFail, error.Error,
3421                   "POST_rename_file_no_replace_queryarg",
3422                   "409 Conflict",
3423                   "There was already a child by that name, and you asked me "
3424                   "to not replace it")
3425         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3426         d.addCallback(self.failUnlessIsEmptyJSON)
3427         return d
3428
3429     def test_POST_rename_file_no_replace_field(self):
3430         # rename a file and replace a directory with it
3431         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
3432                       from_name="bar.txt", to_name='empty')
3433         d.addBoth(self.shouldFail, error.Error,
3434                   "POST_rename_file_no_replace_field",
3435                   "409 Conflict",
3436                   "There was already a child by that name, and you asked me "
3437                   "to not replace it")
3438         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3439         d.addCallback(self.failUnlessIsEmptyJSON)
3440         return d
3441
3442     def failUnlessIsEmptyJSON(self, res):
3443         data = simplejson.loads(res)
3444         self.failUnlessEqual(data[0], "dirnode", data)
3445         self.failUnlessReallyEqual(len(data[1]["children"]), 0)
3446
3447     def test_POST_rename_file_slash_fail(self):
3448         d = self.POST(self.public_url + "/foo", t="rename",
3449                       from_name="bar.txt", to_name='kirk/spock.txt')
3450         d.addBoth(self.shouldFail, error.Error,
3451                   "test_POST_rename_file_slash_fail",
3452                   "400 Bad Request",
3453                   "to_name= may not contain a slash",
3454                   )
3455         d.addCallback(lambda res:
3456                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3457         return d
3458
3459     def test_POST_rename_dir(self):
3460         d = self.POST(self.public_url, t="rename",
3461                       from_name="foo", to_name='plunk')
3462         d.addCallback(lambda res:
3463                       self.failIfNodeHasChild(self.public_root, u"foo"))
3464         d.addCallback(lambda res:
3465                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
3466         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
3467         d.addCallback(self.failUnlessIsFooJSON)
3468         return d
3469
3470     def test_POST_move_file(self):
3471         d = self.POST(self.public_url + "/foo", t="move",
3472                       from_name="bar.txt", to_dir="sub")
3473         d.addCallback(lambda res:
3474                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3475         d.addCallback(lambda res:
3476                       self.failUnlessNodeHasChild(self._sub_node, u"bar.txt"))
3477         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3478         d.addCallback(self.failUnlessIsBarDotTxt)
3479         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3480         d.addCallback(self.failUnlessIsBarJSON)
3481         return d
3482
3483     def test_POST_move_file_new_name(self):
3484         d = self.POST(self.public_url + "/foo", t="move",
3485                       from_name="bar.txt", to_name="wibble.txt", to_dir="sub")
3486         d.addCallback(lambda res:
3487                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3488         d.addCallback(lambda res:
3489                       self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3490         d.addCallback(lambda res:
3491                       self.failUnlessNodeHasChild(self._sub_node, u"wibble.txt"))
3492         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt"))
3493         d.addCallback(self.failUnlessIsBarDotTxt)
3494         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt?t=json"))
3495         d.addCallback(self.failUnlessIsBarJSON)
3496         return d
3497
3498     def test_POST_move_file_replace(self):
3499         d = self.POST(self.public_url + "/foo", t="move",
3500                       from_name="bar.txt", to_name="baz.txt", to_dir="sub")
3501         d.addCallback(lambda res:
3502                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3503         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3504         d.addCallback(self.failUnlessIsBarDotTxt)
3505         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt?t=json"))
3506         d.addCallback(self.failUnlessIsBarJSON)
3507         return d
3508
3509     def test_POST_move_file_no_replace(self):
3510         d = self.shouldFail2(error.Error, "POST_move_file_no_replace",
3511                              "409 Conflict",
3512                              "There was already a child by that name, and you asked me to not replace it",
3513                              self.POST, self.public_url + "/foo", t="move",
3514                              replace="false", from_name="bar.txt",
3515                              to_name="baz.txt", to_dir="sub")
3516         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3517         d.addCallback(self.failUnlessIsBarDotTxt)
3518         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3519         d.addCallback(self.failUnlessIsBarJSON)
3520         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3521         d.addCallback(self.failUnlessIsSubBazDotTxt)
3522         return d
3523
3524     def test_POST_move_file_slash_fail(self):
3525         d = self.shouldFail2(error.Error, "test_POST_rename_file_slash_fail",
3526                              "400 Bad Request",
3527                              "to_name= may not contain a slash",
3528                              self.POST, self.public_url + "/foo", t="move",
3529                              from_name="bar.txt",
3530                              to_name="slash/fail.txt", to_dir="sub")
3531         d.addCallback(lambda res:
3532                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3533         d.addCallback(lambda res:
3534                       self.failIfNodeHasChild(self._sub_node, u"slash/fail.txt"))
3535         d.addCallback(lambda ign:
3536                       self.shouldFail2(error.Error,
3537                                        "test_POST_rename_file_slash_fail2",
3538                                        "400 Bad Request",
3539                                        "from_name= may not contain a slash",
3540                                        self.POST, self.public_url + "/foo",
3541                                        t="move",
3542                                        from_name="nope/bar.txt",
3543                                        to_name="fail.txt", to_dir="sub"))
3544         return d
3545
3546     def test_POST_move_file_no_target(self):
3547         d = self.shouldFail2(error.Error, "POST_move_file_no_target",
3548                              "400 Bad Request",
3549                              "move requires from_name and to_dir",
3550                              self.POST, self.public_url + "/foo", t="move",
3551                              from_name="bar.txt", to_name="baz.txt")
3552         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3553         d.addCallback(self.failUnlessIsBarDotTxt)
3554         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3555         d.addCallback(self.failUnlessIsBarJSON)
3556         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3557         d.addCallback(self.failUnlessIsBazDotTxt)
3558         return d
3559
3560     def test_POST_move_file_bad_target_type(self):
3561         d = self.shouldFail2(error.Error, "test_POST_move_file_bad_target_type",
3562                              "400 Bad Request", "invalid target_type parameter",
3563                              self.POST,
3564                              self.public_url + "/foo", t="move",
3565                              target_type="*D", from_name="bar.txt",
3566                              to_dir="sub")
3567         return d
3568
3569     def test_POST_move_file_multi_level(self):
3570         d = self.POST(self.public_url + "/foo/sub/level2?t=mkdir", "")
3571         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t="move",
3572                       from_name="bar.txt", to_dir="sub/level2"))
3573         d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3574         d.addCallback(lambda res: self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3575         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt"))
3576         d.addCallback(self.failUnlessIsBarDotTxt)
3577         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt?t=json"))
3578         d.addCallback(self.failUnlessIsBarJSON)
3579         return d
3580
3581     def test_POST_move_file_to_uri(self):
3582         d = self.POST(self.public_url + "/foo", t="move", target_type="uri",
3583                       from_name="bar.txt", to_dir=self._sub_uri)
3584         d.addCallback(lambda res:
3585                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3586         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3587         d.addCallback(self.failUnlessIsBarDotTxt)
3588         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3589         d.addCallback(self.failUnlessIsBarJSON)
3590         return d
3591
3592     def test_POST_move_file_to_nonexist_dir(self):
3593         d = self.shouldFail2(error.Error, "POST_move_file_to_nonexist_dir",
3594                             "404 Not Found", "No such child: nopechucktesta",
3595                             self.POST, self.public_url + "/foo", t="move",
3596                             from_name="bar.txt", to_dir="nopechucktesta")
3597         return d
3598
3599     def test_POST_move_file_into_file(self):
3600         d = self.shouldFail2(error.Error, "POST_move_file_into_file",
3601                              "400 Bad Request", "to_dir is not a directory",
3602                              self.POST, self.public_url + "/foo", t="move",
3603                              from_name="bar.txt", to_dir="baz.txt")
3604         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3605         d.addCallback(self.failUnlessIsBazDotTxt)
3606         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3607         d.addCallback(self.failUnlessIsBarDotTxt)
3608         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3609         d.addCallback(self.failUnlessIsBarJSON)
3610         return d
3611
3612     def test_POST_move_file_to_bad_uri(self):
3613         d =  self.shouldFail2(error.Error, "POST_move_file_to_bad_uri",
3614                               "400 Bad Request", "to_dir is not a directory",
3615                               self.POST, self.public_url + "/foo", t="move",
3616                               from_name="bar.txt", target_type="uri",
3617                               to_dir="URI:DIR2:mn5jlyjnrjeuydyswlzyui72i:rmneifcj6k6sycjljjhj3f6majsq2zqffydnnul5hfa4j577arma")
3618         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3619         d.addCallback(self.failUnlessIsBarDotTxt)
3620         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3621         d.addCallback(self.failUnlessIsBarJSON)
3622         return d
3623
3624     def test_POST_move_dir(self):
3625         d = self.POST(self.public_url + "/foo", t="move",
3626                       from_name="bar.txt", to_dir="empty")
3627         d.addCallback(lambda res: self.POST(self.public_url + "/foo",
3628                       t="move", from_name="empty", to_dir="sub"))
3629         d.addCallback(lambda res:
3630                       self.failIfNodeHasChild(self._foo_node, u"empty"))
3631         d.addCallback(lambda res:
3632                       self.failUnlessNodeHasChild(self._sub_node, u"empty"))
3633         d.addCallback(lambda res:
3634                       self._sub_node.get_child_at_path(u"empty"))
3635         d.addCallback(lambda node:
3636                       self.failUnlessNodeHasChild(node, u"bar.txt"))
3637         d.addCallback(lambda res:
3638                       self.GET(self.public_url + "/foo/sub/empty/bar.txt"))
3639         d.addCallback(self.failUnlessIsBarDotTxt)
3640         return d
3641
3642     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
3643         """ If target is not None then the redirection has to go to target.  If
3644         statuscode is not None then the redirection has to be accomplished with
3645         that HTTP status code."""
3646         if not isinstance(res, failure.Failure):
3647             to_where = (target is None) and "somewhere" or ("to " + target)
3648             self.fail("%s: we were expecting to get redirected %s, not get an"
3649                       " actual page: %s" % (which, to_where, res))
3650         res.trap(error.PageRedirect)
3651         if statuscode is not None:
3652             self.failUnlessReallyEqual(res.value.status, statuscode,
3653                                        "%s: not a redirect" % which)
3654         if target is not None:
3655             # the PageRedirect does not seem to capture the uri= query arg
3656             # properly, so we can't check for it.
3657             realtarget = self.webish_url + target
3658             self.failUnlessReallyEqual(res.value.location, realtarget,
3659                                        "%s: wrong target" % which)
3660         return res.value.location
3661
3662     def test_GET_URI_form(self):
3663         base = "/uri?uri=%s" % self._bar_txt_uri
3664         # this is supposed to give us a redirect to /uri/$URI, plus arguments
3665         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
3666         d = self.GET(base)
3667         d.addBoth(self.shouldRedirect, targetbase)
3668         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
3669         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
3670         d.addCallback(lambda res: self.GET(base+"&t=json"))
3671         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
3672         d.addCallback(self.log, "about to get file by uri")
3673         d.addCallback(lambda res: self.GET(base, followRedirect=True))
3674         d.addCallback(self.failUnlessIsBarDotTxt)
3675         d.addCallback(self.log, "got file by uri, about to get dir by uri")
3676         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
3677                                            followRedirect=True))
3678         d.addCallback(self.failUnlessIsFooJSON)
3679         d.addCallback(self.log, "got dir by uri")
3680
3681         return d
3682
3683     def test_GET_URI_form_bad(self):
3684         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
3685                              "400 Bad Request", "GET /uri requires uri=",
3686                              self.GET, "/uri")
3687         return d
3688
3689     def test_GET_rename_form(self):
3690         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
3691                      followRedirect=True)
3692         def _check(res):
3693             self.failUnlessIn('name="when_done" value="."', res)
3694             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
3695             self.failUnlessIn(FAVICON_MARKUP, res)
3696         d.addCallback(_check)
3697         return d
3698
3699     def log(self, res, msg):
3700         #print "MSG: %s  RES: %s" % (msg, res)
3701         log.msg(msg)
3702         return res
3703
3704     def test_GET_URI_URL(self):
3705         base = "/uri/%s" % self._bar_txt_uri
3706         d = self.GET(base)
3707         d.addCallback(self.failUnlessIsBarDotTxt)
3708         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
3709         d.addCallback(self.failUnlessIsBarDotTxt)
3710         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
3711         d.addCallback(self.failUnlessIsBarDotTxt)
3712         return d
3713
3714     def test_GET_URI_URL_dir(self):
3715         base = "/uri/%s?t=json" % self._foo_uri
3716         d = self.GET(base)
3717         d.addCallback(self.failUnlessIsFooJSON)
3718         return d
3719
3720     def test_GET_URI_URL_missing(self):
3721         base = "/uri/%s" % self._bad_file_uri
3722         d = self.shouldHTTPError("test_GET_URI_URL_missing",
3723                                  http.GONE, None, "NotEnoughSharesError",
3724                                  self.GET, base)
3725         # TODO: how can we exercise both sides of WebDownloadTarget.fail
3726         # here? we must arrange for a download to fail after target.open()
3727         # has been called, and then inspect the response to see that it is
3728         # shorter than we expected.
3729         return d
3730
3731     def test_PUT_DIRURL_uri(self):
3732         d = self.s.create_dirnode()
3733         def _made_dir(dn):
3734             new_uri = dn.get_uri()
3735             # replace /foo with a new (empty) directory
3736             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
3737             d.addCallback(lambda res:
3738                           self.failUnlessReallyEqual(res.strip(), new_uri))
3739             d.addCallback(lambda res:
3740                           self.failUnlessRWChildURIIs(self.public_root,
3741                                                       u"foo",
3742                                                       new_uri))
3743             return d
3744         d.addCallback(_made_dir)
3745         return d
3746
3747     def test_PUT_DIRURL_uri_noreplace(self):
3748         d = self.s.create_dirnode()
3749         def _made_dir(dn):
3750             new_uri = dn.get_uri()
3751             # replace /foo with a new (empty) directory, but ask that
3752             # replace=false, so it should fail
3753             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
3754                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
3755                                  self.PUT,
3756                                  self.public_url + "/foo?t=uri&replace=false",
3757                                  new_uri)
3758             d.addCallback(lambda res:
3759                           self.failUnlessRWChildURIIs(self.public_root,
3760                                                       u"foo",
3761                                                       self._foo_uri))
3762             return d
3763         d.addCallback(_made_dir)
3764         return d
3765
3766     def test_PUT_DIRURL_bad_t(self):
3767         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
3768                              "400 Bad Request", "PUT to a directory",
3769                              self.PUT, self.public_url + "/foo?t=BOGUS", "")
3770         d.addCallback(lambda res:
3771                       self.failUnlessRWChildURIIs(self.public_root,
3772                                                   u"foo",
3773                                                   self._foo_uri))
3774         return d
3775
3776     def test_PUT_NEWFILEURL_uri(self):
3777         contents, n, new_uri = self.makefile(8)
3778         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
3779         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3780         d.addCallback(lambda res:
3781                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3782                                                       contents))
3783         return d
3784
3785     def test_PUT_NEWFILEURL_mdmf(self):
3786         new_contents = self.NEWFILE_CONTENTS * 300000
3787         d = self.PUT(self.public_url + \
3788                      "/foo/mdmf.txt?format=mdmf",
3789                      new_contents)
3790         d.addCallback(lambda ignored:
3791             self.GET(self.public_url + "/foo/mdmf.txt?t=json"))
3792         def _got_json(json):
3793             data = simplejson.loads(json)
3794             data = data[1]
3795             self.failUnlessIn("format", data)
3796             self.failUnlessEqual(data["format"], "MDMF")
3797             self.failUnless(data['rw_uri'].startswith("URI:MDMF"))
3798             self.failUnless(data['ro_uri'].startswith("URI:MDMF"))
3799         d.addCallback(_got_json)
3800         return d
3801
3802     def test_PUT_NEWFILEURL_sdmf(self):
3803         new_contents = self.NEWFILE_CONTENTS * 300000
3804         d = self.PUT(self.public_url + \
3805                      "/foo/sdmf.txt?format=sdmf",
3806                      new_contents)
3807         d.addCallback(lambda ignored:
3808             self.GET(self.public_url + "/foo/sdmf.txt?t=json"))
3809         def _got_json(json):
3810             data = simplejson.loads(json)
3811             data = data[1]
3812             self.failUnlessIn("format", data)
3813             self.failUnlessEqual(data["format"], "SDMF")
3814         d.addCallback(_got_json)
3815         return d
3816
3817     def test_PUT_NEWFILEURL_bad_format(self):
3818         new_contents = self.NEWFILE_CONTENTS * 300000
3819         return self.shouldHTTPError("PUT_NEWFILEURL_bad_format",
3820                                     400, "Bad Request", "Unknown format: foo",
3821                                     self.PUT, self.public_url + \
3822                                     "/foo/foo.txt?format=foo",
3823                                     new_contents)
3824
3825     def test_PUT_NEWFILEURL_uri_replace(self):
3826         contents, n, new_uri = self.makefile(8)
3827         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
3828         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3829         d.addCallback(lambda res:
3830                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3831                                                       contents))
3832         return d
3833
3834     def test_PUT_NEWFILEURL_uri_no_replace(self):
3835         contents, n, new_uri = self.makefile(8)
3836         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
3837         d.addBoth(self.shouldFail, error.Error,
3838                   "PUT_NEWFILEURL_uri_no_replace",
3839                   "409 Conflict",
3840                   "There was already a child by that name, and you asked me "
3841                   "to not replace it")
3842         return d
3843
3844     def test_PUT_NEWFILEURL_uri_unknown_bad(self):
3845         d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
3846         d.addBoth(self.shouldFail, error.Error,
3847                   "POST_put_uri_unknown_bad",
3848                   "400 Bad Request",
3849                   "unknown cap in a write slot")
3850         return d
3851
3852     def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
3853         d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
3854         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3855                       u"put-future-ro.txt")
3856         return d
3857
3858     def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
3859         d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
3860         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3861                       u"put-future-imm.txt")
3862         return d
3863
3864     def test_PUT_NEWFILE_URI(self):
3865         file_contents = "New file contents here\n"
3866         d = self.PUT("/uri", file_contents)
3867         def _check(uri):
3868             assert isinstance(uri, str), uri
3869             self.failUnlessIn(uri, self.get_all_contents())
3870             self.failUnlessReallyEqual(self.get_all_contents()[uri],
3871                                        file_contents)
3872             return self.GET("/uri/%s" % uri)
3873         d.addCallback(_check)
3874         def _check2(res):
3875             self.failUnlessReallyEqual(res, file_contents)
3876         d.addCallback(_check2)
3877         return d
3878
3879     def test_PUT_NEWFILE_URI_not_mutable(self):
3880         file_contents = "New file contents here\n"
3881         d = self.PUT("/uri?mutable=false", file_contents)
3882         def _check(uri):
3883             assert isinstance(uri, str), uri
3884             self.failUnlessIn(uri, self.get_all_contents())
3885             self.failUnlessReallyEqual(self.get_all_contents()[uri],
3886                                        file_contents)
3887             return self.GET("/uri/%s" % uri)
3888         d.addCallback(_check)
3889         def _check2(res):
3890             self.failUnlessReallyEqual(res, file_contents)
3891         d.addCallback(_check2)
3892         return d
3893
3894     def test_PUT_NEWFILE_URI_only_PUT(self):
3895         d = self.PUT("/uri?t=bogus", "")
3896         d.addBoth(self.shouldFail, error.Error,
3897                   "PUT_NEWFILE_URI_only_PUT",
3898                   "400 Bad Request",
3899                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
3900         return d
3901
3902     def test_PUT_NEWFILE_URI_mutable(self):
3903         file_contents = "New file contents here\n"
3904         d = self.PUT("/uri?mutable=true", file_contents)
3905         def _check1(filecap):
3906             filecap = filecap.strip()
3907             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
3908             self.filecap = filecap
3909             u = uri.WriteableSSKFileURI.init_from_string(filecap)
3910             self.failUnlessIn(u.get_storage_index(), self.get_all_contents())
3911             n = self.s.create_node_from_uri(filecap)
3912             return n.download_best_version()
3913         d.addCallback(_check1)
3914         def _check2(data):
3915             self.failUnlessReallyEqual(data, file_contents)
3916             return self.GET("/uri/%s" % urllib.quote(self.filecap))
3917         d.addCallback(_check2)
3918         def _check3(res):
3919             self.failUnlessReallyEqual(res, file_contents)
3920         d.addCallback(_check3)
3921         return d
3922
3923     def test_PUT_mkdir(self):
3924         d = self.PUT("/uri?t=mkdir", "")
3925         def _check(uri):
3926             n = self.s.create_node_from_uri(uri.strip())
3927             d2 = self.failUnlessNodeKeysAre(n, [])
3928             d2.addCallback(lambda res:
3929                            self.GET("/uri/%s?t=json" % uri))
3930             return d2
3931         d.addCallback(_check)
3932         d.addCallback(self.failUnlessIsEmptyJSON)
3933         return d
3934
3935     def test_PUT_mkdir_mdmf(self):
3936         d = self.PUT("/uri?t=mkdir&format=mdmf", "")
3937         def _got(res):
3938             u = uri.from_string(res)
3939             # Check that this is an MDMF writecap
3940             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
3941         d.addCallback(_got)
3942         return d
3943
3944     def test_PUT_mkdir_sdmf(self):
3945         d = self.PUT("/uri?t=mkdir&format=sdmf", "")
3946         def _got(res):
3947             u = uri.from_string(res)
3948             self.failUnlessIsInstance(u, uri.DirectoryURI)
3949         d.addCallback(_got)
3950         return d
3951
3952     def test_PUT_mkdir_bad_format(self):
3953         return self.shouldHTTPError("PUT_mkdir_bad_format",
3954                                     400, "Bad Request", "Unknown format: foo",
3955                                     self.PUT, "/uri?t=mkdir&format=foo",
3956                                     "")
3957
3958     def test_POST_check(self):
3959         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
3960         def _done(res):
3961             # this returns a string form of the results, which are probably
3962             # None since we're using fake filenodes.
3963             # TODO: verify that the check actually happened, by changing
3964             # FakeCHKFileNode to count how many times .check() has been
3965             # called.
3966             pass
3967         d.addCallback(_done)
3968         return d
3969
3970
3971     def test_PUT_update_at_offset(self):
3972         file_contents = "test file" * 100000 # about 900 KiB
3973         d = self.PUT("/uri?mutable=true", file_contents)
3974         def _then(filecap):
3975             self.filecap = filecap
3976             new_data = file_contents[:100]
3977             new = "replaced and so on"
3978             new_data += new
3979             new_data += file_contents[len(new_data):]
3980             assert len(new_data) == len(file_contents)
3981             self.new_data = new_data
3982         d.addCallback(_then)
3983         d.addCallback(lambda ignored:
3984             self.PUT("/uri/%s?replace=True&offset=100" % self.filecap,
3985                      "replaced and so on"))
3986         def _get_data(filecap):
3987             n = self.s.create_node_from_uri(filecap)
3988             return n.download_best_version()
3989         d.addCallback(_get_data)
3990         d.addCallback(lambda results:
3991             self.failUnlessEqual(results, self.new_data))
3992         # Now try appending things to the file
3993         d.addCallback(lambda ignored:
3994             self.PUT("/uri/%s?offset=%d" % (self.filecap, len(self.new_data)),
3995                      "puppies" * 100))
3996         d.addCallback(_get_data)
3997         d.addCallback(lambda results:
3998             self.failUnlessEqual(results, self.new_data + ("puppies" * 100)))
3999         # and try replacing the beginning of the file
4000         d.addCallback(lambda ignored:
4001             self.PUT("/uri/%s?offset=0" % self.filecap, "begin"))
4002         d.addCallback(_get_data)
4003         d.addCallback(lambda results:
4004             self.failUnlessEqual(results, "begin"+self.new_data[len("begin"):]+("puppies"*100)))
4005         return d
4006
4007     def test_PUT_update_at_invalid_offset(self):
4008         file_contents = "test file" * 100000 # about 900 KiB
4009         d = self.PUT("/uri?mutable=true", file_contents)
4010         def _then(filecap):
4011             self.filecap = filecap
4012         d.addCallback(_then)
4013         # Negative offsets should cause an error.
4014         d.addCallback(lambda ignored:
4015             self.shouldHTTPError("PUT_update_at_invalid_offset",
4016                                  400, "Bad Request",
4017                                  "Invalid offset",
4018                                  self.PUT,
4019                                  "/uri/%s?offset=-1" % self.filecap,
4020                                  "foo"))
4021         return d
4022
4023     def test_PUT_update_at_offset_immutable(self):
4024         file_contents = "Test file" * 100000
4025         d = self.PUT("/uri", file_contents)
4026         def _then(filecap):
4027             self.filecap = filecap
4028         d.addCallback(_then)
4029         d.addCallback(lambda ignored:
4030             self.shouldHTTPError("PUT_update_at_offset_immutable",
4031                                  400, "Bad Request",
4032                                  "immutable",
4033                                  self.PUT,
4034                                  "/uri/%s?offset=50" % self.filecap,
4035                                  "foo"))
4036         return d
4037
4038
4039     def test_bad_method(self):
4040         url = self.webish_url + self.public_url + "/foo/bar.txt"
4041         d = self.shouldHTTPError("bad_method",
4042                                  501, "Not Implemented",
4043                                  "I don't know how to treat a BOGUS request.",
4044                                  client.getPage, url, method="BOGUS")
4045         return d
4046
4047     def test_short_url(self):
4048         url = self.webish_url + "/uri"
4049         d = self.shouldHTTPError("short_url", 501, "Not Implemented",
4050                                  "I don't know how to treat a DELETE request.",
4051                                  client.getPage, url, method="DELETE")
4052         return d
4053
4054     def test_ophandle_bad(self):
4055         url = self.webish_url + "/operations/bogus?t=status"
4056         d = self.shouldHTTPError("ophandle_bad", 404, "404 Not Found",
4057                                  "unknown/expired handle 'bogus'",
4058                                  client.getPage, url)
4059         return d
4060
4061     def test_ophandle_cancel(self):
4062         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
4063                       followRedirect=True)
4064         d.addCallback(lambda ignored:
4065                       self.GET("/operations/128?t=status&output=JSON"))
4066         def _check1(res):
4067             data = simplejson.loads(res)
4068             self.failUnless("finished" in data, res)
4069             monitor = self.ws.root.child_operations.handles["128"][0]
4070             d = self.POST("/operations/128?t=cancel&output=JSON")
4071             def _check2(res):
4072                 data = simplejson.loads(res)
4073                 self.failUnless("finished" in data, res)
4074                 # t=cancel causes the handle to be forgotten
4075                 self.failUnless(monitor.is_cancelled())
4076             d.addCallback(_check2)
4077             return d
4078         d.addCallback(_check1)
4079         d.addCallback(lambda ignored:
4080                       self.shouldHTTPError("ophandle_cancel",
4081                                            404, "404 Not Found",
4082                                            "unknown/expired handle '128'",
4083                                            self.GET,
4084                                            "/operations/128?t=status&output=JSON"))
4085         return d
4086
4087     def test_ophandle_retainfor(self):
4088         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
4089                       followRedirect=True)
4090         d.addCallback(lambda ignored:
4091                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
4092         def _check1(res):
4093             data = simplejson.loads(res)
4094             self.failUnless("finished" in data, res)
4095         d.addCallback(_check1)
4096         # the retain-for=0 will cause the handle to be expired very soon
4097         d.addCallback(lambda ign:
4098             self.clock.advance(2.0))
4099         d.addCallback(lambda ignored:
4100                       self.shouldHTTPError("ophandle_retainfor",
4101                                            404, "404 Not Found",
4102                                            "unknown/expired handle '129'",
4103                                            self.GET,
4104                                            "/operations/129?t=status&output=JSON"))
4105         return d
4106
4107     def test_ophandle_release_after_complete(self):
4108         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
4109                       followRedirect=True)
4110         d.addCallback(self.wait_for_operation, "130")
4111         d.addCallback(lambda ignored:
4112                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
4113         # the release-after-complete=true will cause the handle to be expired
4114         d.addCallback(lambda ignored:
4115                       self.shouldHTTPError("ophandle_release_after_complete",
4116                                            404, "404 Not Found",
4117                                            "unknown/expired handle '130'",
4118                                            self.GET,
4119                                            "/operations/130?t=status&output=JSON"))
4120         return d
4121
4122     def test_uncollected_ophandle_expiration(self):
4123         # uncollected ophandles should expire after 4 days
4124         def _make_uncollected_ophandle(ophandle):
4125             d = self.POST(self.public_url +
4126                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
4127                           followRedirect=False)
4128             # When we start the operation, the webapi server will want
4129             # to redirect us to the page for the ophandle, so we get
4130             # confirmation that the operation has started. If the
4131             # manifest operation has finished by the time we get there,
4132             # following that redirect (by setting followRedirect=True
4133             # above) has the side effect of collecting the ophandle that
4134             # we've just created, which means that we can't use the
4135             # ophandle to test the uncollected timeout anymore. So,
4136             # instead, catch the 302 here and don't follow it.
4137             d.addBoth(self.should302, "uncollected_ophandle_creation")
4138             return d
4139         # Create an ophandle, don't collect it, then advance the clock by
4140         # 4 days - 1 second and make sure that the ophandle is still there.
4141         d = _make_uncollected_ophandle(131)
4142         d.addCallback(lambda ign:
4143             self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
4144         d.addCallback(lambda ign:
4145             self.GET("/operations/131?t=status&output=JSON"))
4146         def _check1(res):
4147             data = simplejson.loads(res)
4148             self.failUnless("finished" in data, res)
4149         d.addCallback(_check1)
4150         # Create an ophandle, don't collect it, then try to collect it
4151         # after 4 days. It should be gone.
4152         d.addCallback(lambda ign:
4153             _make_uncollected_ophandle(132))
4154         d.addCallback(lambda ign:
4155             self.clock.advance(96*60*60))
4156         d.addCallback(lambda ign:
4157             self.shouldHTTPError("uncollected_ophandle_expired_after_100_hours",
4158                                  404, "404 Not Found",
4159                                  "unknown/expired handle '132'",
4160                                  self.GET,
4161                                  "/operations/132?t=status&output=JSON"))
4162         return d
4163
4164     def test_collected_ophandle_expiration(self):
4165         # collected ophandles should expire after 1 day
4166         def _make_collected_ophandle(ophandle):
4167             d = self.POST(self.public_url +
4168                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
4169                           followRedirect=True)
4170             # By following the initial redirect, we collect the ophandle
4171             # we've just created.
4172             return d
4173         # Create a collected ophandle, then collect it after 23 hours
4174         # and 59 seconds to make sure that it is still there.
4175         d = _make_collected_ophandle(133)
4176         d.addCallback(lambda ign:
4177             self.clock.advance((24*60*60) - 1))
4178         d.addCallback(lambda ign:
4179             self.GET("/operations/133?t=status&output=JSON"))
4180         def _check1(res):
4181             data = simplejson.loads(res)
4182             self.failUnless("finished" in data, res)
4183         d.addCallback(_check1)
4184         # Create another uncollected ophandle, then try to collect it
4185         # after 24 hours to make sure that it is gone.
4186         d.addCallback(lambda ign:
4187             _make_collected_ophandle(134))
4188         d.addCallback(lambda ign:
4189             self.clock.advance(24*60*60))
4190         d.addCallback(lambda ign:
4191             self.shouldHTTPError("collected_ophandle_expired_after_1_day",
4192                                  404, "404 Not Found",
4193                                  "unknown/expired handle '134'",
4194                                  self.GET,
4195                                  "/operations/134?t=status&output=JSON"))
4196         return d
4197
4198     def test_incident(self):
4199         d = self.POST("/report_incident", details="eek")
4200         def _done(res):
4201             self.failIfIn("<html>", res)
4202             self.failUnlessIn("Thank you for your report!", res)
4203         d.addCallback(_done)
4204         return d
4205
4206     def test_static(self):
4207         webdir = os.path.join(self.staticdir, "subdir")
4208         fileutil.make_dirs(webdir)
4209         f = open(os.path.join(webdir, "hello.txt"), "wb")
4210         f.write("hello")
4211         f.close()
4212
4213         d = self.GET("/static/subdir/hello.txt")
4214         def _check(res):
4215             self.failUnlessReallyEqual(res, "hello")
4216         d.addCallback(_check)
4217         return d
4218
4219
4220 class IntroducerWeb(unittest.TestCase):
4221     def setUp(self):
4222         self.node = None
4223
4224     def tearDown(self):
4225         d = defer.succeed(None)
4226         if self.node:
4227             d.addCallback(lambda ign: self.node.stopService())
4228         d.addCallback(flushEventualQueue)
4229         return d
4230
4231     def test_welcome(self):
4232         basedir = "web.IntroducerWeb.test_welcome"
4233         os.mkdir(basedir)
4234         fileutil.write(os.path.join(basedir, "tahoe.cfg"), "[node]\nweb.port = tcp:0\n")
4235         self.node = IntroducerNode(basedir)
4236         self.ws = self.node.getServiceNamed("webish")
4237
4238         d = fireEventually(None)
4239         d.addCallback(lambda ign: self.node.startService())
4240         d.addCallback(lambda ign: self.node.when_tub_ready())
4241
4242         d.addCallback(lambda ign: self.GET("/"))
4243         def _check(res):
4244             self.failUnlessIn('Welcome to the Tahoe-LAFS Introducer', res)
4245             self.failUnlessIn(FAVICON_MARKUP, res)
4246         d.addCallback(_check)
4247         return d
4248
4249     def GET(self, urlpath, followRedirect=False, return_response=False,
4250             **kwargs):
4251         # if return_response=True, this fires with (data, statuscode,
4252         # respheaders) instead of just data.
4253         assert not isinstance(urlpath, unicode)
4254         url = self.ws.getURL().rstrip('/') + urlpath
4255         factory = HTTPClientGETFactory(url, method="GET",
4256                                        followRedirect=followRedirect, **kwargs)
4257         reactor.connectTCP("localhost", self.ws.getPortnum(), factory)
4258         d = factory.deferred
4259         def _got_data(data):
4260             return (data, factory.status, factory.response_headers)
4261         if return_response:
4262             d.addCallback(_got_data)
4263         return factory.deferred
4264
4265
4266 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4267     def test_load_file(self):
4268         # This will raise an exception unless a well-formed XML file is found under that name.
4269         common.getxmlfile('directory.xhtml').load()
4270
4271     def test_parse_replace_arg(self):
4272         self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
4273         self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
4274         self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
4275                                    "only-files")
4276         self.shouldFail(AssertionError, "test_parse_replace_arg", "",
4277                         common.parse_replace_arg, "only_fles")
4278
4279     def test_abbreviate_time(self):
4280         self.failUnlessReallyEqual(common.abbreviate_time(None), "")
4281         self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
4282         self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
4283         self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
4284         self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
4285         self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
4286
4287     def test_compute_rate(self):
4288         self.failUnlessReallyEqual(common.compute_rate(None, None), None)
4289         self.failUnlessReallyEqual(common.compute_rate(None, 1), None)
4290         self.failUnlessReallyEqual(common.compute_rate(250000, None), None)
4291         self.failUnlessReallyEqual(common.compute_rate(250000, 0), None)
4292         self.failUnlessReallyEqual(common.compute_rate(250000, 10), 25000.0)
4293         self.failUnlessReallyEqual(common.compute_rate(0, 10), 0.0)
4294         self.shouldFail(AssertionError, "test_compute_rate", "",
4295                         common.compute_rate, -100, 10)
4296         self.shouldFail(AssertionError, "test_compute_rate", "",
4297                         common.compute_rate, 100, -10)
4298
4299         # Sanity check
4300         rate = common.compute_rate(10*1000*1000, 1)
4301         self.failUnlessReallyEqual(common.abbreviate_rate(rate), "10.00MBps")
4302
4303     def test_abbreviate_rate(self):
4304         self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
4305         self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
4306         self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
4307         self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
4308
4309     def test_abbreviate_size(self):
4310         self.failUnlessReallyEqual(common.abbreviate_size(None), "")
4311         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
4312         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
4313         self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
4314         self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
4315
4316     def test_plural(self):
4317         def convert(s):
4318             return "%d second%s" % (s, status.plural(s))
4319         self.failUnlessReallyEqual(convert(0), "0 seconds")
4320         self.failUnlessReallyEqual(convert(1), "1 second")
4321         self.failUnlessReallyEqual(convert(2), "2 seconds")
4322         def convert2(s):
4323             return "has share%s: %s" % (status.plural(s), ",".join(s))
4324         self.failUnlessReallyEqual(convert2([]), "has shares: ")
4325         self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
4326         self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
4327
4328
4329 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4330
4331     def CHECK(self, ign, which, args, clientnum=0):
4332         fileurl = self.fileurls[which]
4333         url = fileurl + "?" + args
4334         return self.GET(url, method="POST", clientnum=clientnum)
4335
4336     def test_filecheck(self):
4337         self.basedir = "web/Grid/filecheck"
4338         self.set_up_grid()
4339         c0 = self.g.clients[0]
4340         self.uris = {}
4341         DATA = "data" * 100
4342         d = c0.upload(upload.Data(DATA, convergence=""))
4343         def _stash_uri(ur, which):
4344             self.uris[which] = ur.get_uri()
4345         d.addCallback(_stash_uri, "good")
4346         d.addCallback(lambda ign:
4347                       c0.upload(upload.Data(DATA+"1", convergence="")))
4348         d.addCallback(_stash_uri, "sick")
4349         d.addCallback(lambda ign:
4350                       c0.upload(upload.Data(DATA+"2", convergence="")))
4351         d.addCallback(_stash_uri, "dead")
4352         def _stash_mutable_uri(n, which):
4353             self.uris[which] = n.get_uri()
4354             assert isinstance(self.uris[which], str)
4355         d.addCallback(lambda ign:
4356             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4357         d.addCallback(_stash_mutable_uri, "corrupt")
4358         d.addCallback(lambda ign:
4359                       c0.upload(upload.Data("literal", convergence="")))
4360         d.addCallback(_stash_uri, "small")
4361         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
4362         d.addCallback(_stash_mutable_uri, "smalldir")
4363
4364         def _compute_fileurls(ignored):
4365             self.fileurls = {}
4366             for which in self.uris:
4367                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4368         d.addCallback(_compute_fileurls)
4369
4370         def _clobber_shares(ignored):
4371             good_shares = self.find_uri_shares(self.uris["good"])
4372             self.failUnlessReallyEqual(len(good_shares), 10)
4373             sick_shares = self.find_uri_shares(self.uris["sick"])
4374             os.unlink(sick_shares[0][2])
4375             dead_shares = self.find_uri_shares(self.uris["dead"])
4376             for i in range(1, 10):
4377                 os.unlink(dead_shares[i][2])
4378             c_shares = self.find_uri_shares(self.uris["corrupt"])
4379             cso = CorruptShareOptions()
4380             cso.stdout = StringIO()
4381             cso.parseOptions([c_shares[0][2]])
4382             corrupt_share(cso)
4383         d.addCallback(_clobber_shares)
4384
4385         d.addCallback(self.CHECK, "good", "t=check")
4386         def _got_html_good(res):
4387             self.failUnlessIn("Healthy", res)
4388             self.failIfIn("Not Healthy", res)
4389             self.failUnlessIn(FAVICON_MARKUP, res)
4390         d.addCallback(_got_html_good)
4391         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
4392         def _got_html_good_return_to(res):
4393             self.failUnlessIn("Healthy", res)
4394             self.failIfIn("Not Healthy", res)
4395             self.failUnlessIn('<a href="somewhere">Return to file', res)
4396         d.addCallback(_got_html_good_return_to)
4397         d.addCallback(self.CHECK, "good", "t=check&output=json")
4398         def _got_json_good(res):
4399             r = simplejson.loads(res)
4400             self.failUnlessEqual(r["summary"], "Healthy")
4401             self.failUnless(r["results"]["healthy"])
4402             self.failIf(r["results"]["needs-rebalancing"])
4403             self.failUnless(r["results"]["recoverable"])
4404         d.addCallback(_got_json_good)
4405
4406         d.addCallback(self.CHECK, "small", "t=check")
4407         def _got_html_small(res):
4408             self.failUnlessIn("Literal files are always healthy", res)
4409             self.failIfIn("Not Healthy", res)
4410         d.addCallback(_got_html_small)
4411         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
4412         def _got_html_small_return_to(res):
4413             self.failUnlessIn("Literal files are always healthy", res)
4414             self.failIfIn("Not Healthy", res)
4415             self.failUnlessIn('<a href="somewhere">Return to file', res)
4416         d.addCallback(_got_html_small_return_to)
4417         d.addCallback(self.CHECK, "small", "t=check&output=json")
4418         def _got_json_small(res):
4419             r = simplejson.loads(res)
4420             self.failUnlessEqual(r["storage-index"], "")
4421             self.failUnless(r["results"]["healthy"])
4422         d.addCallback(_got_json_small)
4423
4424         d.addCallback(self.CHECK, "smalldir", "t=check")
4425         def _got_html_smalldir(res):
4426             self.failUnlessIn("Literal files are always healthy", res)
4427             self.failIfIn("Not Healthy", res)
4428         d.addCallback(_got_html_smalldir)
4429         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
4430         def _got_json_smalldir(res):
4431             r = simplejson.loads(res)
4432             self.failUnlessEqual(r["storage-index"], "")
4433             self.failUnless(r["results"]["healthy"])
4434         d.addCallback(_got_json_smalldir)
4435
4436         d.addCallback(self.CHECK, "sick", "t=check")
4437         def _got_html_sick(res):
4438             self.failUnlessIn("Not Healthy", res)
4439         d.addCallback(_got_html_sick)
4440         d.addCallback(self.CHECK, "sick", "t=check&output=json")
4441         def _got_json_sick(res):
4442             r = simplejson.loads(res)
4443             self.failUnlessEqual(r["summary"],
4444                                  "Not Healthy: 9 shares (enc 3-of-10)")
4445             self.failIf(r["results"]["healthy"])
4446             self.failIf(r["results"]["needs-rebalancing"])
4447             self.failUnless(r["results"]["recoverable"])
4448         d.addCallback(_got_json_sick)
4449
4450         d.addCallback(self.CHECK, "dead", "t=check")
4451         def _got_html_dead(res):
4452             self.failUnlessIn("Not Healthy", res)
4453         d.addCallback(_got_html_dead)
4454         d.addCallback(self.CHECK, "dead", "t=check&output=json")
4455         def _got_json_dead(res):
4456             r = simplejson.loads(res)
4457             self.failUnlessEqual(r["summary"],
4458                                  "Not Healthy: 1 shares (enc 3-of-10)")
4459             self.failIf(r["results"]["healthy"])
4460             self.failIf(r["results"]["needs-rebalancing"])
4461             self.failIf(r["results"]["recoverable"])
4462         d.addCallback(_got_json_dead)
4463
4464         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
4465         def _got_html_corrupt(res):
4466             self.failUnlessIn("Not Healthy! : Unhealthy", res)
4467         d.addCallback(_got_html_corrupt)
4468         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
4469         def _got_json_corrupt(res):
4470             r = simplejson.loads(res)
4471             self.failUnlessIn("Unhealthy: 9 shares (enc 3-of-10)", r["summary"])
4472             self.failIf(r["results"]["healthy"])
4473             self.failUnless(r["results"]["recoverable"])
4474             self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
4475             self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
4476         d.addCallback(_got_json_corrupt)
4477
4478         d.addErrback(self.explain_web_error)
4479         return d
4480
4481     def test_repair_html(self):
4482         self.basedir = "web/Grid/repair_html"
4483         self.set_up_grid()
4484         c0 = self.g.clients[0]
4485         self.uris = {}
4486         DATA = "data" * 100
4487         d = c0.upload(upload.Data(DATA, convergence=""))
4488         def _stash_uri(ur, which):
4489             self.uris[which] = ur.get_uri()
4490         d.addCallback(_stash_uri, "good")
4491         d.addCallback(lambda ign:
4492                       c0.upload(upload.Data(DATA+"1", convergence="")))
4493         d.addCallback(_stash_uri, "sick")
4494         d.addCallback(lambda ign:
4495                       c0.upload(upload.Data(DATA+"2", convergence="")))
4496         d.addCallback(_stash_uri, "dead")
4497         def _stash_mutable_uri(n, which):
4498             self.uris[which] = n.get_uri()
4499             assert isinstance(self.uris[which], str)
4500         d.addCallback(lambda ign:
4501             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4502         d.addCallback(_stash_mutable_uri, "corrupt")
4503
4504         def _compute_fileurls(ignored):
4505             self.fileurls = {}
4506             for which in self.uris:
4507                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4508         d.addCallback(_compute_fileurls)
4509
4510         def _clobber_shares(ignored):
4511             good_shares = self.find_uri_shares(self.uris["good"])
4512             self.failUnlessReallyEqual(len(good_shares), 10)
4513             sick_shares = self.find_uri_shares(self.uris["sick"])
4514             os.unlink(sick_shares[0][2])
4515             dead_shares = self.find_uri_shares(self.uris["dead"])
4516             for i in range(1, 10):
4517                 os.unlink(dead_shares[i][2])
4518             c_shares = self.find_uri_shares(self.uris["corrupt"])
4519             cso = CorruptShareOptions()
4520             cso.stdout = StringIO()
4521             cso.parseOptions([c_shares[0][2]])
4522             corrupt_share(cso)
4523         d.addCallback(_clobber_shares)
4524
4525         d.addCallback(self.CHECK, "good", "t=check&repair=true")
4526         def _got_html_good(res):
4527             self.failUnlessIn("Healthy", res)
4528             self.failIfIn("Not Healthy", res)
4529             self.failUnlessIn("No repair necessary", res)
4530             self.failUnlessIn(FAVICON_MARKUP, res)
4531         d.addCallback(_got_html_good)
4532
4533         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
4534         def _got_html_sick(res):
4535             self.failUnlessIn("Healthy : healthy", res)
4536             self.failIfIn("Not Healthy", res)
4537             self.failUnlessIn("Repair successful", res)
4538         d.addCallback(_got_html_sick)
4539
4540         # repair of a dead file will fail, of course, but it isn't yet
4541         # clear how this should be reported. Right now it shows up as
4542         # a "410 Gone".
4543         #
4544         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
4545         #def _got_html_dead(res):
4546         #    print res
4547         #    self.failUnlessIn("Healthy : healthy", res)
4548         #    self.failIfIn("Not Healthy", res)
4549         #    self.failUnlessIn("No repair necessary", res)
4550         #d.addCallback(_got_html_dead)
4551
4552         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
4553         def _got_html_corrupt(res):
4554             self.failUnlessIn("Healthy : Healthy", res)
4555             self.failIfIn("Not Healthy", res)
4556             self.failUnlessIn("Repair successful", res)
4557         d.addCallback(_got_html_corrupt)
4558
4559         d.addErrback(self.explain_web_error)
4560         return d
4561
4562     def test_repair_json(self):
4563         self.basedir = "web/Grid/repair_json"
4564         self.set_up_grid()
4565         c0 = self.g.clients[0]
4566         self.uris = {}
4567         DATA = "data" * 100
4568         d = c0.upload(upload.Data(DATA+"1", convergence=""))
4569         def _stash_uri(ur, which):
4570             self.uris[which] = ur.get_uri()
4571         d.addCallback(_stash_uri, "sick")
4572
4573         def _compute_fileurls(ignored):
4574             self.fileurls = {}
4575             for which in self.uris:
4576                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4577         d.addCallback(_compute_fileurls)
4578
4579         def _clobber_shares(ignored):
4580             sick_shares = self.find_uri_shares(self.uris["sick"])
4581             os.unlink(sick_shares[0][2])
4582         d.addCallback(_clobber_shares)
4583
4584         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
4585         def _got_json_sick(res):
4586             r = simplejson.loads(res)
4587             self.failUnlessReallyEqual(r["repair-attempted"], True)
4588             self.failUnlessReallyEqual(r["repair-successful"], True)
4589             self.failUnlessEqual(r["pre-repair-results"]["summary"],
4590                                  "Not Healthy: 9 shares (enc 3-of-10)")
4591             self.failIf(r["pre-repair-results"]["results"]["healthy"])
4592             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
4593             self.failUnless(r["post-repair-results"]["results"]["healthy"])
4594         d.addCallback(_got_json_sick)
4595
4596         d.addErrback(self.explain_web_error)
4597         return d
4598
4599     def test_unknown(self, immutable=False):
4600         self.basedir = "web/Grid/unknown"
4601         if immutable:
4602             self.basedir = "web/Grid/unknown-immutable"
4603
4604         self.set_up_grid()
4605         c0 = self.g.clients[0]
4606         self.uris = {}
4607         self.fileurls = {}
4608
4609         # the future cap format may contain slashes, which must be tolerated
4610         expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
4611                                                            safe="")
4612
4613         if immutable:
4614             name = u"future-imm"
4615             future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
4616             d = c0.create_immutable_dirnode({name: (future_node, {})})
4617         else:
4618             name = u"future"
4619             future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4620             d = c0.create_dirnode()
4621
4622         def _stash_root_and_create_file(n):
4623             self.rootnode = n
4624             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
4625             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
4626             if not immutable:
4627                 return self.rootnode.set_node(name, future_node)
4628         d.addCallback(_stash_root_and_create_file)
4629
4630         # make sure directory listing tolerates unknown nodes
4631         d.addCallback(lambda ign: self.GET(self.rooturl))
4632         def _check_directory_html(res, expected_type_suffix):
4633             pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
4634                                   '<td>%s</td>' % (expected_type_suffix, str(name)),
4635                                  re.DOTALL)
4636             self.failUnless(re.search(pattern, res), res)
4637             # find the More Info link for name, should be relative
4638             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4639             info_url = mo.group(1)
4640             self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
4641         if immutable:
4642             d.addCallback(_check_directory_html, "-IMM")
4643         else:
4644             d.addCallback(_check_directory_html, "")
4645
4646         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4647         def _check_directory_json(res, expect_rw_uri):
4648             data = simplejson.loads(res)
4649             self.failUnlessEqual(data[0], "dirnode")
4650             f = data[1]["children"][name]
4651             self.failUnlessEqual(f[0], "unknown")
4652             if expect_rw_uri:
4653                 self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data)
4654             else:
4655                 self.failIfIn("rw_uri", f[1])
4656             if immutable:
4657                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
4658             else:
4659                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
4660             self.failUnlessIn("metadata", f[1])
4661         d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
4662
4663         def _check_info(res, expect_rw_uri, expect_ro_uri):
4664             self.failUnlessIn("Object Type: <span>unknown</span>", res)
4665             if expect_rw_uri:
4666                 self.failUnlessIn(unknown_rwcap, res)
4667             if expect_ro_uri:
4668                 if immutable:
4669                     self.failUnlessIn(unknown_immcap, res)
4670                 else:
4671                     self.failUnlessIn(unknown_rocap, res)
4672             else:
4673                 self.failIfIn(unknown_rocap, res)
4674             self.failIfIn("Raw data as", res)
4675             self.failIfIn("Directory writecap", res)
4676             self.failIfIn("Checker Operations", res)
4677             self.failIfIn("Mutable File Operations", res)
4678             self.failIfIn("Directory Operations", res)
4679
4680         # FIXME: these should have expect_rw_uri=not immutable; I don't know
4681         # why they fail. Possibly related to ticket #922.
4682
4683         d.addCallback(lambda ign: self.GET(expected_info_url))
4684         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
4685         d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
4686         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
4687
4688         def _check_json(res, expect_rw_uri):
4689             data = simplejson.loads(res)
4690             self.failUnlessEqual(data[0], "unknown")
4691             if expect_rw_uri:
4692                 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data)
4693             else:
4694                 self.failIfIn("rw_uri", data[1])
4695
4696             if immutable:
4697                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data)
4698                 self.failUnlessReallyEqual(data[1]["mutable"], False)
4699             elif expect_rw_uri:
4700                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4701                 self.failUnlessReallyEqual(data[1]["mutable"], True)
4702             else:
4703                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4704                 self.failIfIn("mutable", data[1])
4705
4706             # TODO: check metadata contents
4707             self.failUnlessIn("metadata", data[1])
4708
4709         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
4710         d.addCallback(_check_json, expect_rw_uri=not immutable)
4711
4712         # and make sure that a read-only version of the directory can be
4713         # rendered too. This version will not have unknown_rwcap, whether
4714         # or not future_node was immutable.
4715         d.addCallback(lambda ign: self.GET(self.rourl))
4716         if immutable:
4717             d.addCallback(_check_directory_html, "-IMM")
4718         else:
4719             d.addCallback(_check_directory_html, "-RO")
4720
4721         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
4722         d.addCallback(_check_directory_json, expect_rw_uri=False)
4723
4724         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
4725         d.addCallback(_check_json, expect_rw_uri=False)
4726
4727         # TODO: check that getting t=info from the Info link in the ro directory
4728         # works, and does not include the writecap URI.
4729         return d
4730
4731     def test_immutable_unknown(self):
4732         return self.test_unknown(immutable=True)
4733
4734     def test_mutant_dirnodes_are_omitted(self):
4735         self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
4736
4737         self.set_up_grid()
4738         c = self.g.clients[0]
4739         nm = c.nodemaker
4740         self.uris = {}
4741         self.fileurls = {}
4742
4743         lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
4744         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
4745         mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
4746
4747         # This method tests mainly dirnode, but we'd have to duplicate code in order to
4748         # test the dirnode and web layers separately.
4749
4750         # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
4751         # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
4752         # When the directory is read, the mutants should be silently disposed of, leaving
4753         # their lonely sibling.
4754         # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
4755         # because immutable directories don't have a writecap and therefore that field
4756         # isn't (and can't be) decrypted.
4757         # TODO: The field still exists in the netstring. Technically we should check what
4758         # happens if something is put there (_unpack_contents should raise ValueError),
4759         # but that can wait.
4760
4761         lonely_child = nm.create_from_cap(lonely_uri)
4762         mutant_ro_child = nm.create_from_cap(mut_read_uri)
4763         mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
4764
4765         def _by_hook_or_by_crook():
4766             return True
4767         for n in [mutant_ro_child, mutant_write_in_ro_child]:
4768             n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
4769
4770         mutant_write_in_ro_child.get_write_uri    = lambda: None
4771         mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
4772
4773         kids = {u"lonely":      (lonely_child, {}),
4774                 u"ro":          (mutant_ro_child, {}),
4775                 u"write-in-ro": (mutant_write_in_ro_child, {}),
4776                 }
4777         d = c.create_immutable_dirnode(kids)
4778
4779         def _created(dn):
4780             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
4781             self.failIf(dn.is_mutable())
4782             self.failUnless(dn.is_readonly())
4783             # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
4784             self.failIf(hasattr(dn._node, 'get_writekey'))
4785             rep = str(dn)
4786             self.failUnlessIn("RO-IMM", rep)
4787             cap = dn.get_cap()
4788             self.failUnlessIn("CHK", cap.to_string())
4789             self.cap = cap
4790             self.rootnode = dn
4791             self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
4792             return download_to_data(dn._node)
4793         d.addCallback(_created)
4794
4795         def _check_data(data):
4796             # Decode the netstring representation of the directory to check that all children
4797             # are present. This is a bit of an abstraction violation, but there's not really
4798             # any other way to do it given that the real DirectoryNode._unpack_contents would
4799             # strip the mutant children out (which is what we're trying to test, later).
4800             position = 0
4801             numkids = 0
4802             while position < len(data):
4803                 entries, position = split_netstring(data, 1, position)
4804                 entry = entries[0]
4805                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
4806                 name = name_utf8.decode("utf-8")
4807                 self.failUnlessEqual(rwcapdata, "")
4808                 self.failUnlessIn(name, kids)
4809                 (expected_child, ign) = kids[name]
4810                 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
4811                 numkids += 1
4812
4813             self.failUnlessReallyEqual(numkids, 3)
4814             return self.rootnode.list()
4815         d.addCallback(_check_data)
4816
4817         # Now when we use the real directory listing code, the mutants should be absent.
4818         def _check_kids(children):
4819             self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
4820             lonely_node, lonely_metadata = children[u"lonely"]
4821
4822             self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
4823             self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
4824         d.addCallback(_check_kids)
4825
4826         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
4827         d.addCallback(lambda n: n.list())
4828         d.addCallback(_check_kids)  # again with dirnode recreated from cap
4829
4830         # Make sure the lonely child can be listed in HTML...
4831         d.addCallback(lambda ign: self.GET(self.rooturl))
4832         def _check_html(res):
4833             self.failIfIn("URI:SSK", res)
4834             get_lonely = "".join([r'<td>FILE</td>',
4835                                   r'\s+<td>',
4836                                   r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
4837                                   r'</td>',
4838                                   r'\s+<td align="right">%d</td>' % len("one"),
4839                                   ])
4840             self.failUnless(re.search(get_lonely, res), res)
4841
4842             # find the More Info link for name, should be relative
4843             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4844             info_url = mo.group(1)
4845             self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
4846         d.addCallback(_check_html)
4847
4848         # ... and in JSON.
4849         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4850         def _check_json(res):
4851             data = simplejson.loads(res)
4852             self.failUnlessEqual(data[0], "dirnode")
4853             listed_children = data[1]["children"]
4854             self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
4855             ll_type, ll_data = listed_children[u"lonely"]
4856             self.failUnlessEqual(ll_type, "filenode")
4857             self.failIfIn("rw_uri", ll_data)
4858             self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
4859         d.addCallback(_check_json)
4860         return d
4861
4862     def test_deep_check(self):
4863         self.basedir = "web/Grid/deep_check"
4864         self.set_up_grid()
4865         c0 = self.g.clients[0]
4866         self.uris = {}
4867         self.fileurls = {}
4868         DATA = "data" * 100
4869         d = c0.create_dirnode()
4870         def _stash_root_and_create_file(n):
4871             self.rootnode = n
4872             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4873             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4874         d.addCallback(_stash_root_and_create_file)
4875         def _stash_uri(fn, which):
4876             self.uris[which] = fn.get_uri()
4877             return fn
4878         d.addCallback(_stash_uri, "good")
4879         d.addCallback(lambda ign:
4880                       self.rootnode.add_file(u"small",
4881                                              upload.Data("literal",
4882                                                         convergence="")))
4883         d.addCallback(_stash_uri, "small")
4884         d.addCallback(lambda ign:
4885                       self.rootnode.add_file(u"sick",
4886                                              upload.Data(DATA+"1",
4887                                                         convergence="")))
4888         d.addCallback(_stash_uri, "sick")
4889
4890         # this tests that deep-check and stream-manifest will ignore
4891         # UnknownNode instances. Hopefully this will also cover deep-stats.
4892         future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4893         d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
4894
4895         def _clobber_shares(ignored):
4896             self.delete_shares_numbered(self.uris["sick"], [0,1])
4897         d.addCallback(_clobber_shares)
4898
4899         # root
4900         # root/good
4901         # root/small
4902         # root/sick
4903         # root/future
4904
4905         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4906         def _done(res):
4907             try:
4908                 units = [simplejson.loads(line)
4909                          for line in res.splitlines()
4910                          if line]
4911             except ValueError:
4912                 print "response is:", res
4913                 print "undecodeable line was '%s'" % line
4914                 raise
4915             self.failUnlessReallyEqual(len(units), 5+1)
4916             # should be parent-first
4917             u0 = units[0]
4918             self.failUnlessEqual(u0["path"], [])
4919             self.failUnlessEqual(u0["type"], "directory")
4920             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4921             u0cr = u0["check-results"]
4922             self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
4923
4924             ugood = [u for u in units
4925                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4926             self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
4927             ugoodcr = ugood["check-results"]
4928             self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
4929
4930             stats = units[-1]
4931             self.failUnlessEqual(stats["type"], "stats")
4932             s = stats["stats"]
4933             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4934             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4935             self.failUnlessReallyEqual(s["count-directories"], 1)
4936             self.failUnlessReallyEqual(s["count-unknown"], 1)
4937         d.addCallback(_done)
4938
4939         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4940         def _check_manifest(res):
4941             self.failUnless(res.endswith("\n"))
4942             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
4943             self.failUnlessReallyEqual(len(units), 5+1)
4944             self.failUnlessEqual(units[-1]["type"], "stats")
4945             first = units[0]
4946             self.failUnlessEqual(first["path"], [])
4947             self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri())
4948             self.failUnlessEqual(first["type"], "directory")
4949             stats = units[-1]["stats"]
4950             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4951             self.failUnlessReallyEqual(stats["count-literal-files"], 1)
4952             self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
4953             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4954             self.failUnlessReallyEqual(stats["count-unknown"], 1)
4955         d.addCallback(_check_manifest)
4956
4957         # now add root/subdir and root/subdir/grandchild, then make subdir
4958         # unrecoverable, then see what happens
4959
4960         d.addCallback(lambda ign:
4961                       self.rootnode.create_subdirectory(u"subdir"))
4962         d.addCallback(_stash_uri, "subdir")
4963         d.addCallback(lambda subdir_node:
4964                       subdir_node.add_file(u"grandchild",
4965                                            upload.Data(DATA+"2",
4966                                                        convergence="")))
4967         d.addCallback(_stash_uri, "grandchild")
4968
4969         d.addCallback(lambda ign:
4970                       self.delete_shares_numbered(self.uris["subdir"],
4971                                                   range(1, 10)))
4972
4973         # root
4974         # root/good
4975         # root/small
4976         # root/sick
4977         # root/future
4978         # root/subdir [unrecoverable]
4979         # root/subdir/grandchild
4980
4981         # how should a streaming-JSON API indicate fatal error?
4982         # answer: emit ERROR: instead of a JSON string
4983
4984         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4985         def _check_broken_manifest(res):
4986             lines = res.splitlines()
4987             error_lines = [i
4988                            for (i,line) in enumerate(lines)
4989                            if line.startswith("ERROR:")]
4990             if not error_lines:
4991                 self.fail("no ERROR: in output: %s" % (res,))
4992             first_error = error_lines[0]
4993             error_line = lines[first_error]
4994             error_msg = lines[first_error+1:]
4995             error_msg_s = "\n".join(error_msg) + "\n"
4996             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4997                               error_line)
4998             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4999             units = [simplejson.loads(line) for line in lines[:first_error]]
5000             self.failUnlessReallyEqual(len(units), 6) # includes subdir
5001             last_unit = units[-1]
5002             self.failUnlessEqual(last_unit["path"], ["subdir"])
5003         d.addCallback(_check_broken_manifest)
5004
5005         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
5006         def _check_broken_deepcheck(res):
5007             lines = res.splitlines()
5008             error_lines = [i
5009                            for (i,line) in enumerate(lines)
5010                            if line.startswith("ERROR:")]
5011             if not error_lines:
5012                 self.fail("no ERROR: in output: %s" % (res,))
5013             first_error = error_lines[0]
5014             error_line = lines[first_error]
5015             error_msg = lines[first_error+1:]
5016             error_msg_s = "\n".join(error_msg) + "\n"
5017             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
5018                               error_line)
5019             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
5020             units = [simplejson.loads(line) for line in lines[:first_error]]
5021             self.failUnlessReallyEqual(len(units), 6) # includes subdir
5022             last_unit = units[-1]
5023             self.failUnlessEqual(last_unit["path"], ["subdir"])
5024             r = last_unit["check-results"]["results"]
5025             self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
5026             self.failUnlessReallyEqual(r["count-shares-good"], 1)
5027             self.failUnlessReallyEqual(r["recoverable"], False)
5028         d.addCallback(_check_broken_deepcheck)
5029
5030         d.addErrback(self.explain_web_error)
5031         return d
5032
5033     def test_deep_check_and_repair(self):
5034         self.basedir = "web/Grid/deep_check_and_repair"
5035         self.set_up_grid()
5036         c0 = self.g.clients[0]
5037         self.uris = {}
5038         self.fileurls = {}
5039         DATA = "data" * 100
5040         d = c0.create_dirnode()
5041         def _stash_root_and_create_file(n):
5042             self.rootnode = n
5043             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5044             return n.add_file(u"good", upload.Data(DATA, convergence=""))
5045         d.addCallback(_stash_root_and_create_file)
5046         def _stash_uri(fn, which):
5047             self.uris[which] = fn.get_uri()
5048         d.addCallback(_stash_uri, "good")
5049         d.addCallback(lambda ign:
5050                       self.rootnode.add_file(u"small",
5051                                              upload.Data("literal",
5052                                                         convergence="")))
5053         d.addCallback(_stash_uri, "small")
5054         d.addCallback(lambda ign:
5055                       self.rootnode.add_file(u"sick",
5056                                              upload.Data(DATA+"1",
5057                                                         convergence="")))
5058         d.addCallback(_stash_uri, "sick")
5059         #d.addCallback(lambda ign:
5060         #              self.rootnode.add_file(u"dead",
5061         #                                     upload.Data(DATA+"2",
5062         #                                                convergence="")))
5063         #d.addCallback(_stash_uri, "dead")
5064
5065         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
5066         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
5067         #d.addCallback(_stash_uri, "corrupt")
5068
5069         def _clobber_shares(ignored):
5070             good_shares = self.find_uri_shares(self.uris["good"])
5071             self.failUnlessReallyEqual(len(good_shares), 10)
5072             sick_shares = self.find_uri_shares(self.uris["sick"])
5073             os.unlink(sick_shares[0][2])
5074             #dead_shares = self.find_uri_shares(self.uris["dead"])
5075             #for i in range(1, 10):
5076             #    os.unlink(dead_shares[i][2])
5077
5078             #c_shares = self.find_uri_shares(self.uris["corrupt"])
5079             #cso = CorruptShareOptions()
5080             #cso.stdout = StringIO()
5081             #cso.parseOptions([c_shares[0][2]])
5082             #corrupt_share(cso)
5083         d.addCallback(_clobber_shares)
5084
5085         # root
5086         # root/good   CHK, 10 shares
5087         # root/small  LIT
5088         # root/sick   CHK, 9 shares
5089
5090         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
5091         def _done(res):
5092             units = [simplejson.loads(line)
5093                      for line in res.splitlines()
5094                      if line]
5095             self.failUnlessReallyEqual(len(units), 4+1)
5096             # should be parent-first
5097             u0 = units[0]
5098             self.failUnlessEqual(u0["path"], [])
5099             self.failUnlessEqual(u0["type"], "directory")
5100             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
5101             u0crr = u0["check-and-repair-results"]
5102             self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
5103             self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
5104
5105             ugood = [u for u in units
5106                      if u["type"] == "file" and u["path"] == [u"good"]][0]
5107             self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
5108             ugoodcrr = ugood["check-and-repair-results"]
5109             self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
5110             self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
5111
5112             usick = [u for u in units
5113                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
5114             self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"])
5115             usickcrr = usick["check-and-repair-results"]
5116             self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
5117             self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
5118             self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
5119             self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
5120
5121             stats = units[-1]
5122             self.failUnlessEqual(stats["type"], "stats")
5123             s = stats["stats"]
5124             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
5125             self.failUnlessReallyEqual(s["count-literal-files"], 1)
5126             self.failUnlessReallyEqual(s["count-directories"], 1)
5127         d.addCallback(_done)
5128
5129         d.addErrback(self.explain_web_error)
5130         return d
5131
5132     def _count_leases(self, ignored, which):
5133         u = self.uris[which]
5134         shares = self.find_uri_shares(u)
5135         lease_counts = []
5136         for shnum, serverid, fn in shares:
5137             sf = get_share_file(fn)
5138             num_leases = len(list(sf.get_leases()))
5139             lease_counts.append( (fn, num_leases) )
5140         return lease_counts
5141
5142     def _assert_leasecount(self, lease_counts, expected):
5143         for (fn, num_leases) in lease_counts:
5144             if num_leases != expected:
5145                 self.fail("expected %d leases, have %d, on %s" %
5146                           (expected, num_leases, fn))
5147
5148     def test_add_lease(self):
5149         self.basedir = "web/Grid/add_lease"
5150         self.set_up_grid(num_clients=2)
5151         c0 = self.g.clients[0]
5152         self.uris = {}
5153         DATA = "data" * 100
5154         d = c0.upload(upload.Data(DATA, convergence=""))
5155         def _stash_uri(ur, which):
5156             self.uris[which] = ur.get_uri()
5157         d.addCallback(_stash_uri, "one")
5158         d.addCallback(lambda ign:
5159                       c0.upload(upload.Data(DATA+"1", convergence="")))
5160         d.addCallback(_stash_uri, "two")
5161         def _stash_mutable_uri(n, which):
5162             self.uris[which] = n.get_uri()
5163             assert isinstance(self.uris[which], str)
5164         d.addCallback(lambda ign:
5165             c0.create_mutable_file(publish.MutableData(DATA+"2")))
5166         d.addCallback(_stash_mutable_uri, "mutable")
5167
5168         def _compute_fileurls(ignored):
5169             self.fileurls = {}
5170             for which in self.uris:
5171                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
5172         d.addCallback(_compute_fileurls)
5173
5174         d.addCallback(self._count_leases, "one")
5175         d.addCallback(self._assert_leasecount, 1)
5176         d.addCallback(self._count_leases, "two")
5177         d.addCallback(self._assert_leasecount, 1)
5178         d.addCallback(self._count_leases, "mutable")
5179         d.addCallback(self._assert_leasecount, 1)
5180
5181         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
5182         def _got_html_good(res):
5183             self.failUnlessIn("Healthy", res)
5184             self.failIfIn("Not Healthy", res)
5185         d.addCallback(_got_html_good)
5186
5187         d.addCallback(self._count_leases, "one")
5188         d.addCallback(self._assert_leasecount, 1)
5189         d.addCallback(self._count_leases, "two")
5190         d.addCallback(self._assert_leasecount, 1)
5191         d.addCallback(self._count_leases, "mutable")
5192         d.addCallback(self._assert_leasecount, 1)
5193
5194         # this CHECK uses the original client, which uses the same
5195         # lease-secrets, so it will just renew the original lease
5196         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
5197         d.addCallback(_got_html_good)
5198
5199         d.addCallback(self._count_leases, "one")
5200         d.addCallback(self._assert_leasecount, 1)
5201         d.addCallback(self._count_leases, "two")
5202         d.addCallback(self._assert_leasecount, 1)
5203         d.addCallback(self._count_leases, "mutable")
5204         d.addCallback(self._assert_leasecount, 1)
5205
5206         # this CHECK uses an alternate client, which adds a second lease
5207         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
5208         d.addCallback(_got_html_good)
5209
5210         d.addCallback(self._count_leases, "one")
5211         d.addCallback(self._assert_leasecount, 2)
5212         d.addCallback(self._count_leases, "two")
5213         d.addCallback(self._assert_leasecount, 1)
5214         d.addCallback(self._count_leases, "mutable")
5215         d.addCallback(self._assert_leasecount, 1)
5216
5217         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
5218         d.addCallback(_got_html_good)
5219
5220         d.addCallback(self._count_leases, "one")
5221         d.addCallback(self._assert_leasecount, 2)
5222         d.addCallback(self._count_leases, "two")
5223         d.addCallback(self._assert_leasecount, 1)
5224         d.addCallback(self._count_leases, "mutable")
5225         d.addCallback(self._assert_leasecount, 1)
5226
5227         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
5228                       clientnum=1)
5229         d.addCallback(_got_html_good)
5230
5231         d.addCallback(self._count_leases, "one")
5232         d.addCallback(self._assert_leasecount, 2)
5233         d.addCallback(self._count_leases, "two")
5234         d.addCallback(self._assert_leasecount, 1)
5235         d.addCallback(self._count_leases, "mutable")
5236         d.addCallback(self._assert_leasecount, 2)
5237
5238         d.addErrback(self.explain_web_error)
5239         return d
5240
5241     def test_deep_add_lease(self):
5242         self.basedir = "web/Grid/deep_add_lease"
5243         self.set_up_grid(num_clients=2)
5244         c0 = self.g.clients[0]
5245         self.uris = {}
5246         self.fileurls = {}
5247         DATA = "data" * 100
5248         d = c0.create_dirnode()
5249         def _stash_root_and_create_file(n):
5250             self.rootnode = n
5251             self.uris["root"] = n.get_uri()
5252             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5253             return n.add_file(u"one", upload.Data(DATA, convergence=""))
5254         d.addCallback(_stash_root_and_create_file)
5255         def _stash_uri(fn, which):
5256             self.uris[which] = fn.get_uri()
5257         d.addCallback(_stash_uri, "one")
5258         d.addCallback(lambda ign:
5259                       self.rootnode.add_file(u"small",
5260                                              upload.Data("literal",
5261                                                         convergence="")))
5262         d.addCallback(_stash_uri, "small")
5263
5264         d.addCallback(lambda ign:
5265             c0.create_mutable_file(publish.MutableData("mutable")))
5266         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
5267         d.addCallback(_stash_uri, "mutable")
5268
5269         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
5270         def _done(res):
5271             units = [simplejson.loads(line)
5272                      for line in res.splitlines()
5273                      if line]
5274             # root, one, small, mutable,   stats
5275             self.failUnlessReallyEqual(len(units), 4+1)
5276         d.addCallback(_done)
5277
5278         d.addCallback(self._count_leases, "root")
5279         d.addCallback(self._assert_leasecount, 1)
5280         d.addCallback(self._count_leases, "one")
5281         d.addCallback(self._assert_leasecount, 1)
5282         d.addCallback(self._count_leases, "mutable")
5283         d.addCallback(self._assert_leasecount, 1)
5284
5285         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
5286         d.addCallback(_done)
5287
5288         d.addCallback(self._count_leases, "root")
5289         d.addCallback(self._assert_leasecount, 1)
5290         d.addCallback(self._count_leases, "one")
5291         d.addCallback(self._assert_leasecount, 1)
5292         d.addCallback(self._count_leases, "mutable")
5293         d.addCallback(self._assert_leasecount, 1)
5294
5295         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
5296                       clientnum=1)
5297         d.addCallback(_done)
5298
5299         d.addCallback(self._count_leases, "root")
5300         d.addCallback(self._assert_leasecount, 2)
5301         d.addCallback(self._count_leases, "one")
5302         d.addCallback(self._assert_leasecount, 2)
5303         d.addCallback(self._count_leases, "mutable")
5304         d.addCallback(self._assert_leasecount, 2)
5305
5306         d.addErrback(self.explain_web_error)
5307         return d
5308
5309
5310     def test_exceptions(self):
5311         self.basedir = "web/Grid/exceptions"
5312         self.set_up_grid(num_clients=1, num_servers=2)
5313         c0 = self.g.clients[0]
5314         c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 2
5315         self.fileurls = {}
5316         DATA = "data" * 100
5317         d = c0.create_dirnode()
5318         def _stash_root(n):
5319             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5320             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
5321             return n
5322         d.addCallback(_stash_root)
5323         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
5324         def _stash_bad(ur):
5325             self.fileurls["1share"] = "uri/" + urllib.quote(ur.get_uri())
5326             self.delete_shares_numbered(ur.get_uri(), range(1,10))
5327
5328             u = uri.from_string(ur.get_uri())
5329             u.key = testutil.flip_bit(u.key, 0)
5330             baduri = u.to_string()
5331             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
5332         d.addCallback(_stash_bad)
5333         d.addCallback(lambda ign: c0.create_dirnode())
5334         def _mangle_dirnode_1share(n):
5335             u = n.get_uri()
5336             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
5337             self.fileurls["dir-1share-json"] = url + "?t=json"
5338             self.delete_shares_numbered(u, range(1,10))
5339         d.addCallback(_mangle_dirnode_1share)
5340         d.addCallback(lambda ign: c0.create_dirnode())
5341         def _mangle_dirnode_0share(n):
5342             u = n.get_uri()
5343             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
5344             self.fileurls["dir-0share-json"] = url + "?t=json"
5345             self.delete_shares_numbered(u, range(0,10))
5346         d.addCallback(_mangle_dirnode_0share)
5347
5348         # NotEnoughSharesError should be reported sensibly, with a
5349         # text/plain explanation of the problem, and perhaps some
5350         # information on which shares *could* be found.
5351
5352         d.addCallback(lambda ignored:
5353                       self.shouldHTTPError("GET unrecoverable",
5354                                            410, "Gone", "NoSharesError",
5355                                            self.GET, self.fileurls["0shares"]))
5356         def _check_zero_shares(body):
5357             self.failIfIn("<html>", body)
5358             body = " ".join(body.strip().split())
5359             exp = ("NoSharesError: no shares could be found. "
5360                    "Zero shares usually indicates a corrupt URI, or that "
5361                    "no servers were connected, but it might also indicate "
5362                    "severe corruption. You should perform a filecheck on "
5363                    "this object to learn more. The full error message is: "
5364                    "no shares (need 3). Last failure: None")
5365             self.failUnlessReallyEqual(exp, body)
5366         d.addCallback(_check_zero_shares)
5367
5368
5369         d.addCallback(lambda ignored:
5370                       self.shouldHTTPError("GET 1share",
5371                                            410, "Gone", "NotEnoughSharesError",
5372                                            self.GET, self.fileurls["1share"]))
5373         def _check_one_share(body):
5374             self.failIfIn("<html>", body)
5375             body = " ".join(body.strip().split())
5376             msgbase = ("NotEnoughSharesError: This indicates that some "
5377                        "servers were unavailable, or that shares have been "
5378                        "lost to server departure, hard drive failure, or disk "
5379                        "corruption. You should perform a filecheck on "
5380                        "this object to learn more. The full error message is:"
5381                        )
5382             msg1 = msgbase + (" ran out of shares:"
5383                               " complete=sh0"
5384                               " pending="
5385                               " overdue= unused= need 3. Last failure: None")
5386             msg2 = msgbase + (" ran out of shares:"
5387                               " complete="
5388                               " pending=Share(sh0-on-xgru5)"
5389                               " overdue= unused= need 3. Last failure: None")
5390             self.failUnless(body == msg1 or body == msg2, body)
5391         d.addCallback(_check_one_share)
5392
5393         d.addCallback(lambda ignored:
5394                       self.shouldHTTPError("GET imaginary",
5395                                            404, "Not Found", None,
5396                                            self.GET, self.fileurls["imaginary"]))
5397         def _missing_child(body):
5398             self.failUnlessIn("No such child: imaginary", body)
5399         d.addCallback(_missing_child)
5400
5401         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
5402         def _check_0shares_dir_html(body):
5403             self.failUnlessIn("<html>", body)
5404             # we should see the regular page, but without the child table or
5405             # the dirops forms
5406             body = " ".join(body.strip().split())
5407             self.failUnlessIn('href="?t=info">More info on this directory',
5408                               body)
5409             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5410                    "could not be retrieved, because there were insufficient "
5411                    "good shares. This might indicate that no servers were "
5412                    "connected, insufficient servers were connected, the URI "
5413                    "was corrupt, or that shares have been lost due to server "
5414                    "departure, hard drive failure, or disk corruption. You "
5415                    "should perform a filecheck on this object to learn more.")
5416             self.failUnlessIn(exp, body)
5417             self.failUnlessIn("No upload forms: directory is unreadable", body)
5418         d.addCallback(_check_0shares_dir_html)
5419
5420         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
5421         def _check_1shares_dir_html(body):
5422             # at some point, we'll split UnrecoverableFileError into 0-shares
5423             # and some-shares like we did for immutable files (since there
5424             # are different sorts of advice to offer in each case). For now,
5425             # they present the same way.
5426             self.failUnlessIn("<html>", body)
5427             body = " ".join(body.strip().split())
5428             self.failUnlessIn('href="?t=info">More info on this directory',
5429                               body)
5430             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5431                    "could not be retrieved, because there were insufficient "
5432                    "good shares. This might indicate that no servers were "
5433                    "connected, insufficient servers were connected, the URI "
5434                    "was corrupt, or that shares have been lost due to server "
5435                    "departure, hard drive failure, or disk corruption. You "
5436                    "should perform a filecheck on this object to learn more.")
5437             self.failUnlessIn(exp, body)
5438             self.failUnlessIn("No upload forms: directory is unreadable", body)
5439         d.addCallback(_check_1shares_dir_html)
5440
5441         d.addCallback(lambda ignored:
5442                       self.shouldHTTPError("GET dir-0share-json",
5443                                            410, "Gone", "UnrecoverableFileError",
5444                                            self.GET,
5445                                            self.fileurls["dir-0share-json"]))
5446         def _check_unrecoverable_file(body):
5447             self.failIfIn("<html>", body)
5448             body = " ".join(body.strip().split())
5449             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5450                    "could not be retrieved, because there were insufficient "
5451                    "good shares. This might indicate that no servers were "
5452                    "connected, insufficient servers were connected, the URI "
5453                    "was corrupt, or that shares have been lost due to server "
5454                    "departure, hard drive failure, or disk corruption. You "
5455                    "should perform a filecheck on this object to learn more.")
5456             self.failUnlessReallyEqual(exp, body)
5457         d.addCallback(_check_unrecoverable_file)
5458
5459         d.addCallback(lambda ignored:
5460                       self.shouldHTTPError("GET dir-1share-json",
5461                                            410, "Gone", "UnrecoverableFileError",
5462                                            self.GET,
5463                                            self.fileurls["dir-1share-json"]))
5464         d.addCallback(_check_unrecoverable_file)
5465
5466         d.addCallback(lambda ignored:
5467                       self.shouldHTTPError("GET imaginary",
5468                                            404, "Not Found", None,
5469                                            self.GET, self.fileurls["imaginary"]))
5470
5471         # attach a webapi child that throws a random error, to test how it
5472         # gets rendered.
5473         w = c0.getServiceNamed("webish")
5474         w.root.putChild("ERRORBOOM", ErrorBoom())
5475
5476         # "Accept: */*" :        should get a text/html stack trace
5477         # "Accept: text/plain" : should get a text/plain stack trace
5478         # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
5479         # no Accept header:      should get a text/html stack trace
5480
5481         d.addCallback(lambda ignored:
5482                       self.shouldHTTPError("GET errorboom_html",
5483                                            500, "Internal Server Error", None,
5484                                            self.GET, "ERRORBOOM",
5485                                            headers={"accept": "*/*"}))
5486         def _internal_error_html1(body):
5487             self.failUnlessIn("<html>", "expected HTML, not '%s'" % body)
5488         d.addCallback(_internal_error_html1)
5489
5490         d.addCallback(lambda ignored:
5491                       self.shouldHTTPError("GET errorboom_text",
5492                                            500, "Internal Server Error", None,
5493                                            self.GET, "ERRORBOOM",
5494                                            headers={"accept": "text/plain"}))
5495         def _internal_error_text2(body):
5496             self.failIfIn("<html>", body)
5497             self.failUnless(body.startswith("Traceback "), body)
5498         d.addCallback(_internal_error_text2)
5499
5500         CLI_accepts = "text/plain, application/octet-stream"
5501         d.addCallback(lambda ignored:
5502                       self.shouldHTTPError("GET errorboom_text",
5503                                            500, "Internal Server Error", None,
5504                                            self.GET, "ERRORBOOM",
5505                                            headers={"accept": CLI_accepts}))
5506         def _internal_error_text3(body):
5507             self.failIfIn("<html>", body)
5508             self.failUnless(body.startswith("Traceback "), body)
5509         d.addCallback(_internal_error_text3)
5510
5511         d.addCallback(lambda ignored:
5512                       self.shouldHTTPError("GET errorboom_text",
5513                                            500, "Internal Server Error", None,
5514                                            self.GET, "ERRORBOOM"))
5515         def _internal_error_html4(body):
5516             self.failUnlessIn("<html>", body)
5517         d.addCallback(_internal_error_html4)
5518
5519         def _flush_errors(res):
5520             # Trial: please ignore the CompletelyUnhandledError in the logs
5521             self.flushLoggedErrors(CompletelyUnhandledError)
5522             return res
5523         d.addBoth(_flush_errors)
5524
5525         return d
5526
5527     def test_blacklist(self):
5528         # download from a blacklisted URI, get an error
5529         self.basedir = "web/Grid/blacklist"
5530         self.set_up_grid()
5531         c0 = self.g.clients[0]
5532         c0_basedir = c0.basedir
5533         fn = os.path.join(c0_basedir, "access.blacklist")
5534         self.uris = {}
5535         DATA = "off-limits " * 50
5536
5537         d = c0.upload(upload.Data(DATA, convergence=""))
5538         def _stash_uri_and_create_dir(ur):
5539             self.uri = ur.get_uri()
5540             self.url = "uri/"+self.uri
5541             u = uri.from_string_filenode(self.uri)
5542             self.si = u.get_storage_index()
5543             childnode = c0.create_node_from_uri(self.uri, None)
5544             return c0.create_dirnode({u"blacklisted.txt": (childnode,{}) })
5545         d.addCallback(_stash_uri_and_create_dir)
5546         def _stash_dir(node):
5547             self.dir_node = node
5548             self.dir_uri = node.get_uri()
5549             self.dir_url = "uri/"+self.dir_uri
5550         d.addCallback(_stash_dir)
5551         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5552         def _check_dir_html(body):
5553             self.failUnlessIn("<html>", body)
5554             self.failUnlessIn("blacklisted.txt</a>", body)
5555         d.addCallback(_check_dir_html)
5556         d.addCallback(lambda ign: self.GET(self.url))
5557         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5558
5559         def _blacklist(ign):
5560             f = open(fn, "w")
5561             f.write(" # this is a comment\n")
5562             f.write(" \n")
5563             f.write("\n") # also exercise blank lines
5564             f.write("%s %s\n" % (base32.b2a(self.si), "off-limits to you"))
5565             f.close()
5566             # clients should be checking the blacklist each time, so we don't
5567             # need to restart the client
5568         d.addCallback(_blacklist)
5569         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_uri",
5570                                                        403, "Forbidden",
5571                                                        "Access Prohibited: off-limits",
5572                                                        self.GET, self.url))
5573
5574         # We should still be able to list the parent directory, in HTML...
5575         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5576         def _check_dir_html2(body):
5577             self.failUnlessIn("<html>", body)
5578             self.failUnlessIn("blacklisted.txt</strike>", body)
5579         d.addCallback(_check_dir_html2)
5580
5581         # ... and in JSON (used by CLI).
5582         d.addCallback(lambda ign: self.GET(self.dir_url+"?t=json", followRedirect=True))
5583         def _check_dir_json(res):
5584             data = simplejson.loads(res)
5585             self.failUnless(isinstance(data, list), data)
5586             self.failUnlessEqual(data[0], "dirnode")
5587             self.failUnless(isinstance(data[1], dict), data)
5588             self.failUnlessIn("children", data[1])
5589             self.failUnlessIn("blacklisted.txt", data[1]["children"])
5590             childdata = data[1]["children"]["blacklisted.txt"]
5591             self.failUnless(isinstance(childdata, list), data)
5592             self.failUnlessEqual(childdata[0], "filenode")
5593             self.failUnless(isinstance(childdata[1], dict), data)
5594         d.addCallback(_check_dir_json)
5595
5596         def _unblacklist(ign):
5597             open(fn, "w").close()
5598             # the Blacklist object watches mtime to tell when the file has
5599             # changed, but on windows this test will run faster than the
5600             # filesystem's mtime resolution. So we edit Blacklist.last_mtime
5601             # to force a reload.
5602             self.g.clients[0].blacklist.last_mtime -= 2.0
5603         d.addCallback(_unblacklist)
5604
5605         # now a read should work
5606         d.addCallback(lambda ign: self.GET(self.url))
5607         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5608
5609         # read again to exercise the blacklist-is-unchanged logic
5610         d.addCallback(lambda ign: self.GET(self.url))
5611         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5612
5613         # now add a blacklisted directory, and make sure files under it are
5614         # refused too
5615         def _add_dir(ign):
5616             childnode = c0.create_node_from_uri(self.uri, None)
5617             return c0.create_dirnode({u"child": (childnode,{}) })
5618         d.addCallback(_add_dir)
5619         def _get_dircap(dn):
5620             self.dir_si_b32 = base32.b2a(dn.get_storage_index())
5621             self.dir_url_base = "uri/"+dn.get_write_uri()
5622             self.dir_url_json1 = "uri/"+dn.get_write_uri()+"?t=json"
5623             self.dir_url_json2 = "uri/"+dn.get_write_uri()+"/?t=json"
5624             self.dir_url_json_ro = "uri/"+dn.get_readonly_uri()+"/?t=json"
5625             self.child_url = "uri/"+dn.get_readonly_uri()+"/child"
5626         d.addCallback(_get_dircap)
5627         d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True))
5628         d.addCallback(lambda body: self.failUnlessIn("<html>", body))
5629         d.addCallback(lambda ign: self.GET(self.dir_url_json1))
5630         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5631         d.addCallback(lambda ign: self.GET(self.dir_url_json2))
5632         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5633         d.addCallback(lambda ign: self.GET(self.dir_url_json_ro))
5634         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5635         d.addCallback(lambda ign: self.GET(self.child_url))
5636         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5637
5638         def _block_dir(ign):
5639             f = open(fn, "w")
5640             f.write("%s %s\n" % (self.dir_si_b32, "dir-off-limits to you"))
5641             f.close()
5642             self.g.clients[0].blacklist.last_mtime -= 2.0
5643         d.addCallback(_block_dir)
5644         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir base",
5645                                                        403, "Forbidden",
5646                                                        "Access Prohibited: dir-off-limits",
5647                                                        self.GET, self.dir_url_base))
5648         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json1",
5649                                                        403, "Forbidden",
5650                                                        "Access Prohibited: dir-off-limits",
5651                                                        self.GET, self.dir_url_json1))
5652         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json2",
5653                                                        403, "Forbidden",
5654                                                        "Access Prohibited: dir-off-limits",
5655                                                        self.GET, self.dir_url_json2))
5656         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json_ro",
5657                                                        403, "Forbidden",
5658                                                        "Access Prohibited: dir-off-limits",
5659                                                        self.GET, self.dir_url_json_ro))
5660         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir child",
5661                                                        403, "Forbidden",
5662                                                        "Access Prohibited: dir-off-limits",
5663                                                        self.GET, self.child_url))
5664         return d
5665
5666
5667 class CompletelyUnhandledError(Exception):
5668     pass
5669 class ErrorBoom(rend.Page):
5670     def beforeRender(self, ctx):
5671         raise CompletelyUnhandledError("whoops")