src/allmydata/test/test_web.py

   1 import os.path, re, urllib, time, cgi
   2 import simplejson
   3 from StringIO import StringIO
   4
   5 from twisted.application import service
   6 from twisted.trial import unittest
   7 from twisted.internet import defer, reactor
   8 from twisted.internet.task import Clock
   9 from twisted.web import client, error, http
  10 from twisted.python import failure, log
  11
  12 from foolscap.api import fireEventually, flushEventualQueue
  13
  14 from nevow.util import escapeToXML
  15 from nevow import rend
  16
  17 from allmydata import interfaces, uri, webish, dirnode
  18 from allmydata.storage.shares import get_share_file
  19 from allmydata.storage_client import StorageFarmBroker, StubServer
  20 from allmydata.immutable import upload
  21 from allmydata.immutable.downloader.status import DownloadStatus
  22 from allmydata.dirnode import DirectoryNode
  23 from allmydata.nodemaker import NodeMaker
  24 from allmydata.unknown import UnknownNode
  25 from allmydata.web import status, common
  26 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  27 from allmydata.util import fileutil, base32, hashutil
  28 from allmydata.util.consumer import download_to_data
  29 from allmydata.util.netstring import split_netstring
  30 from allmydata.util.encodingutil import to_str
  31 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  32      create_chk_filenode, WebErrorMixin, ShouldFailMixin, \
  33      make_mutable_file_uri, create_mutable_filenode
  34 from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
  35 from allmydata.mutable import servermap, publish, retrieve
  36 import allmydata.test.common_util as testutil
  37 from allmydata.test.no_network import GridTestMixin
  38 from allmydata.test.common_web import HTTPClientGETFactory, \
  39      HTTPClientHEADFactory
  40 from allmydata.client import Client, SecretHolder
  41 from allmydata.introducer import IntroducerNode
  42
  43 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  44 # create a webserver that works against them
  45
  46 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  47
  48 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
  49 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
  50 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
  51
  52 FAVICON_MARKUP = '<link href="/icon.png" rel="shortcut icon" />'
  53
  54
  55 class FakeStatsProvider:
  56     def get_stats(self):
  57         stats = {'stats': {}, 'counters': {}}
  58         return stats
  59
  60 class FakeNodeMaker(NodeMaker):
  61     encoding_params = {
  62         'k': 3,
  63         'n': 10,
  64         'happy': 7,
  65         'max_segment_size':128*1024 # 1024=KiB
  66     }
  67     def _create_lit(self, cap):
  68         return FakeCHKFileNode(cap, self.all_contents)
  69     def _create_immutable(self, cap):
  70         return FakeCHKFileNode(cap, self.all_contents)
  71     def _create_mutable(self, cap):
  72         return FakeMutableFileNode(None, None,
  73                                    self.encoding_params, None,
  74                                    self.all_contents).init_from_cap(cap)
  75     def create_mutable_file(self, contents="", keysize=None,
  76                             version=SDMF_VERSION):
  77         n = FakeMutableFileNode(None, None, self.encoding_params, None,
  78                                 self.all_contents)
  79         return n.create(contents, version=version)
  80
  81 class FakeUploader(service.Service):
  82     name = "uploader"
  83     helper_furl = None
  84     helper_connected = False
  85
  86     def upload(self, uploadable):
  87         d = uploadable.get_size()
  88         d.addCallback(lambda size: uploadable.read(size))
  89         def _got_data(datav):
  90             data = "".join(datav)
  91             n = create_chk_filenode(data, self.all_contents)
  92             ur = upload.UploadResults(file_size=len(data),
  93                                       ciphertext_fetched=0,
  94                                       preexisting_shares=0,
  95                                       pushed_shares=10,
  96                                       sharemap={},
  97                                       servermap={},
  98                                       timings={},
  99                                       uri_extension_data={},
 100                                       uri_extension_hash="fake",
 101                                       verifycapstr="fakevcap")
 102             ur.set_uri(n.get_uri())
 103             return ur
 104         d.addCallback(_got_data)
 105         return d
 106
 107     def get_helper_info(self):
 108         return (self.helper_furl, self.helper_connected)
 109
 110
 111 def build_one_ds():
 112     ds = DownloadStatus("storage_index", 1234)
 113     now = time.time()
 114
 115     serverA = StubServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
 116     serverB = StubServer(hashutil.tagged_hash("foo", "serverid_b")[:20])
 117     storage_index = hashutil.storage_index_hash("SI")
 118     e0 = ds.add_segment_request(0, now)
 119     e0.activate(now+0.5)
 120     e0.deliver(now+1, 0, 100, 0.5) # when, start,len, decodetime
 121     e1 = ds.add_segment_request(1, now+2)
 122     e1.error(now+3)
 123     # two outstanding requests
 124     e2 = ds.add_segment_request(2, now+4)
 125     e3 = ds.add_segment_request(3, now+5)
 126     del e2,e3 # hush pyflakes
 127
 128     # simulate a segment which gets delivered faster than a system clock tick (ticket #1166)
 129     e = ds.add_segment_request(4, now)
 130     e.activate(now)
 131     e.deliver(now, 0, 140, 0.5)
 132
 133     e = ds.add_dyhb_request(serverA, now)
 134     e.finished([1,2], now+1)
 135     e = ds.add_dyhb_request(serverB, now+2) # left unfinished
 136
 137     e = ds.add_read_event(0, 120, now)
 138     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 139     e.finished(now+1)
 140     e = ds.add_read_event(120, 30, now+2) # left unfinished
 141
 142     e = ds.add_block_request(serverA, 1, 100, 20, now)
 143     e.finished(20, now+1)
 144     e = ds.add_block_request(serverB, 1, 120, 30, now+1) # left unfinished
 145
 146     # make sure that add_read_event() can come first too
 147     ds1 = DownloadStatus(storage_index, 1234)
 148     e = ds1.add_read_event(0, 120, now)
 149     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 150     e.finished(now+1)
 151
 152     return ds
 153
 154 class FakeHistory:
 155     _all_upload_status = [upload.UploadStatus()]
 156     _all_download_status = [build_one_ds()]
 157     _all_mapupdate_statuses = [servermap.UpdateStatus()]
 158     _all_publish_statuses = [publish.PublishStatus()]
 159     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
 160
 161     def list_all_upload_statuses(self):
 162         return self._all_upload_status
 163     def list_all_download_statuses(self):
 164         return self._all_download_status
 165     def list_all_mapupdate_statuses(self):
 166         return self._all_mapupdate_statuses
 167     def list_all_publish_statuses(self):
 168         return self._all_publish_statuses
 169     def list_all_retrieve_statuses(self):
 170         return self._all_retrieve_statuses
 171     def list_all_helper_statuses(self):
 172         return []
 173
 174 class FakeDisplayableServer(StubServer):
 175     def __init__(self, serverid, nickname):
 176         StubServer.__init__(self, serverid)
 177         self.announcement = {"my-version": "allmydata-tahoe-fake",
 178                              "service-name": "storage",
 179                              "nickname": nickname}
 180     def is_connected(self):
 181         return True
 182     def get_permutation_seed(self):
 183         return ""
 184     def get_remote_host(self):
 185         return ""
 186     def get_last_loss_time(self):
 187         return None
 188     def get_announcement_time(self):
 189         return None
 190     def get_announcement(self):
 191         return self.announcement
 192     def get_nickname(self):
 193         return self.announcement["nickname"]
 194
 195 class FakeBucketCounter(object):
 196     def get_state(self):
 197         return {"last-complete-bucket-count": 0}
 198     def get_progress(self):
 199         return {"estimated-time-per-cycle": 0,
 200                 "cycle-in-progress": False,
 201                 "remaining-wait-time": 0}
 202
 203 class FakeLeaseChecker(object):
 204     def __init__(self):
 205         self.expiration_enabled = False
 206         self.mode = "age"
 207         self.override_lease_duration = None
 208         self.sharetypes_to_expire = {}
 209     def get_state(self):
 210         return {"history": None}
 211     def get_progress(self):
 212         return {"estimated-time-per-cycle": 0,
 213                 "cycle-in-progress": False,
 214                 "remaining-wait-time": 0}
 215
 216 class FakeStorageServer(service.MultiService):
 217     name = 'storage'
 218     def __init__(self, nodeid, nickname):
 219         service.MultiService.__init__(self)
 220         self.my_nodeid = nodeid
 221         self.nickname = nickname
 222         self.bucket_counter = FakeBucketCounter()
 223         self.lease_checker = FakeLeaseChecker()
 224     def get_stats(self):
 225         return {"storage_server.accepting_immutable_shares": False}
 226
 227 class FakeClient(Client):
 228     def __init__(self):
 229         # don't upcall to Client.__init__, since we only want to initialize a
 230         # minimal subset
 231         service.MultiService.__init__(self)
 232         self.all_contents = {}
 233         self.nodeid = "fake_nodeid"
 234         self.nickname = u"fake_nickname \u263A"
 235         self.introducer_furl = "None"
 236         self.stats_provider = FakeStatsProvider()
 237         self._secret_holder = SecretHolder("lease secret", "convergence secret")
 238         self.helper = None
 239         self.convergence = "some random string"
 240         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 241         # fake knowledge of another server
 242         self.storage_broker.test_add_server("other_nodeid",
 243                                             FakeDisplayableServer("other_nodeid", u"other_nickname \u263B"))
 244         self.introducer_client = None
 245         self.history = FakeHistory()
 246         self.uploader = FakeUploader()
 247         self.uploader.all_contents = self.all_contents
 248         self.uploader.setServiceParent(self)
 249         self.blacklist = None
 250         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 251                                        self.uploader, None,
 252                                        None, None, None)
 253         self.nodemaker.all_contents = self.all_contents
 254         self.mutable_file_default = SDMF_VERSION
 255         self.addService(FakeStorageServer(self.nodeid, self.nickname))
 256
 257     def startService(self):
 258         return service.MultiService.startService(self)
 259     def stopService(self):
 260         return service.MultiService.stopService(self)
 261
 262     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 263
 264 class WebMixin(object):
 265     def setUp(self):
 266         self.s = FakeClient()
 267         self.s.startService()
 268         self.staticdir = self.mktemp()
 269         self.clock = Clock()
 270         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
 271                                       clock=self.clock)
 272         self.ws.setServiceParent(self.s)
 273         self.webish_port = self.ws.getPortnum()
 274         self.webish_url = self.ws.getURL()
 275         assert self.webish_url.endswith("/")
 276         self.webish_url = self.webish_url[:-1] # these tests add their own /
 277
 278         l = [ self.s.create_dirnode() for x in range(6) ]
 279         d = defer.DeferredList(l)
 280         def _then(res):
 281             self.public_root = res[0][1]
 282             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 283             self.public_url = "/uri/" + self.public_root.get_uri()
 284             self.private_root = res[1][1]
 285
 286             foo = res[2][1]
 287             self._foo_node = foo
 288             self._foo_uri = foo.get_uri()
 289             self._foo_readonly_uri = foo.get_readonly_uri()
 290             self._foo_verifycap = foo.get_verify_cap().to_string()
 291             # NOTE: we ignore the deferred on all set_uri() calls, because we
 292             # know the fake nodes do these synchronously
 293             self.public_root.set_uri(u"foo", foo.get_uri(),
 294                                      foo.get_readonly_uri())
 295
 296             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 297             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 298             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 299
 300             # sdmf
 301             # XXX: Do we ever use this?
 302             self.BAZ_CONTENTS, n, self._baz_txt_uri, self._baz_txt_readonly_uri = self.makefile_mutable(0)
 303
 304             foo.set_uri(u"baz.txt", self._baz_txt_uri, self._baz_txt_readonly_uri)
 305
 306             # mdmf
 307             self.QUUX_CONTENTS, n, self._quux_txt_uri, self._quux_txt_readonly_uri = self.makefile_mutable(0, mdmf=True)
 308             assert self._quux_txt_uri.startswith("URI:MDMF")
 309             foo.set_uri(u"quux.txt", self._quux_txt_uri, self._quux_txt_readonly_uri)
 310
 311             foo.set_uri(u"empty", res[3][1].get_uri(),
 312                         res[3][1].get_readonly_uri())
 313             sub_uri = res[4][1].get_uri()
 314             self._sub_uri = sub_uri
 315             foo.set_uri(u"sub", sub_uri, sub_uri)
 316             sub = self.s.create_node_from_uri(sub_uri)
 317             self._sub_node = sub
 318
 319             _ign, n, blocking_uri = self.makefile(1)
 320             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 321
 322             # filenode to test for html encoding issues
 323             self._htmlname_unicode = u"<&weirdly'named\"file>>>_<iframe />.txt"
 324             self._htmlname_raw = self._htmlname_unicode.encode('utf-8')
 325             self._htmlname_urlencoded = urllib.quote(self._htmlname_raw, '')
 326             self._htmlname_escaped = escapeToXML(self._htmlname_raw)
 327             self._htmlname_escaped_attr = cgi.escape(self._htmlname_raw, quote=True)
 328             self._htmlname_escaped_double = escapeToXML(cgi.escape(self._htmlname_raw, quote=True))
 329             self.HTMLNAME_CONTENTS, n, self._htmlname_txt_uri = self.makefile(0)
 330             foo.set_uri(self._htmlname_unicode, self._htmlname_txt_uri, self._htmlname_txt_uri)
 331
 332             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 333             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 334             # still think of it as an umlaut
 335             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 336
 337             self.SUBBAZ_CONTENTS, n, baz_file = self.makefile(2)
 338             self._baz_file_uri = baz_file
 339             sub.set_uri(u"baz.txt", baz_file, baz_file)
 340
 341             _ign, n, self._bad_file_uri = self.makefile(3)
 342             # this uri should not be downloadable
 343             del self.s.all_contents[self._bad_file_uri]
 344
 345             rodir = res[5][1]
 346             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 347                                      rodir.get_readonly_uri())
 348             rodir.set_uri(u"nor", baz_file, baz_file)
 349
 350             # public/
 351             # public/foo/
 352             # public/foo/bar.txt
 353             # public/foo/baz.txt
 354             # public/foo/quux.txt
 355             # public/foo/blockingfile
 356             # public/foo/<&weirdly'named\"file>>>_<iframe />.txt
 357             # public/foo/empty/
 358             # public/foo/sub/
 359             # public/foo/sub/baz.txt
 360             # public/reedownlee/
 361             # public/reedownlee/nor
 362             self.NEWFILE_CONTENTS = "newfile contents\n"
 363
 364             return foo.get_metadata_for(u"bar.txt")
 365         d.addCallback(_then)
 366         def _got_metadata(metadata):
 367             self._bar_txt_metadata = metadata
 368         d.addCallback(_got_metadata)
 369         return d
 370
 371     def get_all_contents(self):
 372         return self.s.all_contents
 373
 374     def makefile(self, number):
 375         contents = "contents of file %s\n" % number
 376         n = create_chk_filenode(contents, self.get_all_contents())
 377         return contents, n, n.get_uri()
 378
 379     def makefile_mutable(self, number, mdmf=False):
 380         contents = "contents of mutable file %s\n" % number
 381         n = create_mutable_filenode(contents, mdmf, self.s.all_contents)
 382         return contents, n, n.get_uri(), n.get_readonly_uri()
 383
 384     def tearDown(self):
 385         return self.s.stopService()
 386
 387     def failUnlessIsBarDotTxt(self, res):
 388         self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
 389
 390     def failUnlessIsQuuxDotTxt(self, res):
 391         self.failUnlessReallyEqual(res, self.QUUX_CONTENTS, res)
 392
 393     def failUnlessIsBazDotTxt(self, res):
 394         self.failUnlessReallyEqual(res, self.BAZ_CONTENTS, res)
 395
 396     def failUnlessIsSubBazDotTxt(self, res):
 397         self.failUnlessReallyEqual(res, self.SUBBAZ_CONTENTS, res)
 398
 399     def failUnlessIsBarJSON(self, res):
 400         data = simplejson.loads(res)
 401         self.failUnless(isinstance(data, list))
 402         self.failUnlessEqual(data[0], "filenode")
 403         self.failUnless(isinstance(data[1], dict))
 404         self.failIf(data[1]["mutable"])
 405         self.failIfIn("rw_uri", data[1]) # immutable
 406         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
 407         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
 408         self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
 409
 410     def failUnlessIsQuuxJSON(self, res, readonly=False):
 411         data = simplejson.loads(res)
 412         self.failUnless(isinstance(data, list))
 413         self.failUnlessEqual(data[0], "filenode")
 414         self.failUnless(isinstance(data[1], dict))
 415         metadata = data[1]
 416         return self.failUnlessIsQuuxDotTxtMetadata(metadata, readonly)
 417
 418     def failUnlessIsQuuxDotTxtMetadata(self, metadata, readonly):
 419         self.failUnless(metadata['mutable'])
 420         if readonly:
 421             self.failIfIn("rw_uri", metadata)
 422         else:
 423             self.failUnlessIn("rw_uri", metadata)
 424             self.failUnlessEqual(metadata['rw_uri'], self._quux_txt_uri)
 425         self.failUnlessIn("ro_uri", metadata)
 426         self.failUnlessEqual(metadata['ro_uri'], self._quux_txt_readonly_uri)
 427         self.failUnlessReallyEqual(metadata['size'], len(self.QUUX_CONTENTS))
 428
 429     def failUnlessIsFooJSON(self, res):
 430         data = simplejson.loads(res)
 431         self.failUnless(isinstance(data, list))
 432         self.failUnlessEqual(data[0], "dirnode", res)
 433         self.failUnless(isinstance(data[1], dict))
 434         self.failUnless(data[1]["mutable"])
 435         self.failUnlessIn("rw_uri", data[1]) # mutable
 436         self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
 437         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
 438         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
 439
 440         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 441         self.failUnlessEqual(kidnames,
 442                              [self._htmlname_unicode, u"bar.txt", u"baz.txt",
 443                               u"blockingfile", u"empty", u"n\u00fc.txt", u"quux.txt", u"sub"])
 444         kids = dict( [(unicode(name),value)
 445                       for (name,value)
 446                       in data[1]["children"].iteritems()] )
 447         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 448         self.failUnlessIn("metadata", kids[u"sub"][1])
 449         self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
 450         tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
 451         self.failUnlessIn("linkcrtime", tahoe_md)
 452         self.failUnlessIn("linkmotime", tahoe_md)
 453         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 454         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 455         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri)
 456         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]),
 457                                    self._bar_txt_verifycap)
 458         self.failUnlessIn("metadata", kids[u"bar.txt"][1])
 459         self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
 460         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
 461                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
 462         self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
 463                                    self._bar_txt_uri)
 464         self.failUnlessIn("quux.txt", kids)
 465         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["rw_uri"]),
 466                                    self._quux_txt_uri)
 467         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["ro_uri"]),
 468                                    self._quux_txt_readonly_uri)
 469
 470     def GET(self, urlpath, followRedirect=False, return_response=False,
 471             **kwargs):
 472         # if return_response=True, this fires with (data, statuscode,
 473         # respheaders) instead of just data.
 474         assert not isinstance(urlpath, unicode)
 475         url = self.webish_url + urlpath
 476         factory = HTTPClientGETFactory(url, method="GET",
 477                                        followRedirect=followRedirect, **kwargs)
 478         reactor.connectTCP("localhost", self.webish_port, factory)
 479         d = factory.deferred
 480         def _got_data(data):
 481             return (data, factory.status, factory.response_headers)
 482         if return_response:
 483             d.addCallback(_got_data)
 484         return factory.deferred
 485
 486     def HEAD(self, urlpath, return_response=False, **kwargs):
 487         # this requires some surgery, because twisted.web.client doesn't want
 488         # to give us back the response headers.
 489         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 490         reactor.connectTCP("localhost", self.webish_port, factory)
 491         d = factory.deferred
 492         def _got_data(data):
 493             return (data, factory.status, factory.response_headers)
 494         if return_response:
 495             d.addCallback(_got_data)
 496         return factory.deferred
 497
 498     def PUT(self, urlpath, data, **kwargs):
 499         url = self.webish_url + urlpath
 500         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 501
 502     def DELETE(self, urlpath):
 503         url = self.webish_url + urlpath
 504         return client.getPage(url, method="DELETE")
 505
 506     def POST(self, urlpath, followRedirect=False, **fields):
 507         sepbase = "boogabooga"
 508         sep = "--" + sepbase
 509         form = []
 510         form.append(sep)
 511         form.append('Content-Disposition: form-data; name="_charset"')
 512         form.append('')
 513         form.append('UTF-8')
 514         form.append(sep)
 515         for name, value in fields.iteritems():
 516             if isinstance(value, tuple):
 517                 filename, value = value
 518                 form.append('Content-Disposition: form-data; name="%s"; '
 519                             'filename="%s"' % (name, filename.encode("utf-8")))
 520             else:
 521                 form.append('Content-Disposition: form-data; name="%s"' % name)
 522             form.append('')
 523             if isinstance(value, unicode):
 524                 value = value.encode("utf-8")
 525             else:
 526                 value = str(value)
 527             assert isinstance(value, str)
 528             form.append(value)
 529             form.append(sep)
 530         form[-1] += "--"
 531         body = ""
 532         headers = {}
 533         if fields:
 534             body = "\r\n".join(form) + "\r\n"
 535             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 536         return self.POST2(urlpath, body, headers, followRedirect)
 537
 538     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 539         url = self.webish_url + urlpath
 540         return client.getPage(url, method="POST", postdata=body,
 541                               headers=headers, followRedirect=followRedirect)
 542
 543     def shouldFail(self, res, expected_failure, which,
 544                    substring=None, response_substring=None):
 545         if isinstance(res, failure.Failure):
 546             res.trap(expected_failure)
 547             if substring:
 548                 self.failUnlessIn(substring, str(res), which)
 549             if response_substring:
 550                 self.failUnlessIn(response_substring, res.value.response, which)
 551         else:
 552             self.fail("%s was supposed to raise %s, not get '%s'" %
 553                       (which, expected_failure, res))
 554
 555     def shouldFail2(self, expected_failure, which, substring,
 556                     response_substring,
 557                     callable, *args, **kwargs):
 558         assert substring is None or isinstance(substring, str)
 559         assert response_substring is None or isinstance(response_substring, str)
 560         d = defer.maybeDeferred(callable, *args, **kwargs)
 561         def done(res):
 562             if isinstance(res, failure.Failure):
 563                 res.trap(expected_failure)
 564                 if substring:
 565                     self.failUnlessIn(substring, str(res),
 566                                       "'%s' not in '%s' for test '%s'" % \
 567                                       (substring, str(res), which))
 568                 if response_substring:
 569                     self.failUnlessIn(response_substring, res.value.response,
 570                                       "'%s' not in '%s' for test '%s'" % \
 571                                       (response_substring, res.value.response,
 572                                        which))
 573             else:
 574                 self.fail("%s was supposed to raise %s, not get '%s'" %
 575                           (which, expected_failure, res))
 576         d.addBoth(done)
 577         return d
 578
 579     def should404(self, res, which):
 580         if isinstance(res, failure.Failure):
 581             res.trap(error.Error)
 582             self.failUnlessReallyEqual(res.value.status, "404")
 583         else:
 584             self.fail("%s was supposed to Error(404), not get '%s'" %
 585                       (which, res))
 586
 587     def should302(self, res, which):
 588         if isinstance(res, failure.Failure):
 589             res.trap(error.Error)
 590             self.failUnlessReallyEqual(res.value.status, "302")
 591         else:
 592             self.fail("%s was supposed to Error(302), not get '%s'" %
 593                       (which, res))
 594
 595
 596 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
 597     def test_create(self):
 598         pass
 599
 600     def test_welcome(self):
 601         d = self.GET("/")
 602         def _check(res):
 603             self.failUnlessIn('<title>Tahoe-LAFS - Welcome</title>', res)
 604             self.failUnlessIn(FAVICON_MARKUP, res)
 605             self.failUnlessIn('<a href="status">Recent and Active Operations</a>', res)
 606             self.failUnlessIn('<a href="statistics">Operational Statistics</a>', res)
 607             self.failUnlessIn('<input type="hidden" name="t" value="report-incident" />', res)
 608             res_u = res.decode('utf-8')
 609             self.failUnlessIn(u'<td>fake_nickname \u263A</td>', res_u)
 610             self.failUnlessIn(u'<div class="nickname">other_nickname \u263B</div>', res_u)
 611             self.failUnlessIn(u'\u00A9 <a href="https://tahoe-lafs.org/">Tahoe-LAFS Software Foundation', res_u)
 612
 613             self.s.basedir = 'web/test_welcome'
 614             fileutil.make_dirs("web/test_welcome")
 615             fileutil.make_dirs("web/test_welcome/private")
 616             return self.GET("/")
 617         d.addCallback(_check)
 618         return d
 619
 620     def test_helper_status(self):
 621         d = defer.succeed(None)
 622
 623         # set helper furl to None
 624         def _set_no_helper(ign):
 625             self.s.uploader.helper_furl = None
 626             return self.GET("/")
 627         d.addCallback(_set_no_helper)
 628         def _check_no_helper(res):
 629             html = res.replace('\n', ' ')
 630             self.failUnless(re.search('<div class="status-indicator connected-not-configured"></div>[ ]*Helper', html), res)
 631         d.addCallback(_check_no_helper)
 632
 633         # enable helper, not connected
 634         def _set_helper_not_connected(ign):
 635             self.s.uploader.helper_furl = "pb://someHelper"
 636             self.s.uploader.helper_connected = False
 637             return self.GET("/")
 638         d.addCallback(_set_helper_not_connected)
 639         def _check_helper_not_connected(res):
 640             html = res.replace('\n', ' ')
 641             self.failUnless(re.search('<div class="status-indicator connected-no"></div>[ ]*Helper', html), res)
 642         d.addCallback(_check_helper_not_connected)
 643
 644         # enable helper, connected
 645         def _set_helper_connected(ign):
 646             self.s.uploader.helper_furl = "pb://someHelper"
 647             self.s.uploader.helper_connected = True
 648             return self.GET("/")
 649         d.addCallback(_set_helper_connected)
 650         def _check_helper_connected(res):
 651             html = res.replace('\n', ' ')
 652             self.failUnless(re.search('<div class="status-indicator connected-yes"></div>[ ]*Helper', html), res)
 653         d.addCallback(_check_helper_connected)
 654         return d
 655
 656     def test_storage(self):
 657         d = self.GET("/storage")
 658         def _check(res):
 659             self.failUnlessIn('Storage Server Status', res)
 660             self.failUnlessIn(FAVICON_MARKUP, res)
 661             res_u = res.decode('utf-8')
 662             self.failUnlessIn(u'<li>Server Nickname: <span class="nickname mine">fake_nickname \u263A</span></li>', res_u)
 663         d.addCallback(_check)
 664         return d
 665
 666     def test_status(self):
 667         h = self.s.get_history()
 668         dl_num = h.list_all_download_statuses()[0].get_counter()
 669         ul_num = h.list_all_upload_statuses()[0].get_counter()
 670         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 671         pub_num = h.list_all_publish_statuses()[0].get_counter()
 672         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 673         d = self.GET("/status", followRedirect=True)
 674         def _check(res):
 675             self.failUnlessIn('Recent and Active Operations', res)
 676             self.failUnlessIn('"down-%d"' % dl_num, res)
 677             self.failUnlessIn('"up-%d"' % ul_num, res)
 678             self.failUnlessIn('"mapupdate-%d"' % mu_num, res)
 679             self.failUnlessIn('"publish-%d"' % pub_num, res)
 680             self.failUnlessIn('"retrieve-%d"' % ret_num, res)
 681         d.addCallback(_check)
 682         d.addCallback(lambda res: self.GET("/status/?t=json"))
 683         def _check_json(res):
 684             data = simplejson.loads(res)
 685             self.failUnless(isinstance(data, dict))
 686             #active = data["active"]
 687             # TODO: test more. We need a way to fake an active operation
 688             # here.
 689         d.addCallback(_check_json)
 690
 691         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 692         def _check_dl(res):
 693             self.failUnlessIn("File Download Status", res)
 694         d.addCallback(_check_dl)
 695         d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
 696         def _check_dl_json(res):
 697             data = simplejson.loads(res)
 698             self.failUnless(isinstance(data, dict))
 699             self.failUnlessIn("read", data)
 700             self.failUnlessEqual(data["read"][0]["length"], 120)
 701             self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
 702             self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
 703             self.failUnlessEqual(data["segment"][2]["finish_time"], None)
 704             phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
 705             cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
 706             # serverids[] keys are strings, since that's what JSON does, but
 707             # we'd really like them to be ints
 708             self.failUnlessEqual(data["serverids"]["0"], "phwrsjte")
 709             self.failUnless(data["serverids"].has_key("1"),
 710                             str(data["serverids"]))
 711             self.failUnlessEqual(data["serverids"]["1"], "cmpuvkjm",
 712                                  str(data["serverids"]))
 713             self.failUnlessEqual(data["server_info"][phwr_id]["short"],
 714                                  "phwrsjte")
 715             self.failUnlessEqual(data["server_info"][cmpu_id]["short"],
 716                                  "cmpuvkjm")
 717             self.failUnlessIn("dyhb", data)
 718             self.failUnlessIn("misc", data)
 719         d.addCallback(_check_dl_json)
 720         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 721         def _check_ul(res):
 722             self.failUnlessIn("File Upload Status", res)
 723         d.addCallback(_check_ul)
 724         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 725         def _check_mapupdate(res):
 726             self.failUnlessIn("Mutable File Servermap Update Status", res)
 727         d.addCallback(_check_mapupdate)
 728         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 729         def _check_publish(res):
 730             self.failUnlessIn("Mutable File Publish Status", res)
 731         d.addCallback(_check_publish)
 732         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 733         def _check_retrieve(res):
 734             self.failUnlessIn("Mutable File Retrieve Status", res)
 735         d.addCallback(_check_retrieve)
 736
 737         return d
 738
 739     def test_status_numbers(self):
 740         drrm = status.DownloadResultsRendererMixin()
 741         self.failUnlessReallyEqual(drrm.render_time(None, None), "")
 742         self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
 743         self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
 744         self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
 745         self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
 746         self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
 747         self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 748         self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
 749         self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
 750
 751         urrm = status.UploadResultsRendererMixin()
 752         self.failUnlessReallyEqual(urrm.render_time(None, None), "")
 753         self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
 754         self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
 755         self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
 756         self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
 757         self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
 758         self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 759         self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
 760         self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
 761
 762     def test_GET_FILEURL(self):
 763         d = self.GET(self.public_url + "/foo/bar.txt")
 764         d.addCallback(self.failUnlessIsBarDotTxt)
 765         return d
 766
 767     def test_GET_FILEURL_range(self):
 768         headers = {"range": "bytes=1-10"}
 769         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 770                      return_response=True)
 771         def _got((res, status, headers)):
 772             self.failUnlessReallyEqual(int(status), 206)
 773             self.failUnless(headers.has_key("content-range"))
 774             self.failUnlessReallyEqual(headers["content-range"][0],
 775                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 776             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
 777         d.addCallback(_got)
 778         return d
 779
 780     def test_GET_FILEURL_partial_range(self):
 781         headers = {"range": "bytes=5-"}
 782         length  = len(self.BAR_CONTENTS)
 783         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 784                      return_response=True)
 785         def _got((res, status, headers)):
 786             self.failUnlessReallyEqual(int(status), 206)
 787             self.failUnless(headers.has_key("content-range"))
 788             self.failUnlessReallyEqual(headers["content-range"][0],
 789                                        "bytes 5-%d/%d" % (length-1, length))
 790             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
 791         d.addCallback(_got)
 792         return d
 793
 794     def test_GET_FILEURL_partial_end_range(self):
 795         headers = {"range": "bytes=-5"}
 796         length  = len(self.BAR_CONTENTS)
 797         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 798                      return_response=True)
 799         def _got((res, status, headers)):
 800             self.failUnlessReallyEqual(int(status), 206)
 801             self.failUnless(headers.has_key("content-range"))
 802             self.failUnlessReallyEqual(headers["content-range"][0],
 803                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 804             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
 805         d.addCallback(_got)
 806         return d
 807
 808     def test_GET_FILEURL_partial_range_overrun(self):
 809         headers = {"range": "bytes=100-200"}
 810         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
 811                              "416 Requested Range not satisfiable",
 812                              "First beyond end of file",
 813                              self.GET, self.public_url + "/foo/bar.txt",
 814                              headers=headers)
 815         return d
 816
 817     def test_HEAD_FILEURL_range(self):
 818         headers = {"range": "bytes=1-10"}
 819         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 820                      return_response=True)
 821         def _got((res, status, headers)):
 822             self.failUnlessReallyEqual(res, "")
 823             self.failUnlessReallyEqual(int(status), 206)
 824             self.failUnless(headers.has_key("content-range"))
 825             self.failUnlessReallyEqual(headers["content-range"][0],
 826                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 827         d.addCallback(_got)
 828         return d
 829
 830     def test_HEAD_FILEURL_partial_range(self):
 831         headers = {"range": "bytes=5-"}
 832         length  = len(self.BAR_CONTENTS)
 833         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 834                      return_response=True)
 835         def _got((res, status, headers)):
 836             self.failUnlessReallyEqual(int(status), 206)
 837             self.failUnless(headers.has_key("content-range"))
 838             self.failUnlessReallyEqual(headers["content-range"][0],
 839                                        "bytes 5-%d/%d" % (length-1, length))
 840         d.addCallback(_got)
 841         return d
 842
 843     def test_HEAD_FILEURL_partial_end_range(self):
 844         headers = {"range": "bytes=-5"}
 845         length  = len(self.BAR_CONTENTS)
 846         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 847                      return_response=True)
 848         def _got((res, status, headers)):
 849             self.failUnlessReallyEqual(int(status), 206)
 850             self.failUnless(headers.has_key("content-range"))
 851             self.failUnlessReallyEqual(headers["content-range"][0],
 852                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 853         d.addCallback(_got)
 854         return d
 855
 856     def test_HEAD_FILEURL_partial_range_overrun(self):
 857         headers = {"range": "bytes=100-200"}
 858         d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
 859                              "416 Requested Range not satisfiable",
 860                              "",
 861                              self.HEAD, self.public_url + "/foo/bar.txt",
 862                              headers=headers)
 863         return d
 864
 865     def test_GET_FILEURL_range_bad(self):
 866         headers = {"range": "BOGUS=fizbop-quarnak"}
 867         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 868                      return_response=True)
 869         def _got((res, status, headers)):
 870             self.failUnlessReallyEqual(int(status), 200)
 871             self.failUnless(not headers.has_key("content-range"))
 872             self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
 873         d.addCallback(_got)
 874         return d
 875
 876     def test_HEAD_FILEURL(self):
 877         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 878         def _got((res, status, headers)):
 879             self.failUnlessReallyEqual(res, "")
 880             self.failUnlessReallyEqual(headers["content-length"][0],
 881                                        str(len(self.BAR_CONTENTS)))
 882             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
 883         d.addCallback(_got)
 884         return d
 885
 886     def test_GET_FILEURL_named(self):
 887         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 888         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 889         d = self.GET(base + "/@@name=/blah.txt")
 890         d.addCallback(self.failUnlessIsBarDotTxt)
 891         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 892         d.addCallback(self.failUnlessIsBarDotTxt)
 893         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 894         d.addCallback(self.failUnlessIsBarDotTxt)
 895         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 896         d.addCallback(self.failUnlessIsBarDotTxt)
 897         save_url = base + "?save=true&filename=blah.txt"
 898         d.addCallback(lambda res: self.GET(save_url))
 899         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 900         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 901         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 902         u_url = base + "?save=true&filename=" + u_fn_e
 903         d.addCallback(lambda res: self.GET(u_url))
 904         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 905         return d
 906
 907     def test_PUT_FILEURL_named_bad(self):
 908         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 909         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 910                              "400 Bad Request",
 911                              "/file can only be used with GET or HEAD",
 912                              self.PUT, base + "/@@name=/blah.txt", "")
 913         return d
 914
 915
 916     def test_GET_DIRURL_named_bad(self):
 917         base = "/file/%s" % urllib.quote(self._foo_uri)
 918         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 919                              "400 Bad Request",
 920                              "is not a file-cap",
 921                              self.GET, base + "/@@name=/blah.txt")
 922         return d
 923
 924     def test_GET_slash_file_bad(self):
 925         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 926                              "404 Not Found",
 927                              "/file must be followed by a file-cap and a name",
 928                              self.GET, "/file")
 929         return d
 930
 931     def test_GET_unhandled_URI_named(self):
 932         contents, n, newuri = self.makefile(12)
 933         verifier_cap = n.get_verify_cap().to_string()
 934         base = "/file/%s" % urllib.quote(verifier_cap)
 935         # client.create_node_from_uri() can't handle verify-caps
 936         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 937                              "400 Bad Request", "is not a file-cap",
 938                              self.GET, base)
 939         return d
 940
 941     def test_GET_unhandled_URI(self):
 942         contents, n, newuri = self.makefile(12)
 943         verifier_cap = n.get_verify_cap().to_string()
 944         base = "/uri/%s" % urllib.quote(verifier_cap)
 945         # client.create_node_from_uri() can't handle verify-caps
 946         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 947                              "400 Bad Request",
 948                              "GET unknown URI type: can only do t=info",
 949                              self.GET, base)
 950         return d
 951
 952     def test_GET_FILE_URI(self):
 953         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 954         d = self.GET(base)
 955         d.addCallback(self.failUnlessIsBarDotTxt)
 956         return d
 957
 958     def test_GET_FILE_URI_mdmf(self):
 959         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 960         d = self.GET(base)
 961         d.addCallback(self.failUnlessIsQuuxDotTxt)
 962         return d
 963
 964     def test_GET_FILE_URI_mdmf_extensions(self):
 965         base = "/uri/%s" % urllib.quote("%s:RANDOMSTUFF" % self._quux_txt_uri)
 966         d = self.GET(base)
 967         d.addCallback(self.failUnlessIsQuuxDotTxt)
 968         return d
 969
 970     def test_GET_FILE_URI_mdmf_readonly(self):
 971         base = "/uri/%s" % urllib.quote(self._quux_txt_readonly_uri)
 972         d = self.GET(base)
 973         d.addCallback(self.failUnlessIsQuuxDotTxt)
 974         return d
 975
 976     def test_GET_FILE_URI_badchild(self):
 977         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 978         errmsg = "Files have no children, certainly not named 'boguschild'"
 979         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 980                              "400 Bad Request", errmsg,
 981                              self.GET, base)
 982         return d
 983
 984     def test_PUT_FILE_URI_badchild(self):
 985         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 986         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 987         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 988                              "400 Bad Request", errmsg,
 989                              self.PUT, base, "")
 990         return d
 991
 992     def test_PUT_FILE_URI_mdmf(self):
 993         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 994         self._quux_new_contents = "new_contents"
 995         d = self.GET(base)
 996         d.addCallback(lambda res:
 997             self.failUnlessIsQuuxDotTxt(res))
 998         d.addCallback(lambda ignored:
 999             self.PUT(base, self._quux_new_contents))
1000         d.addCallback(lambda ignored:
1001             self.GET(base))
1002         d.addCallback(lambda res:
1003             self.failUnlessReallyEqual(res, self._quux_new_contents))
1004         return d
1005
1006     def test_PUT_FILE_URI_mdmf_extensions(self):
1007         base = "/uri/%s" % urllib.quote("%s:EXTENSIONSTUFF" % self._quux_txt_uri)
1008         self._quux_new_contents = "new_contents"
1009         d = self.GET(base)
1010         d.addCallback(lambda res: self.failUnlessIsQuuxDotTxt(res))
1011         d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
1012         d.addCallback(lambda ignored: self.GET(base))
1013         d.addCallback(lambda res: self.failUnlessEqual(self._quux_new_contents,
1014                                                        res))
1015         return d
1016
1017     def test_PUT_FILE_URI_mdmf_readonly(self):
1018         # We're not allowed to PUT things to a readonly cap.
1019         base = "/uri/%s" % self._quux_txt_readonly_uri
1020         d = self.GET(base)
1021         d.addCallback(lambda res:
1022             self.failUnlessIsQuuxDotTxt(res))
1023         # What should we get here? We get a 500 error now; that's not right.
1024         d.addCallback(lambda ignored:
1025             self.shouldFail2(error.Error, "test_PUT_FILE_URI_mdmf_readonly",
1026                              "400 Bad Request", "read-only cap",
1027                              self.PUT, base, "new data"))
1028         return d
1029
1030     def test_PUT_FILE_URI_sdmf_readonly(self):
1031         # We're not allowed to put things to a readonly cap.
1032         base = "/uri/%s" % self._baz_txt_readonly_uri
1033         d = self.GET(base)
1034         d.addCallback(lambda res:
1035             self.failUnlessIsBazDotTxt(res))
1036         d.addCallback(lambda ignored:
1037             self.shouldFail2(error.Error, "test_PUT_FILE_URI_sdmf_readonly",
1038                              "400 Bad Request", "read-only cap",
1039                              self.PUT, base, "new_data"))
1040         return d
1041
1042     def test_GET_etags(self):
1043
1044         def _check_etags(uri):
1045             d1 = _get_etag(uri)
1046             d2 = _get_etag(uri, 'json')
1047             d = defer.DeferredList([d1, d2], consumeErrors=True)
1048             def _check(results):
1049                 # All deferred must succeed
1050                 self.failUnless(all([r[0] for r in results]))
1051                 # the etag for the t=json form should be just like the etag
1052                 # fo the default t='' form, but with a 'json' suffix
1053                 self.failUnlessEqual(results[0][1] + 'json', results[1][1])
1054             d.addCallback(_check)
1055             return d
1056
1057         def _get_etag(uri, t=''):
1058             targetbase = "/uri/%s?t=%s" % (urllib.quote(uri.strip()), t)
1059             d = self.GET(targetbase, return_response=True, followRedirect=True)
1060             def _just_the_etag(result):
1061                 data, response, headers = result
1062                 etag = headers['etag'][0]
1063                 if uri.startswith('URI:DIR'):
1064                     self.failUnless(etag.startswith('DIR:'), etag)
1065                 return etag
1066             return d.addCallback(_just_the_etag)
1067
1068         # Check that etags work with immutable directories
1069         (newkids, caps) = self._create_immutable_children()
1070         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1071                       simplejson.dumps(newkids))
1072         def _stash_immdir_uri(uri):
1073             self._immdir_uri = uri
1074             return uri
1075         d.addCallback(_stash_immdir_uri)
1076         d.addCallback(_check_etags)
1077
1078         # Check that etags work with immutable files
1079         d.addCallback(lambda _: _check_etags(self._bar_txt_uri))
1080
1081         # use the ETag on GET
1082         def _check_match(ign):
1083             uri = "/uri/%s" % self._bar_txt_uri
1084             d = self.GET(uri, return_response=True)
1085             # extract the ETag
1086             d.addCallback(lambda (data, code, headers):
1087                           headers['etag'][0])
1088             # do a GET that's supposed to match the ETag
1089             d.addCallback(lambda etag:
1090                           self.GET(uri, return_response=True,
1091                                    headers={"If-None-Match": etag}))
1092             # make sure it short-circuited (304 instead of 200)
1093             d.addCallback(lambda (data, code, headers):
1094                           self.failUnlessEqual(int(code), http.NOT_MODIFIED))
1095             return d
1096         d.addCallback(_check_match)
1097
1098         def _no_etag(uri, t):
1099             target = "/uri/%s?t=%s" % (uri, t)
1100             d = self.GET(target, return_response=True, followRedirect=True)
1101             d.addCallback(lambda (data, code, headers):
1102                           self.failIf("etag" in headers, target))
1103             return d
1104         def _yes_etag(uri, t):
1105             target = "/uri/%s?t=%s" % (uri, t)
1106             d = self.GET(target, return_response=True, followRedirect=True)
1107             d.addCallback(lambda (data, code, headers):
1108                           self.failUnless("etag" in headers, target))
1109             return d
1110
1111         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, ""))
1112         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "json"))
1113         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "uri"))
1114         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "readonly-uri"))
1115         d.addCallback(lambda ign: _no_etag(self._bar_txt_uri, "info"))
1116
1117         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, ""))
1118         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "json"))
1119         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "uri"))
1120         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "readonly-uri"))
1121         d.addCallback(lambda ign: _no_etag(self._immdir_uri, "info"))
1122         d.addCallback(lambda ign: _no_etag(self._immdir_uri, "rename-form"))
1123
1124         return d
1125
1126     # TODO: version of this with a Unicode filename
1127     def test_GET_FILEURL_save(self):
1128         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
1129                      return_response=True)
1130         def _got((res, statuscode, headers)):
1131             content_disposition = headers["content-disposition"][0]
1132             self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
1133             self.failUnlessIsBarDotTxt(res)
1134         d.addCallback(_got)
1135         return d
1136
1137     def test_GET_FILEURL_missing(self):
1138         d = self.GET(self.public_url + "/foo/missing")
1139         d.addBoth(self.should404, "test_GET_FILEURL_missing")
1140         return d
1141
1142     def test_GET_FILEURL_info_mdmf(self):
1143         d = self.GET("/uri/%s?t=info" % self._quux_txt_uri)
1144         def _got(res):
1145             self.failUnlessIn("mutable file (mdmf)", res)
1146             self.failUnlessIn(self._quux_txt_uri, res)
1147             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1148         d.addCallback(_got)
1149         return d
1150
1151     def test_GET_FILEURL_info_mdmf_readonly(self):
1152         d = self.GET("/uri/%s?t=info" % self._quux_txt_readonly_uri)
1153         def _got(res):
1154             self.failUnlessIn("mutable file (mdmf)", res)
1155             self.failIfIn(self._quux_txt_uri, res)
1156             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1157         d.addCallback(_got)
1158         return d
1159
1160     def test_GET_FILEURL_info_sdmf(self):
1161         d = self.GET("/uri/%s?t=info" % self._baz_txt_uri)
1162         def _got(res):
1163             self.failUnlessIn("mutable file (sdmf)", res)
1164             self.failUnlessIn(self._baz_txt_uri, res)
1165         d.addCallback(_got)
1166         return d
1167
1168     def test_GET_FILEURL_info_mdmf_extensions(self):
1169         d = self.GET("/uri/%s:STUFF?t=info" % self._quux_txt_uri)
1170         def _got(res):
1171             self.failUnlessIn("mutable file (mdmf)", res)
1172             self.failUnlessIn(self._quux_txt_uri, res)
1173             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1174         d.addCallback(_got)
1175         return d
1176
1177     def test_PUT_overwrite_only_files(self):
1178         # create a directory, put a file in that directory.
1179         contents, n, filecap = self.makefile(8)
1180         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
1181         d.addCallback(lambda res:
1182             self.PUT(self.public_url + "/foo/dir/file1.txt",
1183                      self.NEWFILE_CONTENTS))
1184         # try to overwrite the file with replace=only-files
1185         # (this should work)
1186         d.addCallback(lambda res:
1187             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
1188                      filecap))
1189         d.addCallback(lambda res:
1190             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
1191                  "There was already a child by that name, and you asked me "
1192                  "to not replace it",
1193                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
1194                  filecap))
1195         return d
1196
1197     def test_PUT_NEWFILEURL(self):
1198         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
1199         # TODO: we lose the response code, so we can't check this
1200         #self.failUnlessReallyEqual(responsecode, 201)
1201         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1202         d.addCallback(lambda res:
1203                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1204                                                       self.NEWFILE_CONTENTS))
1205         return d
1206
1207     def test_PUT_NEWFILEURL_not_mutable(self):
1208         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
1209                      self.NEWFILE_CONTENTS)
1210         # TODO: we lose the response code, so we can't check this
1211         #self.failUnlessReallyEqual(responsecode, 201)
1212         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1213         d.addCallback(lambda res:
1214                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1215                                                       self.NEWFILE_CONTENTS))
1216         return d
1217
1218     def test_PUT_NEWFILEURL_unlinked_mdmf(self):
1219         # this should get us a few segments of an MDMF mutable file,
1220         # which we can then test for.
1221         contents = self.NEWFILE_CONTENTS * 300000
1222         d = self.PUT("/uri?format=mdmf",
1223                      contents)
1224         def _got_filecap(filecap):
1225             self.failUnless(filecap.startswith("URI:MDMF"))
1226             return filecap
1227         d.addCallback(_got_filecap)
1228         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1229         d.addCallback(lambda json: self.failUnlessIn("MDMF", json))
1230         return d
1231
1232     def test_PUT_NEWFILEURL_unlinked_sdmf(self):
1233         contents = self.NEWFILE_CONTENTS * 300000
1234         d = self.PUT("/uri?format=sdmf",
1235                      contents)
1236         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1237         d.addCallback(lambda json: self.failUnlessIn("SDMF", json))
1238         return d
1239
1240     def test_PUT_NEWFILEURL_unlinked_bad_format(self):
1241         contents = self.NEWFILE_CONTENTS * 300000
1242         return self.shouldHTTPError("PUT_NEWFILEURL_unlinked_bad_format",
1243                                     400, "Bad Request", "Unknown format: foo",
1244                                     self.PUT, "/uri?format=foo",
1245                                     contents)
1246
1247     def test_PUT_NEWFILEURL_range_bad(self):
1248         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
1249         target = self.public_url + "/foo/new.txt"
1250         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
1251                              "501 Not Implemented",
1252                              "Content-Range in PUT not yet supported",
1253                              # (and certainly not for immutable files)
1254                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
1255                              headers=headers)
1256         d.addCallback(lambda res:
1257                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
1258         return d
1259
1260     def test_PUT_NEWFILEURL_mutable(self):
1261         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1262                      self.NEWFILE_CONTENTS)
1263         # TODO: we lose the response code, so we can't check this
1264         #self.failUnlessReallyEqual(responsecode, 201)
1265         def _check_uri(res):
1266             u = uri.from_string_mutable_filenode(res)
1267             self.failUnless(u.is_mutable())
1268             self.failIf(u.is_readonly())
1269             return res
1270         d.addCallback(_check_uri)
1271         d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
1272         d.addCallback(lambda res:
1273                       self.failUnlessMutableChildContentsAre(self._foo_node,
1274                                                              u"new.txt",
1275                                                              self.NEWFILE_CONTENTS))
1276         return d
1277
1278     def test_PUT_NEWFILEURL_mutable_toobig(self):
1279         # It is okay to upload large mutable files, so we should be able
1280         # to do that.
1281         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1282                      "b" * (self.s.MUTABLE_SIZELIMIT + 1))
1283         return d
1284
1285     def test_PUT_NEWFILEURL_replace(self):
1286         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
1287         # TODO: we lose the response code, so we can't check this
1288         #self.failUnlessReallyEqual(responsecode, 200)
1289         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
1290         d.addCallback(lambda res:
1291                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1292                                                       self.NEWFILE_CONTENTS))
1293         return d
1294
1295     def test_PUT_NEWFILEURL_bad_t(self):
1296         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
1297                              "PUT to a file: bad t=bogus",
1298                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
1299                              "contents")
1300         return d
1301
1302     def test_PUT_NEWFILEURL_no_replace(self):
1303         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
1304                      self.NEWFILE_CONTENTS)
1305         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
1306                   "409 Conflict",
1307                   "There was already a child by that name, and you asked me "
1308                   "to not replace it")
1309         return d
1310
1311     def test_PUT_NEWFILEURL_mkdirs(self):
1312         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
1313         fn = self._foo_node
1314         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
1315         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
1316         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
1317         d.addCallback(lambda res:
1318                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
1319                                                       self.NEWFILE_CONTENTS))
1320         return d
1321
1322     def test_PUT_NEWFILEURL_blocked(self):
1323         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
1324                      self.NEWFILE_CONTENTS)
1325         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
1326                   "409 Conflict",
1327                   "Unable to create directory 'blockingfile': a file was in the way")
1328         return d
1329
1330     def test_PUT_NEWFILEURL_emptyname(self):
1331         # an empty pathname component (i.e. a double-slash) is disallowed
1332         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
1333                              "400 Bad Request",
1334                              "The webapi does not allow empty pathname components",
1335                              self.PUT, self.public_url + "/foo//new.txt", "")
1336         return d
1337
1338     def test_DELETE_FILEURL(self):
1339         d = self.DELETE(self.public_url + "/foo/bar.txt")
1340         d.addCallback(lambda res:
1341                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1342         return d
1343
1344     def test_DELETE_FILEURL_missing(self):
1345         d = self.DELETE(self.public_url + "/foo/missing")
1346         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
1347         return d
1348
1349     def test_DELETE_FILEURL_missing2(self):
1350         d = self.DELETE(self.public_url + "/missing/missing")
1351         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
1352         return d
1353
1354     def failUnlessHasBarDotTxtMetadata(self, res):
1355         data = simplejson.loads(res)
1356         self.failUnless(isinstance(data, list))
1357         self.failUnlessIn("metadata", data[1])
1358         self.failUnlessIn("tahoe", data[1]["metadata"])
1359         self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
1360         self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
1361         self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
1362                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
1363
1364     def test_GET_FILEURL_json(self):
1365         # twisted.web.http.parse_qs ignores any query args without an '=', so
1366         # I can't do "GET /path?json", I have to do "GET /path/t=json"
1367         # instead. This may make it tricky to emulate the S3 interface
1368         # completely.
1369         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
1370         def _check1(data):
1371             self.failUnlessIsBarJSON(data)
1372             self.failUnlessHasBarDotTxtMetadata(data)
1373             return
1374         d.addCallback(_check1)
1375         return d
1376
1377     def test_GET_FILEURL_json_mutable_type(self):
1378         # The JSON should include format, which says whether the
1379         # file is SDMF or MDMF
1380         d = self.PUT("/uri?format=mdmf",
1381                      self.NEWFILE_CONTENTS * 300000)
1382         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1383         def _got_json(json, version):
1384             data = simplejson.loads(json)
1385             assert "filenode" == data[0]
1386             data = data[1]
1387             assert isinstance(data, dict)
1388
1389             self.failUnlessIn("format", data)
1390             self.failUnlessEqual(data["format"], version)
1391
1392         d.addCallback(_got_json, "MDMF")
1393         # Now make an SDMF file and check that it is reported correctly.
1394         d.addCallback(lambda ignored:
1395             self.PUT("/uri?format=sdmf",
1396                       self.NEWFILE_CONTENTS * 300000))
1397         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1398         d.addCallback(_got_json, "SDMF")
1399         return d
1400
1401     def test_GET_FILEURL_json_mdmf(self):
1402         d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
1403         d.addCallback(self.failUnlessIsQuuxJSON)
1404         return d
1405
1406     def test_GET_FILEURL_json_missing(self):
1407         d = self.GET(self.public_url + "/foo/missing?json")
1408         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
1409         return d
1410
1411     def test_GET_FILEURL_uri(self):
1412         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
1413         def _check(res):
1414             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1415         d.addCallback(_check)
1416         d.addCallback(lambda res:
1417                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
1418         def _check2(res):
1419             # for now, for files, uris and readonly-uris are the same
1420             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1421         d.addCallback(_check2)
1422         return d
1423
1424     def test_GET_FILEURL_badtype(self):
1425         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
1426                                  "bad t=bogus",
1427                                  self.GET,
1428                                  self.public_url + "/foo/bar.txt?t=bogus")
1429         return d
1430
1431     def test_CSS_FILE(self):
1432         d = self.GET("/tahoe.css", followRedirect=True)
1433         def _check(res):
1434             CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
1435             self.failUnless(CSS_STYLE.search(res), res)
1436         d.addCallback(_check)
1437         return d
1438
1439     def test_GET_FILEURL_uri_missing(self):
1440         d = self.GET(self.public_url + "/foo/missing?t=uri")
1441         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
1442         return d
1443
1444     def _check_upload_and_mkdir_forms(self, html):
1445         # We should have a form to create a file, with radio buttons that allow
1446         # the user to toggle whether it is a CHK/LIT (default), SDMF, or MDMF file.
1447         self.failUnlessIn('name="t" value="upload"', html)
1448         self.failUnlessIn('input checked="checked" type="radio" id="upload-chk" value="chk" name="format"', html)
1449         self.failUnlessIn('input type="radio" id="upload-sdmf" value="sdmf" name="format"', html)
1450         self.failUnlessIn('input type="radio" id="upload-mdmf" value="mdmf" name="format"', html)
1451
1452         # We should also have the ability to create a mutable directory, with
1453         # radio buttons that allow the user to toggle whether it is an SDMF (default)
1454         # or MDMF directory.
1455         self.failUnlessIn('name="t" value="mkdir"', html)
1456         self.failUnlessIn('input checked="checked" type="radio" id="mkdir-sdmf" value="sdmf" name="format"', html)
1457         self.failUnlessIn('input type="radio" id="mkdir-mdmf" value="mdmf" name="format"', html)
1458
1459         self.failUnlessIn(FAVICON_MARKUP, html)
1460
1461     def test_GET_DIRECTORY_html(self):
1462         d = self.GET(self.public_url + "/foo", followRedirect=True)
1463         def _check(html):
1464             self.failUnlessIn('<div class="toolbar-item"><a href="../../..">Return to Welcome page</a></div>', html)
1465             self._check_upload_and_mkdir_forms(html)
1466             self.failUnlessIn("quux", html)
1467         d.addCallback(_check)
1468         return d
1469
1470     def test_GET_DIRECTORY_html_filenode_encoding(self):
1471         d = self.GET(self.public_url + "/foo", followRedirect=True)
1472         def _check(html):
1473             # Check if encoded entries are there
1474             self.failUnlessIn('@@named=/' + self._htmlname_urlencoded + '">'
1475                               + self._htmlname_escaped + '</a>', html)
1476             self.failUnlessIn('value="' + self._htmlname_escaped_attr + '"', html)
1477             self.failIfIn(self._htmlname_escaped_double, html)
1478             # Make sure that Nevow escaping actually works by checking for unsafe characters
1479             # and that '&' is escaped.
1480             for entity in '<>':
1481                 self.failUnlessIn(entity, self._htmlname_raw)
1482                 self.failIfIn(entity, self._htmlname_escaped)
1483             self.failUnlessIn('&', re.sub(r'&(amp|lt|gt|quot|apos);', '', self._htmlname_raw))
1484             self.failIfIn('&', re.sub(r'&(amp|lt|gt|quot|apos);', '', self._htmlname_escaped))
1485         d.addCallback(_check)
1486         return d
1487
1488     def test_GET_root_html(self):
1489         d = self.GET("/")
1490         d.addCallback(self._check_upload_and_mkdir_forms)
1491         return d
1492
1493     def test_GET_DIRURL(self):
1494         # the addSlash means we get a redirect here
1495         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1496         ROOT = "../../.."
1497         d = self.GET(self.public_url + "/foo", followRedirect=True)
1498         def _check(res):
1499             self.failUnlessIn('<a href="%s">Return to Welcome page' % ROOT, res)
1500
1501             # the FILE reference points to a URI, but it should end in bar.txt
1502             bar_url = ("%s/file/%s/@@named=/bar.txt" %
1503                        (ROOT, urllib.quote(self._bar_txt_uri)))
1504             get_bar = "".join([r'<td>FILE</td>',
1505                                r'\s+<td>',
1506                                r'<a href="%s">bar.txt</a>' % bar_url,
1507                                r'</td>',
1508                                r'\s+<td align="right">%d</td>' % len(self.BAR_CONTENTS),
1509                                ])
1510             self.failUnless(re.search(get_bar, res), res)
1511             for label in ['unlink', 'rename/move']:
1512                 for line in res.split("\n"):
1513                     # find the line that contains the relevant button for bar.txt
1514                     if ("form action" in line and
1515                         ('value="%s"' % (label,)) in line and
1516                         'value="bar.txt"' in line):
1517                         # the form target should use a relative URL
1518                         foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1519                         self.failUnlessIn('action="%s"' % foo_url, line)
1520                         # and the when_done= should too
1521                         #done_url = urllib.quote(???)
1522                         #self.failUnlessIn('name="when_done" value="%s"' % done_url, line)
1523
1524                         # 'unlink' needs to use POST because it directly has a side effect
1525                         if label == 'unlink':
1526                             self.failUnlessIn('method="post"', line)
1527                         break
1528                 else:
1529                     self.fail("unable to find '%s bar.txt' line" % (label,))
1530
1531             # the DIR reference just points to a URI
1532             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1533             get_sub = ((r'<td>DIR</td>')
1534                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1535             self.failUnless(re.search(get_sub, res), res)
1536         d.addCallback(_check)
1537
1538         # look at a readonly directory
1539         d.addCallback(lambda res:
1540                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
1541         def _check2(res):
1542             self.failUnlessIn("(read-only)", res)
1543             self.failIfIn("Upload a file", res)
1544         d.addCallback(_check2)
1545
1546         # and at a directory that contains a readonly directory
1547         d.addCallback(lambda res:
1548                       self.GET(self.public_url, followRedirect=True))
1549         def _check3(res):
1550             self.failUnless(re.search('<td>DIR-RO</td>'
1551                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1552         d.addCallback(_check3)
1553
1554         # and an empty directory
1555         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1556         def _check4(res):
1557             self.failUnlessIn("directory is empty", res)
1558             MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
1559             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1560         d.addCallback(_check4)
1561
1562         # and at a literal directory
1563         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1564         d.addCallback(lambda res:
1565                       self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1566         def _check5(res):
1567             self.failUnlessIn('(immutable)', res)
1568             self.failUnless(re.search('<td>FILE</td>'
1569                                       r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1570         d.addCallback(_check5)
1571         return d
1572
1573     def test_GET_DIRURL_badtype(self):
1574         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1575                                  400, "Bad Request",
1576                                  "bad t=bogus",
1577                                  self.GET,
1578                                  self.public_url + "/foo?t=bogus")
1579         return d
1580
1581     def test_GET_DIRURL_json(self):
1582         d = self.GET(self.public_url + "/foo?t=json")
1583         d.addCallback(self.failUnlessIsFooJSON)
1584         return d
1585
1586     def test_GET_DIRURL_json_format(self):
1587         d = self.PUT(self.public_url + \
1588                      "/foo/sdmf.txt?format=sdmf",
1589                      self.NEWFILE_CONTENTS * 300000)
1590         d.addCallback(lambda ignored:
1591             self.PUT(self.public_url + \
1592                      "/foo/mdmf.txt?format=mdmf",
1593                      self.NEWFILE_CONTENTS * 300000))
1594         # Now we have an MDMF and SDMF file in the directory. If we GET
1595         # its JSON, we should see their encodings.
1596         d.addCallback(lambda ignored:
1597             self.GET(self.public_url + "/foo?t=json"))
1598         def _got_json(json):
1599             data = simplejson.loads(json)
1600             assert data[0] == "dirnode"
1601
1602             data = data[1]
1603             kids = data['children']
1604
1605             mdmf_data = kids['mdmf.txt'][1]
1606             self.failUnlessIn("format", mdmf_data)
1607             self.failUnlessEqual(mdmf_data["format"], "MDMF")
1608
1609             sdmf_data = kids['sdmf.txt'][1]
1610             self.failUnlessIn("format", sdmf_data)
1611             self.failUnlessEqual(sdmf_data["format"], "SDMF")
1612         d.addCallback(_got_json)
1613         return d
1614
1615
1616     def test_POST_DIRURL_manifest_no_ophandle(self):
1617         d = self.shouldFail2(error.Error,
1618                              "test_POST_DIRURL_manifest_no_ophandle",
1619                              "400 Bad Request",
1620                              "slow operation requires ophandle=",
1621                              self.POST, self.public_url, t="start-manifest")
1622         return d
1623
1624     def test_POST_DIRURL_manifest(self):
1625         d = defer.succeed(None)
1626         def getman(ignored, output):
1627             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1628                           followRedirect=True)
1629             d.addCallback(self.wait_for_operation, "125")
1630             d.addCallback(self.get_operation_results, "125", output)
1631             return d
1632         d.addCallback(getman, None)
1633         def _got_html(manifest):
1634             self.failUnlessIn("Manifest of SI=", manifest)
1635             self.failUnlessIn("<td>sub</td>", manifest)
1636             self.failUnlessIn(self._sub_uri, manifest)
1637             self.failUnlessIn("<td>sub/baz.txt</td>", manifest)
1638             self.failUnlessIn(FAVICON_MARKUP, manifest)
1639         d.addCallback(_got_html)
1640
1641         # both t=status and unadorned GET should be identical
1642         d.addCallback(lambda res: self.GET("/operations/125"))
1643         d.addCallback(_got_html)
1644
1645         d.addCallback(getman, "html")
1646         d.addCallback(_got_html)
1647         d.addCallback(getman, "text")
1648         def _got_text(manifest):
1649             self.failUnlessIn("\nsub " + self._sub_uri + "\n", manifest)
1650             self.failUnlessIn("\nsub/baz.txt URI:CHK:", manifest)
1651         d.addCallback(_got_text)
1652         d.addCallback(getman, "JSON")
1653         def _got_json(res):
1654             data = res["manifest"]
1655             got = {}
1656             for (path_list, cap) in data:
1657                 got[tuple(path_list)] = cap
1658             self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
1659             self.failUnlessIn((u"sub", u"baz.txt"), got)
1660             self.failUnlessIn("finished", res)
1661             self.failUnlessIn("origin", res)
1662             self.failUnlessIn("storage-index", res)
1663             self.failUnlessIn("verifycaps", res)
1664             self.failUnlessIn("stats", res)
1665         d.addCallback(_got_json)
1666         return d
1667
1668     def test_POST_DIRURL_deepsize_no_ophandle(self):
1669         d = self.shouldFail2(error.Error,
1670                              "test_POST_DIRURL_deepsize_no_ophandle",
1671                              "400 Bad Request",
1672                              "slow operation requires ophandle=",
1673                              self.POST, self.public_url, t="start-deep-size")
1674         return d
1675
1676     def test_POST_DIRURL_deepsize(self):
1677         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1678                       followRedirect=True)
1679         d.addCallback(self.wait_for_operation, "126")
1680         d.addCallback(self.get_operation_results, "126", "json")
1681         def _got_json(data):
1682             self.failUnlessReallyEqual(data["finished"], True)
1683             size = data["size"]
1684             self.failUnless(size > 1000)
1685         d.addCallback(_got_json)
1686         d.addCallback(self.get_operation_results, "126", "text")
1687         def _got_text(res):
1688             mo = re.search(r'^size: (\d+)$', res, re.M)
1689             self.failUnless(mo, res)
1690             size = int(mo.group(1))
1691             # with directories, the size varies.
1692             self.failUnless(size > 1000)
1693         d.addCallback(_got_text)
1694         return d
1695
1696     def test_POST_DIRURL_deepstats_no_ophandle(self):
1697         d = self.shouldFail2(error.Error,
1698                              "test_POST_DIRURL_deepstats_no_ophandle",
1699                              "400 Bad Request",
1700                              "slow operation requires ophandle=",
1701                              self.POST, self.public_url, t="start-deep-stats")
1702         return d
1703
1704     def test_POST_DIRURL_deepstats(self):
1705         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1706                       followRedirect=True)
1707         d.addCallback(self.wait_for_operation, "127")
1708         d.addCallback(self.get_operation_results, "127", "json")
1709         def _got_json(stats):
1710             expected = {"count-immutable-files": 4,
1711                         "count-mutable-files": 2,
1712                         "count-literal-files": 0,
1713                         "count-files": 6,
1714                         "count-directories": 3,
1715                         "size-immutable-files": 76,
1716                         "size-literal-files": 0,
1717                         #"size-directories": 1912, # varies
1718                         #"largest-directory": 1590,
1719                         "largest-directory-children": 8,
1720                         "largest-immutable-file": 19,
1721                         }
1722             for k,v in expected.iteritems():
1723                 self.failUnlessReallyEqual(stats[k], v,
1724                                            "stats[%s] was %s, not %s" %
1725                                            (k, stats[k], v))
1726             self.failUnlessReallyEqual(stats["size-files-histogram"],
1727                                        [ [11, 31, 4] ])
1728         d.addCallback(_got_json)
1729         return d
1730
1731     def test_POST_DIRURL_stream_manifest(self):
1732         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1733         def _check(res):
1734             self.failUnless(res.endswith("\n"))
1735             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1736             self.failUnlessReallyEqual(len(units), 10)
1737             self.failUnlessEqual(units[-1]["type"], "stats")
1738             first = units[0]
1739             self.failUnlessEqual(first["path"], [])
1740             self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri)
1741             self.failUnlessEqual(first["type"], "directory")
1742             baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0]
1743             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1744             self.failIfEqual(baz["storage-index"], None)
1745             self.failIfEqual(baz["verifycap"], None)
1746             self.failIfEqual(baz["repaircap"], None)
1747             # XXX: Add quux and baz to this test.
1748             return
1749         d.addCallback(_check)
1750         return d
1751
1752     def test_GET_DIRURL_uri(self):
1753         d = self.GET(self.public_url + "/foo?t=uri")
1754         def _check(res):
1755             self.failUnlessReallyEqual(to_str(res), self._foo_uri)
1756         d.addCallback(_check)
1757         return d
1758
1759     def test_GET_DIRURL_readonly_uri(self):
1760         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1761         def _check(res):
1762             self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri)
1763         d.addCallback(_check)
1764         return d
1765
1766     def test_PUT_NEWDIRURL(self):
1767         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1768         d.addCallback(lambda res:
1769                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1770         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1771         d.addCallback(self.failUnlessNodeKeysAre, [])
1772         return d
1773
1774     def test_PUT_NEWDIRURL_mdmf(self):
1775         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1776         d.addCallback(lambda res:
1777                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1778         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1779         d.addCallback(lambda node:
1780             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1781         return d
1782
1783     def test_PUT_NEWDIRURL_sdmf(self):
1784         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=sdmf",
1785                      "")
1786         d.addCallback(lambda res:
1787                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1788         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1789         d.addCallback(lambda node:
1790             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1791         return d
1792
1793     def test_PUT_NEWDIRURL_bad_format(self):
1794         return self.shouldHTTPError("PUT_NEWDIRURL_bad_format",
1795                                     400, "Bad Request", "Unknown format: foo",
1796                                     self.PUT, self.public_url +
1797                                     "/foo/newdir=?t=mkdir&format=foo", "")
1798
1799     def test_POST_NEWDIRURL(self):
1800         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1801         d.addCallback(lambda res:
1802                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1803         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1804         d.addCallback(self.failUnlessNodeKeysAre, [])
1805         return d
1806
1807     def test_POST_NEWDIRURL_mdmf(self):
1808         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1809         d.addCallback(lambda res:
1810                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1811         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1812         d.addCallback(lambda node:
1813             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1814         return d
1815
1816     def test_POST_NEWDIRURL_sdmf(self):
1817         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=sdmf", "")
1818         d.addCallback(lambda res:
1819             self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1820         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1821         d.addCallback(lambda node:
1822             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1823         return d
1824
1825     def test_POST_NEWDIRURL_bad_format(self):
1826         return self.shouldHTTPError("POST_NEWDIRURL_bad_format",
1827                                     400, "Bad Request", "Unknown format: foo",
1828                                     self.POST2, self.public_url + \
1829                                     "/foo/newdir?t=mkdir&format=foo", "")
1830
1831     def test_POST_NEWDIRURL_emptyname(self):
1832         # an empty pathname component (i.e. a double-slash) is disallowed
1833         d = self.shouldFail2(error.Error, "POST_NEWDIRURL_emptyname",
1834                              "400 Bad Request",
1835                              "The webapi does not allow empty pathname components, i.e. a double slash",
1836                              self.POST, self.public_url + "//?t=mkdir")
1837         return d
1838
1839     def _do_POST_NEWDIRURL_initial_children_test(self, version=None):
1840         (newkids, caps) = self._create_initial_children()
1841         query = "/foo/newdir?t=mkdir-with-children"
1842         if version == MDMF_VERSION:
1843             query += "&format=mdmf"
1844         elif version == SDMF_VERSION:
1845             query += "&format=sdmf"
1846         else:
1847             version = SDMF_VERSION # for later
1848         d = self.POST2(self.public_url + query,
1849                        simplejson.dumps(newkids))
1850         def _check(uri):
1851             n = self.s.create_node_from_uri(uri.strip())
1852             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1853             self.failUnlessEqual(n._node.get_version(), version)
1854             d2.addCallback(lambda ign:
1855                            self.failUnlessROChildURIIs(n, u"child-imm",
1856                                                        caps['filecap1']))
1857             d2.addCallback(lambda ign:
1858                            self.failUnlessRWChildURIIs(n, u"child-mutable",
1859                                                        caps['filecap2']))
1860             d2.addCallback(lambda ign:
1861                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1862                                                        caps['filecap3']))
1863             d2.addCallback(lambda ign:
1864                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1865                                                        caps['unknown_rocap']))
1866             d2.addCallback(lambda ign:
1867                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1868                                                        caps['unknown_rwcap']))
1869             d2.addCallback(lambda ign:
1870                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1871                                                        caps['unknown_immcap']))
1872             d2.addCallback(lambda ign:
1873                            self.failUnlessRWChildURIIs(n, u"dirchild",
1874                                                        caps['dircap']))
1875             d2.addCallback(lambda ign:
1876                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1877                                                        caps['litdircap']))
1878             d2.addCallback(lambda ign:
1879                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1880                                                        caps['emptydircap']))
1881             return d2
1882         d.addCallback(_check)
1883         d.addCallback(lambda res:
1884                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1885         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1886         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1887         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1888         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1889         return d
1890
1891     def test_POST_NEWDIRURL_initial_children(self):
1892         return self._do_POST_NEWDIRURL_initial_children_test()
1893
1894     def test_POST_NEWDIRURL_initial_children_mdmf(self):
1895         return self._do_POST_NEWDIRURL_initial_children_test(MDMF_VERSION)
1896
1897     def test_POST_NEWDIRURL_initial_children_sdmf(self):
1898         return self._do_POST_NEWDIRURL_initial_children_test(SDMF_VERSION)
1899
1900     def test_POST_NEWDIRURL_initial_children_bad_format(self):
1901         (newkids, caps) = self._create_initial_children()
1902         return self.shouldHTTPError("POST_NEWDIRURL_initial_children_bad_format",
1903                                     400, "Bad Request", "Unknown format: foo",
1904                                     self.POST2, self.public_url + \
1905                                     "/foo/newdir?t=mkdir-with-children&format=foo",
1906                                     simplejson.dumps(newkids))
1907
1908     def test_POST_NEWDIRURL_immutable(self):
1909         (newkids, caps) = self._create_immutable_children()
1910         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1911                        simplejson.dumps(newkids))
1912         def _check(uri):
1913             n = self.s.create_node_from_uri(uri.strip())
1914             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1915             d2.addCallback(lambda ign:
1916                            self.failUnlessROChildURIIs(n, u"child-imm",
1917                                                        caps['filecap1']))
1918             d2.addCallback(lambda ign:
1919                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1920                                                        caps['unknown_immcap']))
1921             d2.addCallback(lambda ign:
1922                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
1923                                                        caps['immdircap']))
1924             d2.addCallback(lambda ign:
1925                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1926                                                        caps['litdircap']))
1927             d2.addCallback(lambda ign:
1928                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1929                                                        caps['emptydircap']))
1930             return d2
1931         d.addCallback(_check)
1932         d.addCallback(lambda res:
1933                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1934         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1935         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1936         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1937         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1938         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1939         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1940         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1941         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
1942         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1943         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
1944         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1945         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
1946         d.addErrback(self.explain_web_error)
1947         return d
1948
1949     def test_POST_NEWDIRURL_immutable_bad(self):
1950         (newkids, caps) = self._create_initial_children()
1951         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
1952                              "400 Bad Request",
1953                              "needed to be immutable but was not",
1954                              self.POST2,
1955                              self.public_url + "/foo/newdir?t=mkdir-immutable",
1956                              simplejson.dumps(newkids))
1957         return d
1958
1959     def test_PUT_NEWDIRURL_exists(self):
1960         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1961         d.addCallback(lambda res:
1962                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1963         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1964         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1965         return d
1966
1967     def test_PUT_NEWDIRURL_blocked(self):
1968         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1969                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1970                              self.PUT,
1971                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1972         d.addCallback(lambda res:
1973                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1974         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1975         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1976         return d
1977
1978     def test_PUT_NEWDIRURL_mkdirs(self):
1979         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1980         d.addCallback(lambda res:
1981                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1982         d.addCallback(lambda res:
1983                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1984         d.addCallback(lambda res:
1985                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1986         d.addCallback(self.failUnlessNodeKeysAre, [])
1987         return d
1988
1989     def test_PUT_NEWDIRURL_mkdirs_mdmf(self):
1990         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=mdmf", "")
1991         d.addCallback(lambda ignored:
1992             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1993         d.addCallback(lambda ignored:
1994             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1995         d.addCallback(lambda ignored:
1996             self._foo_node.get_child_at_path(u"subdir"))
1997         def _got_subdir(subdir):
1998             # XXX: What we want?
1999             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
2000             self.failUnlessNodeHasChild(subdir, u"newdir")
2001             return subdir.get_child_at_path(u"newdir")
2002         d.addCallback(_got_subdir)
2003         d.addCallback(lambda newdir:
2004             self.failUnlessEqual(newdir._node.get_version(), MDMF_VERSION))
2005         return d
2006
2007     def test_PUT_NEWDIRURL_mkdirs_sdmf(self):
2008         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=sdmf", "")
2009         d.addCallback(lambda ignored:
2010             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
2011         d.addCallback(lambda ignored:
2012             self.failIfNodeHasChild(self._foo_node, u"newdir"))
2013         d.addCallback(lambda ignored:
2014             self._foo_node.get_child_at_path(u"subdir"))
2015         def _got_subdir(subdir):
2016             # XXX: What we want?
2017             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
2018             self.failUnlessNodeHasChild(subdir, u"newdir")
2019             return subdir.get_child_at_path(u"newdir")
2020         d.addCallback(_got_subdir)
2021         d.addCallback(lambda newdir:
2022             self.failUnlessEqual(newdir._node.get_version(), SDMF_VERSION))
2023         return d
2024
2025     def test_PUT_NEWDIRURL_mkdirs_bad_format(self):
2026         return self.shouldHTTPError("PUT_NEWDIRURL_mkdirs_bad_format",
2027                                     400, "Bad Request", "Unknown format: foo",
2028                                     self.PUT, self.public_url + \
2029                                     "/foo/subdir/newdir?t=mkdir&format=foo",
2030                                     "")
2031
2032     def test_DELETE_DIRURL(self):
2033         d = self.DELETE(self.public_url + "/foo")
2034         d.addCallback(lambda res:
2035                       self.failIfNodeHasChild(self.public_root, u"foo"))
2036         return d
2037
2038     def test_DELETE_DIRURL_missing(self):
2039         d = self.DELETE(self.public_url + "/foo/missing")
2040         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
2041         d.addCallback(lambda res:
2042                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
2043         return d
2044
2045     def test_DELETE_DIRURL_missing2(self):
2046         d = self.DELETE(self.public_url + "/missing")
2047         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
2048         return d
2049
2050     def dump_root(self):
2051         print "NODEWALK"
2052         w = webish.DirnodeWalkerMixin()
2053         def visitor(childpath, childnode, metadata):
2054             print childpath
2055         d = w.walk(self.public_root, visitor)
2056         return d
2057
2058     def failUnlessNodeKeysAre(self, node, expected_keys):
2059         for k in expected_keys:
2060             assert isinstance(k, unicode)
2061         d = node.list()
2062         def _check(children):
2063             self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
2064         d.addCallback(_check)
2065         return d
2066     def failUnlessNodeHasChild(self, node, name):
2067         assert isinstance(name, unicode)
2068         d = node.list()
2069         def _check(children):
2070             self.failUnlessIn(name, children)
2071         d.addCallback(_check)
2072         return d
2073     def failIfNodeHasChild(self, node, name):
2074         assert isinstance(name, unicode)
2075         d = node.list()
2076         def _check(children):
2077             self.failIfIn(name, children)
2078         d.addCallback(_check)
2079         return d
2080
2081     def failUnlessChildContentsAre(self, node, name, expected_contents):
2082         assert isinstance(name, unicode)
2083         d = node.get_child_at_path(name)
2084         d.addCallback(lambda node: download_to_data(node))
2085         def _check(contents):
2086             self.failUnlessReallyEqual(contents, expected_contents)
2087         d.addCallback(_check)
2088         return d
2089
2090     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
2091         assert isinstance(name, unicode)
2092         d = node.get_child_at_path(name)
2093         d.addCallback(lambda node: node.download_best_version())
2094         def _check(contents):
2095             self.failUnlessReallyEqual(contents, expected_contents)
2096         d.addCallback(_check)
2097         return d
2098
2099     def failUnlessRWChildURIIs(self, node, name, expected_uri):
2100         assert isinstance(name, unicode)
2101         d = node.get_child_at_path(name)
2102         def _check(child):
2103             self.failUnless(child.is_unknown() or not child.is_readonly())
2104             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
2105             self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
2106             expected_ro_uri = self._make_readonly(expected_uri)
2107             if expected_ro_uri:
2108                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
2109         d.addCallback(_check)
2110         return d
2111
2112     def failUnlessROChildURIIs(self, node, name, expected_uri):
2113         assert isinstance(name, unicode)
2114         d = node.get_child_at_path(name)
2115         def _check(child):
2116             self.failUnless(child.is_unknown() or child.is_readonly())
2117             self.failUnlessReallyEqual(child.get_write_uri(), None)
2118             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
2119             self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
2120         d.addCallback(_check)
2121         return d
2122
2123     def failUnlessURIMatchesRWChild(self, got_uri, node, name):
2124         assert isinstance(name, unicode)
2125         d = node.get_child_at_path(name)
2126         def _check(child):
2127             self.failUnless(child.is_unknown() or not child.is_readonly())
2128             self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
2129             self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
2130             expected_ro_uri = self._make_readonly(got_uri)
2131             if expected_ro_uri:
2132                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
2133         d.addCallback(_check)
2134         return d
2135
2136     def failUnlessURIMatchesROChild(self, got_uri, node, name):
2137         assert isinstance(name, unicode)
2138         d = node.get_child_at_path(name)
2139         def _check(child):
2140             self.failUnless(child.is_unknown() or child.is_readonly())
2141             self.failUnlessReallyEqual(child.get_write_uri(), None)
2142             self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
2143             self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
2144         d.addCallback(_check)
2145         return d
2146
2147     def failUnlessCHKURIHasContents(self, got_uri, contents):
2148         self.failUnless(self.get_all_contents()[got_uri] == contents)
2149
2150     def test_POST_upload(self):
2151         d = self.POST(self.public_url + "/foo", t="upload",
2152                       file=("new.txt", self.NEWFILE_CONTENTS))
2153         fn = self._foo_node
2154         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2155         d.addCallback(lambda res:
2156                       self.failUnlessChildContentsAre(fn, u"new.txt",
2157                                                       self.NEWFILE_CONTENTS))
2158         return d
2159
2160     def test_POST_upload_unicode(self):
2161         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2162         d = self.POST(self.public_url + "/foo", t="upload",
2163                       file=(filename, self.NEWFILE_CONTENTS))
2164         fn = self._foo_node
2165         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2166         d.addCallback(lambda res:
2167                       self.failUnlessChildContentsAre(fn, filename,
2168                                                       self.NEWFILE_CONTENTS))
2169         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2170         d.addCallback(lambda res: self.GET(target_url))
2171         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2172                                                                   self.NEWFILE_CONTENTS,
2173                                                                   contents))
2174         return d
2175
2176     def test_POST_upload_unicode_named(self):
2177         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2178         d = self.POST(self.public_url + "/foo", t="upload",
2179                       name=filename,
2180                       file=("overridden", self.NEWFILE_CONTENTS))
2181         fn = self._foo_node
2182         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2183         d.addCallback(lambda res:
2184                       self.failUnlessChildContentsAre(fn, filename,
2185                                                       self.NEWFILE_CONTENTS))
2186         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2187         d.addCallback(lambda res: self.GET(target_url))
2188         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2189                                                                   self.NEWFILE_CONTENTS,
2190                                                                   contents))
2191         return d
2192
2193     def test_POST_upload_no_link(self):
2194         d = self.POST("/uri", t="upload",
2195                       file=("new.txt", self.NEWFILE_CONTENTS))
2196         def _check_upload_results(page):
2197             # this should be a page which describes the results of the upload
2198             # that just finished.
2199             self.failUnlessIn("Upload Results:", page)
2200             self.failUnlessIn("URI:", page)
2201             uri_re = re.compile("URI: <tt><span>(.*)</span>")
2202             mo = uri_re.search(page)
2203             self.failUnless(mo, page)
2204             new_uri = mo.group(1)
2205             return new_uri
2206         d.addCallback(_check_upload_results)
2207         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
2208         return d
2209
2210     def test_POST_upload_no_link_whendone(self):
2211         d = self.POST("/uri", t="upload", when_done="/",
2212                       file=("new.txt", self.NEWFILE_CONTENTS))
2213         d.addBoth(self.shouldRedirect, "/")
2214         return d
2215
2216     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
2217         d = defer.maybeDeferred(callable, *args, **kwargs)
2218         def done(res):
2219             if isinstance(res, failure.Failure):
2220                 res.trap(error.PageRedirect)
2221                 statuscode = res.value.status
2222                 target = res.value.location
2223                 return checker(statuscode, target)
2224             self.fail("%s: callable was supposed to redirect, not return '%s'"
2225                       % (which, res))
2226         d.addBoth(done)
2227         return d
2228
2229     def test_POST_upload_no_link_whendone_results(self):
2230         def check(statuscode, target):
2231             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2232             self.failUnless(target.startswith(self.webish_url), target)
2233             return client.getPage(target, method="GET")
2234         # We encode "uri" as "%75ri" to exercise a case affected by ticket #1860.
2235         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
2236                                  check,
2237                                  self.POST, "/uri", t="upload",
2238                                  when_done="/%75ri/%(uri)s",
2239                                  file=("new.txt", self.NEWFILE_CONTENTS))
2240         d.addCallback(lambda res:
2241                       self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
2242         return d
2243
2244     def test_POST_upload_no_link_mutable(self):
2245         d = self.POST("/uri", t="upload", mutable="true",
2246                       file=("new.txt", self.NEWFILE_CONTENTS))
2247         def _check(filecap):
2248             filecap = filecap.strip()
2249             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2250             self.filecap = filecap
2251             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2252             self.failUnlessIn(u.get_storage_index(), self.get_all_contents())
2253             n = self.s.create_node_from_uri(filecap)
2254             return n.download_best_version()
2255         d.addCallback(_check)
2256         def _check2(data):
2257             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2258             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2259         d.addCallback(_check2)
2260         def _check3(data):
2261             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2262             return self.GET("/file/%s" % urllib.quote(self.filecap))
2263         d.addCallback(_check3)
2264         def _check4(data):
2265             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2266         d.addCallback(_check4)
2267         return d
2268
2269     def test_POST_upload_no_link_mutable_toobig(self):
2270         # The SDMF size limit is no longer in place, so we should be
2271         # able to upload mutable files that are as large as we want them
2272         # to be.
2273         d = self.POST("/uri", t="upload", mutable="true",
2274                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2275         return d
2276
2277
2278     def test_POST_upload_format_unlinked(self):
2279         def _check_upload_unlinked(ign, format, uri_prefix):
2280             filename = format + ".txt"
2281             d = self.POST("/uri?t=upload&format=" + format,
2282                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2283             def _got_results(results):
2284                 if format.upper() in ("SDMF", "MDMF"):
2285                     # webapi.rst says this returns a filecap
2286                     filecap = results
2287                 else:
2288                     # for immutable, it returns an "upload results page", and
2289                     # the filecap is buried inside
2290                     line = [l for l in results.split("\n") if "URI: " in l][0]
2291                     mo = re.search(r'<span>([^<]+)</span>', line)
2292                     filecap = mo.group(1)
2293                 self.failUnless(filecap.startswith(uri_prefix),
2294                                 (uri_prefix, filecap))
2295                 return self.GET("/uri/%s?t=json" % filecap)
2296             d.addCallback(_got_results)
2297             def _got_json(json):
2298                 data = simplejson.loads(json)
2299                 data = data[1]
2300                 self.failUnlessIn("format", data)
2301                 self.failUnlessEqual(data["format"], format.upper())
2302             d.addCallback(_got_json)
2303             return d
2304         d = defer.succeed(None)
2305         d.addCallback(_check_upload_unlinked, "chk", "URI:CHK")
2306         d.addCallback(_check_upload_unlinked, "CHK", "URI:CHK")
2307         d.addCallback(_check_upload_unlinked, "sdmf", "URI:SSK")
2308         d.addCallback(_check_upload_unlinked, "mdmf", "URI:MDMF")
2309         return d
2310
2311     def test_POST_upload_bad_format_unlinked(self):
2312         return self.shouldHTTPError("POST_upload_bad_format_unlinked",
2313                                     400, "Bad Request", "Unknown format: foo",
2314                                     self.POST,
2315                                     "/uri?t=upload&format=foo",
2316                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2317
2318     def test_POST_upload_format(self):
2319         def _check_upload(ign, format, uri_prefix, fn=None):
2320             filename = format + ".txt"
2321             d = self.POST(self.public_url +
2322                           "/foo?t=upload&format=" + format,
2323                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2324             def _got_filecap(filecap):
2325                 if fn is not None:
2326                     filenameu = unicode(filename)
2327                     self.failUnlessURIMatchesRWChild(filecap, fn, filenameu)
2328                 self.failUnless(filecap.startswith(uri_prefix))
2329                 return self.GET(self.public_url + "/foo/%s?t=json" % filename)
2330             d.addCallback(_got_filecap)
2331             def _got_json(json):
2332                 data = simplejson.loads(json)
2333                 data = data[1]
2334                 self.failUnlessIn("format", data)
2335                 self.failUnlessEqual(data["format"], format.upper())
2336             d.addCallback(_got_json)
2337             return d
2338
2339         d = defer.succeed(None)
2340         d.addCallback(_check_upload, "chk", "URI:CHK")
2341         d.addCallback(_check_upload, "sdmf", "URI:SSK", self._foo_node)
2342         d.addCallback(_check_upload, "mdmf", "URI:MDMF")
2343         d.addCallback(_check_upload, "MDMF", "URI:MDMF")
2344         return d
2345
2346     def test_POST_upload_bad_format(self):
2347         return self.shouldHTTPError("POST_upload_bad_format",
2348                                     400, "Bad Request", "Unknown format: foo",
2349                                     self.POST, self.public_url + \
2350                                     "/foo?t=upload&format=foo",
2351                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2352
2353     def test_POST_upload_mutable(self):
2354         # this creates a mutable file
2355         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
2356                       file=("new.txt", self.NEWFILE_CONTENTS))
2357         fn = self._foo_node
2358         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2359         d.addCallback(lambda res:
2360                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2361                                                              self.NEWFILE_CONTENTS))
2362         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2363         def _got(newnode):
2364             self.failUnless(IMutableFileNode.providedBy(newnode))
2365             self.failUnless(newnode.is_mutable())
2366             self.failIf(newnode.is_readonly())
2367             self._mutable_node = newnode
2368             self._mutable_uri = newnode.get_uri()
2369         d.addCallback(_got)
2370
2371         # now upload it again and make sure that the URI doesn't change
2372         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
2373         d.addCallback(lambda res:
2374                       self.POST(self.public_url + "/foo", t="upload",
2375                                 mutable="true",
2376                                 file=("new.txt", NEWER_CONTENTS)))
2377         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2378         d.addCallback(lambda res:
2379                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2380                                                              NEWER_CONTENTS))
2381         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2382         def _got2(newnode):
2383             self.failUnless(IMutableFileNode.providedBy(newnode))
2384             self.failUnless(newnode.is_mutable())
2385             self.failIf(newnode.is_readonly())
2386             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2387         d.addCallback(_got2)
2388
2389         # upload a second time, using PUT instead of POST
2390         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
2391         d.addCallback(lambda res:
2392                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
2393         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2394         d.addCallback(lambda res:
2395                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2396                                                              NEW2_CONTENTS))
2397
2398         # finally list the directory, since mutable files are displayed
2399         # slightly differently
2400
2401         d.addCallback(lambda res:
2402                       self.GET(self.public_url + "/foo/",
2403                                followRedirect=True))
2404         def _check_page(res):
2405             # TODO: assert more about the contents
2406             self.failUnlessIn("SSK", res)
2407             return res
2408         d.addCallback(_check_page)
2409
2410         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2411         def _got3(newnode):
2412             self.failUnless(IMutableFileNode.providedBy(newnode))
2413             self.failUnless(newnode.is_mutable())
2414             self.failIf(newnode.is_readonly())
2415             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2416         d.addCallback(_got3)
2417
2418         # look at the JSON form of the enclosing directory
2419         d.addCallback(lambda res:
2420                       self.GET(self.public_url + "/foo/?t=json",
2421                                followRedirect=True))
2422         def _check_page_json(res):
2423             parsed = simplejson.loads(res)
2424             self.failUnlessEqual(parsed[0], "dirnode")
2425             children = dict( [(unicode(name),value)
2426                               for (name,value)
2427                               in parsed[1]["children"].iteritems()] )
2428             self.failUnlessIn(u"new.txt", children)
2429             new_json = children[u"new.txt"]
2430             self.failUnlessEqual(new_json[0], "filenode")
2431             self.failUnless(new_json[1]["mutable"])
2432             self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri)
2433             ro_uri = self._mutable_node.get_readonly().to_string()
2434             self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri)
2435         d.addCallback(_check_page_json)
2436
2437         # and the JSON form of the file
2438         d.addCallback(lambda res:
2439                       self.GET(self.public_url + "/foo/new.txt?t=json"))
2440         def _check_file_json(res):
2441             parsed = simplejson.loads(res)
2442             self.failUnlessEqual(parsed[0], "filenode")
2443             self.failUnless(parsed[1]["mutable"])
2444             self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri)
2445             ro_uri = self._mutable_node.get_readonly().to_string()
2446             self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri)
2447         d.addCallback(_check_file_json)
2448
2449         # and look at t=uri and t=readonly-uri
2450         d.addCallback(lambda res:
2451                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
2452         d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
2453         d.addCallback(lambda res:
2454                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
2455         def _check_ro_uri(res):
2456             ro_uri = self._mutable_node.get_readonly().to_string()
2457             self.failUnlessReallyEqual(res, ro_uri)
2458         d.addCallback(_check_ro_uri)
2459
2460         # make sure we can get to it from /uri/URI
2461         d.addCallback(lambda res:
2462                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
2463         d.addCallback(lambda res:
2464                       self.failUnlessReallyEqual(res, NEW2_CONTENTS))
2465
2466         # and that HEAD computes the size correctly
2467         d.addCallback(lambda res:
2468                       self.HEAD(self.public_url + "/foo/new.txt",
2469                                 return_response=True))
2470         def _got_headers((res, status, headers)):
2471             self.failUnlessReallyEqual(res, "")
2472             self.failUnlessReallyEqual(headers["content-length"][0],
2473                                        str(len(NEW2_CONTENTS)))
2474             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
2475         d.addCallback(_got_headers)
2476
2477         # make sure that outdated size limits aren't enforced anymore.
2478         d.addCallback(lambda ignored:
2479             self.POST(self.public_url + "/foo", t="upload",
2480                       mutable="true",
2481                       file=("new.txt",
2482                             "b" * (self.s.MUTABLE_SIZELIMIT+1))))
2483         d.addErrback(self.dump_error)
2484         return d
2485
2486     def test_POST_upload_mutable_toobig(self):
2487         # SDMF had a size limti that was removed a while ago. MDMF has
2488         # never had a size limit. Test to make sure that we do not
2489         # encounter errors when trying to upload large mutable files,
2490         # since there should be no coded prohibitions regarding large
2491         # mutable files.
2492         d = self.POST(self.public_url + "/foo",
2493                       t="upload", mutable="true",
2494                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2495         return d
2496
2497     def dump_error(self, f):
2498         # if the web server returns an error code (like 400 Bad Request),
2499         # web.client.getPage puts the HTTP response body into the .response
2500         # attribute of the exception object that it gives back. It does not
2501         # appear in the Failure's repr(), so the ERROR that trial displays
2502         # will be rather terse and unhelpful. addErrback this method to the
2503         # end of your chain to get more information out of these errors.
2504         if f.check(error.Error):
2505             print "web.error.Error:"
2506             print f
2507             print f.value.response
2508         return f
2509
2510     def test_POST_upload_replace(self):
2511         d = self.POST(self.public_url + "/foo", t="upload",
2512                       file=("bar.txt", self.NEWFILE_CONTENTS))
2513         fn = self._foo_node
2514         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
2515         d.addCallback(lambda res:
2516                       self.failUnlessChildContentsAre(fn, u"bar.txt",
2517                                                       self.NEWFILE_CONTENTS))
2518         return d
2519
2520     def test_POST_upload_no_replace_ok(self):
2521         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2522                       file=("new.txt", self.NEWFILE_CONTENTS))
2523         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
2524         d.addCallback(lambda res: self.failUnlessReallyEqual(res,
2525                                                              self.NEWFILE_CONTENTS))
2526         return d
2527
2528     def test_POST_upload_no_replace_queryarg(self):
2529         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2530                       file=("bar.txt", self.NEWFILE_CONTENTS))
2531         d.addBoth(self.shouldFail, error.Error,
2532                   "POST_upload_no_replace_queryarg",
2533                   "409 Conflict",
2534                   "There was already a child by that name, and you asked me "
2535                   "to not replace it")
2536         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2537         d.addCallback(self.failUnlessIsBarDotTxt)
2538         return d
2539
2540     def test_POST_upload_no_replace_field(self):
2541         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
2542                       file=("bar.txt", self.NEWFILE_CONTENTS))
2543         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
2544                   "409 Conflict",
2545                   "There was already a child by that name, and you asked me "
2546                   "to not replace it")
2547         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2548         d.addCallback(self.failUnlessIsBarDotTxt)
2549         return d
2550
2551     def test_POST_upload_whendone(self):
2552         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
2553                       file=("new.txt", self.NEWFILE_CONTENTS))
2554         d.addBoth(self.shouldRedirect, "/THERE")
2555         fn = self._foo_node
2556         d.addCallback(lambda res:
2557                       self.failUnlessChildContentsAre(fn, u"new.txt",
2558                                                       self.NEWFILE_CONTENTS))
2559         return d
2560
2561     def test_POST_upload_named(self):
2562         fn = self._foo_node
2563         d = self.POST(self.public_url + "/foo", t="upload",
2564                       name="new.txt", file=self.NEWFILE_CONTENTS)
2565         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2566         d.addCallback(lambda res:
2567                       self.failUnlessChildContentsAre(fn, u"new.txt",
2568                                                       self.NEWFILE_CONTENTS))
2569         return d
2570
2571     def test_POST_upload_named_badfilename(self):
2572         d = self.POST(self.public_url + "/foo", t="upload",
2573                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
2574         d.addBoth(self.shouldFail, error.Error,
2575                   "test_POST_upload_named_badfilename",
2576                   "400 Bad Request",
2577                   "name= may not contain a slash",
2578                   )
2579         # make sure that nothing was added
2580         d.addCallback(lambda res:
2581                       self.failUnlessNodeKeysAre(self._foo_node,
2582                                                  [self._htmlname_unicode,
2583                                                   u"bar.txt", u"baz.txt", u"blockingfile",
2584                                                   u"empty", u"n\u00fc.txt", u"quux.txt",
2585                                                   u"sub"]))
2586         return d
2587
2588     def test_POST_FILEURL_check(self):
2589         bar_url = self.public_url + "/foo/bar.txt"
2590         d = self.POST(bar_url, t="check")
2591         def _check(res):
2592             self.failUnlessIn("Healthy :", res)
2593         d.addCallback(_check)
2594         redir_url = "http://allmydata.org/TARGET"
2595         def _check2(statuscode, target):
2596             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2597             self.failUnlessReallyEqual(target, redir_url)
2598         d.addCallback(lambda res:
2599                       self.shouldRedirect2("test_POST_FILEURL_check",
2600                                            _check2,
2601                                            self.POST, bar_url,
2602                                            t="check",
2603                                            when_done=redir_url))
2604         d.addCallback(lambda res:
2605                       self.POST(bar_url, t="check", return_to=redir_url))
2606         def _check3(res):
2607             self.failUnlessIn("Healthy :", res)
2608             self.failUnlessIn("Return to file", res)
2609             self.failUnlessIn(redir_url, res)
2610         d.addCallback(_check3)
2611
2612         d.addCallback(lambda res:
2613                       self.POST(bar_url, t="check", output="JSON"))
2614         def _check_json(res):
2615             data = simplejson.loads(res)
2616             self.failUnlessIn("storage-index", data)
2617             self.failUnless(data["results"]["healthy"])
2618         d.addCallback(_check_json)
2619
2620         return d
2621
2622     def test_POST_FILEURL_check_and_repair(self):
2623         bar_url = self.public_url + "/foo/bar.txt"
2624         d = self.POST(bar_url, t="check", repair="true")
2625         def _check(res):
2626             self.failUnlessIn("Healthy :", res)
2627         d.addCallback(_check)
2628         redir_url = "http://allmydata.org/TARGET"
2629         def _check2(statuscode, target):
2630             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2631             self.failUnlessReallyEqual(target, redir_url)
2632         d.addCallback(lambda res:
2633                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
2634                                            _check2,
2635                                            self.POST, bar_url,
2636                                            t="check", repair="true",
2637                                            when_done=redir_url))
2638         d.addCallback(lambda res:
2639                       self.POST(bar_url, t="check", return_to=redir_url))
2640         def _check3(res):
2641             self.failUnlessIn("Healthy :", res)
2642             self.failUnlessIn("Return to file", res)
2643             self.failUnlessIn(redir_url, res)
2644         d.addCallback(_check3)
2645         return d
2646
2647     def test_POST_DIRURL_check(self):
2648         foo_url = self.public_url + "/foo/"
2649         d = self.POST(foo_url, t="check")
2650         def _check(res):
2651             self.failUnlessIn("Healthy :", res)
2652         d.addCallback(_check)
2653         redir_url = "http://allmydata.org/TARGET"
2654         def _check2(statuscode, target):
2655             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2656             self.failUnlessReallyEqual(target, redir_url)
2657         d.addCallback(lambda res:
2658                       self.shouldRedirect2("test_POST_DIRURL_check",
2659                                            _check2,
2660                                            self.POST, foo_url,
2661                                            t="check",
2662                                            when_done=redir_url))
2663         d.addCallback(lambda res:
2664                       self.POST(foo_url, t="check", return_to=redir_url))
2665         def _check3(res):
2666             self.failUnlessIn("Healthy :", res)
2667             self.failUnlessIn("Return to file/directory", res)
2668             self.failUnlessIn(redir_url, res)
2669         d.addCallback(_check3)
2670
2671         d.addCallback(lambda res:
2672                       self.POST(foo_url, t="check", output="JSON"))
2673         def _check_json(res):
2674             data = simplejson.loads(res)
2675             self.failUnlessIn("storage-index", data)
2676             self.failUnless(data["results"]["healthy"])
2677         d.addCallback(_check_json)
2678
2679         return d
2680
2681     def test_POST_DIRURL_check_and_repair(self):
2682         foo_url = self.public_url + "/foo/"
2683         d = self.POST(foo_url, t="check", repair="true")
2684         def _check(res):
2685             self.failUnlessIn("Healthy :", res)
2686         d.addCallback(_check)
2687         redir_url = "http://allmydata.org/TARGET"
2688         def _check2(statuscode, target):
2689             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2690             self.failUnlessReallyEqual(target, redir_url)
2691         d.addCallback(lambda res:
2692                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2693                                            _check2,
2694                                            self.POST, foo_url,
2695                                            t="check", repair="true",
2696                                            when_done=redir_url))
2697         d.addCallback(lambda res:
2698                       self.POST(foo_url, t="check", return_to=redir_url))
2699         def _check3(res):
2700             self.failUnlessIn("Healthy :", res)
2701             self.failUnlessIn("Return to file/directory", res)
2702             self.failUnlessIn(redir_url, res)
2703         d.addCallback(_check3)
2704         return d
2705
2706     def test_POST_FILEURL_mdmf_check(self):
2707         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2708         d = self.POST(quux_url, t="check")
2709         def _check(res):
2710             self.failUnlessIn("Healthy", res)
2711         d.addCallback(_check)
2712         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2713         d.addCallback(lambda ignored:
2714                       self.POST(quux_extension_url, t="check"))
2715         d.addCallback(_check)
2716         return d
2717
2718     def test_POST_FILEURL_mdmf_check_and_repair(self):
2719         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2720         d = self.POST(quux_url, t="check", repair="true")
2721         def _check(res):
2722             self.failUnlessIn("Healthy", res)
2723         d.addCallback(_check)
2724         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2725         d.addCallback(lambda ignored:
2726                       self.POST(quux_extension_url, t="check", repair="true"))
2727         d.addCallback(_check)
2728         return d
2729
2730     def wait_for_operation(self, ignored, ophandle):
2731         url = "/operations/" + ophandle
2732         url += "?t=status&output=JSON"
2733         d = self.GET(url)
2734         def _got(res):
2735             data = simplejson.loads(res)
2736             if not data["finished"]:
2737                 d = self.stall(delay=1.0)
2738                 d.addCallback(self.wait_for_operation, ophandle)
2739                 return d
2740             return data
2741         d.addCallback(_got)
2742         return d
2743
2744     def get_operation_results(self, ignored, ophandle, output=None):
2745         url = "/operations/" + ophandle
2746         url += "?t=status"
2747         if output:
2748             url += "&output=" + output
2749         d = self.GET(url)
2750         def _got(res):
2751             if output and output.lower() == "json":
2752                 return simplejson.loads(res)
2753             return res
2754         d.addCallback(_got)
2755         return d
2756
2757     def test_POST_DIRURL_deepcheck_no_ophandle(self):
2758         d = self.shouldFail2(error.Error,
2759                              "test_POST_DIRURL_deepcheck_no_ophandle",
2760                              "400 Bad Request",
2761                              "slow operation requires ophandle=",
2762                              self.POST, self.public_url, t="start-deep-check")
2763         return d
2764
2765     def test_POST_DIRURL_deepcheck(self):
2766         def _check_redirect(statuscode, target):
2767             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2768             self.failUnless(target.endswith("/operations/123"))
2769         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2770                                  self.POST, self.public_url,
2771                                  t="start-deep-check", ophandle="123")
2772         d.addCallback(self.wait_for_operation, "123")
2773         def _check_json(data):
2774             self.failUnlessReallyEqual(data["finished"], True)
2775             self.failUnlessReallyEqual(data["count-objects-checked"], 11)
2776             self.failUnlessReallyEqual(data["count-objects-healthy"], 11)
2777         d.addCallback(_check_json)
2778         d.addCallback(self.get_operation_results, "123", "html")
2779         def _check_html(res):
2780             self.failUnlessIn("Objects Checked: <span>11</span>", res)
2781             self.failUnlessIn("Objects Healthy: <span>11</span>", res)
2782             self.failUnlessIn(FAVICON_MARKUP, res)
2783         d.addCallback(_check_html)
2784
2785         d.addCallback(lambda res:
2786                       self.GET("/operations/123/"))
2787         d.addCallback(_check_html) # should be the same as without the slash
2788
2789         d.addCallback(lambda res:
2790                       self.shouldFail2(error.Error, "one", "404 Not Found",
2791                                        "No detailed results for SI bogus",
2792                                        self.GET, "/operations/123/bogus"))
2793
2794         foo_si = self._foo_node.get_storage_index()
2795         foo_si_s = base32.b2a(foo_si)
2796         d.addCallback(lambda res:
2797                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2798         def _check_foo_json(res):
2799             data = simplejson.loads(res)
2800             self.failUnlessEqual(data["storage-index"], foo_si_s)
2801             self.failUnless(data["results"]["healthy"])
2802         d.addCallback(_check_foo_json)
2803         return d
2804
2805     def test_POST_DIRURL_deepcheck_and_repair(self):
2806         d = self.POST(self.public_url, t="start-deep-check", repair="true",
2807                       ophandle="124", output="json", followRedirect=True)
2808         d.addCallback(self.wait_for_operation, "124")
2809         def _check_json(data):
2810             self.failUnlessReallyEqual(data["finished"], True)
2811             self.failUnlessReallyEqual(data["count-objects-checked"], 11)
2812             self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 11)
2813             self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2814             self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2815             self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2816             self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2817             self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2818             self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 11)
2819             self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2820             self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2821         d.addCallback(_check_json)
2822         d.addCallback(self.get_operation_results, "124", "html")
2823         def _check_html(res):
2824             self.failUnlessIn("Objects Checked: <span>11</span>", res)
2825
2826             self.failUnlessIn("Objects Healthy (before repair): <span>11</span>", res)
2827             self.failUnlessIn("Objects Unhealthy (before repair): <span>0</span>", res)
2828             self.failUnlessIn("Corrupt Shares (before repair): <span>0</span>", res)
2829
2830             self.failUnlessIn("Repairs Attempted: <span>0</span>", res)
2831             self.failUnlessIn("Repairs Successful: <span>0</span>", res)
2832             self.failUnlessIn("Repairs Unsuccessful: <span>0</span>", res)
2833
2834             self.failUnlessIn("Objects Healthy (after repair): <span>11</span>", res)
2835             self.failUnlessIn("Objects Unhealthy (after repair): <span>0</span>", res)
2836             self.failUnlessIn("Corrupt Shares (after repair): <span>0</span>", res)
2837
2838             self.failUnlessIn(FAVICON_MARKUP, res)
2839         d.addCallback(_check_html)
2840         return d
2841
2842     def test_POST_FILEURL_bad_t(self):
2843         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2844                              "POST to file: bad t=bogus",
2845                              self.POST, self.public_url + "/foo/bar.txt",
2846                              t="bogus")
2847         return d
2848
2849     def test_POST_mkdir(self): # return value?
2850         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2851         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2852         d.addCallback(self.failUnlessNodeKeysAre, [])
2853         return d
2854
2855     def test_POST_mkdir_mdmf(self):
2856         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=mdmf")
2857         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2858         d.addCallback(lambda node:
2859             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2860         return d
2861
2862     def test_POST_mkdir_sdmf(self):
2863         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=sdmf")
2864         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2865         d.addCallback(lambda node:
2866             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2867         return d
2868
2869     def test_POST_mkdir_bad_format(self):
2870         return self.shouldHTTPError("POST_mkdir_bad_format",
2871                                     400, "Bad Request", "Unknown format: foo",
2872                                     self.POST, self.public_url +
2873                                     "/foo?t=mkdir&name=newdir&format=foo")
2874
2875     def test_POST_mkdir_initial_children(self):
2876         (newkids, caps) = self._create_initial_children()
2877         d = self.POST2(self.public_url +
2878                        "/foo?t=mkdir-with-children&name=newdir",
2879                        simplejson.dumps(newkids))
2880         d.addCallback(lambda res:
2881                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2882         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2883         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2884         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2885         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2886         return d
2887
2888     def test_POST_mkdir_initial_children_mdmf(self):
2889         (newkids, caps) = self._create_initial_children()
2890         d = self.POST2(self.public_url +
2891                        "/foo?t=mkdir-with-children&name=newdir&format=mdmf",
2892                        simplejson.dumps(newkids))
2893         d.addCallback(lambda res:
2894                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2895         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2896         d.addCallback(lambda node:
2897             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2898         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2899         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2900                        caps['filecap1'])
2901         return d
2902
2903     # XXX: Duplication.
2904     def test_POST_mkdir_initial_children_sdmf(self):
2905         (newkids, caps) = self._create_initial_children()
2906         d = self.POST2(self.public_url +
2907                        "/foo?t=mkdir-with-children&name=newdir&format=sdmf",
2908                        simplejson.dumps(newkids))
2909         d.addCallback(lambda res:
2910                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2911         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2912         d.addCallback(lambda node:
2913             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2914         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2915         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2916                        caps['filecap1'])
2917         return d
2918
2919     def test_POST_mkdir_initial_children_bad_format(self):
2920         (newkids, caps) = self._create_initial_children()
2921         return self.shouldHTTPError("POST_mkdir_initial_children_bad_format",
2922                                     400, "Bad Request", "Unknown format: foo",
2923                                     self.POST, self.public_url + \
2924                                     "/foo?t=mkdir-with-children&name=newdir&format=foo",
2925                                     simplejson.dumps(newkids))
2926
2927     def test_POST_mkdir_immutable(self):
2928         (newkids, caps) = self._create_immutable_children()
2929         d = self.POST2(self.public_url +
2930                        "/foo?t=mkdir-immutable&name=newdir",
2931                        simplejson.dumps(newkids))
2932         d.addCallback(lambda res:
2933                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2934         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2935         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2936         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2937         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2938         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2939         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2940         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2941         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2942         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2943         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2944         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2945         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2946         return d
2947
2948     def test_POST_mkdir_immutable_bad(self):
2949         (newkids, caps) = self._create_initial_children()
2950         d = self.shouldFail2(error.Error, "POST_mkdir_immutable_bad",
2951                              "400 Bad Request",
2952                              "needed to be immutable but was not",
2953                              self.POST2,
2954                              self.public_url +
2955                              "/foo?t=mkdir-immutable&name=newdir",
2956                              simplejson.dumps(newkids))
2957         return d
2958
2959     def test_POST_mkdir_2(self):
2960         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
2961         d.addCallback(lambda res:
2962                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2963         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2964         d.addCallback(self.failUnlessNodeKeysAre, [])
2965         return d
2966
2967     def test_POST_mkdirs_2(self):
2968         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
2969         d.addCallback(lambda res:
2970                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
2971         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
2972         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
2973         d.addCallback(self.failUnlessNodeKeysAre, [])
2974         return d
2975
2976     def test_POST_mkdir_no_parentdir_noredirect(self):
2977         d = self.POST("/uri?t=mkdir")
2978         def _after_mkdir(res):
2979             uri.DirectoryURI.init_from_string(res)
2980         d.addCallback(_after_mkdir)
2981         return d
2982
2983     def test_POST_mkdir_no_parentdir_noredirect_mdmf(self):
2984         d = self.POST("/uri?t=mkdir&format=mdmf")
2985         def _after_mkdir(res):
2986             u = uri.from_string(res)
2987             # Check that this is an MDMF writecap
2988             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
2989         d.addCallback(_after_mkdir)
2990         return d
2991
2992     def test_POST_mkdir_no_parentdir_noredirect_sdmf(self):
2993         d = self.POST("/uri?t=mkdir&format=sdmf")
2994         def _after_mkdir(res):
2995             u = uri.from_string(res)
2996             self.failUnlessIsInstance(u, uri.DirectoryURI)
2997         d.addCallback(_after_mkdir)
2998         return d
2999
3000     def test_POST_mkdir_no_parentdir_noredirect_bad_format(self):
3001         return self.shouldHTTPError("POST_mkdir_no_parentdir_noredirect_bad_format",
3002                                     400, "Bad Request", "Unknown format: foo",
3003                                     self.POST, self.public_url +
3004                                     "/uri?t=mkdir&format=foo")
3005
3006     def test_POST_mkdir_no_parentdir_noredirect2(self):
3007         # make sure form-based arguments (as on the welcome page) still work
3008         d = self.POST("/uri", t="mkdir")
3009         def _after_mkdir(res):
3010             uri.DirectoryURI.init_from_string(res)
3011         d.addCallback(_after_mkdir)
3012         d.addErrback(self.explain_web_error)
3013         return d
3014
3015     def test_POST_mkdir_no_parentdir_redirect(self):
3016         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
3017         d.addBoth(self.shouldRedirect, None, statuscode='303')
3018         def _check_target(target):
3019             target = urllib.unquote(target)
3020             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
3021         d.addCallback(_check_target)
3022         return d
3023
3024     def test_POST_mkdir_no_parentdir_redirect2(self):
3025         d = self.POST("/uri", t="mkdir", redirect_to_result="true")
3026         d.addBoth(self.shouldRedirect, None, statuscode='303')
3027         def _check_target(target):
3028             target = urllib.unquote(target)
3029             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
3030         d.addCallback(_check_target)
3031         d.addErrback(self.explain_web_error)
3032         return d
3033
3034     def _make_readonly(self, u):
3035         ro_uri = uri.from_string(u).get_readonly()
3036         if ro_uri is None:
3037             return None
3038         return ro_uri.to_string()
3039
3040     def _create_initial_children(self):
3041         contents, n, filecap1 = self.makefile(12)
3042         md1 = {"metakey1": "metavalue1"}
3043         filecap2 = make_mutable_file_uri()
3044         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
3045         filecap3 = node3.get_readonly_uri()
3046         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
3047         dircap = DirectoryNode(node4, None, None).get_uri()
3048         mdmfcap = make_mutable_file_uri(mdmf=True)
3049         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
3050         emptydircap = "URI:DIR2-LIT:"
3051         newkids = {u"child-imm":        ["filenode", {"rw_uri": filecap1,
3052                                                       "ro_uri": self._make_readonly(filecap1),
3053                                                       "metadata": md1, }],
3054                    u"child-mutable":    ["filenode", {"rw_uri": filecap2,
3055                                                       "ro_uri": self._make_readonly(filecap2)}],
3056                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
3057                    u"unknownchild-rw":  ["unknown",  {"rw_uri": unknown_rwcap,
3058                                                       "ro_uri": unknown_rocap}],
3059                    u"unknownchild-ro":  ["unknown",  {"ro_uri": unknown_rocap}],
3060                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
3061                    u"dirchild":         ["dirnode",  {"rw_uri": dircap,
3062                                                       "ro_uri": self._make_readonly(dircap)}],
3063                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
3064                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
3065                    u"child-mutable-mdmf": ["filenode", {"rw_uri": mdmfcap,
3066                                                         "ro_uri": self._make_readonly(mdmfcap)}],
3067                    }
3068         return newkids, {'filecap1': filecap1,
3069                          'filecap2': filecap2,
3070                          'filecap3': filecap3,
3071                          'unknown_rwcap': unknown_rwcap,
3072                          'unknown_rocap': unknown_rocap,
3073                          'unknown_immcap': unknown_immcap,
3074                          'dircap': dircap,
3075                          'litdircap': litdircap,
3076                          'emptydircap': emptydircap,
3077                          'mdmfcap': mdmfcap}
3078
3079     def _create_immutable_children(self):
3080         contents, n, filecap1 = self.makefile(12)
3081         md1 = {"metakey1": "metavalue1"}
3082         tnode = create_chk_filenode("immutable directory contents\n"*10,
3083                                     self.get_all_contents())
3084         dnode = DirectoryNode(tnode, None, None)
3085         assert not dnode.is_mutable()
3086         immdircap = dnode.get_uri()
3087         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
3088         emptydircap = "URI:DIR2-LIT:"
3089         newkids = {u"child-imm":        ["filenode", {"ro_uri": filecap1,
3090                                                       "metadata": md1, }],
3091                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
3092                    u"dirchild-imm":     ["dirnode",  {"ro_uri": immdircap}],
3093                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
3094                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
3095                    }
3096         return newkids, {'filecap1': filecap1,
3097                          'unknown_immcap': unknown_immcap,
3098                          'immdircap': immdircap,
3099                          'litdircap': litdircap,
3100                          'emptydircap': emptydircap}
3101
3102     def test_POST_mkdir_no_parentdir_initial_children(self):
3103         (newkids, caps) = self._create_initial_children()
3104         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
3105         def _after_mkdir(res):
3106             self.failUnless(res.startswith("URI:DIR"), res)
3107             n = self.s.create_node_from_uri(res)
3108             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3109             d2.addCallback(lambda ign:
3110                            self.failUnlessROChildURIIs(n, u"child-imm",
3111                                                        caps['filecap1']))
3112             d2.addCallback(lambda ign:
3113                            self.failUnlessRWChildURIIs(n, u"child-mutable",
3114                                                        caps['filecap2']))
3115             d2.addCallback(lambda ign:
3116                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
3117                                                        caps['filecap3']))
3118             d2.addCallback(lambda ign:
3119                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
3120                                                        caps['unknown_rwcap']))
3121             d2.addCallback(lambda ign:
3122                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
3123                                                        caps['unknown_rocap']))
3124             d2.addCallback(lambda ign:
3125                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3126                                                        caps['unknown_immcap']))
3127             d2.addCallback(lambda ign:
3128                            self.failUnlessRWChildURIIs(n, u"dirchild",
3129                                                        caps['dircap']))
3130             return d2
3131         d.addCallback(_after_mkdir)
3132         return d
3133
3134     def test_POST_mkdir_no_parentdir_unexpected_children(self):
3135         # the regular /uri?t=mkdir operation is specified to ignore its body.
3136         # Only t=mkdir-with-children pays attention to it.
3137         (newkids, caps) = self._create_initial_children()
3138         d = self.shouldHTTPError("POST_mkdir_no_parentdir_unexpected_children",
3139                                  400, "Bad Request",
3140                                  "t=mkdir does not accept children=, "
3141                                  "try t=mkdir-with-children instead",
3142                                  self.POST2, "/uri?t=mkdir", # without children
3143                                  simplejson.dumps(newkids))
3144         return d
3145
3146     def test_POST_noparent_bad(self):
3147         d = self.shouldHTTPError("POST_noparent_bad",
3148                                  400, "Bad Request",
3149                                  "/uri accepts only PUT, PUT?t=mkdir, "
3150                                  "POST?t=upload, and POST?t=mkdir",
3151                                  self.POST, "/uri?t=bogus")
3152         return d
3153
3154     def test_POST_mkdir_no_parentdir_immutable(self):
3155         (newkids, caps) = self._create_immutable_children()
3156         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
3157         def _after_mkdir(res):
3158             self.failUnless(res.startswith("URI:DIR"), res)
3159             n = self.s.create_node_from_uri(res)
3160             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3161             d2.addCallback(lambda ign:
3162                            self.failUnlessROChildURIIs(n, u"child-imm",
3163                                                           caps['filecap1']))
3164             d2.addCallback(lambda ign:
3165                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3166                                                           caps['unknown_immcap']))
3167             d2.addCallback(lambda ign:
3168                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
3169                                                           caps['immdircap']))
3170             d2.addCallback(lambda ign:
3171                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
3172                                                           caps['litdircap']))
3173             d2.addCallback(lambda ign:
3174                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
3175                                                           caps['emptydircap']))
3176             return d2
3177         d.addCallback(_after_mkdir)
3178         return d
3179
3180     def test_POST_mkdir_no_parentdir_immutable_bad(self):
3181         (newkids, caps) = self._create_initial_children()
3182         d = self.shouldFail2(error.Error,
3183                              "test_POST_mkdir_no_parentdir_immutable_bad",
3184                              "400 Bad Request",
3185                              "needed to be immutable but was not",
3186                              self.POST2,
3187                              "/uri?t=mkdir-immutable",
3188                              simplejson.dumps(newkids))
3189         return d
3190
3191     def test_welcome_page_mkdir_button(self):
3192         # Fetch the welcome page.
3193         d = self.GET("/")
3194         def _after_get_welcome_page(res):
3195             MKDIR_BUTTON_RE = re.compile(
3196                 '<form action="([^"]*)" method="post".*'
3197                 '<input type="hidden" name="t" value="([^"]*)" />[ ]*'
3198                 '<input type="hidden" name="([^"]*)" value="([^"]*)" />[ ]*'
3199                 '<input type="submit" class="btn" value="Create a directory[^"]*" />')
3200             html = res.replace('\n', ' ')
3201             mo = MKDIR_BUTTON_RE.search(html)
3202             self.failUnless(mo, html)
3203             formaction = mo.group(1)
3204             formt = mo.group(2)
3205             formaname = mo.group(3)
3206             formavalue = mo.group(4)
3207             return (formaction, formt, formaname, formavalue)
3208         d.addCallback(_after_get_welcome_page)
3209         def _after_parse_form(res):
3210             (formaction, formt, formaname, formavalue) = res
3211             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
3212         d.addCallback(_after_parse_form)
3213         d.addBoth(self.shouldRedirect, None, statuscode='303')
3214         return d
3215
3216     def test_POST_mkdir_replace(self): # return value?
3217         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
3218         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3219         d.addCallback(self.failUnlessNodeKeysAre, [])
3220         return d
3221
3222     def test_POST_mkdir_no_replace_queryarg(self): # return value?
3223         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
3224         d.addBoth(self.shouldFail, error.Error,
3225                   "POST_mkdir_no_replace_queryarg",
3226                   "409 Conflict",
3227                   "There was already a child by that name, and you asked me "
3228                   "to not replace it")
3229         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3230         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3231         return d
3232
3233     def test_POST_mkdir_no_replace_field(self): # return value?
3234         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
3235                       replace="false")
3236         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
3237                   "409 Conflict",
3238                   "There was already a child by that name, and you asked me "
3239                   "to not replace it")
3240         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3241         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3242         return d
3243
3244     def test_POST_mkdir_whendone_field(self):
3245         d = self.POST(self.public_url + "/foo",
3246                       t="mkdir", name="newdir", when_done="/THERE")
3247         d.addBoth(self.shouldRedirect, "/THERE")
3248         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3249         d.addCallback(self.failUnlessNodeKeysAre, [])
3250         return d
3251
3252     def test_POST_mkdir_whendone_queryarg(self):
3253         d = self.POST(self.public_url + "/foo?when_done=/THERE",
3254                       t="mkdir", name="newdir")
3255         d.addBoth(self.shouldRedirect, "/THERE")
3256         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3257         d.addCallback(self.failUnlessNodeKeysAre, [])
3258         return d
3259
3260     def test_POST_bad_t(self):
3261         d = self.shouldFail2(error.Error, "POST_bad_t",
3262                              "400 Bad Request",
3263                              "POST to a directory with bad t=BOGUS",
3264                              self.POST, self.public_url + "/foo", t="BOGUS")
3265         return d
3266
3267     def test_POST_set_children(self, command_name="set_children"):
3268         contents9, n9, newuri9 = self.makefile(9)
3269         contents10, n10, newuri10 = self.makefile(10)
3270         contents11, n11, newuri11 = self.makefile(11)
3271
3272         reqbody = """{
3273                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
3274                                                 "size": 0,
3275                                                 "metadata": {
3276                                                   "ctime": 1002777696.7564139,
3277                                                   "mtime": 1002777696.7564139
3278                                                  }
3279                                                } ],
3280                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
3281                                                 "size": 1,
3282                                                 "metadata": {
3283                                                   "ctime": 1002777696.7564139,
3284                                                   "mtime": 1002777696.7564139
3285                                                  }
3286                                                } ],
3287                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
3288                                                 "size": 2,
3289                                                 "metadata": {
3290                                                   "ctime": 1002777696.7564139,
3291                                                   "mtime": 1002777696.7564139
3292                                                  }
3293                                                } ]
3294                     }""" % (newuri9, newuri10, newuri11)
3295
3296         url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
3297
3298         d = client.getPage(url, method="POST", postdata=reqbody)
3299         def _then(res):
3300             self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
3301             self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
3302             self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
3303
3304         d.addCallback(_then)
3305         d.addErrback(self.dump_error)
3306         return d
3307
3308     def test_POST_set_children_with_hyphen(self):
3309         return self.test_POST_set_children(command_name="set-children")
3310
3311     def test_POST_link_uri(self):
3312         contents, n, newuri = self.makefile(8)
3313         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
3314         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
3315         d.addCallback(lambda res:
3316                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3317                                                       contents))
3318         return d
3319
3320     def test_POST_link_uri_replace(self):
3321         contents, n, newuri = self.makefile(8)
3322         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
3323         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
3324         d.addCallback(lambda res:
3325                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3326                                                       contents))
3327         return d
3328
3329     def test_POST_link_uri_unknown_bad(self):
3330         d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
3331         d.addBoth(self.shouldFail, error.Error,
3332                   "POST_link_uri_unknown_bad",
3333                   "400 Bad Request",
3334                   "unknown cap in a write slot")
3335         return d
3336
3337     def test_POST_link_uri_unknown_ro_good(self):
3338         d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
3339         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
3340         return d
3341
3342     def test_POST_link_uri_unknown_imm_good(self):
3343         d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
3344         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
3345         return d
3346
3347     def test_POST_link_uri_no_replace_queryarg(self):
3348         contents, n, newuri = self.makefile(8)
3349         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
3350                       name="bar.txt", uri=newuri)
3351         d.addBoth(self.shouldFail, error.Error,
3352                   "POST_link_uri_no_replace_queryarg",
3353                   "409 Conflict",
3354                   "There was already a child by that name, and you asked me "
3355                   "to not replace it")
3356         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3357         d.addCallback(self.failUnlessIsBarDotTxt)
3358         return d
3359
3360     def test_POST_link_uri_no_replace_field(self):
3361         contents, n, newuri = self.makefile(8)
3362         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
3363                       name="bar.txt", uri=newuri)
3364         d.addBoth(self.shouldFail, error.Error,
3365                   "POST_link_uri_no_replace_field",
3366                   "409 Conflict",
3367                   "There was already a child by that name, and you asked me "
3368                   "to not replace it")
3369         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3370         d.addCallback(self.failUnlessIsBarDotTxt)
3371         return d
3372
3373     def test_POST_delete(self, command_name='delete'):
3374         d = self._foo_node.list()
3375         def _check_before(children):
3376             self.failUnlessIn(u"bar.txt", children)
3377         d.addCallback(_check_before)
3378         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t=command_name, name="bar.txt"))
3379         d.addCallback(lambda res: self._foo_node.list())
3380         def _check_after(children):
3381             self.failIfIn(u"bar.txt", children)
3382         d.addCallback(_check_after)
3383         return d
3384
3385     def test_POST_unlink(self):
3386         return self.test_POST_delete(command_name='unlink')
3387
3388     def test_POST_rename_file(self):
3389         d = self.POST(self.public_url + "/foo", t="rename",
3390                       from_name="bar.txt", to_name='wibble.txt')
3391         d.addCallback(lambda res:
3392                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3393         d.addCallback(lambda res:
3394                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
3395         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
3396         d.addCallback(self.failUnlessIsBarDotTxt)
3397         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
3398         d.addCallback(self.failUnlessIsBarJSON)
3399         return d
3400
3401     def test_POST_rename_file_redundant(self):
3402         d = self.POST(self.public_url + "/foo", t="rename",
3403                       from_name="bar.txt", to_name='bar.txt')
3404         d.addCallback(lambda res:
3405                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3406         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3407         d.addCallback(self.failUnlessIsBarDotTxt)
3408         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3409         d.addCallback(self.failUnlessIsBarJSON)
3410         return d
3411
3412     def test_POST_rename_file_replace(self):
3413         # rename a file and replace a directory with it
3414         d = self.POST(self.public_url + "/foo", t="rename",
3415                       from_name="bar.txt", to_name='empty')
3416         d.addCallback(lambda res:
3417                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3418         d.addCallback(lambda res:
3419                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
3420         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
3421         d.addCallback(self.failUnlessIsBarDotTxt)
3422         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3423         d.addCallback(self.failUnlessIsBarJSON)
3424         return d
3425
3426     def test_POST_rename_file_no_replace_queryarg(self):
3427         # rename a file and replace a directory with it
3428         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
3429                       from_name="bar.txt", to_name='empty')
3430         d.addBoth(self.shouldFail, error.Error,
3431                   "POST_rename_file_no_replace_queryarg",
3432                   "409 Conflict",
3433                   "There was already a child by that name, and you asked me "
3434                   "to not replace it")
3435         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3436         d.addCallback(self.failUnlessIsEmptyJSON)
3437         return d
3438
3439     def test_POST_rename_file_no_replace_field(self):
3440         # rename a file and replace a directory with it
3441         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
3442                       from_name="bar.txt", to_name='empty')
3443         d.addBoth(self.shouldFail, error.Error,
3444                   "POST_rename_file_no_replace_field",
3445                   "409 Conflict",
3446                   "There was already a child by that name, and you asked me "
3447                   "to not replace it")
3448         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3449         d.addCallback(self.failUnlessIsEmptyJSON)
3450         return d
3451
3452     def failUnlessIsEmptyJSON(self, res):
3453         data = simplejson.loads(res)
3454         self.failUnlessEqual(data[0], "dirnode", data)
3455         self.failUnlessReallyEqual(len(data[1]["children"]), 0)
3456
3457     def test_POST_rename_file_slash_fail(self):
3458         d = self.POST(self.public_url + "/foo", t="rename",
3459                       from_name="bar.txt", to_name='kirk/spock.txt')
3460         d.addBoth(self.shouldFail, error.Error,
3461                   "test_POST_rename_file_slash_fail",
3462                   "400 Bad Request",
3463                   "to_name= may not contain a slash",
3464                   )
3465         d.addCallback(lambda res:
3466                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3467         return d
3468
3469     def test_POST_rename_dir(self):
3470         d = self.POST(self.public_url, t="rename",
3471                       from_name="foo", to_name='plunk')
3472         d.addCallback(lambda res:
3473                       self.failIfNodeHasChild(self.public_root, u"foo"))
3474         d.addCallback(lambda res:
3475                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
3476         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
3477         d.addCallback(self.failUnlessIsFooJSON)
3478         return d
3479
3480     def test_POST_move_file(self):
3481         d = self.POST(self.public_url + "/foo", t="move",
3482                       from_name="bar.txt", to_dir="sub")
3483         d.addCallback(lambda res:
3484                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3485         d.addCallback(lambda res:
3486                       self.failUnlessNodeHasChild(self._sub_node, u"bar.txt"))
3487         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3488         d.addCallback(self.failUnlessIsBarDotTxt)
3489         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3490         d.addCallback(self.failUnlessIsBarJSON)
3491         return d
3492
3493     def test_POST_move_file_new_name(self):
3494         d = self.POST(self.public_url + "/foo", t="move",
3495                       from_name="bar.txt", to_name="wibble.txt", to_dir="sub")
3496         d.addCallback(lambda res:
3497                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3498         d.addCallback(lambda res:
3499                       self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3500         d.addCallback(lambda res:
3501                       self.failUnlessNodeHasChild(self._sub_node, u"wibble.txt"))
3502         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt"))
3503         d.addCallback(self.failUnlessIsBarDotTxt)
3504         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt?t=json"))
3505         d.addCallback(self.failUnlessIsBarJSON)
3506         return d
3507
3508     def test_POST_move_file_replace(self):
3509         d = self.POST(self.public_url + "/foo", t="move",
3510                       from_name="bar.txt", to_name="baz.txt", to_dir="sub")
3511         d.addCallback(lambda res:
3512                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3513         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3514         d.addCallback(self.failUnlessIsBarDotTxt)
3515         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt?t=json"))
3516         d.addCallback(self.failUnlessIsBarJSON)
3517         return d
3518
3519     def test_POST_move_file_no_replace(self):
3520         d = self.shouldFail2(error.Error, "POST_move_file_no_replace",
3521                              "409 Conflict",
3522                              "There was already a child by that name, and you asked me to not replace it",
3523                              self.POST, self.public_url + "/foo", t="move",
3524                              replace="false", from_name="bar.txt",
3525                              to_name="baz.txt", to_dir="sub")
3526         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3527         d.addCallback(self.failUnlessIsBarDotTxt)
3528         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3529         d.addCallback(self.failUnlessIsBarJSON)
3530         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3531         d.addCallback(self.failUnlessIsSubBazDotTxt)
3532         return d
3533
3534     def test_POST_move_file_slash_fail(self):
3535         d = self.shouldFail2(error.Error, "test_POST_rename_file_slash_fail",
3536                              "400 Bad Request",
3537                              "to_name= may not contain a slash",
3538                              self.POST, self.public_url + "/foo", t="move",
3539                              from_name="bar.txt",
3540                              to_name="slash/fail.txt", to_dir="sub")
3541         d.addCallback(lambda res:
3542                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3543         d.addCallback(lambda res:
3544                       self.failIfNodeHasChild(self._sub_node, u"slash/fail.txt"))
3545         d.addCallback(lambda ign:
3546                       self.shouldFail2(error.Error,
3547                                        "test_POST_rename_file_slash_fail2",
3548                                        "400 Bad Request",
3549                                        "from_name= may not contain a slash",
3550                                        self.POST, self.public_url + "/foo",
3551                                        t="move",
3552                                        from_name="nope/bar.txt",
3553                                        to_name="fail.txt", to_dir="sub"))
3554         return d
3555
3556     def test_POST_move_file_no_target(self):
3557         d = self.shouldFail2(error.Error, "POST_move_file_no_target",
3558                              "400 Bad Request",
3559                              "move requires from_name and to_dir",
3560                              self.POST, self.public_url + "/foo", t="move",
3561                              from_name="bar.txt", to_name="baz.txt")
3562         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3563         d.addCallback(self.failUnlessIsBarDotTxt)
3564         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3565         d.addCallback(self.failUnlessIsBarJSON)
3566         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3567         d.addCallback(self.failUnlessIsBazDotTxt)
3568         return d
3569
3570     def test_POST_move_file_bad_target_type(self):
3571         d = self.shouldFail2(error.Error, "test_POST_move_file_bad_target_type",
3572                              "400 Bad Request", "invalid target_type parameter",
3573                              self.POST,
3574                              self.public_url + "/foo", t="move",
3575                              target_type="*D", from_name="bar.txt",
3576                              to_dir="sub")
3577         return d
3578
3579     def test_POST_move_file_multi_level(self):
3580         d = self.POST(self.public_url + "/foo/sub/level2?t=mkdir", "")
3581         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t="move",
3582                       from_name="bar.txt", to_dir="sub/level2"))
3583         d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3584         d.addCallback(lambda res: self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3585         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt"))
3586         d.addCallback(self.failUnlessIsBarDotTxt)
3587         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt?t=json"))
3588         d.addCallback(self.failUnlessIsBarJSON)
3589         return d
3590
3591     def test_POST_move_file_to_uri(self):
3592         d = self.POST(self.public_url + "/foo", t="move", target_type="uri",
3593                       from_name="bar.txt", to_dir=self._sub_uri)
3594         d.addCallback(lambda res:
3595                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3596         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3597         d.addCallback(self.failUnlessIsBarDotTxt)
3598         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3599         d.addCallback(self.failUnlessIsBarJSON)
3600         return d
3601
3602     def test_POST_move_file_to_nonexist_dir(self):
3603         d = self.shouldFail2(error.Error, "POST_move_file_to_nonexist_dir",
3604                             "404 Not Found", "No such child: nopechucktesta",
3605                             self.POST, self.public_url + "/foo", t="move",
3606                             from_name="bar.txt", to_dir="nopechucktesta")
3607         return d
3608
3609     def test_POST_move_file_into_file(self):
3610         d = self.shouldFail2(error.Error, "POST_move_file_into_file",
3611                              "400 Bad Request", "to_dir is not a directory",
3612                              self.POST, self.public_url + "/foo", t="move",
3613                              from_name="bar.txt", to_dir="baz.txt")
3614         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3615         d.addCallback(self.failUnlessIsBazDotTxt)
3616         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3617         d.addCallback(self.failUnlessIsBarDotTxt)
3618         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3619         d.addCallback(self.failUnlessIsBarJSON)
3620         return d
3621
3622     def test_POST_move_file_to_bad_uri(self):
3623         d =  self.shouldFail2(error.Error, "POST_move_file_to_bad_uri",
3624                               "400 Bad Request", "to_dir is not a directory",
3625                               self.POST, self.public_url + "/foo", t="move",
3626                               from_name="bar.txt", target_type="uri",
3627                               to_dir="URI:DIR2:mn5jlyjnrjeuydyswlzyui72i:rmneifcj6k6sycjljjhj3f6majsq2zqffydnnul5hfa4j577arma")
3628         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3629         d.addCallback(self.failUnlessIsBarDotTxt)
3630         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3631         d.addCallback(self.failUnlessIsBarJSON)
3632         return d
3633
3634     def test_POST_move_dir(self):
3635         d = self.POST(self.public_url + "/foo", t="move",
3636                       from_name="bar.txt", to_dir="empty")
3637         d.addCallback(lambda res: self.POST(self.public_url + "/foo",
3638                       t="move", from_name="empty", to_dir="sub"))
3639         d.addCallback(lambda res:
3640                       self.failIfNodeHasChild(self._foo_node, u"empty"))
3641         d.addCallback(lambda res:
3642                       self.failUnlessNodeHasChild(self._sub_node, u"empty"))
3643         d.addCallback(lambda res:
3644                       self._sub_node.get_child_at_path(u"empty"))
3645         d.addCallback(lambda node:
3646                       self.failUnlessNodeHasChild(node, u"bar.txt"))
3647         d.addCallback(lambda res:
3648                       self.GET(self.public_url + "/foo/sub/empty/bar.txt"))
3649         d.addCallback(self.failUnlessIsBarDotTxt)
3650         return d
3651
3652     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
3653         """ If target is not None then the redirection has to go to target.  If
3654         statuscode is not None then the redirection has to be accomplished with
3655         that HTTP status code."""
3656         if not isinstance(res, failure.Failure):
3657             to_where = (target is None) and "somewhere" or ("to " + target)
3658             self.fail("%s: we were expecting to get redirected %s, not get an"
3659                       " actual page: %s" % (which, to_where, res))
3660         res.trap(error.PageRedirect)
3661         if statuscode is not None:
3662             self.failUnlessReallyEqual(res.value.status, statuscode,
3663                                        "%s: not a redirect" % which)
3664         if target is not None:
3665             # the PageRedirect does not seem to capture the uri= query arg
3666             # properly, so we can't check for it.
3667             realtarget = self.webish_url + target
3668             self.failUnlessReallyEqual(res.value.location, realtarget,
3669                                        "%s: wrong target" % which)
3670         return res.value.location
3671
3672     def test_GET_URI_form(self):
3673         base = "/uri?uri=%s" % self._bar_txt_uri
3674         # this is supposed to give us a redirect to /uri/$URI, plus arguments
3675         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
3676         d = self.GET(base)
3677         d.addBoth(self.shouldRedirect, targetbase)
3678         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
3679         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
3680         d.addCallback(lambda res: self.GET(base+"&t=json"))
3681         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
3682         d.addCallback(self.log, "about to get file by uri")
3683         d.addCallback(lambda res: self.GET(base, followRedirect=True))
3684         d.addCallback(self.failUnlessIsBarDotTxt)
3685         d.addCallback(self.log, "got file by uri, about to get dir by uri")
3686         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
3687                                            followRedirect=True))
3688         d.addCallback(self.failUnlessIsFooJSON)
3689         d.addCallback(self.log, "got dir by uri")
3690
3691         return d
3692
3693     def test_GET_URI_form_bad(self):
3694         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
3695                              "400 Bad Request", "GET /uri requires uri=",
3696                              self.GET, "/uri")
3697         return d
3698
3699     def test_GET_rename_form(self):
3700         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
3701                      followRedirect=True)
3702         def _check(res):
3703             self.failUnlessIn('name="when_done" value="."', res)
3704             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
3705             self.failUnlessIn(FAVICON_MARKUP, res)
3706         d.addCallback(_check)
3707         return d
3708
3709     def log(self, res, msg):
3710         #print "MSG: %s  RES: %s" % (msg, res)
3711         log.msg(msg)
3712         return res
3713
3714     def test_GET_URI_URL(self):
3715         base = "/uri/%s" % self._bar_txt_uri
3716         d = self.GET(base)
3717         d.addCallback(self.failUnlessIsBarDotTxt)
3718         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
3719         d.addCallback(self.failUnlessIsBarDotTxt)
3720         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
3721         d.addCallback(self.failUnlessIsBarDotTxt)
3722         return d
3723
3724     def test_GET_URI_URL_dir(self):
3725         base = "/uri/%s?t=json" % self._foo_uri
3726         d = self.GET(base)
3727         d.addCallback(self.failUnlessIsFooJSON)
3728         return d
3729
3730     def test_GET_URI_URL_missing(self):
3731         base = "/uri/%s" % self._bad_file_uri
3732         d = self.shouldHTTPError("test_GET_URI_URL_missing",
3733                                  http.GONE, None, "NotEnoughSharesError",
3734                                  self.GET, base)
3735         # TODO: how can we exercise both sides of WebDownloadTarget.fail
3736         # here? we must arrange for a download to fail after target.open()
3737         # has been called, and then inspect the response to see that it is
3738         # shorter than we expected.
3739         return d
3740
3741     def test_PUT_DIRURL_uri(self):
3742         d = self.s.create_dirnode()
3743         def _made_dir(dn):
3744             new_uri = dn.get_uri()
3745             # replace /foo with a new (empty) directory
3746             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
3747             d.addCallback(lambda res:
3748                           self.failUnlessReallyEqual(res.strip(), new_uri))
3749             d.addCallback(lambda res:
3750                           self.failUnlessRWChildURIIs(self.public_root,
3751                                                       u"foo",
3752                                                       new_uri))
3753             return d
3754         d.addCallback(_made_dir)
3755         return d
3756
3757     def test_PUT_DIRURL_uri_noreplace(self):
3758         d = self.s.create_dirnode()
3759         def _made_dir(dn):
3760             new_uri = dn.get_uri()
3761             # replace /foo with a new (empty) directory, but ask that
3762             # replace=false, so it should fail
3763             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
3764                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
3765                                  self.PUT,
3766                                  self.public_url + "/foo?t=uri&replace=false",
3767                                  new_uri)
3768             d.addCallback(lambda res:
3769                           self.failUnlessRWChildURIIs(self.public_root,
3770                                                       u"foo",
3771                                                       self._foo_uri))
3772             return d
3773         d.addCallback(_made_dir)
3774         return d
3775
3776     def test_PUT_DIRURL_bad_t(self):
3777         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
3778                              "400 Bad Request", "PUT to a directory",
3779                              self.PUT, self.public_url + "/foo?t=BOGUS", "")
3780         d.addCallback(lambda res:
3781                       self.failUnlessRWChildURIIs(self.public_root,
3782                                                   u"foo",
3783                                                   self._foo_uri))
3784         return d
3785
3786     def test_PUT_NEWFILEURL_uri(self):
3787         contents, n, new_uri = self.makefile(8)
3788         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
3789         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3790         d.addCallback(lambda res:
3791                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3792                                                       contents))
3793         return d
3794
3795     def test_PUT_NEWFILEURL_mdmf(self):
3796         new_contents = self.NEWFILE_CONTENTS * 300000
3797         d = self.PUT(self.public_url + \
3798                      "/foo/mdmf.txt?format=mdmf",
3799                      new_contents)
3800         d.addCallback(lambda ignored:
3801             self.GET(self.public_url + "/foo/mdmf.txt?t=json"))
3802         def _got_json(json):
3803             data = simplejson.loads(json)
3804             data = data[1]
3805             self.failUnlessIn("format", data)
3806             self.failUnlessEqual(data["format"], "MDMF")
3807             self.failUnless(data['rw_uri'].startswith("URI:MDMF"))
3808             self.failUnless(data['ro_uri'].startswith("URI:MDMF"))
3809         d.addCallback(_got_json)
3810         return d
3811
3812     def test_PUT_NEWFILEURL_sdmf(self):
3813         new_contents = self.NEWFILE_CONTENTS * 300000
3814         d = self.PUT(self.public_url + \
3815                      "/foo/sdmf.txt?format=sdmf",
3816                      new_contents)
3817         d.addCallback(lambda ignored:
3818             self.GET(self.public_url + "/foo/sdmf.txt?t=json"))
3819         def _got_json(json):
3820             data = simplejson.loads(json)
3821             data = data[1]
3822             self.failUnlessIn("format", data)
3823             self.failUnlessEqual(data["format"], "SDMF")
3824         d.addCallback(_got_json)
3825         return d
3826
3827     def test_PUT_NEWFILEURL_bad_format(self):
3828         new_contents = self.NEWFILE_CONTENTS * 300000
3829         return self.shouldHTTPError("PUT_NEWFILEURL_bad_format",
3830                                     400, "Bad Request", "Unknown format: foo",
3831                                     self.PUT, self.public_url + \
3832                                     "/foo/foo.txt?format=foo",
3833                                     new_contents)
3834
3835     def test_PUT_NEWFILEURL_uri_replace(self):
3836         contents, n, new_uri = self.makefile(8)
3837         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
3838         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3839         d.addCallback(lambda res:
3840                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3841                                                       contents))
3842         return d
3843
3844     def test_PUT_NEWFILEURL_uri_no_replace(self):
3845         contents, n, new_uri = self.makefile(8)
3846         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
3847         d.addBoth(self.shouldFail, error.Error,
3848                   "PUT_NEWFILEURL_uri_no_replace",
3849                   "409 Conflict",
3850                   "There was already a child by that name, and you asked me "
3851                   "to not replace it")
3852         return d
3853
3854     def test_PUT_NEWFILEURL_uri_unknown_bad(self):
3855         d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
3856         d.addBoth(self.shouldFail, error.Error,
3857                   "POST_put_uri_unknown_bad",
3858                   "400 Bad Request",
3859                   "unknown cap in a write slot")
3860         return d
3861
3862     def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
3863         d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
3864         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3865                       u"put-future-ro.txt")
3866         return d
3867
3868     def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
3869         d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
3870         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3871                       u"put-future-imm.txt")
3872         return d
3873
3874     def test_PUT_NEWFILE_URI(self):
3875         file_contents = "New file contents here\n"
3876         d = self.PUT("/uri", file_contents)
3877         def _check(uri):
3878             assert isinstance(uri, str), uri
3879             self.failUnlessIn(uri, self.get_all_contents())
3880             self.failUnlessReallyEqual(self.get_all_contents()[uri],
3881                                        file_contents)
3882             return self.GET("/uri/%s" % uri)
3883         d.addCallback(_check)
3884         def _check2(res):
3885             self.failUnlessReallyEqual(res, file_contents)
3886         d.addCallback(_check2)
3887         return d
3888
3889     def test_PUT_NEWFILE_URI_not_mutable(self):
3890         file_contents = "New file contents here\n"
3891         d = self.PUT("/uri?mutable=false", file_contents)
3892         def _check(uri):
3893             assert isinstance(uri, str), uri
3894             self.failUnlessIn(uri, self.get_all_contents())
3895             self.failUnlessReallyEqual(self.get_all_contents()[uri],
3896                                        file_contents)
3897             return self.GET("/uri/%s" % uri)
3898         d.addCallback(_check)
3899         def _check2(res):
3900             self.failUnlessReallyEqual(res, file_contents)
3901         d.addCallback(_check2)
3902         return d
3903
3904     def test_PUT_NEWFILE_URI_only_PUT(self):
3905         d = self.PUT("/uri?t=bogus", "")
3906         d.addBoth(self.shouldFail, error.Error,
3907                   "PUT_NEWFILE_URI_only_PUT",
3908                   "400 Bad Request",
3909                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
3910         return d
3911
3912     def test_PUT_NEWFILE_URI_mutable(self):
3913         file_contents = "New file contents here\n"
3914         d = self.PUT("/uri?mutable=true", file_contents)
3915         def _check1(filecap):
3916             filecap = filecap.strip()
3917             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
3918             self.filecap = filecap
3919             u = uri.WriteableSSKFileURI.init_from_string(filecap)
3920             self.failUnlessIn(u.get_storage_index(), self.get_all_contents())
3921             n = self.s.create_node_from_uri(filecap)
3922             return n.download_best_version()
3923         d.addCallback(_check1)
3924         def _check2(data):
3925             self.failUnlessReallyEqual(data, file_contents)
3926             return self.GET("/uri/%s" % urllib.quote(self.filecap))
3927         d.addCallback(_check2)
3928         def _check3(res):
3929             self.failUnlessReallyEqual(res, file_contents)
3930         d.addCallback(_check3)
3931         return d
3932
3933     def test_PUT_mkdir(self):
3934         d = self.PUT("/uri?t=mkdir", "")
3935         def _check(uri):
3936             n = self.s.create_node_from_uri(uri.strip())
3937             d2 = self.failUnlessNodeKeysAre(n, [])
3938             d2.addCallback(lambda res:
3939                            self.GET("/uri/%s?t=json" % uri))
3940             return d2
3941         d.addCallback(_check)
3942         d.addCallback(self.failUnlessIsEmptyJSON)
3943         return d
3944
3945     def test_PUT_mkdir_mdmf(self):
3946         d = self.PUT("/uri?t=mkdir&format=mdmf", "")
3947         def _got(res):
3948             u = uri.from_string(res)
3949             # Check that this is an MDMF writecap
3950             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
3951         d.addCallback(_got)
3952         return d
3953
3954     def test_PUT_mkdir_sdmf(self):
3955         d = self.PUT("/uri?t=mkdir&format=sdmf", "")
3956         def _got(res):
3957             u = uri.from_string(res)
3958             self.failUnlessIsInstance(u, uri.DirectoryURI)
3959         d.addCallback(_got)
3960         return d
3961
3962     def test_PUT_mkdir_bad_format(self):
3963         return self.shouldHTTPError("PUT_mkdir_bad_format",
3964                                     400, "Bad Request", "Unknown format: foo",
3965                                     self.PUT, "/uri?t=mkdir&format=foo",
3966                                     "")
3967
3968     def test_POST_check(self):
3969         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
3970         def _done(res):
3971             # this returns a string form of the results, which are probably
3972             # None since we're using fake filenodes.
3973             # TODO: verify that the check actually happened, by changing
3974             # FakeCHKFileNode to count how many times .check() has been
3975             # called.
3976             pass
3977         d.addCallback(_done)
3978         return d
3979
3980
3981     def test_PUT_update_at_offset(self):
3982         file_contents = "test file" * 100000 # about 900 KiB
3983         d = self.PUT("/uri?mutable=true", file_contents)
3984         def _then(filecap):
3985             self.filecap = filecap
3986             new_data = file_contents[:100]
3987             new = "replaced and so on"
3988             new_data += new
3989             new_data += file_contents[len(new_data):]
3990             assert len(new_data) == len(file_contents)
3991             self.new_data = new_data
3992         d.addCallback(_then)
3993         d.addCallback(lambda ignored:
3994             self.PUT("/uri/%s?replace=True&offset=100" % self.filecap,
3995                      "replaced and so on"))
3996         def _get_data(filecap):
3997             n = self.s.create_node_from_uri(filecap)
3998             return n.download_best_version()
3999         d.addCallback(_get_data)
4000         d.addCallback(lambda results:
4001             self.failUnlessEqual(results, self.new_data))
4002         # Now try appending things to the file
4003         d.addCallback(lambda ignored:
4004             self.PUT("/uri/%s?offset=%d" % (self.filecap, len(self.new_data)),
4005                      "puppies" * 100))
4006         d.addCallback(_get_data)
4007         d.addCallback(lambda results:
4008             self.failUnlessEqual(results, self.new_data + ("puppies" * 100)))
4009         # and try replacing the beginning of the file
4010         d.addCallback(lambda ignored:
4011             self.PUT("/uri/%s?offset=0" % self.filecap, "begin"))
4012         d.addCallback(_get_data)
4013         d.addCallback(lambda results:
4014             self.failUnlessEqual(results, "begin"+self.new_data[len("begin"):]+("puppies"*100)))
4015         return d
4016
4017     def test_PUT_update_at_invalid_offset(self):
4018         file_contents = "test file" * 100000 # about 900 KiB
4019         d = self.PUT("/uri?mutable=true", file_contents)
4020         def _then(filecap):
4021             self.filecap = filecap
4022         d.addCallback(_then)
4023         # Negative offsets should cause an error.
4024         d.addCallback(lambda ignored:
4025             self.shouldHTTPError("PUT_update_at_invalid_offset",
4026                                  400, "Bad Request",
4027                                  "Invalid offset",
4028                                  self.PUT,
4029                                  "/uri/%s?offset=-1" % self.filecap,
4030                                  "foo"))
4031         return d
4032
4033     def test_PUT_update_at_offset_immutable(self):
4034         file_contents = "Test file" * 100000
4035         d = self.PUT("/uri", file_contents)
4036         def _then(filecap):
4037             self.filecap = filecap
4038         d.addCallback(_then)
4039         d.addCallback(lambda ignored:
4040             self.shouldHTTPError("PUT_update_at_offset_immutable",
4041                                  400, "Bad Request",
4042                                  "immutable",
4043                                  self.PUT,
4044                                  "/uri/%s?offset=50" % self.filecap,
4045                                  "foo"))
4046         return d
4047
4048
4049     def test_bad_method(self):
4050         url = self.webish_url + self.public_url + "/foo/bar.txt"
4051         d = self.shouldHTTPError("bad_method",
4052                                  501, "Not Implemented",
4053                                  "I don't know how to treat a BOGUS request.",
4054                                  client.getPage, url, method="BOGUS")
4055         return d
4056
4057     def test_short_url(self):
4058         url = self.webish_url + "/uri"
4059         d = self.shouldHTTPError("short_url", 501, "Not Implemented",
4060                                  "I don't know how to treat a DELETE request.",
4061                                  client.getPage, url, method="DELETE")
4062         return d
4063
4064     def test_ophandle_bad(self):
4065         url = self.webish_url + "/operations/bogus?t=status"
4066         d = self.shouldHTTPError("ophandle_bad", 404, "404 Not Found",
4067                                  "unknown/expired handle 'bogus'",
4068                                  client.getPage, url)
4069         return d
4070
4071     def test_ophandle_cancel(self):
4072         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
4073                       followRedirect=True)
4074         d.addCallback(lambda ignored:
4075                       self.GET("/operations/128?t=status&output=JSON"))
4076         def _check1(res):
4077             data = simplejson.loads(res)
4078             self.failUnless("finished" in data, res)
4079             monitor = self.ws.root.child_operations.handles["128"][0]
4080             d = self.POST("/operations/128?t=cancel&output=JSON")
4081             def _check2(res):
4082                 data = simplejson.loads(res)
4083                 self.failUnless("finished" in data, res)
4084                 # t=cancel causes the handle to be forgotten
4085                 self.failUnless(monitor.is_cancelled())
4086             d.addCallback(_check2)
4087             return d
4088         d.addCallback(_check1)
4089         d.addCallback(lambda ignored:
4090                       self.shouldHTTPError("ophandle_cancel",
4091                                            404, "404 Not Found",
4092                                            "unknown/expired handle '128'",
4093                                            self.GET,
4094                                            "/operations/128?t=status&output=JSON"))
4095         return d
4096
4097     def test_ophandle_retainfor(self):
4098         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
4099                       followRedirect=True)
4100         d.addCallback(lambda ignored:
4101                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
4102         def _check1(res):
4103             data = simplejson.loads(res)
4104             self.failUnless("finished" in data, res)
4105         d.addCallback(_check1)
4106         # the retain-for=0 will cause the handle to be expired very soon
4107         d.addCallback(lambda ign:
4108             self.clock.advance(2.0))
4109         d.addCallback(lambda ignored:
4110                       self.shouldHTTPError("ophandle_retainfor",
4111                                            404, "404 Not Found",
4112                                            "unknown/expired handle '129'",
4113                                            self.GET,
4114                                            "/operations/129?t=status&output=JSON"))
4115         return d
4116
4117     def test_ophandle_release_after_complete(self):
4118         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
4119                       followRedirect=True)
4120         d.addCallback(self.wait_for_operation, "130")
4121         d.addCallback(lambda ignored:
4122                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
4123         # the release-after-complete=true will cause the handle to be expired
4124         d.addCallback(lambda ignored:
4125                       self.shouldHTTPError("ophandle_release_after_complete",
4126                                            404, "404 Not Found",
4127                                            "unknown/expired handle '130'",
4128                                            self.GET,
4129                                            "/operations/130?t=status&output=JSON"))
4130         return d
4131
4132     def test_uncollected_ophandle_expiration(self):
4133         # uncollected ophandles should expire after 4 days
4134         def _make_uncollected_ophandle(ophandle):
4135             d = self.POST(self.public_url +
4136                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
4137                           followRedirect=False)
4138             # When we start the operation, the webapi server will want
4139             # to redirect us to the page for the ophandle, so we get
4140             # confirmation that the operation has started. If the
4141             # manifest operation has finished by the time we get there,
4142             # following that redirect (by setting followRedirect=True
4143             # above) has the side effect of collecting the ophandle that
4144             # we've just created, which means that we can't use the
4145             # ophandle to test the uncollected timeout anymore. So,
4146             # instead, catch the 302 here and don't follow it.
4147             d.addBoth(self.should302, "uncollected_ophandle_creation")
4148             return d
4149         # Create an ophandle, don't collect it, then advance the clock by
4150         # 4 days - 1 second and make sure that the ophandle is still there.
4151         d = _make_uncollected_ophandle(131)
4152         d.addCallback(lambda ign:
4153             self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
4154         d.addCallback(lambda ign:
4155             self.GET("/operations/131?t=status&output=JSON"))
4156         def _check1(res):
4157             data = simplejson.loads(res)
4158             self.failUnless("finished" in data, res)
4159         d.addCallback(_check1)
4160         # Create an ophandle, don't collect it, then try to collect it
4161         # after 4 days. It should be gone.
4162         d.addCallback(lambda ign:
4163             _make_uncollected_ophandle(132))
4164         d.addCallback(lambda ign:
4165             self.clock.advance(96*60*60))
4166         d.addCallback(lambda ign:
4167             self.shouldHTTPError("uncollected_ophandle_expired_after_100_hours",
4168                                  404, "404 Not Found",
4169                                  "unknown/expired handle '132'",
4170                                  self.GET,
4171                                  "/operations/132?t=status&output=JSON"))
4172         return d
4173
4174     def test_collected_ophandle_expiration(self):
4175         # collected ophandles should expire after 1 day
4176         def _make_collected_ophandle(ophandle):
4177             d = self.POST(self.public_url +
4178                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
4179                           followRedirect=True)
4180             # By following the initial redirect, we collect the ophandle
4181             # we've just created.
4182             return d
4183         # Create a collected ophandle, then collect it after 23 hours
4184         # and 59 seconds to make sure that it is still there.
4185         d = _make_collected_ophandle(133)
4186         d.addCallback(lambda ign:
4187             self.clock.advance((24*60*60) - 1))
4188         d.addCallback(lambda ign:
4189             self.GET("/operations/133?t=status&output=JSON"))
4190         def _check1(res):
4191             data = simplejson.loads(res)
4192             self.failUnless("finished" in data, res)
4193         d.addCallback(_check1)
4194         # Create another uncollected ophandle, then try to collect it
4195         # after 24 hours to make sure that it is gone.
4196         d.addCallback(lambda ign:
4197             _make_collected_ophandle(134))
4198         d.addCallback(lambda ign:
4199             self.clock.advance(24*60*60))
4200         d.addCallback(lambda ign:
4201             self.shouldHTTPError("collected_ophandle_expired_after_1_day",
4202                                  404, "404 Not Found",
4203                                  "unknown/expired handle '134'",
4204                                  self.GET,
4205                                  "/operations/134?t=status&output=JSON"))
4206         return d
4207
4208     def test_incident(self):
4209         d = self.POST("/report_incident", details="eek")
4210         def _done(res):
4211             self.failIfIn("<html>", res)
4212             self.failUnlessIn("Thank you for your report!", res)
4213         d.addCallback(_done)
4214         return d
4215
4216     def test_static(self):
4217         webdir = os.path.join(self.staticdir, "subdir")
4218         fileutil.make_dirs(webdir)
4219         f = open(os.path.join(webdir, "hello.txt"), "wb")
4220         f.write("hello")
4221         f.close()
4222
4223         d = self.GET("/static/subdir/hello.txt")
4224         def _check(res):
4225             self.failUnlessReallyEqual(res, "hello")
4226         d.addCallback(_check)
4227         return d
4228
4229
4230 class IntroducerWeb(unittest.TestCase):
4231     def setUp(self):
4232         self.node = None
4233
4234     def tearDown(self):
4235         d = defer.succeed(None)
4236         if self.node:
4237             d.addCallback(lambda ign: self.node.stopService())
4238         d.addCallback(flushEventualQueue)
4239         return d
4240
4241     def test_welcome(self):
4242         basedir = "web.IntroducerWeb.test_welcome"
4243         os.mkdir(basedir)
4244         fileutil.write(os.path.join(basedir, "tahoe.cfg"), "[node]\nweb.port = tcp:0\n")
4245         self.node = IntroducerNode(basedir)
4246         self.ws = self.node.getServiceNamed("webish")
4247
4248         d = fireEventually(None)
4249         d.addCallback(lambda ign: self.node.startService())
4250         d.addCallback(lambda ign: self.node.when_tub_ready())
4251
4252         d.addCallback(lambda ign: self.GET("/"))
4253         def _check(res):
4254             self.failUnlessIn('Welcome to the Tahoe-LAFS Introducer', res)
4255             self.failUnlessIn(FAVICON_MARKUP, res)
4256         d.addCallback(_check)
4257         return d
4258
4259     def GET(self, urlpath, followRedirect=False, return_response=False,
4260             **kwargs):
4261         # if return_response=True, this fires with (data, statuscode,
4262         # respheaders) instead of just data.
4263         assert not isinstance(urlpath, unicode)
4264         url = self.ws.getURL().rstrip('/') + urlpath
4265         factory = HTTPClientGETFactory(url, method="GET",
4266                                        followRedirect=followRedirect, **kwargs)
4267         reactor.connectTCP("localhost", self.ws.getPortnum(), factory)
4268         d = factory.deferred
4269         def _got_data(data):
4270             return (data, factory.status, factory.response_headers)
4271         if return_response:
4272             d.addCallback(_got_data)
4273         return factory.deferred
4274
4275
4276 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4277     def test_load_file(self):
4278         # This will raise an exception unless a well-formed XML file is found under that name.
4279         common.getxmlfile('directory.xhtml').load()
4280
4281     def test_parse_replace_arg(self):
4282         self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
4283         self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
4284         self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
4285                                    "only-files")
4286         self.shouldFail(AssertionError, "test_parse_replace_arg", "",
4287                         common.parse_replace_arg, "only_fles")
4288
4289     def test_abbreviate_time(self):
4290         self.failUnlessReallyEqual(common.abbreviate_time(None), "")
4291         self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
4292         self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
4293         self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
4294         self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
4295         self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
4296
4297     def test_compute_rate(self):
4298         self.failUnlessReallyEqual(common.compute_rate(None, None), None)
4299         self.failUnlessReallyEqual(common.compute_rate(None, 1), None)
4300         self.failUnlessReallyEqual(common.compute_rate(250000, None), None)
4301         self.failUnlessReallyEqual(common.compute_rate(250000, 0), None)
4302         self.failUnlessReallyEqual(common.compute_rate(250000, 10), 25000.0)
4303         self.failUnlessReallyEqual(common.compute_rate(0, 10), 0.0)
4304         self.shouldFail(AssertionError, "test_compute_rate", "",
4305                         common.compute_rate, -100, 10)
4306         self.shouldFail(AssertionError, "test_compute_rate", "",
4307                         common.compute_rate, 100, -10)
4308
4309         # Sanity check
4310         rate = common.compute_rate(10*1000*1000, 1)
4311         self.failUnlessReallyEqual(common.abbreviate_rate(rate), "10.00MBps")
4312
4313     def test_abbreviate_rate(self):
4314         self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
4315         self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
4316         self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
4317         self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
4318
4319     def test_abbreviate_size(self):
4320         self.failUnlessReallyEqual(common.abbreviate_size(None), "")
4321         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
4322         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
4323         self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
4324         self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
4325
4326     def test_plural(self):
4327         def convert(s):
4328             return "%d second%s" % (s, status.plural(s))
4329         self.failUnlessReallyEqual(convert(0), "0 seconds")
4330         self.failUnlessReallyEqual(convert(1), "1 second")
4331         self.failUnlessReallyEqual(convert(2), "2 seconds")
4332         def convert2(s):
4333             return "has share%s: %s" % (status.plural(s), ",".join(s))
4334         self.failUnlessReallyEqual(convert2([]), "has shares: ")
4335         self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
4336         self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
4337
4338
4339 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4340
4341     def CHECK(self, ign, which, args, clientnum=0):
4342         fileurl = self.fileurls[which]
4343         url = fileurl + "?" + args
4344         return self.GET(url, method="POST", clientnum=clientnum)
4345
4346     def test_filecheck(self):
4347         self.basedir = "web/Grid/filecheck"
4348         self.set_up_grid()
4349         c0 = self.g.clients[0]
4350         self.uris = {}
4351         DATA = "data" * 100
4352         d = c0.upload(upload.Data(DATA, convergence=""))
4353         def _stash_uri(ur, which):
4354             self.uris[which] = ur.get_uri()
4355         d.addCallback(_stash_uri, "good")
4356         d.addCallback(lambda ign:
4357                       c0.upload(upload.Data(DATA+"1", convergence="")))
4358         d.addCallback(_stash_uri, "sick")
4359         d.addCallback(lambda ign:
4360                       c0.upload(upload.Data(DATA+"2", convergence="")))
4361         d.addCallback(_stash_uri, "dead")
4362         def _stash_mutable_uri(n, which):
4363             self.uris[which] = n.get_uri()
4364             assert isinstance(self.uris[which], str)
4365         d.addCallback(lambda ign:
4366             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4367         d.addCallback(_stash_mutable_uri, "corrupt")
4368         d.addCallback(lambda ign:
4369                       c0.upload(upload.Data("literal", convergence="")))
4370         d.addCallback(_stash_uri, "small")
4371         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
4372         d.addCallback(_stash_mutable_uri, "smalldir")
4373
4374         def _compute_fileurls(ignored):
4375             self.fileurls = {}
4376             for which in self.uris:
4377                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4378         d.addCallback(_compute_fileurls)
4379
4380         def _clobber_shares(ignored):
4381             good_shares = self.find_uri_shares(self.uris["good"])
4382             self.failUnlessReallyEqual(len(good_shares), 10)
4383             sick_shares = self.find_uri_shares(self.uris["sick"])
4384             os.unlink(sick_shares[0][2])
4385             dead_shares = self.find_uri_shares(self.uris["dead"])
4386             for i in range(1, 10):
4387                 os.unlink(dead_shares[i][2])
4388             c_shares = self.find_uri_shares(self.uris["corrupt"])
4389             cso = CorruptShareOptions()
4390             cso.stdout = StringIO()
4391             cso.parseOptions([c_shares[0][2]])
4392             corrupt_share(cso)
4393         d.addCallback(_clobber_shares)
4394
4395         d.addCallback(self.CHECK, "good", "t=check")
4396         def _got_html_good(res):
4397             self.failUnlessIn("Healthy", res)
4398             self.failIfIn("Not Healthy", res)
4399             self.failUnlessIn(FAVICON_MARKUP, res)
4400         d.addCallback(_got_html_good)
4401         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
4402         def _got_html_good_return_to(res):
4403             self.failUnlessIn("Healthy", res)
4404             self.failIfIn("Not Healthy", res)
4405             self.failUnlessIn('<a href="somewhere">Return to file', res)
4406         d.addCallback(_got_html_good_return_to)
4407         d.addCallback(self.CHECK, "good", "t=check&output=json")
4408         def _got_json_good(res):
4409             r = simplejson.loads(res)
4410             self.failUnlessEqual(r["summary"], "Healthy")
4411             self.failUnless(r["results"]["healthy"])
4412             self.failIf(r["results"]["needs-rebalancing"])
4413             self.failUnless(r["results"]["recoverable"])
4414         d.addCallback(_got_json_good)
4415
4416         d.addCallback(self.CHECK, "small", "t=check")
4417         def _got_html_small(res):
4418             self.failUnlessIn("Literal files are always healthy", res)
4419             self.failIfIn("Not Healthy", res)
4420         d.addCallback(_got_html_small)
4421         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
4422         def _got_html_small_return_to(res):
4423             self.failUnlessIn("Literal files are always healthy", res)
4424             self.failIfIn("Not Healthy", res)
4425             self.failUnlessIn('<a href="somewhere">Return to file', res)
4426         d.addCallback(_got_html_small_return_to)
4427         d.addCallback(self.CHECK, "small", "t=check&output=json")
4428         def _got_json_small(res):
4429             r = simplejson.loads(res)
4430             self.failUnlessEqual(r["storage-index"], "")
4431             self.failUnless(r["results"]["healthy"])
4432         d.addCallback(_got_json_small)
4433
4434         d.addCallback(self.CHECK, "smalldir", "t=check")
4435         def _got_html_smalldir(res):
4436             self.failUnlessIn("Literal files are always healthy", res)
4437             self.failIfIn("Not Healthy", res)
4438         d.addCallback(_got_html_smalldir)
4439         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
4440         def _got_json_smalldir(res):
4441             r = simplejson.loads(res)
4442             self.failUnlessEqual(r["storage-index"], "")
4443             self.failUnless(r["results"]["healthy"])
4444         d.addCallback(_got_json_smalldir)
4445
4446         d.addCallback(self.CHECK, "sick", "t=check")
4447         def _got_html_sick(res):
4448             self.failUnlessIn("Not Healthy", res)
4449         d.addCallback(_got_html_sick)
4450         d.addCallback(self.CHECK, "sick", "t=check&output=json")
4451         def _got_json_sick(res):
4452             r = simplejson.loads(res)
4453             self.failUnlessEqual(r["summary"],
4454                                  "Not Healthy: 9 shares (enc 3-of-10)")
4455             self.failIf(r["results"]["healthy"])
4456             self.failIf(r["results"]["needs-rebalancing"])
4457             self.failUnless(r["results"]["recoverable"])
4458         d.addCallback(_got_json_sick)
4459
4460         d.addCallback(self.CHECK, "dead", "t=check")
4461         def _got_html_dead(res):
4462             self.failUnlessIn("Not Healthy", res)
4463         d.addCallback(_got_html_dead)
4464         d.addCallback(self.CHECK, "dead", "t=check&output=json")
4465         def _got_json_dead(res):
4466             r = simplejson.loads(res)
4467             self.failUnlessEqual(r["summary"],
4468                                  "Not Healthy: 1 shares (enc 3-of-10)")
4469             self.failIf(r["results"]["healthy"])
4470             self.failIf(r["results"]["needs-rebalancing"])
4471             self.failIf(r["results"]["recoverable"])
4472         d.addCallback(_got_json_dead)
4473
4474         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
4475         def _got_html_corrupt(res):
4476             self.failUnlessIn("Not Healthy! : Unhealthy", res)
4477         d.addCallback(_got_html_corrupt)
4478         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
4479         def _got_json_corrupt(res):
4480             r = simplejson.loads(res)
4481             self.failUnlessIn("Unhealthy: 9 shares (enc 3-of-10)", r["summary"])
4482             self.failIf(r["results"]["healthy"])
4483             self.failUnless(r["results"]["recoverable"])
4484             self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
4485             self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
4486         d.addCallback(_got_json_corrupt)
4487
4488         d.addErrback(self.explain_web_error)
4489         return d
4490
4491     def test_repair_html(self):
4492         self.basedir = "web/Grid/repair_html"
4493         self.set_up_grid()
4494         c0 = self.g.clients[0]
4495         self.uris = {}
4496         DATA = "data" * 100
4497         d = c0.upload(upload.Data(DATA, convergence=""))
4498         def _stash_uri(ur, which):
4499             self.uris[which] = ur.get_uri()
4500         d.addCallback(_stash_uri, "good")
4501         d.addCallback(lambda ign:
4502                       c0.upload(upload.Data(DATA+"1", convergence="")))
4503         d.addCallback(_stash_uri, "sick")
4504         d.addCallback(lambda ign:
4505                       c0.upload(upload.Data(DATA+"2", convergence="")))
4506         d.addCallback(_stash_uri, "dead")
4507         def _stash_mutable_uri(n, which):
4508             self.uris[which] = n.get_uri()
4509             assert isinstance(self.uris[which], str)
4510         d.addCallback(lambda ign:
4511             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4512         d.addCallback(_stash_mutable_uri, "corrupt")
4513
4514         def _compute_fileurls(ignored):
4515             self.fileurls = {}
4516             for which in self.uris:
4517                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4518         d.addCallback(_compute_fileurls)
4519
4520         def _clobber_shares(ignored):
4521             good_shares = self.find_uri_shares(self.uris["good"])
4522             self.failUnlessReallyEqual(len(good_shares), 10)
4523             sick_shares = self.find_uri_shares(self.uris["sick"])
4524             os.unlink(sick_shares[0][2])
4525             dead_shares = self.find_uri_shares(self.uris["dead"])
4526             for i in range(1, 10):
4527                 os.unlink(dead_shares[i][2])
4528             c_shares = self.find_uri_shares(self.uris["corrupt"])
4529             cso = CorruptShareOptions()
4530             cso.stdout = StringIO()
4531             cso.parseOptions([c_shares[0][2]])
4532             corrupt_share(cso)
4533         d.addCallback(_clobber_shares)
4534
4535         d.addCallback(self.CHECK, "good", "t=check&repair=true")
4536         def _got_html_good(res):
4537             self.failUnlessIn("Healthy", res)
4538             self.failIfIn("Not Healthy", res)
4539             self.failUnlessIn("No repair necessary", res)
4540             self.failUnlessIn(FAVICON_MARKUP, res)
4541         d.addCallback(_got_html_good)
4542
4543         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
4544         def _got_html_sick(res):
4545             self.failUnlessIn("Healthy : healthy", res)
4546             self.failIfIn("Not Healthy", res)
4547             self.failUnlessIn("Repair successful", res)
4548         d.addCallback(_got_html_sick)
4549
4550         # repair of a dead file will fail, of course, but it isn't yet
4551         # clear how this should be reported. Right now it shows up as
4552         # a "410 Gone".
4553         #
4554         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
4555         #def _got_html_dead(res):
4556         #    print res
4557         #    self.failUnlessIn("Healthy : healthy", res)
4558         #    self.failIfIn("Not Healthy", res)
4559         #    self.failUnlessIn("No repair necessary", res)
4560         #d.addCallback(_got_html_dead)
4561
4562         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
4563         def _got_html_corrupt(res):
4564             self.failUnlessIn("Healthy : Healthy", res)
4565             self.failIfIn("Not Healthy", res)
4566             self.failUnlessIn("Repair successful", res)
4567         d.addCallback(_got_html_corrupt)
4568
4569         d.addErrback(self.explain_web_error)
4570         return d
4571
4572     def test_repair_json(self):
4573         self.basedir = "web/Grid/repair_json"
4574         self.set_up_grid()
4575         c0 = self.g.clients[0]
4576         self.uris = {}
4577         DATA = "data" * 100
4578         d = c0.upload(upload.Data(DATA+"1", convergence=""))
4579         def _stash_uri(ur, which):
4580             self.uris[which] = ur.get_uri()
4581         d.addCallback(_stash_uri, "sick")
4582
4583         def _compute_fileurls(ignored):
4584             self.fileurls = {}
4585             for which in self.uris:
4586                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4587         d.addCallback(_compute_fileurls)
4588
4589         def _clobber_shares(ignored):
4590             sick_shares = self.find_uri_shares(self.uris["sick"])
4591             os.unlink(sick_shares[0][2])
4592         d.addCallback(_clobber_shares)
4593
4594         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
4595         def _got_json_sick(res):
4596             r = simplejson.loads(res)
4597             self.failUnlessReallyEqual(r["repair-attempted"], True)
4598             self.failUnlessReallyEqual(r["repair-successful"], True)
4599             self.failUnlessEqual(r["pre-repair-results"]["summary"],
4600                                  "Not Healthy: 9 shares (enc 3-of-10)")
4601             self.failIf(r["pre-repair-results"]["results"]["healthy"])
4602             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
4603             self.failUnless(r["post-repair-results"]["results"]["healthy"])
4604         d.addCallback(_got_json_sick)
4605
4606         d.addErrback(self.explain_web_error)
4607         return d
4608
4609     def test_unknown(self, immutable=False):
4610         self.basedir = "web/Grid/unknown"
4611         if immutable:
4612             self.basedir = "web/Grid/unknown-immutable"
4613
4614         self.set_up_grid()
4615         c0 = self.g.clients[0]
4616         self.uris = {}
4617         self.fileurls = {}
4618
4619         # the future cap format may contain slashes, which must be tolerated
4620         expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
4621                                                            safe="")
4622
4623         if immutable:
4624             name = u"future-imm"
4625             future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
4626             d = c0.create_immutable_dirnode({name: (future_node, {})})
4627         else:
4628             name = u"future"
4629             future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4630             d = c0.create_dirnode()
4631
4632         def _stash_root_and_create_file(n):
4633             self.rootnode = n
4634             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
4635             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
4636             if not immutable:
4637                 return self.rootnode.set_node(name, future_node)
4638         d.addCallback(_stash_root_and_create_file)
4639
4640         # make sure directory listing tolerates unknown nodes
4641         d.addCallback(lambda ign: self.GET(self.rooturl))
4642         def _check_directory_html(res, expected_type_suffix):
4643             pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
4644                                   '<td>%s</td>' % (expected_type_suffix, str(name)),
4645                                  re.DOTALL)
4646             self.failUnless(re.search(pattern, res), res)
4647             # find the More Info link for name, should be relative
4648             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4649             info_url = mo.group(1)
4650             self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
4651         if immutable:
4652             d.addCallback(_check_directory_html, "-IMM")
4653         else:
4654             d.addCallback(_check_directory_html, "")
4655
4656         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4657         def _check_directory_json(res, expect_rw_uri):
4658             data = simplejson.loads(res)
4659             self.failUnlessEqual(data[0], "dirnode")
4660             f = data[1]["children"][name]
4661             self.failUnlessEqual(f[0], "unknown")
4662             if expect_rw_uri:
4663                 self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data)
4664             else:
4665                 self.failIfIn("rw_uri", f[1])
4666             if immutable:
4667                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
4668             else:
4669                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
4670             self.failUnlessIn("metadata", f[1])
4671         d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
4672
4673         def _check_info(res, expect_rw_uri, expect_ro_uri):
4674             self.failUnlessIn("Object Type: <span>unknown</span>", res)
4675             if expect_rw_uri:
4676                 self.failUnlessIn(unknown_rwcap, res)
4677             if expect_ro_uri:
4678                 if immutable:
4679                     self.failUnlessIn(unknown_immcap, res)
4680                 else:
4681                     self.failUnlessIn(unknown_rocap, res)
4682             else:
4683                 self.failIfIn(unknown_rocap, res)
4684             self.failIfIn("Raw data as", res)
4685             self.failIfIn("Directory writecap", res)
4686             self.failIfIn("Checker Operations", res)
4687             self.failIfIn("Mutable File Operations", res)
4688             self.failIfIn("Directory Operations", res)
4689
4690         # FIXME: these should have expect_rw_uri=not immutable; I don't know
4691         # why they fail. Possibly related to ticket #922.
4692
4693         d.addCallback(lambda ign: self.GET(expected_info_url))
4694         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
4695         d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
4696         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
4697
4698         def _check_json(res, expect_rw_uri):
4699             data = simplejson.loads(res)
4700             self.failUnlessEqual(data[0], "unknown")
4701             if expect_rw_uri:
4702                 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data)
4703             else:
4704                 self.failIfIn("rw_uri", data[1])
4705
4706             if immutable:
4707                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data)
4708                 self.failUnlessReallyEqual(data[1]["mutable"], False)
4709             elif expect_rw_uri:
4710                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4711                 self.failUnlessReallyEqual(data[1]["mutable"], True)
4712             else:
4713                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4714                 self.failIfIn("mutable", data[1])
4715
4716             # TODO: check metadata contents
4717             self.failUnlessIn("metadata", data[1])
4718
4719         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
4720         d.addCallback(_check_json, expect_rw_uri=not immutable)
4721
4722         # and make sure that a read-only version of the directory can be
4723         # rendered too. This version will not have unknown_rwcap, whether
4724         # or not future_node was immutable.
4725         d.addCallback(lambda ign: self.GET(self.rourl))
4726         if immutable:
4727             d.addCallback(_check_directory_html, "-IMM")
4728         else:
4729             d.addCallback(_check_directory_html, "-RO")
4730
4731         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
4732         d.addCallback(_check_directory_json, expect_rw_uri=False)
4733
4734         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
4735         d.addCallback(_check_json, expect_rw_uri=False)
4736
4737         # TODO: check that getting t=info from the Info link in the ro directory
4738         # works, and does not include the writecap URI.
4739         return d
4740
4741     def test_immutable_unknown(self):
4742         return self.test_unknown(immutable=True)
4743
4744     def test_mutant_dirnodes_are_omitted(self):
4745         self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
4746
4747         self.set_up_grid()
4748         c = self.g.clients[0]
4749         nm = c.nodemaker
4750         self.uris = {}
4751         self.fileurls = {}
4752
4753         lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
4754         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
4755         mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
4756
4757         # This method tests mainly dirnode, but we'd have to duplicate code in order to
4758         # test the dirnode and web layers separately.
4759
4760         # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
4761         # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
4762         # When the directory is read, the mutants should be silently disposed of, leaving
4763         # their lonely sibling.
4764         # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
4765         # because immutable directories don't have a writecap and therefore that field
4766         # isn't (and can't be) decrypted.
4767         # TODO: The field still exists in the netstring. Technically we should check what
4768         # happens if something is put there (_unpack_contents should raise ValueError),
4769         # but that can wait.
4770
4771         lonely_child = nm.create_from_cap(lonely_uri)
4772         mutant_ro_child = nm.create_from_cap(mut_read_uri)
4773         mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
4774
4775         def _by_hook_or_by_crook():
4776             return True
4777         for n in [mutant_ro_child, mutant_write_in_ro_child]:
4778             n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
4779
4780         mutant_write_in_ro_child.get_write_uri    = lambda: None
4781         mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
4782
4783         kids = {u"lonely":      (lonely_child, {}),
4784                 u"ro":          (mutant_ro_child, {}),
4785                 u"write-in-ro": (mutant_write_in_ro_child, {}),
4786                 }
4787         d = c.create_immutable_dirnode(kids)
4788
4789         def _created(dn):
4790             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
4791             self.failIf(dn.is_mutable())
4792             self.failUnless(dn.is_readonly())
4793             # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
4794             self.failIf(hasattr(dn._node, 'get_writekey'))
4795             rep = str(dn)
4796             self.failUnlessIn("RO-IMM", rep)
4797             cap = dn.get_cap()
4798             self.failUnlessIn("CHK", cap.to_string())
4799             self.cap = cap
4800             self.rootnode = dn
4801             self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
4802             return download_to_data(dn._node)
4803         d.addCallback(_created)
4804
4805         def _check_data(data):
4806             # Decode the netstring representation of the directory to check that all children
4807             # are present. This is a bit of an abstraction violation, but there's not really
4808             # any other way to do it given that the real DirectoryNode._unpack_contents would
4809             # strip the mutant children out (which is what we're trying to test, later).
4810             position = 0
4811             numkids = 0
4812             while position < len(data):
4813                 entries, position = split_netstring(data, 1, position)
4814                 entry = entries[0]
4815                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
4816                 name = name_utf8.decode("utf-8")
4817                 self.failUnlessEqual(rwcapdata, "")
4818                 self.failUnlessIn(name, kids)
4819                 (expected_child, ign) = kids[name]
4820                 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
4821                 numkids += 1
4822
4823             self.failUnlessReallyEqual(numkids, 3)
4824             return self.rootnode.list()
4825         d.addCallback(_check_data)
4826
4827         # Now when we use the real directory listing code, the mutants should be absent.
4828         def _check_kids(children):
4829             self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
4830             lonely_node, lonely_metadata = children[u"lonely"]
4831
4832             self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
4833             self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
4834         d.addCallback(_check_kids)
4835
4836         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
4837         d.addCallback(lambda n: n.list())
4838         d.addCallback(_check_kids)  # again with dirnode recreated from cap
4839
4840         # Make sure the lonely child can be listed in HTML...
4841         d.addCallback(lambda ign: self.GET(self.rooturl))
4842         def _check_html(res):
4843             self.failIfIn("URI:SSK", res)
4844             get_lonely = "".join([r'<td>FILE</td>',
4845                                   r'\s+<td>',
4846                                   r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
4847                                   r'</td>',
4848                                   r'\s+<td align="right">%d</td>' % len("one"),
4849                                   ])
4850             self.failUnless(re.search(get_lonely, res), res)
4851
4852             # find the More Info link for name, should be relative
4853             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4854             info_url = mo.group(1)
4855             self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
4856         d.addCallback(_check_html)
4857
4858         # ... and in JSON.
4859         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4860         def _check_json(res):
4861             data = simplejson.loads(res)
4862             self.failUnlessEqual(data[0], "dirnode")
4863             listed_children = data[1]["children"]
4864             self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
4865             ll_type, ll_data = listed_children[u"lonely"]
4866             self.failUnlessEqual(ll_type, "filenode")
4867             self.failIfIn("rw_uri", ll_data)
4868             self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
4869         d.addCallback(_check_json)
4870         return d
4871
4872     def test_deep_check(self):
4873         self.basedir = "web/Grid/deep_check"
4874         self.set_up_grid()
4875         c0 = self.g.clients[0]
4876         self.uris = {}
4877         self.fileurls = {}
4878         DATA = "data" * 100
4879         d = c0.create_dirnode()
4880         def _stash_root_and_create_file(n):
4881             self.rootnode = n
4882             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4883             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4884         d.addCallback(_stash_root_and_create_file)
4885         def _stash_uri(fn, which):
4886             self.uris[which] = fn.get_uri()
4887             return fn
4888         d.addCallback(_stash_uri, "good")
4889         d.addCallback(lambda ign:
4890                       self.rootnode.add_file(u"small",
4891                                              upload.Data("literal",
4892                                                         convergence="")))
4893         d.addCallback(_stash_uri, "small")
4894         d.addCallback(lambda ign:
4895                       self.rootnode.add_file(u"sick",
4896                                              upload.Data(DATA+"1",
4897                                                         convergence="")))
4898         d.addCallback(_stash_uri, "sick")
4899
4900         # this tests that deep-check and stream-manifest will ignore
4901         # UnknownNode instances. Hopefully this will also cover deep-stats.
4902         future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4903         d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
4904
4905         def _clobber_shares(ignored):
4906             self.delete_shares_numbered(self.uris["sick"], [0,1])
4907         d.addCallback(_clobber_shares)
4908
4909         # root
4910         # root/good
4911         # root/small
4912         # root/sick
4913         # root/future
4914
4915         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4916         def _done(res):
4917             try:
4918                 units = [simplejson.loads(line)
4919                          for line in res.splitlines()
4920                          if line]
4921             except ValueError:
4922                 print "response is:", res
4923                 print "undecodeable line was '%s'" % line
4924                 raise
4925             self.failUnlessReallyEqual(len(units), 5+1)
4926             # should be parent-first
4927             u0 = units[0]
4928             self.failUnlessEqual(u0["path"], [])
4929             self.failUnlessEqual(u0["type"], "directory")
4930             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4931             u0cr = u0["check-results"]
4932             self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
4933
4934             ugood = [u for u in units
4935                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4936             self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
4937             ugoodcr = ugood["check-results"]
4938             self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
4939
4940             stats = units[-1]
4941             self.failUnlessEqual(stats["type"], "stats")
4942             s = stats["stats"]
4943             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4944             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4945             self.failUnlessReallyEqual(s["count-directories"], 1)
4946             self.failUnlessReallyEqual(s["count-unknown"], 1)
4947         d.addCallback(_done)
4948
4949         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4950         def _check_manifest(res):
4951             self.failUnless(res.endswith("\n"))
4952             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
4953             self.failUnlessReallyEqual(len(units), 5+1)
4954             self.failUnlessEqual(units[-1]["type"], "stats")
4955             first = units[0]
4956             self.failUnlessEqual(first["path"], [])
4957             self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri())
4958             self.failUnlessEqual(first["type"], "directory")
4959             stats = units[-1]["stats"]
4960             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4961             self.failUnlessReallyEqual(stats["count-literal-files"], 1)
4962             self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
4963             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4964             self.failUnlessReallyEqual(stats["count-unknown"], 1)
4965         d.addCallback(_check_manifest)
4966
4967         # now add root/subdir and root/subdir/grandchild, then make subdir
4968         # unrecoverable, then see what happens
4969
4970         d.addCallback(lambda ign:
4971                       self.rootnode.create_subdirectory(u"subdir"))
4972         d.addCallback(_stash_uri, "subdir")
4973         d.addCallback(lambda subdir_node:
4974                       subdir_node.add_file(u"grandchild",
4975                                            upload.Data(DATA+"2",
4976                                                        convergence="")))
4977         d.addCallback(_stash_uri, "grandchild")
4978
4979         d.addCallback(lambda ign:
4980                       self.delete_shares_numbered(self.uris["subdir"],
4981                                                   range(1, 10)))
4982
4983         # root
4984         # root/good
4985         # root/small
4986         # root/sick
4987         # root/future
4988         # root/subdir [unrecoverable]
4989         # root/subdir/grandchild
4990
4991         # how should a streaming-JSON API indicate fatal error?
4992         # answer: emit ERROR: instead of a JSON string
4993
4994         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4995         def _check_broken_manifest(res):
4996             lines = res.splitlines()
4997             error_lines = [i
4998                            for (i,line) in enumerate(lines)
4999                            if line.startswith("ERROR:")]
5000             if not error_lines:
5001                 self.fail("no ERROR: in output: %s" % (res,))
5002             first_error = error_lines[0]
5003             error_line = lines[first_error]
5004             error_msg = lines[first_error+1:]
5005             error_msg_s = "\n".join(error_msg) + "\n"
5006             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
5007                               error_line)
5008             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
5009             units = [simplejson.loads(line) for line in lines[:first_error]]
5010             self.failUnlessReallyEqual(len(units), 6) # includes subdir
5011             last_unit = units[-1]
5012             self.failUnlessEqual(last_unit["path"], ["subdir"])
5013         d.addCallback(_check_broken_manifest)
5014
5015         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
5016         def _check_broken_deepcheck(res):
5017             lines = res.splitlines()
5018             error_lines = [i
5019                            for (i,line) in enumerate(lines)
5020                            if line.startswith("ERROR:")]
5021             if not error_lines:
5022                 self.fail("no ERROR: in output: %s" % (res,))
5023             first_error = error_lines[0]
5024             error_line = lines[first_error]
5025             error_msg = lines[first_error+1:]
5026             error_msg_s = "\n".join(error_msg) + "\n"
5027             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
5028                               error_line)
5029             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
5030             units = [simplejson.loads(line) for line in lines[:first_error]]
5031             self.failUnlessReallyEqual(len(units), 6) # includes subdir
5032             last_unit = units[-1]
5033             self.failUnlessEqual(last_unit["path"], ["subdir"])
5034             r = last_unit["check-results"]["results"]
5035             self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
5036             self.failUnlessReallyEqual(r["count-shares-good"], 1)
5037             self.failUnlessReallyEqual(r["recoverable"], False)
5038         d.addCallback(_check_broken_deepcheck)
5039
5040         d.addErrback(self.explain_web_error)
5041         return d
5042
5043     def test_deep_check_and_repair(self):
5044         self.basedir = "web/Grid/deep_check_and_repair"
5045         self.set_up_grid()
5046         c0 = self.g.clients[0]
5047         self.uris = {}
5048         self.fileurls = {}
5049         DATA = "data" * 100
5050         d = c0.create_dirnode()
5051         def _stash_root_and_create_file(n):
5052             self.rootnode = n
5053             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5054             return n.add_file(u"good", upload.Data(DATA, convergence=""))
5055         d.addCallback(_stash_root_and_create_file)
5056         def _stash_uri(fn, which):
5057             self.uris[which] = fn.get_uri()
5058         d.addCallback(_stash_uri, "good")
5059         d.addCallback(lambda ign:
5060                       self.rootnode.add_file(u"small",
5061                                              upload.Data("literal",
5062                                                         convergence="")))
5063         d.addCallback(_stash_uri, "small")
5064         d.addCallback(lambda ign:
5065                       self.rootnode.add_file(u"sick",
5066                                              upload.Data(DATA+"1",
5067                                                         convergence="")))
5068         d.addCallback(_stash_uri, "sick")
5069         #d.addCallback(lambda ign:
5070         #              self.rootnode.add_file(u"dead",
5071         #                                     upload.Data(DATA+"2",
5072         #                                                convergence="")))
5073         #d.addCallback(_stash_uri, "dead")
5074
5075         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
5076         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
5077         #d.addCallback(_stash_uri, "corrupt")
5078
5079         def _clobber_shares(ignored):
5080             good_shares = self.find_uri_shares(self.uris["good"])
5081             self.failUnlessReallyEqual(len(good_shares), 10)
5082             sick_shares = self.find_uri_shares(self.uris["sick"])
5083             os.unlink(sick_shares[0][2])
5084             #dead_shares = self.find_uri_shares(self.uris["dead"])
5085             #for i in range(1, 10):
5086             #    os.unlink(dead_shares[i][2])
5087
5088             #c_shares = self.find_uri_shares(self.uris["corrupt"])
5089             #cso = CorruptShareOptions()
5090             #cso.stdout = StringIO()
5091             #cso.parseOptions([c_shares[0][2]])
5092             #corrupt_share(cso)
5093         d.addCallback(_clobber_shares)
5094
5095         # root
5096         # root/good   CHK, 10 shares
5097         # root/small  LIT
5098         # root/sick   CHK, 9 shares
5099
5100         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
5101         def _done(res):
5102             units = [simplejson.loads(line)
5103                      for line in res.splitlines()
5104                      if line]
5105             self.failUnlessReallyEqual(len(units), 4+1)
5106             # should be parent-first
5107             u0 = units[0]
5108             self.failUnlessEqual(u0["path"], [])
5109             self.failUnlessEqual(u0["type"], "directory")
5110             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
5111             u0crr = u0["check-and-repair-results"]
5112             self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
5113             self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
5114
5115             ugood = [u for u in units
5116                      if u["type"] == "file" and u["path"] == [u"good"]][0]
5117             self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
5118             ugoodcrr = ugood["check-and-repair-results"]
5119             self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
5120             self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
5121
5122             usick = [u for u in units
5123                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
5124             self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"])
5125             usickcrr = usick["check-and-repair-results"]
5126             self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
5127             self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
5128             self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
5129             self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
5130
5131             stats = units[-1]
5132             self.failUnlessEqual(stats["type"], "stats")
5133             s = stats["stats"]
5134             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
5135             self.failUnlessReallyEqual(s["count-literal-files"], 1)
5136             self.failUnlessReallyEqual(s["count-directories"], 1)
5137         d.addCallback(_done)
5138
5139         d.addErrback(self.explain_web_error)
5140         return d
5141
5142     def _count_leases(self, ignored, which):
5143         u = self.uris[which]
5144         shares = self.find_uri_shares(u)
5145         lease_counts = []
5146         for shnum, serverid, fn in shares:
5147             sf = get_share_file(fn)
5148             num_leases = len(list(sf.get_leases()))
5149             lease_counts.append( (fn, num_leases) )
5150         return lease_counts
5151
5152     def _assert_leasecount(self, lease_counts, expected):
5153         for (fn, num_leases) in lease_counts:
5154             if num_leases != expected:
5155                 self.fail("expected %d leases, have %d, on %s" %
5156                           (expected, num_leases, fn))
5157
5158     def test_add_lease(self):
5159         self.basedir = "web/Grid/add_lease"
5160         self.set_up_grid(num_clients=2)
5161         c0 = self.g.clients[0]
5162         self.uris = {}
5163         DATA = "data" * 100
5164         d = c0.upload(upload.Data(DATA, convergence=""))
5165         def _stash_uri(ur, which):
5166             self.uris[which] = ur.get_uri()
5167         d.addCallback(_stash_uri, "one")
5168         d.addCallback(lambda ign:
5169                       c0.upload(upload.Data(DATA+"1", convergence="")))
5170         d.addCallback(_stash_uri, "two")
5171         def _stash_mutable_uri(n, which):
5172             self.uris[which] = n.get_uri()
5173             assert isinstance(self.uris[which], str)
5174         d.addCallback(lambda ign:
5175             c0.create_mutable_file(publish.MutableData(DATA+"2")))
5176         d.addCallback(_stash_mutable_uri, "mutable")
5177
5178         def _compute_fileurls(ignored):
5179             self.fileurls = {}
5180             for which in self.uris:
5181                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
5182         d.addCallback(_compute_fileurls)
5183
5184         d.addCallback(self._count_leases, "one")
5185         d.addCallback(self._assert_leasecount, 1)
5186         d.addCallback(self._count_leases, "two")
5187         d.addCallback(self._assert_leasecount, 1)
5188         d.addCallback(self._count_leases, "mutable")
5189         d.addCallback(self._assert_leasecount, 1)
5190
5191         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
5192         def _got_html_good(res):
5193             self.failUnlessIn("Healthy", res)
5194             self.failIfIn("Not Healthy", res)
5195         d.addCallback(_got_html_good)
5196
5197         d.addCallback(self._count_leases, "one")
5198         d.addCallback(self._assert_leasecount, 1)
5199         d.addCallback(self._count_leases, "two")
5200         d.addCallback(self._assert_leasecount, 1)
5201         d.addCallback(self._count_leases, "mutable")
5202         d.addCallback(self._assert_leasecount, 1)
5203
5204         # this CHECK uses the original client, which uses the same
5205         # lease-secrets, so it will just renew the original lease
5206         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
5207         d.addCallback(_got_html_good)
5208
5209         d.addCallback(self._count_leases, "one")
5210         d.addCallback(self._assert_leasecount, 1)
5211         d.addCallback(self._count_leases, "two")
5212         d.addCallback(self._assert_leasecount, 1)
5213         d.addCallback(self._count_leases, "mutable")
5214         d.addCallback(self._assert_leasecount, 1)
5215
5216         # this CHECK uses an alternate client, which adds a second lease
5217         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
5218         d.addCallback(_got_html_good)
5219
5220         d.addCallback(self._count_leases, "one")
5221         d.addCallback(self._assert_leasecount, 2)
5222         d.addCallback(self._count_leases, "two")
5223         d.addCallback(self._assert_leasecount, 1)
5224         d.addCallback(self._count_leases, "mutable")
5225         d.addCallback(self._assert_leasecount, 1)
5226
5227         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
5228         d.addCallback(_got_html_good)
5229
5230         d.addCallback(self._count_leases, "one")
5231         d.addCallback(self._assert_leasecount, 2)
5232         d.addCallback(self._count_leases, "two")
5233         d.addCallback(self._assert_leasecount, 1)
5234         d.addCallback(self._count_leases, "mutable")
5235         d.addCallback(self._assert_leasecount, 1)
5236
5237         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
5238                       clientnum=1)
5239         d.addCallback(_got_html_good)
5240
5241         d.addCallback(self._count_leases, "one")
5242         d.addCallback(self._assert_leasecount, 2)
5243         d.addCallback(self._count_leases, "two")
5244         d.addCallback(self._assert_leasecount, 1)
5245         d.addCallback(self._count_leases, "mutable")
5246         d.addCallback(self._assert_leasecount, 2)
5247
5248         d.addErrback(self.explain_web_error)
5249         return d
5250
5251     def test_deep_add_lease(self):
5252         self.basedir = "web/Grid/deep_add_lease"
5253         self.set_up_grid(num_clients=2)
5254         c0 = self.g.clients[0]
5255         self.uris = {}
5256         self.fileurls = {}
5257         DATA = "data" * 100
5258         d = c0.create_dirnode()
5259         def _stash_root_and_create_file(n):
5260             self.rootnode = n
5261             self.uris["root"] = n.get_uri()
5262             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5263             return n.add_file(u"one", upload.Data(DATA, convergence=""))
5264         d.addCallback(_stash_root_and_create_file)
5265         def _stash_uri(fn, which):
5266             self.uris[which] = fn.get_uri()
5267         d.addCallback(_stash_uri, "one")
5268         d.addCallback(lambda ign:
5269                       self.rootnode.add_file(u"small",
5270                                              upload.Data("literal",
5271                                                         convergence="")))
5272         d.addCallback(_stash_uri, "small")
5273
5274         d.addCallback(lambda ign:
5275             c0.create_mutable_file(publish.MutableData("mutable")))
5276         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
5277         d.addCallback(_stash_uri, "mutable")
5278
5279         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
5280         def _done(res):
5281             units = [simplejson.loads(line)
5282                      for line in res.splitlines()
5283                      if line]
5284             # root, one, small, mutable,   stats
5285             self.failUnlessReallyEqual(len(units), 4+1)
5286         d.addCallback(_done)
5287
5288         d.addCallback(self._count_leases, "root")
5289         d.addCallback(self._assert_leasecount, 1)
5290         d.addCallback(self._count_leases, "one")
5291         d.addCallback(self._assert_leasecount, 1)
5292         d.addCallback(self._count_leases, "mutable")
5293         d.addCallback(self._assert_leasecount, 1)
5294
5295         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
5296         d.addCallback(_done)
5297
5298         d.addCallback(self._count_leases, "root")
5299         d.addCallback(self._assert_leasecount, 1)
5300         d.addCallback(self._count_leases, "one")
5301         d.addCallback(self._assert_leasecount, 1)
5302         d.addCallback(self._count_leases, "mutable")
5303         d.addCallback(self._assert_leasecount, 1)
5304
5305         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
5306                       clientnum=1)
5307         d.addCallback(_done)
5308
5309         d.addCallback(self._count_leases, "root")
5310         d.addCallback(self._assert_leasecount, 2)
5311         d.addCallback(self._count_leases, "one")
5312         d.addCallback(self._assert_leasecount, 2)
5313         d.addCallback(self._count_leases, "mutable")
5314         d.addCallback(self._assert_leasecount, 2)
5315
5316         d.addErrback(self.explain_web_error)
5317         return d
5318
5319
5320     def test_exceptions(self):
5321         self.basedir = "web/Grid/exceptions"
5322         self.set_up_grid(num_clients=1, num_servers=2)
5323         c0 = self.g.clients[0]
5324         c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 2
5325         self.fileurls = {}
5326         DATA = "data" * 100
5327         d = c0.create_dirnode()
5328         def _stash_root(n):
5329             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5330             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
5331             return n
5332         d.addCallback(_stash_root)
5333         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
5334         def _stash_bad(ur):
5335             self.fileurls["1share"] = "uri/" + urllib.quote(ur.get_uri())
5336             self.delete_shares_numbered(ur.get_uri(), range(1,10))
5337
5338             u = uri.from_string(ur.get_uri())
5339             u.key = testutil.flip_bit(u.key, 0)
5340             baduri = u.to_string()
5341             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
5342         d.addCallback(_stash_bad)
5343         d.addCallback(lambda ign: c0.create_dirnode())
5344         def _mangle_dirnode_1share(n):
5345             u = n.get_uri()
5346             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
5347             self.fileurls["dir-1share-json"] = url + "?t=json"
5348             self.delete_shares_numbered(u, range(1,10))
5349         d.addCallback(_mangle_dirnode_1share)
5350         d.addCallback(lambda ign: c0.create_dirnode())
5351         def _mangle_dirnode_0share(n):
5352             u = n.get_uri()
5353             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
5354             self.fileurls["dir-0share-json"] = url + "?t=json"
5355             self.delete_shares_numbered(u, range(0,10))
5356         d.addCallback(_mangle_dirnode_0share)
5357
5358         # NotEnoughSharesError should be reported sensibly, with a
5359         # text/plain explanation of the problem, and perhaps some
5360         # information on which shares *could* be found.
5361
5362         d.addCallback(lambda ignored:
5363                       self.shouldHTTPError("GET unrecoverable",
5364                                            410, "Gone", "NoSharesError",
5365                                            self.GET, self.fileurls["0shares"]))
5366         def _check_zero_shares(body):
5367             self.failIfIn("<html>", body)
5368             body = " ".join(body.strip().split())
5369             exp = ("NoSharesError: no shares could be found. "
5370                    "Zero shares usually indicates a corrupt URI, or that "
5371                    "no servers were connected, but it might also indicate "
5372                    "severe corruption. You should perform a filecheck on "
5373                    "this object to learn more. The full error message is: "
5374                    "no shares (need 3). Last failure: None")
5375             self.failUnlessReallyEqual(exp, body)
5376         d.addCallback(_check_zero_shares)
5377
5378
5379         d.addCallback(lambda ignored:
5380                       self.shouldHTTPError("GET 1share",
5381                                            410, "Gone", "NotEnoughSharesError",
5382                                            self.GET, self.fileurls["1share"]))
5383         def _check_one_share(body):
5384             self.failIfIn("<html>", body)
5385             body = " ".join(body.strip().split())
5386             msgbase = ("NotEnoughSharesError: This indicates that some "
5387                        "servers were unavailable, or that shares have been "
5388                        "lost to server departure, hard drive failure, or disk "
5389                        "corruption. You should perform a filecheck on "
5390                        "this object to learn more. The full error message is:"
5391                        )
5392             msg1 = msgbase + (" ran out of shares:"
5393                               " complete=sh0"
5394                               " pending="
5395                               " overdue= unused= need 3. Last failure: None")
5396             msg2 = msgbase + (" ran out of shares:"
5397                               " complete="
5398                               " pending=Share(sh0-on-xgru5)"
5399                               " overdue= unused= need 3. Last failure: None")
5400             self.failUnless(body == msg1 or body == msg2, body)
5401         d.addCallback(_check_one_share)
5402
5403         d.addCallback(lambda ignored:
5404                       self.shouldHTTPError("GET imaginary",
5405                                            404, "Not Found", None,
5406                                            self.GET, self.fileurls["imaginary"]))
5407         def _missing_child(body):
5408             self.failUnlessIn("No such child: imaginary", body)
5409         d.addCallback(_missing_child)
5410
5411         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
5412         def _check_0shares_dir_html(body):
5413             self.failUnlessIn("<html>", body)
5414             # we should see the regular page, but without the child table or
5415             # the dirops forms
5416             body = " ".join(body.strip().split())
5417             self.failUnlessIn('href="?t=info">More info on this directory',
5418                               body)
5419             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5420                    "could not be retrieved, because there were insufficient "
5421                    "good shares. This might indicate that no servers were "
5422                    "connected, insufficient servers were connected, the URI "
5423                    "was corrupt, or that shares have been lost due to server "
5424                    "departure, hard drive failure, or disk corruption. You "
5425                    "should perform a filecheck on this object to learn more.")
5426             self.failUnlessIn(exp, body)
5427             self.failUnlessIn("No upload forms: directory is unreadable", body)
5428         d.addCallback(_check_0shares_dir_html)
5429
5430         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
5431         def _check_1shares_dir_html(body):
5432             # at some point, we'll split UnrecoverableFileError into 0-shares
5433             # and some-shares like we did for immutable files (since there
5434             # are different sorts of advice to offer in each case). For now,
5435             # they present the same way.
5436             self.failUnlessIn("<html>", body)
5437             body = " ".join(body.strip().split())
5438             self.failUnlessIn('href="?t=info">More info on this directory',
5439                               body)
5440             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5441                    "could not be retrieved, because there were insufficient "
5442                    "good shares. This might indicate that no servers were "
5443                    "connected, insufficient servers were connected, the URI "
5444                    "was corrupt, or that shares have been lost due to server "
5445                    "departure, hard drive failure, or disk corruption. You "
5446                    "should perform a filecheck on this object to learn more.")
5447             self.failUnlessIn(exp, body)
5448             self.failUnlessIn("No upload forms: directory is unreadable", body)
5449         d.addCallback(_check_1shares_dir_html)
5450
5451         d.addCallback(lambda ignored:
5452                       self.shouldHTTPError("GET dir-0share-json",
5453                                            410, "Gone", "UnrecoverableFileError",
5454                                            self.GET,
5455                                            self.fileurls["dir-0share-json"]))
5456         def _check_unrecoverable_file(body):
5457             self.failIfIn("<html>", body)
5458             body = " ".join(body.strip().split())
5459             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5460                    "could not be retrieved, because there were insufficient "
5461                    "good shares. This might indicate that no servers were "
5462                    "connected, insufficient servers were connected, the URI "
5463                    "was corrupt, or that shares have been lost due to server "
5464                    "departure, hard drive failure, or disk corruption. You "
5465                    "should perform a filecheck on this object to learn more.")
5466             self.failUnlessReallyEqual(exp, body)
5467         d.addCallback(_check_unrecoverable_file)
5468
5469         d.addCallback(lambda ignored:
5470                       self.shouldHTTPError("GET dir-1share-json",
5471                                            410, "Gone", "UnrecoverableFileError",
5472                                            self.GET,
5473                                            self.fileurls["dir-1share-json"]))
5474         d.addCallback(_check_unrecoverable_file)
5475
5476         d.addCallback(lambda ignored:
5477                       self.shouldHTTPError("GET imaginary",
5478                                            404, "Not Found", None,
5479                                            self.GET, self.fileurls["imaginary"]))
5480
5481         # attach a webapi child that throws a random error, to test how it
5482         # gets rendered.
5483         w = c0.getServiceNamed("webish")
5484         w.root.putChild("ERRORBOOM", ErrorBoom())
5485
5486         # "Accept: */*" :        should get a text/html stack trace
5487         # "Accept: text/plain" : should get a text/plain stack trace
5488         # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
5489         # no Accept header:      should get a text/html stack trace
5490
5491         d.addCallback(lambda ignored:
5492                       self.shouldHTTPError("GET errorboom_html",
5493                                            500, "Internal Server Error", None,
5494                                            self.GET, "ERRORBOOM",
5495                                            headers={"accept": "*/*"}))
5496         def _internal_error_html1(body):
5497             self.failUnlessIn("<html>", "expected HTML, not '%s'" % body)
5498         d.addCallback(_internal_error_html1)
5499
5500         d.addCallback(lambda ignored:
5501                       self.shouldHTTPError("GET errorboom_text",
5502                                            500, "Internal Server Error", None,
5503                                            self.GET, "ERRORBOOM",
5504                                            headers={"accept": "text/plain"}))
5505         def _internal_error_text2(body):
5506             self.failIfIn("<html>", body)
5507             self.failUnless(body.startswith("Traceback "), body)
5508         d.addCallback(_internal_error_text2)
5509
5510         CLI_accepts = "text/plain, application/octet-stream"
5511         d.addCallback(lambda ignored:
5512                       self.shouldHTTPError("GET errorboom_text",
5513                                            500, "Internal Server Error", None,
5514                                            self.GET, "ERRORBOOM",
5515                                            headers={"accept": CLI_accepts}))
5516         def _internal_error_text3(body):
5517             self.failIfIn("<html>", body)
5518             self.failUnless(body.startswith("Traceback "), body)
5519         d.addCallback(_internal_error_text3)
5520
5521         d.addCallback(lambda ignored:
5522                       self.shouldHTTPError("GET errorboom_text",
5523                                            500, "Internal Server Error", None,
5524                                            self.GET, "ERRORBOOM"))
5525         def _internal_error_html4(body):
5526             self.failUnlessIn("<html>", body)
5527         d.addCallback(_internal_error_html4)
5528
5529         def _flush_errors(res):
5530             # Trial: please ignore the CompletelyUnhandledError in the logs
5531             self.flushLoggedErrors(CompletelyUnhandledError)
5532             return res
5533         d.addBoth(_flush_errors)
5534
5535         return d
5536
5537     def test_blacklist(self):
5538         # download from a blacklisted URI, get an error
5539         self.basedir = "web/Grid/blacklist"
5540         self.set_up_grid()
5541         c0 = self.g.clients[0]
5542         c0_basedir = c0.basedir
5543         fn = os.path.join(c0_basedir, "access.blacklist")
5544         self.uris = {}
5545         DATA = "off-limits " * 50
5546
5547         d = c0.upload(upload.Data(DATA, convergence=""))
5548         def _stash_uri_and_create_dir(ur):
5549             self.uri = ur.get_uri()
5550             self.url = "uri/"+self.uri
5551             u = uri.from_string_filenode(self.uri)
5552             self.si = u.get_storage_index()
5553             childnode = c0.create_node_from_uri(self.uri, None)
5554             return c0.create_dirnode({u"blacklisted.txt": (childnode,{}) })
5555         d.addCallback(_stash_uri_and_create_dir)
5556         def _stash_dir(node):
5557             self.dir_node = node
5558             self.dir_uri = node.get_uri()
5559             self.dir_url = "uri/"+self.dir_uri
5560         d.addCallback(_stash_dir)
5561         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5562         def _check_dir_html(body):
5563             self.failUnlessIn("<html>", body)
5564             self.failUnlessIn("blacklisted.txt</a>", body)
5565         d.addCallback(_check_dir_html)
5566         d.addCallback(lambda ign: self.GET(self.url))
5567         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5568
5569         def _blacklist(ign):
5570             f = open(fn, "w")
5571             f.write(" # this is a comment\n")
5572             f.write(" \n")
5573             f.write("\n") # also exercise blank lines
5574             f.write("%s %s\n" % (base32.b2a(self.si), "off-limits to you"))
5575             f.close()
5576             # clients should be checking the blacklist each time, so we don't
5577             # need to restart the client
5578         d.addCallback(_blacklist)
5579         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_uri",
5580                                                        403, "Forbidden",
5581                                                        "Access Prohibited: off-limits",
5582                                                        self.GET, self.url))
5583
5584         # We should still be able to list the parent directory, in HTML...
5585         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5586         def _check_dir_html2(body):
5587             self.failUnlessIn("<html>", body)
5588             self.failUnlessIn("blacklisted.txt</strike>", body)
5589         d.addCallback(_check_dir_html2)
5590
5591         # ... and in JSON (used by CLI).
5592         d.addCallback(lambda ign: self.GET(self.dir_url+"?t=json", followRedirect=True))
5593         def _check_dir_json(res):
5594             data = simplejson.loads(res)
5595             self.failUnless(isinstance(data, list), data)
5596             self.failUnlessEqual(data[0], "dirnode")
5597             self.failUnless(isinstance(data[1], dict), data)
5598             self.failUnlessIn("children", data[1])
5599             self.failUnlessIn("blacklisted.txt", data[1]["children"])
5600             childdata = data[1]["children"]["blacklisted.txt"]
5601             self.failUnless(isinstance(childdata, list), data)
5602             self.failUnlessEqual(childdata[0], "filenode")
5603             self.failUnless(isinstance(childdata[1], dict), data)
5604         d.addCallback(_check_dir_json)
5605
5606         def _unblacklist(ign):
5607             open(fn, "w").close()
5608             # the Blacklist object watches mtime to tell when the file has
5609             # changed, but on windows this test will run faster than the
5610             # filesystem's mtime resolution. So we edit Blacklist.last_mtime
5611             # to force a reload.
5612             self.g.clients[0].blacklist.last_mtime -= 2.0
5613         d.addCallback(_unblacklist)
5614
5615         # now a read should work
5616         d.addCallback(lambda ign: self.GET(self.url))
5617         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5618
5619         # read again to exercise the blacklist-is-unchanged logic
5620         d.addCallback(lambda ign: self.GET(self.url))
5621         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5622
5623         # now add a blacklisted directory, and make sure files under it are
5624         # refused too
5625         def _add_dir(ign):
5626             childnode = c0.create_node_from_uri(self.uri, None)
5627             return c0.create_dirnode({u"child": (childnode,{}) })
5628         d.addCallback(_add_dir)
5629         def _get_dircap(dn):
5630             self.dir_si_b32 = base32.b2a(dn.get_storage_index())
5631             self.dir_url_base = "uri/"+dn.get_write_uri()
5632             self.dir_url_json1 = "uri/"+dn.get_write_uri()+"?t=json"
5633             self.dir_url_json2 = "uri/"+dn.get_write_uri()+"/?t=json"
5634             self.dir_url_json_ro = "uri/"+dn.get_readonly_uri()+"/?t=json"
5635             self.child_url = "uri/"+dn.get_readonly_uri()+"/child"
5636         d.addCallback(_get_dircap)
5637         d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True))
5638         d.addCallback(lambda body: self.failUnlessIn("<html>", body))
5639         d.addCallback(lambda ign: self.GET(self.dir_url_json1))
5640         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5641         d.addCallback(lambda ign: self.GET(self.dir_url_json2))
5642         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5643         d.addCallback(lambda ign: self.GET(self.dir_url_json_ro))
5644         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5645         d.addCallback(lambda ign: self.GET(self.child_url))
5646         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5647
5648         def _block_dir(ign):
5649             f = open(fn, "w")
5650             f.write("%s %s\n" % (self.dir_si_b32, "dir-off-limits to you"))
5651             f.close()
5652             self.g.clients[0].blacklist.last_mtime -= 2.0
5653         d.addCallback(_block_dir)
5654         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir base",
5655                                                        403, "Forbidden",
5656                                                        "Access Prohibited: dir-off-limits",
5657                                                        self.GET, self.dir_url_base))
5658         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json1",
5659                                                        403, "Forbidden",
5660                                                        "Access Prohibited: dir-off-limits",
5661                                                        self.GET, self.dir_url_json1))
5662         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json2",
5663                                                        403, "Forbidden",
5664                                                        "Access Prohibited: dir-off-limits",
5665                                                        self.GET, self.dir_url_json2))
5666         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json_ro",
5667                                                        403, "Forbidden",
5668                                                        "Access Prohibited: dir-off-limits",
5669                                                        self.GET, self.dir_url_json_ro))
5670         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir child",
5671                                                        403, "Forbidden",
5672                                                        "Access Prohibited: dir-off-limits",
5673                                                        self.GET, self.child_url))
5674         return d
5675
5676
5677 class CompletelyUnhandledError(Exception):
5678     pass
5679 class ErrorBoom(rend.Page):
5680     def beforeRender(self, ctx):
5681         raise CompletelyUnhandledError("whoops")