src/allmydata/test/test_web.py

   1 import os.path, re, urllib, time
   2 import simplejson
   3 from StringIO import StringIO
   4
   5 from twisted.application import service
   6 from twisted.trial import unittest
   7 from twisted.internet import defer, reactor
   8 from twisted.internet.task import Clock
   9 from twisted.web import client, error, http, html
  10 from twisted.python import failure, log
  11
  12 from foolscap.api import fireEventually, flushEventualQueue
  13
  14 from nevow.util import escapeToXML
  15 from nevow import rend
  16
  17 from allmydata import interfaces, uri, webish, dirnode
  18 from allmydata.storage.shares import get_share_file
  19 from allmydata.storage_client import StorageFarmBroker, StubServer
  20 from allmydata.immutable import upload
  21 from allmydata.immutable.downloader.status import DownloadStatus
  22 from allmydata.dirnode import DirectoryNode
  23 from allmydata.nodemaker import NodeMaker
  24 from allmydata.unknown import UnknownNode
  25 from allmydata.web import status, common
  26 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  27 from allmydata.util import fileutil, base32, hashutil
  28 from allmydata.util.consumer import download_to_data
  29 from allmydata.util.netstring import split_netstring
  30 from allmydata.util.encodingutil import to_str
  31 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  32      create_chk_filenode, WebErrorMixin, ShouldFailMixin, \
  33      make_mutable_file_uri, create_mutable_filenode
  34 from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
  35 from allmydata.mutable import servermap, publish, retrieve
  36 import allmydata.test.common_util as testutil
  37 from allmydata.test.no_network import GridTestMixin
  38 from allmydata.test.common_web import HTTPClientGETFactory, \
  39      HTTPClientHEADFactory
  40 from allmydata.client import Client, SecretHolder
  41 from allmydata.introducer import IntroducerNode
  42
  43 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  44 # create a webserver that works against them
  45
  46 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  47
  48 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
  49 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
  50 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
  51
  52 FAVICON_MARKUP = '<link href="/icon.png" rel="shortcut icon" />'
  53
  54
  55 class FakeStatsProvider:
  56     def get_stats(self):
  57         stats = {'stats': {}, 'counters': {}}
  58         return stats
  59
  60 class FakeNodeMaker(NodeMaker):
  61     encoding_params = {
  62         'k': 3,
  63         'n': 10,
  64         'happy': 7,
  65         'max_segment_size':128*1024 # 1024=KiB
  66     }
  67     def _create_lit(self, cap):
  68         return FakeCHKFileNode(cap, self.all_contents)
  69     def _create_immutable(self, cap):
  70         return FakeCHKFileNode(cap, self.all_contents)
  71     def _create_mutable(self, cap):
  72         return FakeMutableFileNode(None, None,
  73                                    self.encoding_params, None,
  74                                    self.all_contents).init_from_cap(cap)
  75     def create_mutable_file(self, contents="", keysize=None,
  76                             version=SDMF_VERSION):
  77         n = FakeMutableFileNode(None, None, self.encoding_params, None,
  78                                 self.all_contents)
  79         return n.create(contents, version=version)
  80
  81 class FakeUploader(service.Service):
  82     name = "uploader"
  83     helper_furl = None
  84     helper_connected = False
  85
  86     def upload(self, uploadable):
  87         d = uploadable.get_size()
  88         d.addCallback(lambda size: uploadable.read(size))
  89         def _got_data(datav):
  90             data = "".join(datav)
  91             n = create_chk_filenode(data, self.all_contents)
  92             ur = upload.UploadResults(file_size=len(data),
  93                                       ciphertext_fetched=0,
  94                                       preexisting_shares=0,
  95                                       pushed_shares=10,
  96                                       sharemap={},
  97                                       servermap={},
  98                                       timings={},
  99                                       uri_extension_data={},
 100                                       uri_extension_hash="fake",
 101                                       verifycapstr="fakevcap")
 102             ur.set_uri(n.get_uri())
 103             return ur
 104         d.addCallback(_got_data)
 105         return d
 106
 107     def get_helper_info(self):
 108         return (self.helper_furl, self.helper_connected)
 109
 110
 111 def build_one_ds():
 112     ds = DownloadStatus("storage_index", 1234)
 113     now = time.time()
 114
 115     serverA = StubServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
 116     serverB = StubServer(hashutil.tagged_hash("foo", "serverid_b")[:20])
 117     storage_index = hashutil.storage_index_hash("SI")
 118     e0 = ds.add_segment_request(0, now)
 119     e0.activate(now+0.5)
 120     e0.deliver(now+1, 0, 100, 0.5) # when, start,len, decodetime
 121     e1 = ds.add_segment_request(1, now+2)
 122     e1.error(now+3)
 123     # two outstanding requests
 124     e2 = ds.add_segment_request(2, now+4)
 125     e3 = ds.add_segment_request(3, now+5)
 126     del e2,e3 # hush pyflakes
 127
 128     # simulate a segment which gets delivered faster than a system clock tick (ticket #1166)
 129     e = ds.add_segment_request(4, now)
 130     e.activate(now)
 131     e.deliver(now, 0, 140, 0.5)
 132
 133     e = ds.add_dyhb_request(serverA, now)
 134     e.finished([1,2], now+1)
 135     e = ds.add_dyhb_request(serverB, now+2) # left unfinished
 136
 137     e = ds.add_read_event(0, 120, now)
 138     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 139     e.finished(now+1)
 140     e = ds.add_read_event(120, 30, now+2) # left unfinished
 141
 142     e = ds.add_block_request(serverA, 1, 100, 20, now)
 143     e.finished(20, now+1)
 144     e = ds.add_block_request(serverB, 1, 120, 30, now+1) # left unfinished
 145
 146     # make sure that add_read_event() can come first too
 147     ds1 = DownloadStatus(storage_index, 1234)
 148     e = ds1.add_read_event(0, 120, now)
 149     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 150     e.finished(now+1)
 151
 152     return ds
 153
 154 class FakeHistory:
 155     _all_upload_status = [upload.UploadStatus()]
 156     _all_download_status = [build_one_ds()]
 157     _all_mapupdate_statuses = [servermap.UpdateStatus()]
 158     _all_publish_statuses = [publish.PublishStatus()]
 159     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
 160
 161     def list_all_upload_statuses(self):
 162         return self._all_upload_status
 163     def list_all_download_statuses(self):
 164         return self._all_download_status
 165     def list_all_mapupdate_statuses(self):
 166         return self._all_mapupdate_statuses
 167     def list_all_publish_statuses(self):
 168         return self._all_publish_statuses
 169     def list_all_retrieve_statuses(self):
 170         return self._all_retrieve_statuses
 171     def list_all_helper_statuses(self):
 172         return []
 173
 174 class FakeDisplayableServer(StubServer):
 175     def __init__(self, serverid, nickname):
 176         StubServer.__init__(self, serverid)
 177         self.announcement = {"my-version": "allmydata-tahoe-fake",
 178                              "service-name": "storage",
 179                              "nickname": nickname}
 180     def is_connected(self):
 181         return True
 182     def get_permutation_seed(self):
 183         return ""
 184     def get_remote_host(self):
 185         return ""
 186     def get_last_loss_time(self):
 187         return None
 188     def get_announcement_time(self):
 189         return None
 190     def get_announcement(self):
 191         return self.announcement
 192     def get_nickname(self):
 193         return self.announcement["nickname"]
 194
 195 class FakeBucketCounter(object):
 196     def get_state(self):
 197         return {"last-complete-bucket-count": 0}
 198     def get_progress(self):
 199         return {"estimated-time-per-cycle": 0,
 200                 "cycle-in-progress": False,
 201                 "remaining-wait-time": 0}
 202
 203 class FakeLeaseChecker(object):
 204     def __init__(self):
 205         self.expiration_enabled = False
 206         self.mode = "age"
 207         self.override_lease_duration = None
 208         self.sharetypes_to_expire = {}
 209     def get_state(self):
 210         return {"history": None}
 211     def get_progress(self):
 212         return {"estimated-time-per-cycle": 0,
 213                 "cycle-in-progress": False,
 214                 "remaining-wait-time": 0}
 215
 216 class FakeStorageServer(service.MultiService):
 217     name = 'storage'
 218     def __init__(self, nodeid, nickname):
 219         service.MultiService.__init__(self)
 220         self.my_nodeid = nodeid
 221         self.nickname = nickname
 222         self.bucket_counter = FakeBucketCounter()
 223         self.lease_checker = FakeLeaseChecker()
 224     def get_stats(self):
 225         return {"storage_server.accepting_immutable_shares": False}
 226
 227 class FakeClient(Client):
 228     def __init__(self):
 229         # don't upcall to Client.__init__, since we only want to initialize a
 230         # minimal subset
 231         service.MultiService.__init__(self)
 232         self.all_contents = {}
 233         self.nodeid = "fake_nodeid"
 234         self.nickname = u"fake_nickname \u263A"
 235         self.introducer_furl = "None"
 236         self.stats_provider = FakeStatsProvider()
 237         self._secret_holder = SecretHolder("lease secret", "convergence secret")
 238         self.helper = None
 239         self.convergence = "some random string"
 240         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 241         # fake knowledge of another server
 242         self.storage_broker.test_add_server("other_nodeid",
 243                                             FakeDisplayableServer("other_nodeid", u"other_nickname \u263B"))
 244         self.introducer_client = None
 245         self.history = FakeHistory()
 246         self.uploader = FakeUploader()
 247         self.uploader.all_contents = self.all_contents
 248         self.uploader.setServiceParent(self)
 249         self.blacklist = None
 250         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 251                                        self.uploader, None,
 252                                        None, None, None)
 253         self.nodemaker.all_contents = self.all_contents
 254         self.mutable_file_default = SDMF_VERSION
 255         self.addService(FakeStorageServer(self.nodeid, self.nickname))
 256
 257     def startService(self):
 258         return service.MultiService.startService(self)
 259     def stopService(self):
 260         return service.MultiService.stopService(self)
 261
 262     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 263
 264 class WebMixin(object):
 265     def setUp(self):
 266         self.s = FakeClient()
 267         self.s.startService()
 268         self.staticdir = self.mktemp()
 269         self.clock = Clock()
 270         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
 271                                       clock=self.clock)
 272         self.ws.setServiceParent(self.s)
 273         self.webish_port = self.ws.getPortnum()
 274         self.webish_url = self.ws.getURL()
 275         assert self.webish_url.endswith("/")
 276         self.webish_url = self.webish_url[:-1] # these tests add their own /
 277
 278         l = [ self.s.create_dirnode() for x in range(6) ]
 279         d = defer.DeferredList(l)
 280         def _then(res):
 281             self.public_root = res[0][1]
 282             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 283             self.public_url = "/uri/" + self.public_root.get_uri()
 284             self.private_root = res[1][1]
 285
 286             foo = res[2][1]
 287             self._foo_node = foo
 288             self._foo_uri = foo.get_uri()
 289             self._foo_readonly_uri = foo.get_readonly_uri()
 290             self._foo_verifycap = foo.get_verify_cap().to_string()
 291             # NOTE: we ignore the deferred on all set_uri() calls, because we
 292             # know the fake nodes do these synchronously
 293             self.public_root.set_uri(u"foo", foo.get_uri(),
 294                                      foo.get_readonly_uri())
 295
 296             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 297             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 298             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 299
 300             # sdmf
 301             # XXX: Do we ever use this?
 302             self.BAZ_CONTENTS, n, self._baz_txt_uri, self._baz_txt_readonly_uri = self.makefile_mutable(0)
 303
 304             foo.set_uri(u"baz.txt", self._baz_txt_uri, self._baz_txt_readonly_uri)
 305
 306             # mdmf
 307             self.QUUX_CONTENTS, n, self._quux_txt_uri, self._quux_txt_readonly_uri = self.makefile_mutable(0, mdmf=True)
 308             assert self._quux_txt_uri.startswith("URI:MDMF")
 309             foo.set_uri(u"quux.txt", self._quux_txt_uri, self._quux_txt_readonly_uri)
 310
 311             foo.set_uri(u"empty", res[3][1].get_uri(),
 312                         res[3][1].get_readonly_uri())
 313             sub_uri = res[4][1].get_uri()
 314             self._sub_uri = sub_uri
 315             foo.set_uri(u"sub", sub_uri, sub_uri)
 316             sub = self.s.create_node_from_uri(sub_uri)
 317             self._sub_node = sub
 318
 319             _ign, n, blocking_uri = self.makefile(1)
 320             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 321
 322             # filenode to test for html encoding issues
 323             self._htmlname_unicode = u"<&weirdly'named\"file>>>_<iframe />.txt"
 324             self._htmlname_raw = self._htmlname_unicode.encode('utf-8')
 325             self._htmlname_urlencoded = urllib.quote(self._htmlname_raw, '')
 326             self._htmlname_escaped = escapeToXML(self._htmlname_raw)
 327             self._htmlname_escaped_attr = html.escape(self._htmlname_raw)
 328             self._htmlname_escaped_double = escapeToXML(html.escape(self._htmlname_raw))
 329             self.HTMLNAME_CONTENTS, n, self._htmlname_txt_uri = self.makefile(0)
 330             foo.set_uri(self._htmlname_unicode, self._htmlname_txt_uri, self._htmlname_txt_uri)
 331
 332             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 333             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 334             # still think of it as an umlaut
 335             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 336
 337             self.SUBBAZ_CONTENTS, n, baz_file = self.makefile(2)
 338             self._baz_file_uri = baz_file
 339             sub.set_uri(u"baz.txt", baz_file, baz_file)
 340
 341             _ign, n, self._bad_file_uri = self.makefile(3)
 342             # this uri should not be downloadable
 343             del self.s.all_contents[self._bad_file_uri]
 344
 345             rodir = res[5][1]
 346             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 347                                      rodir.get_readonly_uri())
 348             rodir.set_uri(u"nor", baz_file, baz_file)
 349
 350             # public/
 351             # public/foo/
 352             # public/foo/bar.txt
 353             # public/foo/baz.txt
 354             # public/foo/quux.txt
 355             # public/foo/blockingfile
 356             # public/foo/<&weirdly'named\"file>>>_<iframe />.txt
 357             # public/foo/empty/
 358             # public/foo/sub/
 359             # public/foo/sub/baz.txt
 360             # public/reedownlee/
 361             # public/reedownlee/nor
 362             self.NEWFILE_CONTENTS = "newfile contents\n"
 363
 364             return foo.get_metadata_for(u"bar.txt")
 365         d.addCallback(_then)
 366         def _got_metadata(metadata):
 367             self._bar_txt_metadata = metadata
 368         d.addCallback(_got_metadata)
 369         return d
 370
 371     def get_all_contents(self):
 372         return self.s.all_contents
 373
 374     def makefile(self, number):
 375         contents = "contents of file %s\n" % number
 376         n = create_chk_filenode(contents, self.get_all_contents())
 377         return contents, n, n.get_uri()
 378
 379     def makefile_mutable(self, number, mdmf=False):
 380         contents = "contents of mutable file %s\n" % number
 381         n = create_mutable_filenode(contents, mdmf, self.s.all_contents)
 382         return contents, n, n.get_uri(), n.get_readonly_uri()
 383
 384     def tearDown(self):
 385         return self.s.stopService()
 386
 387     def failUnlessIsBarDotTxt(self, res):
 388         self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
 389
 390     def failUnlessIsQuuxDotTxt(self, res):
 391         self.failUnlessReallyEqual(res, self.QUUX_CONTENTS, res)
 392
 393     def failUnlessIsBazDotTxt(self, res):
 394         self.failUnlessReallyEqual(res, self.BAZ_CONTENTS, res)
 395
 396     def failUnlessIsSubBazDotTxt(self, res):
 397         self.failUnlessReallyEqual(res, self.SUBBAZ_CONTENTS, res)
 398
 399     def failUnlessIsBarJSON(self, res):
 400         data = simplejson.loads(res)
 401         self.failUnless(isinstance(data, list))
 402         self.failUnlessEqual(data[0], "filenode")
 403         self.failUnless(isinstance(data[1], dict))
 404         self.failIf(data[1]["mutable"])
 405         self.failIfIn("rw_uri", data[1]) # immutable
 406         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
 407         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
 408         self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
 409
 410     def failUnlessIsQuuxJSON(self, res, readonly=False):
 411         data = simplejson.loads(res)
 412         self.failUnless(isinstance(data, list))
 413         self.failUnlessEqual(data[0], "filenode")
 414         self.failUnless(isinstance(data[1], dict))
 415         metadata = data[1]
 416         return self.failUnlessIsQuuxDotTxtMetadata(metadata, readonly)
 417
 418     def failUnlessIsQuuxDotTxtMetadata(self, metadata, readonly):
 419         self.failUnless(metadata['mutable'])
 420         if readonly:
 421             self.failIfIn("rw_uri", metadata)
 422         else:
 423             self.failUnlessIn("rw_uri", metadata)
 424             self.failUnlessEqual(metadata['rw_uri'], self._quux_txt_uri)
 425         self.failUnlessIn("ro_uri", metadata)
 426         self.failUnlessEqual(metadata['ro_uri'], self._quux_txt_readonly_uri)
 427         self.failUnlessReallyEqual(metadata['size'], len(self.QUUX_CONTENTS))
 428
 429     def failUnlessIsFooJSON(self, res):
 430         data = simplejson.loads(res)
 431         self.failUnless(isinstance(data, list))
 432         self.failUnlessEqual(data[0], "dirnode", res)
 433         self.failUnless(isinstance(data[1], dict))
 434         self.failUnless(data[1]["mutable"])
 435         self.failUnlessIn("rw_uri", data[1]) # mutable
 436         self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
 437         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
 438         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
 439
 440         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 441         self.failUnlessEqual(kidnames,
 442                              [self._htmlname_unicode, u"bar.txt", u"baz.txt",
 443                               u"blockingfile", u"empty", u"n\u00fc.txt", u"quux.txt", u"sub"])
 444         kids = dict( [(unicode(name),value)
 445                       for (name,value)
 446                       in data[1]["children"].iteritems()] )
 447         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 448         self.failUnlessIn("metadata", kids[u"sub"][1])
 449         self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
 450         tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
 451         self.failUnlessIn("linkcrtime", tahoe_md)
 452         self.failUnlessIn("linkmotime", tahoe_md)
 453         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 454         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 455         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri)
 456         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]),
 457                                    self._bar_txt_verifycap)
 458         self.failUnlessIn("metadata", kids[u"bar.txt"][1])
 459         self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
 460         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
 461                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
 462         self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
 463                                    self._bar_txt_uri)
 464         self.failUnlessIn("quux.txt", kids)
 465         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["rw_uri"]),
 466                                    self._quux_txt_uri)
 467         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["ro_uri"]),
 468                                    self._quux_txt_readonly_uri)
 469
 470     def GET(self, urlpath, followRedirect=False, return_response=False,
 471             **kwargs):
 472         # if return_response=True, this fires with (data, statuscode,
 473         # respheaders) instead of just data.
 474         assert not isinstance(urlpath, unicode)
 475         url = self.webish_url + urlpath
 476         factory = HTTPClientGETFactory(url, method="GET",
 477                                        followRedirect=followRedirect, **kwargs)
 478         reactor.connectTCP("localhost", self.webish_port, factory)
 479         d = factory.deferred
 480         def _got_data(data):
 481             return (data, factory.status, factory.response_headers)
 482         if return_response:
 483             d.addCallback(_got_data)
 484         return factory.deferred
 485
 486     def HEAD(self, urlpath, return_response=False, **kwargs):
 487         # this requires some surgery, because twisted.web.client doesn't want
 488         # to give us back the response headers.
 489         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 490         reactor.connectTCP("localhost", self.webish_port, factory)
 491         d = factory.deferred
 492         def _got_data(data):
 493             return (data, factory.status, factory.response_headers)
 494         if return_response:
 495             d.addCallback(_got_data)
 496         return factory.deferred
 497
 498     def PUT(self, urlpath, data, **kwargs):
 499         url = self.webish_url + urlpath
 500         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 501
 502     def DELETE(self, urlpath):
 503         url = self.webish_url + urlpath
 504         return client.getPage(url, method="DELETE")
 505
 506     def POST(self, urlpath, followRedirect=False, **fields):
 507         sepbase = "boogabooga"
 508         sep = "--" + sepbase
 509         form = []
 510         form.append(sep)
 511         form.append('Content-Disposition: form-data; name="_charset"')
 512         form.append('')
 513         form.append('UTF-8')
 514         form.append(sep)
 515         for name, value in fields.iteritems():
 516             if isinstance(value, tuple):
 517                 filename, value = value
 518                 form.append('Content-Disposition: form-data; name="%s"; '
 519                             'filename="%s"' % (name, filename.encode("utf-8")))
 520             else:
 521                 form.append('Content-Disposition: form-data; name="%s"' % name)
 522             form.append('')
 523             if isinstance(value, unicode):
 524                 value = value.encode("utf-8")
 525             else:
 526                 value = str(value)
 527             assert isinstance(value, str)
 528             form.append(value)
 529             form.append(sep)
 530         form[-1] += "--"
 531         body = ""
 532         headers = {}
 533         if fields:
 534             body = "\r\n".join(form) + "\r\n"
 535             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 536         return self.POST2(urlpath, body, headers, followRedirect)
 537
 538     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 539         url = self.webish_url + urlpath
 540         return client.getPage(url, method="POST", postdata=body,
 541                               headers=headers, followRedirect=followRedirect)
 542
 543     def shouldFail(self, res, expected_failure, which,
 544                    substring=None, response_substring=None):
 545         if isinstance(res, failure.Failure):
 546             res.trap(expected_failure)
 547             if substring:
 548                 self.failUnlessIn(substring, str(res), which)
 549             if response_substring:
 550                 self.failUnlessIn(response_substring, res.value.response, which)
 551         else:
 552             self.fail("%s was supposed to raise %s, not get '%s'" %
 553                       (which, expected_failure, res))
 554
 555     def shouldFail2(self, expected_failure, which, substring,
 556                     response_substring,
 557                     callable, *args, **kwargs):
 558         assert substring is None or isinstance(substring, str)
 559         assert response_substring is None or isinstance(response_substring, str)
 560         d = defer.maybeDeferred(callable, *args, **kwargs)
 561         def done(res):
 562             if isinstance(res, failure.Failure):
 563                 res.trap(expected_failure)
 564                 if substring:
 565                     self.failUnlessIn(substring, str(res),
 566                                       "'%s' not in '%s' for test '%s'" % \
 567                                       (substring, str(res), which))
 568                 if response_substring:
 569                     self.failUnlessIn(response_substring, res.value.response,
 570                                       "'%s' not in '%s' for test '%s'" % \
 571                                       (response_substring, res.value.response,
 572                                        which))
 573             else:
 574                 self.fail("%s was supposed to raise %s, not get '%s'" %
 575                           (which, expected_failure, res))
 576         d.addBoth(done)
 577         return d
 578
 579     def should404(self, res, which):
 580         if isinstance(res, failure.Failure):
 581             res.trap(error.Error)
 582             self.failUnlessReallyEqual(res.value.status, "404")
 583         else:
 584             self.fail("%s was supposed to Error(404), not get '%s'" %
 585                       (which, res))
 586
 587     def should302(self, res, which):
 588         if isinstance(res, failure.Failure):
 589             res.trap(error.Error)
 590             self.failUnlessReallyEqual(res.value.status, "302")
 591         else:
 592             self.fail("%s was supposed to Error(302), not get '%s'" %
 593                       (which, res))
 594
 595
 596 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
 597     def test_create(self):
 598         pass
 599
 600     def test_welcome(self):
 601         d = self.GET("/")
 602         def _check(res):
 603             self.failUnlessIn('Welcome to Tahoe-LAFS', res)
 604             self.failUnlessIn(FAVICON_MARKUP, res)
 605             self.failUnlessIn('href="https://tahoe-lafs.org/"', res)
 606             self.failUnlessIn('<a href="status/">Recent and Active Operations</a>', res)
 607             self.failUnlessIn('<a href="statistics">Operational Statistics</a>', res)
 608             res_u = res.decode('utf-8')
 609             self.failUnlessIn(u'<th>My nickname:</th> <td class="nickname mine">fake_nickname \u263A</td></tr>', res_u)
 610             self.failUnlessIn(u'<div class="nickname">other_nickname \u263B</div>', res_u)
 611
 612             self.s.basedir = 'web/test_welcome'
 613             fileutil.make_dirs("web/test_welcome")
 614             fileutil.make_dirs("web/test_welcome/private")
 615             return self.GET("/")
 616         d.addCallback(_check)
 617         return d
 618
 619     def test_helper_status(self):
 620         d = defer.succeed(None)
 621
 622         # set helper furl to None
 623         def _set_no_helper(ign):
 624             self.s.uploader.helper_furl = None
 625             return self.GET("/")
 626         d.addCallback(_set_no_helper)
 627         d.addCallback(lambda res:
 628                       self.failUnlessIn('Connected to helper?: <span>not configured</span>', res))
 629
 630         # enable helper, not connected
 631         def _set_helper_not_connected(ign):
 632             self.s.uploader.helper_furl = "pb://someHelper"
 633             self.s.uploader.helper_connected = False
 634             return self.GET("/")
 635         d.addCallback(_set_helper_not_connected)
 636         d.addCallback(lambda res:
 637                       self.failUnlessIn('Connected to helper?: <span>no</span>', res))
 638
 639         # enable helper, connected
 640         def _set_helper_connected(ign):
 641             self.s.uploader.helper_furl = "pb://someHelper"
 642             self.s.uploader.helper_connected = True
 643             return self.GET("/")
 644         d.addCallback(_set_helper_connected)
 645         d.addCallback(lambda res:
 646                       self.failUnlessIn('Connected to helper?: <span>yes</span>', res))
 647         return d
 648
 649     def test_storage(self):
 650         d = self.GET("/storage")
 651         def _check(res):
 652             self.failUnlessIn('Storage Server Status', res)
 653             self.failUnlessIn(FAVICON_MARKUP, res)
 654             res_u = res.decode('utf-8')
 655             self.failUnlessIn(u'<li>Server Nickname: <span class="nickname mine">fake_nickname \u263A</span></li>', res_u)
 656         d.addCallback(_check)
 657         return d
 658
 659     def test_status(self):
 660         h = self.s.get_history()
 661         dl_num = h.list_all_download_statuses()[0].get_counter()
 662         ul_num = h.list_all_upload_statuses()[0].get_counter()
 663         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 664         pub_num = h.list_all_publish_statuses()[0].get_counter()
 665         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 666         d = self.GET("/status", followRedirect=True)
 667         def _check(res):
 668             self.failUnlessIn('Recent and Active Operations', res)
 669             self.failUnlessIn('"down-%d"' % dl_num, res)
 670             self.failUnlessIn('"up-%d"' % ul_num, res)
 671             self.failUnlessIn('"mapupdate-%d"' % mu_num, res)
 672             self.failUnlessIn('"publish-%d"' % pub_num, res)
 673             self.failUnlessIn('"retrieve-%d"' % ret_num, res)
 674         d.addCallback(_check)
 675         d.addCallback(lambda res: self.GET("/status/?t=json"))
 676         def _check_json(res):
 677             data = simplejson.loads(res)
 678             self.failUnless(isinstance(data, dict))
 679             #active = data["active"]
 680             # TODO: test more. We need a way to fake an active operation
 681             # here.
 682         d.addCallback(_check_json)
 683
 684         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 685         def _check_dl(res):
 686             self.failUnlessIn("File Download Status", res)
 687         d.addCallback(_check_dl)
 688         d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
 689         def _check_dl_json(res):
 690             data = simplejson.loads(res)
 691             self.failUnless(isinstance(data, dict))
 692             self.failUnlessIn("read", data)
 693             self.failUnlessEqual(data["read"][0]["length"], 120)
 694             self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
 695             self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
 696             self.failUnlessEqual(data["segment"][2]["finish_time"], None)
 697             phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
 698             cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
 699             # serverids[] keys are strings, since that's what JSON does, but
 700             # we'd really like them to be ints
 701             self.failUnlessEqual(data["serverids"]["0"], "phwrsjte")
 702             self.failUnless(data["serverids"].has_key("1"),
 703                             str(data["serverids"]))
 704             self.failUnlessEqual(data["serverids"]["1"], "cmpuvkjm",
 705                                  str(data["serverids"]))
 706             self.failUnlessEqual(data["server_info"][phwr_id]["short"],
 707                                  "phwrsjte")
 708             self.failUnlessEqual(data["server_info"][cmpu_id]["short"],
 709                                  "cmpuvkjm")
 710             self.failUnlessIn("dyhb", data)
 711             self.failUnlessIn("misc", data)
 712         d.addCallback(_check_dl_json)
 713         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 714         def _check_ul(res):
 715             self.failUnlessIn("File Upload Status", res)
 716         d.addCallback(_check_ul)
 717         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 718         def _check_mapupdate(res):
 719             self.failUnlessIn("Mutable File Servermap Update Status", res)
 720         d.addCallback(_check_mapupdate)
 721         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 722         def _check_publish(res):
 723             self.failUnlessIn("Mutable File Publish Status", res)
 724         d.addCallback(_check_publish)
 725         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 726         def _check_retrieve(res):
 727             self.failUnlessIn("Mutable File Retrieve Status", res)
 728         d.addCallback(_check_retrieve)
 729
 730         return d
 731
 732     def test_status_numbers(self):
 733         drrm = status.DownloadResultsRendererMixin()
 734         self.failUnlessReallyEqual(drrm.render_time(None, None), "")
 735         self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
 736         self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
 737         self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
 738         self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
 739         self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
 740         self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 741         self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
 742         self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
 743
 744         urrm = status.UploadResultsRendererMixin()
 745         self.failUnlessReallyEqual(urrm.render_time(None, None), "")
 746         self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
 747         self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
 748         self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
 749         self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
 750         self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
 751         self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 752         self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
 753         self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
 754
 755     def test_GET_FILEURL(self):
 756         d = self.GET(self.public_url + "/foo/bar.txt")
 757         d.addCallback(self.failUnlessIsBarDotTxt)
 758         return d
 759
 760     def test_GET_FILEURL_range(self):
 761         headers = {"range": "bytes=1-10"}
 762         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 763                      return_response=True)
 764         def _got((res, status, headers)):
 765             self.failUnlessReallyEqual(int(status), 206)
 766             self.failUnless(headers.has_key("content-range"))
 767             self.failUnlessReallyEqual(headers["content-range"][0],
 768                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 769             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
 770         d.addCallback(_got)
 771         return d
 772
 773     def test_GET_FILEURL_partial_range(self):
 774         headers = {"range": "bytes=5-"}
 775         length  = len(self.BAR_CONTENTS)
 776         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 777                      return_response=True)
 778         def _got((res, status, headers)):
 779             self.failUnlessReallyEqual(int(status), 206)
 780             self.failUnless(headers.has_key("content-range"))
 781             self.failUnlessReallyEqual(headers["content-range"][0],
 782                                        "bytes 5-%d/%d" % (length-1, length))
 783             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
 784         d.addCallback(_got)
 785         return d
 786
 787     def test_GET_FILEURL_partial_end_range(self):
 788         headers = {"range": "bytes=-5"}
 789         length  = len(self.BAR_CONTENTS)
 790         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 791                      return_response=True)
 792         def _got((res, status, headers)):
 793             self.failUnlessReallyEqual(int(status), 206)
 794             self.failUnless(headers.has_key("content-range"))
 795             self.failUnlessReallyEqual(headers["content-range"][0],
 796                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 797             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
 798         d.addCallback(_got)
 799         return d
 800
 801     def test_GET_FILEURL_partial_range_overrun(self):
 802         headers = {"range": "bytes=100-200"}
 803         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
 804                              "416 Requested Range not satisfiable",
 805                              "First beyond end of file",
 806                              self.GET, self.public_url + "/foo/bar.txt",
 807                              headers=headers)
 808         return d
 809
 810     def test_HEAD_FILEURL_range(self):
 811         headers = {"range": "bytes=1-10"}
 812         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 813                      return_response=True)
 814         def _got((res, status, headers)):
 815             self.failUnlessReallyEqual(res, "")
 816             self.failUnlessReallyEqual(int(status), 206)
 817             self.failUnless(headers.has_key("content-range"))
 818             self.failUnlessReallyEqual(headers["content-range"][0],
 819                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 820         d.addCallback(_got)
 821         return d
 822
 823     def test_HEAD_FILEURL_partial_range(self):
 824         headers = {"range": "bytes=5-"}
 825         length  = len(self.BAR_CONTENTS)
 826         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 827                      return_response=True)
 828         def _got((res, status, headers)):
 829             self.failUnlessReallyEqual(int(status), 206)
 830             self.failUnless(headers.has_key("content-range"))
 831             self.failUnlessReallyEqual(headers["content-range"][0],
 832                                        "bytes 5-%d/%d" % (length-1, length))
 833         d.addCallback(_got)
 834         return d
 835
 836     def test_HEAD_FILEURL_partial_end_range(self):
 837         headers = {"range": "bytes=-5"}
 838         length  = len(self.BAR_CONTENTS)
 839         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 840                      return_response=True)
 841         def _got((res, status, headers)):
 842             self.failUnlessReallyEqual(int(status), 206)
 843             self.failUnless(headers.has_key("content-range"))
 844             self.failUnlessReallyEqual(headers["content-range"][0],
 845                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 846         d.addCallback(_got)
 847         return d
 848
 849     def test_HEAD_FILEURL_partial_range_overrun(self):
 850         headers = {"range": "bytes=100-200"}
 851         d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
 852                              "416 Requested Range not satisfiable",
 853                              "",
 854                              self.HEAD, self.public_url + "/foo/bar.txt",
 855                              headers=headers)
 856         return d
 857
 858     def test_GET_FILEURL_range_bad(self):
 859         headers = {"range": "BOGUS=fizbop-quarnak"}
 860         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 861                      return_response=True)
 862         def _got((res, status, headers)):
 863             self.failUnlessReallyEqual(int(status), 200)
 864             self.failUnless(not headers.has_key("content-range"))
 865             self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
 866         d.addCallback(_got)
 867         return d
 868
 869     def test_HEAD_FILEURL(self):
 870         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 871         def _got((res, status, headers)):
 872             self.failUnlessReallyEqual(res, "")
 873             self.failUnlessReallyEqual(headers["content-length"][0],
 874                                        str(len(self.BAR_CONTENTS)))
 875             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
 876         d.addCallback(_got)
 877         return d
 878
 879     def test_GET_FILEURL_named(self):
 880         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 881         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 882         d = self.GET(base + "/@@name=/blah.txt")
 883         d.addCallback(self.failUnlessIsBarDotTxt)
 884         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 885         d.addCallback(self.failUnlessIsBarDotTxt)
 886         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 887         d.addCallback(self.failUnlessIsBarDotTxt)
 888         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 889         d.addCallback(self.failUnlessIsBarDotTxt)
 890         save_url = base + "?save=true&filename=blah.txt"
 891         d.addCallback(lambda res: self.GET(save_url))
 892         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 893         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 894         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 895         u_url = base + "?save=true&filename=" + u_fn_e
 896         d.addCallback(lambda res: self.GET(u_url))
 897         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 898         return d
 899
 900     def test_PUT_FILEURL_named_bad(self):
 901         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 902         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 903                              "400 Bad Request",
 904                              "/file can only be used with GET or HEAD",
 905                              self.PUT, base + "/@@name=/blah.txt", "")
 906         return d
 907
 908
 909     def test_GET_DIRURL_named_bad(self):
 910         base = "/file/%s" % urllib.quote(self._foo_uri)
 911         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 912                              "400 Bad Request",
 913                              "is not a file-cap",
 914                              self.GET, base + "/@@name=/blah.txt")
 915         return d
 916
 917     def test_GET_slash_file_bad(self):
 918         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 919                              "404 Not Found",
 920                              "/file must be followed by a file-cap and a name",
 921                              self.GET, "/file")
 922         return d
 923
 924     def test_GET_unhandled_URI_named(self):
 925         contents, n, newuri = self.makefile(12)
 926         verifier_cap = n.get_verify_cap().to_string()
 927         base = "/file/%s" % urllib.quote(verifier_cap)
 928         # client.create_node_from_uri() can't handle verify-caps
 929         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 930                              "400 Bad Request", "is not a file-cap",
 931                              self.GET, base)
 932         return d
 933
 934     def test_GET_unhandled_URI(self):
 935         contents, n, newuri = self.makefile(12)
 936         verifier_cap = n.get_verify_cap().to_string()
 937         base = "/uri/%s" % urllib.quote(verifier_cap)
 938         # client.create_node_from_uri() can't handle verify-caps
 939         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 940                              "400 Bad Request",
 941                              "GET unknown URI type: can only do t=info",
 942                              self.GET, base)
 943         return d
 944
 945     def test_GET_FILE_URI(self):
 946         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 947         d = self.GET(base)
 948         d.addCallback(self.failUnlessIsBarDotTxt)
 949         return d
 950
 951     def test_GET_FILE_URI_mdmf(self):
 952         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 953         d = self.GET(base)
 954         d.addCallback(self.failUnlessIsQuuxDotTxt)
 955         return d
 956
 957     def test_GET_FILE_URI_mdmf_extensions(self):
 958         base = "/uri/%s" % urllib.quote("%s:RANDOMSTUFF" % self._quux_txt_uri)
 959         d = self.GET(base)
 960         d.addCallback(self.failUnlessIsQuuxDotTxt)
 961         return d
 962
 963     def test_GET_FILE_URI_mdmf_readonly(self):
 964         base = "/uri/%s" % urllib.quote(self._quux_txt_readonly_uri)
 965         d = self.GET(base)
 966         d.addCallback(self.failUnlessIsQuuxDotTxt)
 967         return d
 968
 969     def test_GET_FILE_URI_badchild(self):
 970         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 971         errmsg = "Files have no children, certainly not named 'boguschild'"
 972         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 973                              "400 Bad Request", errmsg,
 974                              self.GET, base)
 975         return d
 976
 977     def test_PUT_FILE_URI_badchild(self):
 978         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 979         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 980         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 981                              "400 Bad Request", errmsg,
 982                              self.PUT, base, "")
 983         return d
 984
 985     def test_PUT_FILE_URI_mdmf(self):
 986         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 987         self._quux_new_contents = "new_contents"
 988         d = self.GET(base)
 989         d.addCallback(lambda res:
 990             self.failUnlessIsQuuxDotTxt(res))
 991         d.addCallback(lambda ignored:
 992             self.PUT(base, self._quux_new_contents))
 993         d.addCallback(lambda ignored:
 994             self.GET(base))
 995         d.addCallback(lambda res:
 996             self.failUnlessReallyEqual(res, self._quux_new_contents))
 997         return d
 998
 999     def test_PUT_FILE_URI_mdmf_extensions(self):
1000         base = "/uri/%s" % urllib.quote("%s:EXTENSIONSTUFF" % self._quux_txt_uri)
1001         self._quux_new_contents = "new_contents"
1002         d = self.GET(base)
1003         d.addCallback(lambda res: self.failUnlessIsQuuxDotTxt(res))
1004         d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
1005         d.addCallback(lambda ignored: self.GET(base))
1006         d.addCallback(lambda res: self.failUnlessEqual(self._quux_new_contents,
1007                                                        res))
1008         return d
1009
1010     def test_PUT_FILE_URI_mdmf_readonly(self):
1011         # We're not allowed to PUT things to a readonly cap.
1012         base = "/uri/%s" % self._quux_txt_readonly_uri
1013         d = self.GET(base)
1014         d.addCallback(lambda res:
1015             self.failUnlessIsQuuxDotTxt(res))
1016         # What should we get here? We get a 500 error now; that's not right.
1017         d.addCallback(lambda ignored:
1018             self.shouldFail2(error.Error, "test_PUT_FILE_URI_mdmf_readonly",
1019                              "400 Bad Request", "read-only cap",
1020                              self.PUT, base, "new data"))
1021         return d
1022
1023     def test_PUT_FILE_URI_sdmf_readonly(self):
1024         # We're not allowed to put things to a readonly cap.
1025         base = "/uri/%s" % self._baz_txt_readonly_uri
1026         d = self.GET(base)
1027         d.addCallback(lambda res:
1028             self.failUnlessIsBazDotTxt(res))
1029         d.addCallback(lambda ignored:
1030             self.shouldFail2(error.Error, "test_PUT_FILE_URI_sdmf_readonly",
1031                              "400 Bad Request", "read-only cap",
1032                              self.PUT, base, "new_data"))
1033         return d
1034
1035     def test_GET_etags(self):
1036
1037         def _check_etags(uri):
1038             d1 = _get_etag(uri)
1039             d2 = _get_etag(uri, 'json')
1040             d = defer.DeferredList([d1, d2], consumeErrors=True)
1041             def _check(results):
1042                 # All deferred must succeed
1043                 self.failUnless(all([r[0] for r in results]))
1044                 # the etag for the t=json form should be just like the etag
1045                 # fo the default t='' form, but with a 'json' suffix
1046                 self.failUnlessEqual(results[0][1] + 'json', results[1][1])
1047             d.addCallback(_check)
1048             return d
1049
1050         def _get_etag(uri, t=''):
1051             targetbase = "/uri/%s?t=%s" % (urllib.quote(uri.strip()), t)
1052             d = self.GET(targetbase, return_response=True, followRedirect=True)
1053             def _just_the_etag(result):
1054                 data, response, headers = result
1055                 etag = headers['etag'][0]
1056                 if uri.startswith('URI:DIR'):
1057                     self.failUnless(etag.startswith('DIR:'), etag)
1058                 return etag
1059             return d.addCallback(_just_the_etag)
1060
1061         # Check that etags work with immutable directories
1062         (newkids, caps) = self._create_immutable_children()
1063         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1064                       simplejson.dumps(newkids))
1065         def _stash_immdir_uri(uri):
1066             self._immdir_uri = uri
1067             return uri
1068         d.addCallback(_stash_immdir_uri)
1069         d.addCallback(_check_etags)
1070
1071         # Check that etags work with immutable files
1072         d.addCallback(lambda _: _check_etags(self._bar_txt_uri))
1073
1074         # use the ETag on GET
1075         def _check_match(ign):
1076             uri = "/uri/%s" % self._bar_txt_uri
1077             d = self.GET(uri, return_response=True)
1078             # extract the ETag
1079             d.addCallback(lambda (data, code, headers):
1080                           headers['etag'][0])
1081             # do a GET that's supposed to match the ETag
1082             d.addCallback(lambda etag:
1083                           self.GET(uri, return_response=True,
1084                                    headers={"If-None-Match": etag}))
1085             # make sure it short-circuited (304 instead of 200)
1086             d.addCallback(lambda (data, code, headers):
1087                           self.failUnlessEqual(int(code), http.NOT_MODIFIED))
1088             return d
1089         d.addCallback(_check_match)
1090
1091         def _no_etag(uri, t):
1092             target = "/uri/%s?t=%s" % (uri, t)
1093             d = self.GET(target, return_response=True, followRedirect=True)
1094             d.addCallback(lambda (data, code, headers):
1095                           self.failIf("etag" in headers, target))
1096             return d
1097         def _yes_etag(uri, t):
1098             target = "/uri/%s?t=%s" % (uri, t)
1099             d = self.GET(target, return_response=True, followRedirect=True)
1100             d.addCallback(lambda (data, code, headers):
1101                           self.failUnless("etag" in headers, target))
1102             return d
1103
1104         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, ""))
1105         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "json"))
1106         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "uri"))
1107         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "readonly-uri"))
1108         d.addCallback(lambda ign: _no_etag(self._bar_txt_uri, "info"))
1109
1110         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, ""))
1111         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "json"))
1112         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "uri"))
1113         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "readonly-uri"))
1114         d.addCallback(lambda ign: _no_etag(self._immdir_uri, "info"))
1115         d.addCallback(lambda ign: _no_etag(self._immdir_uri, "rename-form"))
1116
1117         return d
1118
1119     # TODO: version of this with a Unicode filename
1120     def test_GET_FILEURL_save(self):
1121         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
1122                      return_response=True)
1123         def _got((res, statuscode, headers)):
1124             content_disposition = headers["content-disposition"][0]
1125             self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
1126             self.failUnlessIsBarDotTxt(res)
1127         d.addCallback(_got)
1128         return d
1129
1130     def test_GET_FILEURL_missing(self):
1131         d = self.GET(self.public_url + "/foo/missing")
1132         d.addBoth(self.should404, "test_GET_FILEURL_missing")
1133         return d
1134
1135     def test_GET_FILEURL_info_mdmf(self):
1136         d = self.GET("/uri/%s?t=info" % self._quux_txt_uri)
1137         def _got(res):
1138             self.failUnlessIn("mutable file (mdmf)", res)
1139             self.failUnlessIn(self._quux_txt_uri, res)
1140             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1141         d.addCallback(_got)
1142         return d
1143
1144     def test_GET_FILEURL_info_mdmf_readonly(self):
1145         d = self.GET("/uri/%s?t=info" % self._quux_txt_readonly_uri)
1146         def _got(res):
1147             self.failUnlessIn("mutable file (mdmf)", res)
1148             self.failIfIn(self._quux_txt_uri, res)
1149             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1150         d.addCallback(_got)
1151         return d
1152
1153     def test_GET_FILEURL_info_sdmf(self):
1154         d = self.GET("/uri/%s?t=info" % self._baz_txt_uri)
1155         def _got(res):
1156             self.failUnlessIn("mutable file (sdmf)", res)
1157             self.failUnlessIn(self._baz_txt_uri, res)
1158         d.addCallback(_got)
1159         return d
1160
1161     def test_GET_FILEURL_info_mdmf_extensions(self):
1162         d = self.GET("/uri/%s:STUFF?t=info" % self._quux_txt_uri)
1163         def _got(res):
1164             self.failUnlessIn("mutable file (mdmf)", res)
1165             self.failUnlessIn(self._quux_txt_uri, res)
1166             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1167         d.addCallback(_got)
1168         return d
1169
1170     def test_PUT_overwrite_only_files(self):
1171         # create a directory, put a file in that directory.
1172         contents, n, filecap = self.makefile(8)
1173         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
1174         d.addCallback(lambda res:
1175             self.PUT(self.public_url + "/foo/dir/file1.txt",
1176                      self.NEWFILE_CONTENTS))
1177         # try to overwrite the file with replace=only-files
1178         # (this should work)
1179         d.addCallback(lambda res:
1180             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
1181                      filecap))
1182         d.addCallback(lambda res:
1183             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
1184                  "There was already a child by that name, and you asked me "
1185                  "to not replace it",
1186                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
1187                  filecap))
1188         return d
1189
1190     def test_PUT_NEWFILEURL(self):
1191         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
1192         # TODO: we lose the response code, so we can't check this
1193         #self.failUnlessReallyEqual(responsecode, 201)
1194         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1195         d.addCallback(lambda res:
1196                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1197                                                       self.NEWFILE_CONTENTS))
1198         return d
1199
1200     def test_PUT_NEWFILEURL_not_mutable(self):
1201         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
1202                      self.NEWFILE_CONTENTS)
1203         # TODO: we lose the response code, so we can't check this
1204         #self.failUnlessReallyEqual(responsecode, 201)
1205         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1206         d.addCallback(lambda res:
1207                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1208                                                       self.NEWFILE_CONTENTS))
1209         return d
1210
1211     def test_PUT_NEWFILEURL_unlinked_mdmf(self):
1212         # this should get us a few segments of an MDMF mutable file,
1213         # which we can then test for.
1214         contents = self.NEWFILE_CONTENTS * 300000
1215         d = self.PUT("/uri?format=mdmf",
1216                      contents)
1217         def _got_filecap(filecap):
1218             self.failUnless(filecap.startswith("URI:MDMF"))
1219             return filecap
1220         d.addCallback(_got_filecap)
1221         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1222         d.addCallback(lambda json: self.failUnlessIn("MDMF", json))
1223         return d
1224
1225     def test_PUT_NEWFILEURL_unlinked_sdmf(self):
1226         contents = self.NEWFILE_CONTENTS * 300000
1227         d = self.PUT("/uri?format=sdmf",
1228                      contents)
1229         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1230         d.addCallback(lambda json: self.failUnlessIn("SDMF", json))
1231         return d
1232
1233     def test_PUT_NEWFILEURL_unlinked_bad_format(self):
1234         contents = self.NEWFILE_CONTENTS * 300000
1235         return self.shouldHTTPError("PUT_NEWFILEURL_unlinked_bad_format",
1236                                     400, "Bad Request", "Unknown format: foo",
1237                                     self.PUT, "/uri?format=foo",
1238                                     contents)
1239
1240     def test_PUT_NEWFILEURL_range_bad(self):
1241         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
1242         target = self.public_url + "/foo/new.txt"
1243         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
1244                              "501 Not Implemented",
1245                              "Content-Range in PUT not yet supported",
1246                              # (and certainly not for immutable files)
1247                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
1248                              headers=headers)
1249         d.addCallback(lambda res:
1250                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
1251         return d
1252
1253     def test_PUT_NEWFILEURL_mutable(self):
1254         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1255                      self.NEWFILE_CONTENTS)
1256         # TODO: we lose the response code, so we can't check this
1257         #self.failUnlessReallyEqual(responsecode, 201)
1258         def _check_uri(res):
1259             u = uri.from_string_mutable_filenode(res)
1260             self.failUnless(u.is_mutable())
1261             self.failIf(u.is_readonly())
1262             return res
1263         d.addCallback(_check_uri)
1264         d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
1265         d.addCallback(lambda res:
1266                       self.failUnlessMutableChildContentsAre(self._foo_node,
1267                                                              u"new.txt",
1268                                                              self.NEWFILE_CONTENTS))
1269         return d
1270
1271     def test_PUT_NEWFILEURL_mutable_toobig(self):
1272         # It is okay to upload large mutable files, so we should be able
1273         # to do that.
1274         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1275                      "b" * (self.s.MUTABLE_SIZELIMIT + 1))
1276         return d
1277
1278     def test_PUT_NEWFILEURL_replace(self):
1279         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
1280         # TODO: we lose the response code, so we can't check this
1281         #self.failUnlessReallyEqual(responsecode, 200)
1282         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
1283         d.addCallback(lambda res:
1284                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1285                                                       self.NEWFILE_CONTENTS))
1286         return d
1287
1288     def test_PUT_NEWFILEURL_bad_t(self):
1289         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
1290                              "PUT to a file: bad t=bogus",
1291                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
1292                              "contents")
1293         return d
1294
1295     def test_PUT_NEWFILEURL_no_replace(self):
1296         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
1297                      self.NEWFILE_CONTENTS)
1298         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
1299                   "409 Conflict",
1300                   "There was already a child by that name, and you asked me "
1301                   "to not replace it")
1302         return d
1303
1304     def test_PUT_NEWFILEURL_mkdirs(self):
1305         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
1306         fn = self._foo_node
1307         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
1308         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
1309         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
1310         d.addCallback(lambda res:
1311                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
1312                                                       self.NEWFILE_CONTENTS))
1313         return d
1314
1315     def test_PUT_NEWFILEURL_blocked(self):
1316         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
1317                      self.NEWFILE_CONTENTS)
1318         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
1319                   "409 Conflict",
1320                   "Unable to create directory 'blockingfile': a file was in the way")
1321         return d
1322
1323     def test_PUT_NEWFILEURL_emptyname(self):
1324         # an empty pathname component (i.e. a double-slash) is disallowed
1325         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
1326                              "400 Bad Request",
1327                              "The webapi does not allow empty pathname components",
1328                              self.PUT, self.public_url + "/foo//new.txt", "")
1329         return d
1330
1331     def test_DELETE_FILEURL(self):
1332         d = self.DELETE(self.public_url + "/foo/bar.txt")
1333         d.addCallback(lambda res:
1334                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1335         return d
1336
1337     def test_DELETE_FILEURL_missing(self):
1338         d = self.DELETE(self.public_url + "/foo/missing")
1339         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
1340         return d
1341
1342     def test_DELETE_FILEURL_missing2(self):
1343         d = self.DELETE(self.public_url + "/missing/missing")
1344         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
1345         return d
1346
1347     def failUnlessHasBarDotTxtMetadata(self, res):
1348         data = simplejson.loads(res)
1349         self.failUnless(isinstance(data, list))
1350         self.failUnlessIn("metadata", data[1])
1351         self.failUnlessIn("tahoe", data[1]["metadata"])
1352         self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
1353         self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
1354         self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
1355                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
1356
1357     def test_GET_FILEURL_json(self):
1358         # twisted.web.http.parse_qs ignores any query args without an '=', so
1359         # I can't do "GET /path?json", I have to do "GET /path/t=json"
1360         # instead. This may make it tricky to emulate the S3 interface
1361         # completely.
1362         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
1363         def _check1(data):
1364             self.failUnlessIsBarJSON(data)
1365             self.failUnlessHasBarDotTxtMetadata(data)
1366             return
1367         d.addCallback(_check1)
1368         return d
1369
1370     def test_GET_FILEURL_json_mutable_type(self):
1371         # The JSON should include format, which says whether the
1372         # file is SDMF or MDMF
1373         d = self.PUT("/uri?format=mdmf",
1374                      self.NEWFILE_CONTENTS * 300000)
1375         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1376         def _got_json(json, version):
1377             data = simplejson.loads(json)
1378             assert "filenode" == data[0]
1379             data = data[1]
1380             assert isinstance(data, dict)
1381
1382             self.failUnlessIn("format", data)
1383             self.failUnlessEqual(data["format"], version)
1384
1385         d.addCallback(_got_json, "MDMF")
1386         # Now make an SDMF file and check that it is reported correctly.
1387         d.addCallback(lambda ignored:
1388             self.PUT("/uri?format=sdmf",
1389                       self.NEWFILE_CONTENTS * 300000))
1390         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1391         d.addCallback(_got_json, "SDMF")
1392         return d
1393
1394     def test_GET_FILEURL_json_mdmf(self):
1395         d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
1396         d.addCallback(self.failUnlessIsQuuxJSON)
1397         return d
1398
1399     def test_GET_FILEURL_json_missing(self):
1400         d = self.GET(self.public_url + "/foo/missing?json")
1401         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
1402         return d
1403
1404     def test_GET_FILEURL_uri(self):
1405         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
1406         def _check(res):
1407             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1408         d.addCallback(_check)
1409         d.addCallback(lambda res:
1410                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
1411         def _check2(res):
1412             # for now, for files, uris and readonly-uris are the same
1413             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1414         d.addCallback(_check2)
1415         return d
1416
1417     def test_GET_FILEURL_badtype(self):
1418         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
1419                                  "bad t=bogus",
1420                                  self.GET,
1421                                  self.public_url + "/foo/bar.txt?t=bogus")
1422         return d
1423
1424     def test_CSS_FILE(self):
1425         d = self.GET("/tahoe.css", followRedirect=True)
1426         def _check(res):
1427             CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
1428             self.failUnless(CSS_STYLE.search(res), res)
1429         d.addCallback(_check)
1430         return d
1431
1432     def test_GET_FILEURL_uri_missing(self):
1433         d = self.GET(self.public_url + "/foo/missing?t=uri")
1434         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
1435         return d
1436
1437     def _check_upload_and_mkdir_forms(self, html):
1438         # We should have a form to create a file, with radio buttons that allow
1439         # the user to toggle whether it is a CHK/LIT (default), SDMF, or MDMF file.
1440         self.failUnlessIn('name="t" value="upload"', html)
1441         self.failUnlessIn('input checked="checked" type="radio" id="upload-chk" value="chk" name="format"', html)
1442         self.failUnlessIn('input type="radio" id="upload-sdmf" value="sdmf" name="format"', html)
1443         self.failUnlessIn('input type="radio" id="upload-mdmf" value="mdmf" name="format"', html)
1444
1445         # We should also have the ability to create a mutable directory, with
1446         # radio buttons that allow the user to toggle whether it is an SDMF (default)
1447         # or MDMF directory.
1448         self.failUnlessIn('name="t" value="mkdir"', html)
1449         self.failUnlessIn('input checked="checked" type="radio" id="mkdir-sdmf" value="sdmf" name="format"', html)
1450         self.failUnlessIn('input type="radio" id="mkdir-mdmf" value="mdmf" name="format"', html)
1451
1452         self.failUnlessIn(FAVICON_MARKUP, html)
1453
1454     def test_GET_DIRECTORY_html(self):
1455         d = self.GET(self.public_url + "/foo", followRedirect=True)
1456         def _check(html):
1457             self.failUnlessIn('<div class="toolbar-item"><a href="../../..">Return to Welcome page</a></div>', html)
1458             self._check_upload_and_mkdir_forms(html)
1459             self.failUnlessIn("quux", html)
1460         d.addCallback(_check)
1461         return d
1462
1463     def test_GET_DIRECTORY_html_filenode_encoding(self):
1464         d = self.GET(self.public_url + "/foo", followRedirect=True)
1465         def _check(html):
1466             # Check if encoded entries are there
1467             self.failUnlessIn('@@named=/' + self._htmlname_urlencoded + '">'
1468                               + self._htmlname_escaped + '</a>', html)
1469             self.failUnlessIn('value="' + self._htmlname_escaped_attr + '"', html)
1470             self.failIfIn(self._htmlname_escaped_double, html)
1471             # Make sure that Nevow escaping actually works by checking for unsafe characters
1472             # and that '&' is escaped.
1473             for entity in '<>':
1474                 self.failUnlessIn(entity, self._htmlname_raw)
1475                 self.failIfIn(entity, self._htmlname_escaped)
1476             self.failUnlessIn('&', re.sub(r'&(amp|lt|gt|quot|apos);', '', self._htmlname_raw))
1477             self.failIfIn('&', re.sub(r'&(amp|lt|gt|quot|apos);', '', self._htmlname_escaped))
1478         d.addCallback(_check)
1479         return d
1480
1481     def test_GET_root_html(self):
1482         d = self.GET("/")
1483         d.addCallback(self._check_upload_and_mkdir_forms)
1484         return d
1485
1486     def test_GET_DIRURL(self):
1487         # the addSlash means we get a redirect here
1488         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1489         ROOT = "../../.."
1490         d = self.GET(self.public_url + "/foo", followRedirect=True)
1491         def _check(res):
1492             self.failUnlessIn('<a href="%s">Return to Welcome page' % ROOT, res)
1493
1494             # the FILE reference points to a URI, but it should end in bar.txt
1495             bar_url = ("%s/file/%s/@@named=/bar.txt" %
1496                        (ROOT, urllib.quote(self._bar_txt_uri)))
1497             get_bar = "".join([r'<td>FILE</td>',
1498                                r'\s+<td>',
1499                                r'<a href="%s">bar.txt</a>' % bar_url,
1500                                r'</td>',
1501                                r'\s+<td align="right">%d</td>' % len(self.BAR_CONTENTS),
1502                                ])
1503             self.failUnless(re.search(get_bar, res), res)
1504             for label in ['unlink', 'rename/move']:
1505                 for line in res.split("\n"):
1506                     # find the line that contains the relevant button for bar.txt
1507                     if ("form action" in line and
1508                         ('value="%s"' % (label,)) in line and
1509                         'value="bar.txt"' in line):
1510                         # the form target should use a relative URL
1511                         foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1512                         self.failUnlessIn('action="%s"' % foo_url, line)
1513                         # and the when_done= should too
1514                         #done_url = urllib.quote(???)
1515                         #self.failUnlessIn('name="when_done" value="%s"' % done_url, line)
1516
1517                         # 'unlink' needs to use POST because it directly has a side effect
1518                         if label == 'unlink':
1519                             self.failUnlessIn('method="post"', line)
1520                         break
1521                 else:
1522                     self.fail("unable to find '%s bar.txt' line" % (label,))
1523
1524             # the DIR reference just points to a URI
1525             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1526             get_sub = ((r'<td>DIR</td>')
1527                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1528             self.failUnless(re.search(get_sub, res), res)
1529         d.addCallback(_check)
1530
1531         # look at a readonly directory
1532         d.addCallback(lambda res:
1533                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
1534         def _check2(res):
1535             self.failUnlessIn("(read-only)", res)
1536             self.failIfIn("Upload a file", res)
1537         d.addCallback(_check2)
1538
1539         # and at a directory that contains a readonly directory
1540         d.addCallback(lambda res:
1541                       self.GET(self.public_url, followRedirect=True))
1542         def _check3(res):
1543             self.failUnless(re.search('<td>DIR-RO</td>'
1544                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1545         d.addCallback(_check3)
1546
1547         # and an empty directory
1548         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1549         def _check4(res):
1550             self.failUnlessIn("directory is empty", res)
1551             MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
1552             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1553         d.addCallback(_check4)
1554
1555         # and at a literal directory
1556         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1557         d.addCallback(lambda res:
1558                       self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1559         def _check5(res):
1560             self.failUnlessIn('(immutable)', res)
1561             self.failUnless(re.search('<td>FILE</td>'
1562                                       r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1563         d.addCallback(_check5)
1564         return d
1565
1566     def test_GET_DIRURL_badtype(self):
1567         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1568                                  400, "Bad Request",
1569                                  "bad t=bogus",
1570                                  self.GET,
1571                                  self.public_url + "/foo?t=bogus")
1572         return d
1573
1574     def test_GET_DIRURL_json(self):
1575         d = self.GET(self.public_url + "/foo?t=json")
1576         d.addCallback(self.failUnlessIsFooJSON)
1577         return d
1578
1579     def test_GET_DIRURL_json_format(self):
1580         d = self.PUT(self.public_url + \
1581                      "/foo/sdmf.txt?format=sdmf",
1582                      self.NEWFILE_CONTENTS * 300000)
1583         d.addCallback(lambda ignored:
1584             self.PUT(self.public_url + \
1585                      "/foo/mdmf.txt?format=mdmf",
1586                      self.NEWFILE_CONTENTS * 300000))
1587         # Now we have an MDMF and SDMF file in the directory. If we GET
1588         # its JSON, we should see their encodings.
1589         d.addCallback(lambda ignored:
1590             self.GET(self.public_url + "/foo?t=json"))
1591         def _got_json(json):
1592             data = simplejson.loads(json)
1593             assert data[0] == "dirnode"
1594
1595             data = data[1]
1596             kids = data['children']
1597
1598             mdmf_data = kids['mdmf.txt'][1]
1599             self.failUnlessIn("format", mdmf_data)
1600             self.failUnlessEqual(mdmf_data["format"], "MDMF")
1601
1602             sdmf_data = kids['sdmf.txt'][1]
1603             self.failUnlessIn("format", sdmf_data)
1604             self.failUnlessEqual(sdmf_data["format"], "SDMF")
1605         d.addCallback(_got_json)
1606         return d
1607
1608
1609     def test_POST_DIRURL_manifest_no_ophandle(self):
1610         d = self.shouldFail2(error.Error,
1611                              "test_POST_DIRURL_manifest_no_ophandle",
1612                              "400 Bad Request",
1613                              "slow operation requires ophandle=",
1614                              self.POST, self.public_url, t="start-manifest")
1615         return d
1616
1617     def test_POST_DIRURL_manifest(self):
1618         d = defer.succeed(None)
1619         def getman(ignored, output):
1620             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1621                           followRedirect=True)
1622             d.addCallback(self.wait_for_operation, "125")
1623             d.addCallback(self.get_operation_results, "125", output)
1624             return d
1625         d.addCallback(getman, None)
1626         def _got_html(manifest):
1627             self.failUnlessIn("Manifest of SI=", manifest)
1628             self.failUnlessIn("<td>sub</td>", manifest)
1629             self.failUnlessIn(self._sub_uri, manifest)
1630             self.failUnlessIn("<td>sub/baz.txt</td>", manifest)
1631             self.failUnlessIn(FAVICON_MARKUP, manifest)
1632         d.addCallback(_got_html)
1633
1634         # both t=status and unadorned GET should be identical
1635         d.addCallback(lambda res: self.GET("/operations/125"))
1636         d.addCallback(_got_html)
1637
1638         d.addCallback(getman, "html")
1639         d.addCallback(_got_html)
1640         d.addCallback(getman, "text")
1641         def _got_text(manifest):
1642             self.failUnlessIn("\nsub " + self._sub_uri + "\n", manifest)
1643             self.failUnlessIn("\nsub/baz.txt URI:CHK:", manifest)
1644         d.addCallback(_got_text)
1645         d.addCallback(getman, "JSON")
1646         def _got_json(res):
1647             data = res["manifest"]
1648             got = {}
1649             for (path_list, cap) in data:
1650                 got[tuple(path_list)] = cap
1651             self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
1652             self.failUnlessIn((u"sub", u"baz.txt"), got)
1653             self.failUnlessIn("finished", res)
1654             self.failUnlessIn("origin", res)
1655             self.failUnlessIn("storage-index", res)
1656             self.failUnlessIn("verifycaps", res)
1657             self.failUnlessIn("stats", res)
1658         d.addCallback(_got_json)
1659         return d
1660
1661     def test_POST_DIRURL_deepsize_no_ophandle(self):
1662         d = self.shouldFail2(error.Error,
1663                              "test_POST_DIRURL_deepsize_no_ophandle",
1664                              "400 Bad Request",
1665                              "slow operation requires ophandle=",
1666                              self.POST, self.public_url, t="start-deep-size")
1667         return d
1668
1669     def test_POST_DIRURL_deepsize(self):
1670         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1671                       followRedirect=True)
1672         d.addCallback(self.wait_for_operation, "126")
1673         d.addCallback(self.get_operation_results, "126", "json")
1674         def _got_json(data):
1675             self.failUnlessReallyEqual(data["finished"], True)
1676             size = data["size"]
1677             self.failUnless(size > 1000)
1678         d.addCallback(_got_json)
1679         d.addCallback(self.get_operation_results, "126", "text")
1680         def _got_text(res):
1681             mo = re.search(r'^size: (\d+)$', res, re.M)
1682             self.failUnless(mo, res)
1683             size = int(mo.group(1))
1684             # with directories, the size varies.
1685             self.failUnless(size > 1000)
1686         d.addCallback(_got_text)
1687         return d
1688
1689     def test_POST_DIRURL_deepstats_no_ophandle(self):
1690         d = self.shouldFail2(error.Error,
1691                              "test_POST_DIRURL_deepstats_no_ophandle",
1692                              "400 Bad Request",
1693                              "slow operation requires ophandle=",
1694                              self.POST, self.public_url, t="start-deep-stats")
1695         return d
1696
1697     def test_POST_DIRURL_deepstats(self):
1698         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1699                       followRedirect=True)
1700         d.addCallback(self.wait_for_operation, "127")
1701         d.addCallback(self.get_operation_results, "127", "json")
1702         def _got_json(stats):
1703             expected = {"count-immutable-files": 4,
1704                         "count-mutable-files": 2,
1705                         "count-literal-files": 0,
1706                         "count-files": 6,
1707                         "count-directories": 3,
1708                         "size-immutable-files": 76,
1709                         "size-literal-files": 0,
1710                         #"size-directories": 1912, # varies
1711                         #"largest-directory": 1590,
1712                         "largest-directory-children": 8,
1713                         "largest-immutable-file": 19,
1714                         }
1715             for k,v in expected.iteritems():
1716                 self.failUnlessReallyEqual(stats[k], v,
1717                                            "stats[%s] was %s, not %s" %
1718                                            (k, stats[k], v))
1719             self.failUnlessReallyEqual(stats["size-files-histogram"],
1720                                        [ [11, 31, 4] ])
1721         d.addCallback(_got_json)
1722         return d
1723
1724     def test_POST_DIRURL_stream_manifest(self):
1725         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1726         def _check(res):
1727             self.failUnless(res.endswith("\n"))
1728             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1729             self.failUnlessReallyEqual(len(units), 10)
1730             self.failUnlessEqual(units[-1]["type"], "stats")
1731             first = units[0]
1732             self.failUnlessEqual(first["path"], [])
1733             self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri)
1734             self.failUnlessEqual(first["type"], "directory")
1735             baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0]
1736             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1737             self.failIfEqual(baz["storage-index"], None)
1738             self.failIfEqual(baz["verifycap"], None)
1739             self.failIfEqual(baz["repaircap"], None)
1740             # XXX: Add quux and baz to this test.
1741             return
1742         d.addCallback(_check)
1743         return d
1744
1745     def test_GET_DIRURL_uri(self):
1746         d = self.GET(self.public_url + "/foo?t=uri")
1747         def _check(res):
1748             self.failUnlessReallyEqual(to_str(res), self._foo_uri)
1749         d.addCallback(_check)
1750         return d
1751
1752     def test_GET_DIRURL_readonly_uri(self):
1753         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1754         def _check(res):
1755             self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri)
1756         d.addCallback(_check)
1757         return d
1758
1759     def test_PUT_NEWDIRURL(self):
1760         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1761         d.addCallback(lambda res:
1762                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1763         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1764         d.addCallback(self.failUnlessNodeKeysAre, [])
1765         return d
1766
1767     def test_PUT_NEWDIRURL_mdmf(self):
1768         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1769         d.addCallback(lambda res:
1770                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1771         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1772         d.addCallback(lambda node:
1773             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1774         return d
1775
1776     def test_PUT_NEWDIRURL_sdmf(self):
1777         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=sdmf",
1778                      "")
1779         d.addCallback(lambda res:
1780                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1781         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1782         d.addCallback(lambda node:
1783             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1784         return d
1785
1786     def test_PUT_NEWDIRURL_bad_format(self):
1787         return self.shouldHTTPError("PUT_NEWDIRURL_bad_format",
1788                                     400, "Bad Request", "Unknown format: foo",
1789                                     self.PUT, self.public_url +
1790                                     "/foo/newdir=?t=mkdir&format=foo", "")
1791
1792     def test_POST_NEWDIRURL(self):
1793         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1794         d.addCallback(lambda res:
1795                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1796         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1797         d.addCallback(self.failUnlessNodeKeysAre, [])
1798         return d
1799
1800     def test_POST_NEWDIRURL_mdmf(self):
1801         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1802         d.addCallback(lambda res:
1803                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1804         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1805         d.addCallback(lambda node:
1806             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1807         return d
1808
1809     def test_POST_NEWDIRURL_sdmf(self):
1810         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=sdmf", "")
1811         d.addCallback(lambda res:
1812             self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1813         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1814         d.addCallback(lambda node:
1815             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1816         return d
1817
1818     def test_POST_NEWDIRURL_bad_format(self):
1819         return self.shouldHTTPError("POST_NEWDIRURL_bad_format",
1820                                     400, "Bad Request", "Unknown format: foo",
1821                                     self.POST2, self.public_url + \
1822                                     "/foo/newdir?t=mkdir&format=foo", "")
1823
1824     def test_POST_NEWDIRURL_emptyname(self):
1825         # an empty pathname component (i.e. a double-slash) is disallowed
1826         d = self.shouldFail2(error.Error, "POST_NEWDIRURL_emptyname",
1827                              "400 Bad Request",
1828                              "The webapi does not allow empty pathname components, i.e. a double slash",
1829                              self.POST, self.public_url + "//?t=mkdir")
1830         return d
1831
1832     def _do_POST_NEWDIRURL_initial_children_test(self, version=None):
1833         (newkids, caps) = self._create_initial_children()
1834         query = "/foo/newdir?t=mkdir-with-children"
1835         if version == MDMF_VERSION:
1836             query += "&format=mdmf"
1837         elif version == SDMF_VERSION:
1838             query += "&format=sdmf"
1839         else:
1840             version = SDMF_VERSION # for later
1841         d = self.POST2(self.public_url + query,
1842                        simplejson.dumps(newkids))
1843         def _check(uri):
1844             n = self.s.create_node_from_uri(uri.strip())
1845             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1846             self.failUnlessEqual(n._node.get_version(), version)
1847             d2.addCallback(lambda ign:
1848                            self.failUnlessROChildURIIs(n, u"child-imm",
1849                                                        caps['filecap1']))
1850             d2.addCallback(lambda ign:
1851                            self.failUnlessRWChildURIIs(n, u"child-mutable",
1852                                                        caps['filecap2']))
1853             d2.addCallback(lambda ign:
1854                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1855                                                        caps['filecap3']))
1856             d2.addCallback(lambda ign:
1857                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1858                                                        caps['unknown_rocap']))
1859             d2.addCallback(lambda ign:
1860                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1861                                                        caps['unknown_rwcap']))
1862             d2.addCallback(lambda ign:
1863                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1864                                                        caps['unknown_immcap']))
1865             d2.addCallback(lambda ign:
1866                            self.failUnlessRWChildURIIs(n, u"dirchild",
1867                                                        caps['dircap']))
1868             d2.addCallback(lambda ign:
1869                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1870                                                        caps['litdircap']))
1871             d2.addCallback(lambda ign:
1872                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1873                                                        caps['emptydircap']))
1874             return d2
1875         d.addCallback(_check)
1876         d.addCallback(lambda res:
1877                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1878         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1879         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1880         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1881         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1882         return d
1883
1884     def test_POST_NEWDIRURL_initial_children(self):
1885         return self._do_POST_NEWDIRURL_initial_children_test()
1886
1887     def test_POST_NEWDIRURL_initial_children_mdmf(self):
1888         return self._do_POST_NEWDIRURL_initial_children_test(MDMF_VERSION)
1889
1890     def test_POST_NEWDIRURL_initial_children_sdmf(self):
1891         return self._do_POST_NEWDIRURL_initial_children_test(SDMF_VERSION)
1892
1893     def test_POST_NEWDIRURL_initial_children_bad_format(self):
1894         (newkids, caps) = self._create_initial_children()
1895         return self.shouldHTTPError("POST_NEWDIRURL_initial_children_bad_format",
1896                                     400, "Bad Request", "Unknown format: foo",
1897                                     self.POST2, self.public_url + \
1898                                     "/foo/newdir?t=mkdir-with-children&format=foo",
1899                                     simplejson.dumps(newkids))
1900
1901     def test_POST_NEWDIRURL_immutable(self):
1902         (newkids, caps) = self._create_immutable_children()
1903         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1904                        simplejson.dumps(newkids))
1905         def _check(uri):
1906             n = self.s.create_node_from_uri(uri.strip())
1907             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1908             d2.addCallback(lambda ign:
1909                            self.failUnlessROChildURIIs(n, u"child-imm",
1910                                                        caps['filecap1']))
1911             d2.addCallback(lambda ign:
1912                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1913                                                        caps['unknown_immcap']))
1914             d2.addCallback(lambda ign:
1915                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
1916                                                        caps['immdircap']))
1917             d2.addCallback(lambda ign:
1918                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1919                                                        caps['litdircap']))
1920             d2.addCallback(lambda ign:
1921                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1922                                                        caps['emptydircap']))
1923             return d2
1924         d.addCallback(_check)
1925         d.addCallback(lambda res:
1926                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1927         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1928         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1929         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1930         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1931         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1932         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1933         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1934         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
1935         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1936         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
1937         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1938         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
1939         d.addErrback(self.explain_web_error)
1940         return d
1941
1942     def test_POST_NEWDIRURL_immutable_bad(self):
1943         (newkids, caps) = self._create_initial_children()
1944         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
1945                              "400 Bad Request",
1946                              "needed to be immutable but was not",
1947                              self.POST2,
1948                              self.public_url + "/foo/newdir?t=mkdir-immutable",
1949                              simplejson.dumps(newkids))
1950         return d
1951
1952     def test_PUT_NEWDIRURL_exists(self):
1953         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1954         d.addCallback(lambda res:
1955                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1956         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1957         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1958         return d
1959
1960     def test_PUT_NEWDIRURL_blocked(self):
1961         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1962                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1963                              self.PUT,
1964                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1965         d.addCallback(lambda res:
1966                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1967         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1968         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1969         return d
1970
1971     def test_PUT_NEWDIRURL_mkdirs(self):
1972         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1973         d.addCallback(lambda res:
1974                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1975         d.addCallback(lambda res:
1976                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1977         d.addCallback(lambda res:
1978                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1979         d.addCallback(self.failUnlessNodeKeysAre, [])
1980         return d
1981
1982     def test_PUT_NEWDIRURL_mkdirs_mdmf(self):
1983         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=mdmf", "")
1984         d.addCallback(lambda ignored:
1985             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1986         d.addCallback(lambda ignored:
1987             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1988         d.addCallback(lambda ignored:
1989             self._foo_node.get_child_at_path(u"subdir"))
1990         def _got_subdir(subdir):
1991             # XXX: What we want?
1992             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1993             self.failUnlessNodeHasChild(subdir, u"newdir")
1994             return subdir.get_child_at_path(u"newdir")
1995         d.addCallback(_got_subdir)
1996         d.addCallback(lambda newdir:
1997             self.failUnlessEqual(newdir._node.get_version(), MDMF_VERSION))
1998         return d
1999
2000     def test_PUT_NEWDIRURL_mkdirs_sdmf(self):
2001         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=sdmf", "")
2002         d.addCallback(lambda ignored:
2003             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
2004         d.addCallback(lambda ignored:
2005             self.failIfNodeHasChild(self._foo_node, u"newdir"))
2006         d.addCallback(lambda ignored:
2007             self._foo_node.get_child_at_path(u"subdir"))
2008         def _got_subdir(subdir):
2009             # XXX: What we want?
2010             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
2011             self.failUnlessNodeHasChild(subdir, u"newdir")
2012             return subdir.get_child_at_path(u"newdir")
2013         d.addCallback(_got_subdir)
2014         d.addCallback(lambda newdir:
2015             self.failUnlessEqual(newdir._node.get_version(), SDMF_VERSION))
2016         return d
2017
2018     def test_PUT_NEWDIRURL_mkdirs_bad_format(self):
2019         return self.shouldHTTPError("PUT_NEWDIRURL_mkdirs_bad_format",
2020                                     400, "Bad Request", "Unknown format: foo",
2021                                     self.PUT, self.public_url + \
2022                                     "/foo/subdir/newdir?t=mkdir&format=foo",
2023                                     "")
2024
2025     def test_DELETE_DIRURL(self):
2026         d = self.DELETE(self.public_url + "/foo")
2027         d.addCallback(lambda res:
2028                       self.failIfNodeHasChild(self.public_root, u"foo"))
2029         return d
2030
2031     def test_DELETE_DIRURL_missing(self):
2032         d = self.DELETE(self.public_url + "/foo/missing")
2033         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
2034         d.addCallback(lambda res:
2035                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
2036         return d
2037
2038     def test_DELETE_DIRURL_missing2(self):
2039         d = self.DELETE(self.public_url + "/missing")
2040         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
2041         return d
2042
2043     def dump_root(self):
2044         print "NODEWALK"
2045         w = webish.DirnodeWalkerMixin()
2046         def visitor(childpath, childnode, metadata):
2047             print childpath
2048         d = w.walk(self.public_root, visitor)
2049         return d
2050
2051     def failUnlessNodeKeysAre(self, node, expected_keys):
2052         for k in expected_keys:
2053             assert isinstance(k, unicode)
2054         d = node.list()
2055         def _check(children):
2056             self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
2057         d.addCallback(_check)
2058         return d
2059     def failUnlessNodeHasChild(self, node, name):
2060         assert isinstance(name, unicode)
2061         d = node.list()
2062         def _check(children):
2063             self.failUnlessIn(name, children)
2064         d.addCallback(_check)
2065         return d
2066     def failIfNodeHasChild(self, node, name):
2067         assert isinstance(name, unicode)
2068         d = node.list()
2069         def _check(children):
2070             self.failIfIn(name, children)
2071         d.addCallback(_check)
2072         return d
2073
2074     def failUnlessChildContentsAre(self, node, name, expected_contents):
2075         assert isinstance(name, unicode)
2076         d = node.get_child_at_path(name)
2077         d.addCallback(lambda node: download_to_data(node))
2078         def _check(contents):
2079             self.failUnlessReallyEqual(contents, expected_contents)
2080         d.addCallback(_check)
2081         return d
2082
2083     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
2084         assert isinstance(name, unicode)
2085         d = node.get_child_at_path(name)
2086         d.addCallback(lambda node: node.download_best_version())
2087         def _check(contents):
2088             self.failUnlessReallyEqual(contents, expected_contents)
2089         d.addCallback(_check)
2090         return d
2091
2092     def failUnlessRWChildURIIs(self, node, name, expected_uri):
2093         assert isinstance(name, unicode)
2094         d = node.get_child_at_path(name)
2095         def _check(child):
2096             self.failUnless(child.is_unknown() or not child.is_readonly())
2097             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
2098             self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
2099             expected_ro_uri = self._make_readonly(expected_uri)
2100             if expected_ro_uri:
2101                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
2102         d.addCallback(_check)
2103         return d
2104
2105     def failUnlessROChildURIIs(self, node, name, expected_uri):
2106         assert isinstance(name, unicode)
2107         d = node.get_child_at_path(name)
2108         def _check(child):
2109             self.failUnless(child.is_unknown() or child.is_readonly())
2110             self.failUnlessReallyEqual(child.get_write_uri(), None)
2111             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
2112             self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
2113         d.addCallback(_check)
2114         return d
2115
2116     def failUnlessURIMatchesRWChild(self, got_uri, node, name):
2117         assert isinstance(name, unicode)
2118         d = node.get_child_at_path(name)
2119         def _check(child):
2120             self.failUnless(child.is_unknown() or not child.is_readonly())
2121             self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
2122             self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
2123             expected_ro_uri = self._make_readonly(got_uri)
2124             if expected_ro_uri:
2125                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
2126         d.addCallback(_check)
2127         return d
2128
2129     def failUnlessURIMatchesROChild(self, got_uri, node, name):
2130         assert isinstance(name, unicode)
2131         d = node.get_child_at_path(name)
2132         def _check(child):
2133             self.failUnless(child.is_unknown() or child.is_readonly())
2134             self.failUnlessReallyEqual(child.get_write_uri(), None)
2135             self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
2136             self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
2137         d.addCallback(_check)
2138         return d
2139
2140     def failUnlessCHKURIHasContents(self, got_uri, contents):
2141         self.failUnless(self.get_all_contents()[got_uri] == contents)
2142
2143     def test_POST_upload(self):
2144         d = self.POST(self.public_url + "/foo", t="upload",
2145                       file=("new.txt", self.NEWFILE_CONTENTS))
2146         fn = self._foo_node
2147         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2148         d.addCallback(lambda res:
2149                       self.failUnlessChildContentsAre(fn, u"new.txt",
2150                                                       self.NEWFILE_CONTENTS))
2151         return d
2152
2153     def test_POST_upload_unicode(self):
2154         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2155         d = self.POST(self.public_url + "/foo", t="upload",
2156                       file=(filename, self.NEWFILE_CONTENTS))
2157         fn = self._foo_node
2158         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2159         d.addCallback(lambda res:
2160                       self.failUnlessChildContentsAre(fn, filename,
2161                                                       self.NEWFILE_CONTENTS))
2162         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2163         d.addCallback(lambda res: self.GET(target_url))
2164         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2165                                                                   self.NEWFILE_CONTENTS,
2166                                                                   contents))
2167         return d
2168
2169     def test_POST_upload_unicode_named(self):
2170         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2171         d = self.POST(self.public_url + "/foo", t="upload",
2172                       name=filename,
2173                       file=("overridden", self.NEWFILE_CONTENTS))
2174         fn = self._foo_node
2175         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2176         d.addCallback(lambda res:
2177                       self.failUnlessChildContentsAre(fn, filename,
2178                                                       self.NEWFILE_CONTENTS))
2179         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2180         d.addCallback(lambda res: self.GET(target_url))
2181         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2182                                                                   self.NEWFILE_CONTENTS,
2183                                                                   contents))
2184         return d
2185
2186     def test_POST_upload_no_link(self):
2187         d = self.POST("/uri", t="upload",
2188                       file=("new.txt", self.NEWFILE_CONTENTS))
2189         def _check_upload_results(page):
2190             # this should be a page which describes the results of the upload
2191             # that just finished.
2192             self.failUnlessIn("Upload Results:", page)
2193             self.failUnlessIn("URI:", page)
2194             uri_re = re.compile("URI: <tt><span>(.*)</span>")
2195             mo = uri_re.search(page)
2196             self.failUnless(mo, page)
2197             new_uri = mo.group(1)
2198             return new_uri
2199         d.addCallback(_check_upload_results)
2200         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
2201         return d
2202
2203     def test_POST_upload_no_link_whendone(self):
2204         d = self.POST("/uri", t="upload", when_done="/",
2205                       file=("new.txt", self.NEWFILE_CONTENTS))
2206         d.addBoth(self.shouldRedirect, "/")
2207         return d
2208
2209     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
2210         d = defer.maybeDeferred(callable, *args, **kwargs)
2211         def done(res):
2212             if isinstance(res, failure.Failure):
2213                 res.trap(error.PageRedirect)
2214                 statuscode = res.value.status
2215                 target = res.value.location
2216                 return checker(statuscode, target)
2217             self.fail("%s: callable was supposed to redirect, not return '%s'"
2218                       % (which, res))
2219         d.addBoth(done)
2220         return d
2221
2222     def test_POST_upload_no_link_whendone_results(self):
2223         def check(statuscode, target):
2224             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2225             self.failUnless(target.startswith(self.webish_url), target)
2226             return client.getPage(target, method="GET")
2227         # We encode "uri" as "%75ri" to exercise a case affected by ticket #1860.
2228         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
2229                                  check,
2230                                  self.POST, "/uri", t="upload",
2231                                  when_done="/%75ri/%(uri)s",
2232                                  file=("new.txt", self.NEWFILE_CONTENTS))
2233         d.addCallback(lambda res:
2234                       self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
2235         return d
2236
2237     def test_POST_upload_no_link_mutable(self):
2238         d = self.POST("/uri", t="upload", mutable="true",
2239                       file=("new.txt", self.NEWFILE_CONTENTS))
2240         def _check(filecap):
2241             filecap = filecap.strip()
2242             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2243             self.filecap = filecap
2244             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2245             self.failUnlessIn(u.get_storage_index(), self.get_all_contents())
2246             n = self.s.create_node_from_uri(filecap)
2247             return n.download_best_version()
2248         d.addCallback(_check)
2249         def _check2(data):
2250             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2251             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2252         d.addCallback(_check2)
2253         def _check3(data):
2254             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2255             return self.GET("/file/%s" % urllib.quote(self.filecap))
2256         d.addCallback(_check3)
2257         def _check4(data):
2258             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2259         d.addCallback(_check4)
2260         return d
2261
2262     def test_POST_upload_no_link_mutable_toobig(self):
2263         # The SDMF size limit is no longer in place, so we should be
2264         # able to upload mutable files that are as large as we want them
2265         # to be.
2266         d = self.POST("/uri", t="upload", mutable="true",
2267                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2268         return d
2269
2270
2271     def test_POST_upload_format_unlinked(self):
2272         def _check_upload_unlinked(ign, format, uri_prefix):
2273             filename = format + ".txt"
2274             d = self.POST("/uri?t=upload&format=" + format,
2275                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2276             def _got_results(results):
2277                 if format.upper() in ("SDMF", "MDMF"):
2278                     # webapi.rst says this returns a filecap
2279                     filecap = results
2280                 else:
2281                     # for immutable, it returns an "upload results page", and
2282                     # the filecap is buried inside
2283                     line = [l for l in results.split("\n") if "URI: " in l][0]
2284                     mo = re.search(r'<span>([^<]+)</span>', line)
2285                     filecap = mo.group(1)
2286                 self.failUnless(filecap.startswith(uri_prefix),
2287                                 (uri_prefix, filecap))
2288                 return self.GET("/uri/%s?t=json" % filecap)
2289             d.addCallback(_got_results)
2290             def _got_json(json):
2291                 data = simplejson.loads(json)
2292                 data = data[1]
2293                 self.failUnlessIn("format", data)
2294                 self.failUnlessEqual(data["format"], format.upper())
2295             d.addCallback(_got_json)
2296             return d
2297         d = defer.succeed(None)
2298         d.addCallback(_check_upload_unlinked, "chk", "URI:CHK")
2299         d.addCallback(_check_upload_unlinked, "CHK", "URI:CHK")
2300         d.addCallback(_check_upload_unlinked, "sdmf", "URI:SSK")
2301         d.addCallback(_check_upload_unlinked, "mdmf", "URI:MDMF")
2302         return d
2303
2304     def test_POST_upload_bad_format_unlinked(self):
2305         return self.shouldHTTPError("POST_upload_bad_format_unlinked",
2306                                     400, "Bad Request", "Unknown format: foo",
2307                                     self.POST,
2308                                     "/uri?t=upload&format=foo",
2309                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2310
2311     def test_POST_upload_format(self):
2312         def _check_upload(ign, format, uri_prefix, fn=None):
2313             filename = format + ".txt"
2314             d = self.POST(self.public_url +
2315                           "/foo?t=upload&format=" + format,
2316                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2317             def _got_filecap(filecap):
2318                 if fn is not None:
2319                     filenameu = unicode(filename)
2320                     self.failUnlessURIMatchesRWChild(filecap, fn, filenameu)
2321                 self.failUnless(filecap.startswith(uri_prefix))
2322                 return self.GET(self.public_url + "/foo/%s?t=json" % filename)
2323             d.addCallback(_got_filecap)
2324             def _got_json(json):
2325                 data = simplejson.loads(json)
2326                 data = data[1]
2327                 self.failUnlessIn("format", data)
2328                 self.failUnlessEqual(data["format"], format.upper())
2329             d.addCallback(_got_json)
2330             return d
2331
2332         d = defer.succeed(None)
2333         d.addCallback(_check_upload, "chk", "URI:CHK")
2334         d.addCallback(_check_upload, "sdmf", "URI:SSK", self._foo_node)
2335         d.addCallback(_check_upload, "mdmf", "URI:MDMF")
2336         d.addCallback(_check_upload, "MDMF", "URI:MDMF")
2337         return d
2338
2339     def test_POST_upload_bad_format(self):
2340         return self.shouldHTTPError("POST_upload_bad_format",
2341                                     400, "Bad Request", "Unknown format: foo",
2342                                     self.POST, self.public_url + \
2343                                     "/foo?t=upload&format=foo",
2344                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2345
2346     def test_POST_upload_mutable(self):
2347         # this creates a mutable file
2348         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
2349                       file=("new.txt", self.NEWFILE_CONTENTS))
2350         fn = self._foo_node
2351         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2352         d.addCallback(lambda res:
2353                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2354                                                              self.NEWFILE_CONTENTS))
2355         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2356         def _got(newnode):
2357             self.failUnless(IMutableFileNode.providedBy(newnode))
2358             self.failUnless(newnode.is_mutable())
2359             self.failIf(newnode.is_readonly())
2360             self._mutable_node = newnode
2361             self._mutable_uri = newnode.get_uri()
2362         d.addCallback(_got)
2363
2364         # now upload it again and make sure that the URI doesn't change
2365         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
2366         d.addCallback(lambda res:
2367                       self.POST(self.public_url + "/foo", t="upload",
2368                                 mutable="true",
2369                                 file=("new.txt", NEWER_CONTENTS)))
2370         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2371         d.addCallback(lambda res:
2372                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2373                                                              NEWER_CONTENTS))
2374         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2375         def _got2(newnode):
2376             self.failUnless(IMutableFileNode.providedBy(newnode))
2377             self.failUnless(newnode.is_mutable())
2378             self.failIf(newnode.is_readonly())
2379             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2380         d.addCallback(_got2)
2381
2382         # upload a second time, using PUT instead of POST
2383         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
2384         d.addCallback(lambda res:
2385                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
2386         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2387         d.addCallback(lambda res:
2388                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2389                                                              NEW2_CONTENTS))
2390
2391         # finally list the directory, since mutable files are displayed
2392         # slightly differently
2393
2394         d.addCallback(lambda res:
2395                       self.GET(self.public_url + "/foo/",
2396                                followRedirect=True))
2397         def _check_page(res):
2398             # TODO: assert more about the contents
2399             self.failUnlessIn("SSK", res)
2400             return res
2401         d.addCallback(_check_page)
2402
2403         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2404         def _got3(newnode):
2405             self.failUnless(IMutableFileNode.providedBy(newnode))
2406             self.failUnless(newnode.is_mutable())
2407             self.failIf(newnode.is_readonly())
2408             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2409         d.addCallback(_got3)
2410
2411         # look at the JSON form of the enclosing directory
2412         d.addCallback(lambda res:
2413                       self.GET(self.public_url + "/foo/?t=json",
2414                                followRedirect=True))
2415         def _check_page_json(res):
2416             parsed = simplejson.loads(res)
2417             self.failUnlessEqual(parsed[0], "dirnode")
2418             children = dict( [(unicode(name),value)
2419                               for (name,value)
2420                               in parsed[1]["children"].iteritems()] )
2421             self.failUnlessIn(u"new.txt", children)
2422             new_json = children[u"new.txt"]
2423             self.failUnlessEqual(new_json[0], "filenode")
2424             self.failUnless(new_json[1]["mutable"])
2425             self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri)
2426             ro_uri = self._mutable_node.get_readonly().to_string()
2427             self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri)
2428         d.addCallback(_check_page_json)
2429
2430         # and the JSON form of the file
2431         d.addCallback(lambda res:
2432                       self.GET(self.public_url + "/foo/new.txt?t=json"))
2433         def _check_file_json(res):
2434             parsed = simplejson.loads(res)
2435             self.failUnlessEqual(parsed[0], "filenode")
2436             self.failUnless(parsed[1]["mutable"])
2437             self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri)
2438             ro_uri = self._mutable_node.get_readonly().to_string()
2439             self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri)
2440         d.addCallback(_check_file_json)
2441
2442         # and look at t=uri and t=readonly-uri
2443         d.addCallback(lambda res:
2444                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
2445         d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
2446         d.addCallback(lambda res:
2447                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
2448         def _check_ro_uri(res):
2449             ro_uri = self._mutable_node.get_readonly().to_string()
2450             self.failUnlessReallyEqual(res, ro_uri)
2451         d.addCallback(_check_ro_uri)
2452
2453         # make sure we can get to it from /uri/URI
2454         d.addCallback(lambda res:
2455                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
2456         d.addCallback(lambda res:
2457                       self.failUnlessReallyEqual(res, NEW2_CONTENTS))
2458
2459         # and that HEAD computes the size correctly
2460         d.addCallback(lambda res:
2461                       self.HEAD(self.public_url + "/foo/new.txt",
2462                                 return_response=True))
2463         def _got_headers((res, status, headers)):
2464             self.failUnlessReallyEqual(res, "")
2465             self.failUnlessReallyEqual(headers["content-length"][0],
2466                                        str(len(NEW2_CONTENTS)))
2467             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
2468         d.addCallback(_got_headers)
2469
2470         # make sure that outdated size limits aren't enforced anymore.
2471         d.addCallback(lambda ignored:
2472             self.POST(self.public_url + "/foo", t="upload",
2473                       mutable="true",
2474                       file=("new.txt",
2475                             "b" * (self.s.MUTABLE_SIZELIMIT+1))))
2476         d.addErrback(self.dump_error)
2477         return d
2478
2479     def test_POST_upload_mutable_toobig(self):
2480         # SDMF had a size limti that was removed a while ago. MDMF has
2481         # never had a size limit. Test to make sure that we do not
2482         # encounter errors when trying to upload large mutable files,
2483         # since there should be no coded prohibitions regarding large
2484         # mutable files.
2485         d = self.POST(self.public_url + "/foo",
2486                       t="upload", mutable="true",
2487                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2488         return d
2489
2490     def dump_error(self, f):
2491         # if the web server returns an error code (like 400 Bad Request),
2492         # web.client.getPage puts the HTTP response body into the .response
2493         # attribute of the exception object that it gives back. It does not
2494         # appear in the Failure's repr(), so the ERROR that trial displays
2495         # will be rather terse and unhelpful. addErrback this method to the
2496         # end of your chain to get more information out of these errors.
2497         if f.check(error.Error):
2498             print "web.error.Error:"
2499             print f
2500             print f.value.response
2501         return f
2502
2503     def test_POST_upload_replace(self):
2504         d = self.POST(self.public_url + "/foo", t="upload",
2505                       file=("bar.txt", self.NEWFILE_CONTENTS))
2506         fn = self._foo_node
2507         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
2508         d.addCallback(lambda res:
2509                       self.failUnlessChildContentsAre(fn, u"bar.txt",
2510                                                       self.NEWFILE_CONTENTS))
2511         return d
2512
2513     def test_POST_upload_no_replace_ok(self):
2514         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2515                       file=("new.txt", self.NEWFILE_CONTENTS))
2516         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
2517         d.addCallback(lambda res: self.failUnlessReallyEqual(res,
2518                                                              self.NEWFILE_CONTENTS))
2519         return d
2520
2521     def test_POST_upload_no_replace_queryarg(self):
2522         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2523                       file=("bar.txt", self.NEWFILE_CONTENTS))
2524         d.addBoth(self.shouldFail, error.Error,
2525                   "POST_upload_no_replace_queryarg",
2526                   "409 Conflict",
2527                   "There was already a child by that name, and you asked me "
2528                   "to not replace it")
2529         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2530         d.addCallback(self.failUnlessIsBarDotTxt)
2531         return d
2532
2533     def test_POST_upload_no_replace_field(self):
2534         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
2535                       file=("bar.txt", self.NEWFILE_CONTENTS))
2536         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
2537                   "409 Conflict",
2538                   "There was already a child by that name, and you asked me "
2539                   "to not replace it")
2540         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2541         d.addCallback(self.failUnlessIsBarDotTxt)
2542         return d
2543
2544     def test_POST_upload_whendone(self):
2545         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
2546                       file=("new.txt", self.NEWFILE_CONTENTS))
2547         d.addBoth(self.shouldRedirect, "/THERE")
2548         fn = self._foo_node
2549         d.addCallback(lambda res:
2550                       self.failUnlessChildContentsAre(fn, u"new.txt",
2551                                                       self.NEWFILE_CONTENTS))
2552         return d
2553
2554     def test_POST_upload_named(self):
2555         fn = self._foo_node
2556         d = self.POST(self.public_url + "/foo", t="upload",
2557                       name="new.txt", file=self.NEWFILE_CONTENTS)
2558         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2559         d.addCallback(lambda res:
2560                       self.failUnlessChildContentsAre(fn, u"new.txt",
2561                                                       self.NEWFILE_CONTENTS))
2562         return d
2563
2564     def test_POST_upload_named_badfilename(self):
2565         d = self.POST(self.public_url + "/foo", t="upload",
2566                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
2567         d.addBoth(self.shouldFail, error.Error,
2568                   "test_POST_upload_named_badfilename",
2569                   "400 Bad Request",
2570                   "name= may not contain a slash",
2571                   )
2572         # make sure that nothing was added
2573         d.addCallback(lambda res:
2574                       self.failUnlessNodeKeysAre(self._foo_node,
2575                                                  [self._htmlname_unicode,
2576                                                   u"bar.txt", u"baz.txt", u"blockingfile",
2577                                                   u"empty", u"n\u00fc.txt", u"quux.txt",
2578                                                   u"sub"]))
2579         return d
2580
2581     def test_POST_FILEURL_check(self):
2582         bar_url = self.public_url + "/foo/bar.txt"
2583         d = self.POST(bar_url, t="check")
2584         def _check(res):
2585             self.failUnlessIn("Healthy :", res)
2586         d.addCallback(_check)
2587         redir_url = "http://allmydata.org/TARGET"
2588         def _check2(statuscode, target):
2589             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2590             self.failUnlessReallyEqual(target, redir_url)
2591         d.addCallback(lambda res:
2592                       self.shouldRedirect2("test_POST_FILEURL_check",
2593                                            _check2,
2594                                            self.POST, bar_url,
2595                                            t="check",
2596                                            when_done=redir_url))
2597         d.addCallback(lambda res:
2598                       self.POST(bar_url, t="check", return_to=redir_url))
2599         def _check3(res):
2600             self.failUnlessIn("Healthy :", res)
2601             self.failUnlessIn("Return to file", res)
2602             self.failUnlessIn(redir_url, res)
2603         d.addCallback(_check3)
2604
2605         d.addCallback(lambda res:
2606                       self.POST(bar_url, t="check", output="JSON"))
2607         def _check_json(res):
2608             data = simplejson.loads(res)
2609             self.failUnlessIn("storage-index", data)
2610             self.failUnless(data["results"]["healthy"])
2611         d.addCallback(_check_json)
2612
2613         return d
2614
2615     def test_POST_FILEURL_check_and_repair(self):
2616         bar_url = self.public_url + "/foo/bar.txt"
2617         d = self.POST(bar_url, t="check", repair="true")
2618         def _check(res):
2619             self.failUnlessIn("Healthy :", res)
2620         d.addCallback(_check)
2621         redir_url = "http://allmydata.org/TARGET"
2622         def _check2(statuscode, target):
2623             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2624             self.failUnlessReallyEqual(target, redir_url)
2625         d.addCallback(lambda res:
2626                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
2627                                            _check2,
2628                                            self.POST, bar_url,
2629                                            t="check", repair="true",
2630                                            when_done=redir_url))
2631         d.addCallback(lambda res:
2632                       self.POST(bar_url, t="check", return_to=redir_url))
2633         def _check3(res):
2634             self.failUnlessIn("Healthy :", res)
2635             self.failUnlessIn("Return to file", res)
2636             self.failUnlessIn(redir_url, res)
2637         d.addCallback(_check3)
2638         return d
2639
2640     def test_POST_DIRURL_check(self):
2641         foo_url = self.public_url + "/foo/"
2642         d = self.POST(foo_url, t="check")
2643         def _check(res):
2644             self.failUnlessIn("Healthy :", res)
2645         d.addCallback(_check)
2646         redir_url = "http://allmydata.org/TARGET"
2647         def _check2(statuscode, target):
2648             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2649             self.failUnlessReallyEqual(target, redir_url)
2650         d.addCallback(lambda res:
2651                       self.shouldRedirect2("test_POST_DIRURL_check",
2652                                            _check2,
2653                                            self.POST, foo_url,
2654                                            t="check",
2655                                            when_done=redir_url))
2656         d.addCallback(lambda res:
2657                       self.POST(foo_url, t="check", return_to=redir_url))
2658         def _check3(res):
2659             self.failUnlessIn("Healthy :", res)
2660             self.failUnlessIn("Return to file/directory", res)
2661             self.failUnlessIn(redir_url, res)
2662         d.addCallback(_check3)
2663
2664         d.addCallback(lambda res:
2665                       self.POST(foo_url, t="check", output="JSON"))
2666         def _check_json(res):
2667             data = simplejson.loads(res)
2668             self.failUnlessIn("storage-index", data)
2669             self.failUnless(data["results"]["healthy"])
2670         d.addCallback(_check_json)
2671
2672         return d
2673
2674     def test_POST_DIRURL_check_and_repair(self):
2675         foo_url = self.public_url + "/foo/"
2676         d = self.POST(foo_url, t="check", repair="true")
2677         def _check(res):
2678             self.failUnlessIn("Healthy :", res)
2679         d.addCallback(_check)
2680         redir_url = "http://allmydata.org/TARGET"
2681         def _check2(statuscode, target):
2682             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2683             self.failUnlessReallyEqual(target, redir_url)
2684         d.addCallback(lambda res:
2685                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2686                                            _check2,
2687                                            self.POST, foo_url,
2688                                            t="check", repair="true",
2689                                            when_done=redir_url))
2690         d.addCallback(lambda res:
2691                       self.POST(foo_url, t="check", return_to=redir_url))
2692         def _check3(res):
2693             self.failUnlessIn("Healthy :", res)
2694             self.failUnlessIn("Return to file/directory", res)
2695             self.failUnlessIn(redir_url, res)
2696         d.addCallback(_check3)
2697         return d
2698
2699     def test_POST_FILEURL_mdmf_check(self):
2700         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2701         d = self.POST(quux_url, t="check")
2702         def _check(res):
2703             self.failUnlessIn("Healthy", res)
2704         d.addCallback(_check)
2705         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2706         d.addCallback(lambda ignored:
2707                       self.POST(quux_extension_url, t="check"))
2708         d.addCallback(_check)
2709         return d
2710
2711     def test_POST_FILEURL_mdmf_check_and_repair(self):
2712         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2713         d = self.POST(quux_url, t="check", repair="true")
2714         def _check(res):
2715             self.failUnlessIn("Healthy", res)
2716         d.addCallback(_check)
2717         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2718         d.addCallback(lambda ignored:
2719                       self.POST(quux_extension_url, t="check", repair="true"))
2720         d.addCallback(_check)
2721         return d
2722
2723     def wait_for_operation(self, ignored, ophandle):
2724         url = "/operations/" + ophandle
2725         url += "?t=status&output=JSON"
2726         d = self.GET(url)
2727         def _got(res):
2728             data = simplejson.loads(res)
2729             if not data["finished"]:
2730                 d = self.stall(delay=1.0)
2731                 d.addCallback(self.wait_for_operation, ophandle)
2732                 return d
2733             return data
2734         d.addCallback(_got)
2735         return d
2736
2737     def get_operation_results(self, ignored, ophandle, output=None):
2738         url = "/operations/" + ophandle
2739         url += "?t=status"
2740         if output:
2741             url += "&output=" + output
2742         d = self.GET(url)
2743         def _got(res):
2744             if output and output.lower() == "json":
2745                 return simplejson.loads(res)
2746             return res
2747         d.addCallback(_got)
2748         return d
2749
2750     def test_POST_DIRURL_deepcheck_no_ophandle(self):
2751         d = self.shouldFail2(error.Error,
2752                              "test_POST_DIRURL_deepcheck_no_ophandle",
2753                              "400 Bad Request",
2754                              "slow operation requires ophandle=",
2755                              self.POST, self.public_url, t="start-deep-check")
2756         return d
2757
2758     def test_POST_DIRURL_deepcheck(self):
2759         def _check_redirect(statuscode, target):
2760             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2761             self.failUnless(target.endswith("/operations/123"))
2762         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2763                                  self.POST, self.public_url,
2764                                  t="start-deep-check", ophandle="123")
2765         d.addCallback(self.wait_for_operation, "123")
2766         def _check_json(data):
2767             self.failUnlessReallyEqual(data["finished"], True)
2768             self.failUnlessReallyEqual(data["count-objects-checked"], 11)
2769             self.failUnlessReallyEqual(data["count-objects-healthy"], 11)
2770         d.addCallback(_check_json)
2771         d.addCallback(self.get_operation_results, "123", "html")
2772         def _check_html(res):
2773             self.failUnlessIn("Objects Checked: <span>11</span>", res)
2774             self.failUnlessIn("Objects Healthy: <span>11</span>", res)
2775             self.failUnlessIn(FAVICON_MARKUP, res)
2776         d.addCallback(_check_html)
2777
2778         d.addCallback(lambda res:
2779                       self.GET("/operations/123/"))
2780         d.addCallback(_check_html) # should be the same as without the slash
2781
2782         d.addCallback(lambda res:
2783                       self.shouldFail2(error.Error, "one", "404 Not Found",
2784                                        "No detailed results for SI bogus",
2785                                        self.GET, "/operations/123/bogus"))
2786
2787         foo_si = self._foo_node.get_storage_index()
2788         foo_si_s = base32.b2a(foo_si)
2789         d.addCallback(lambda res:
2790                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2791         def _check_foo_json(res):
2792             data = simplejson.loads(res)
2793             self.failUnlessEqual(data["storage-index"], foo_si_s)
2794             self.failUnless(data["results"]["healthy"])
2795         d.addCallback(_check_foo_json)
2796         return d
2797
2798     def test_POST_DIRURL_deepcheck_and_repair(self):
2799         d = self.POST(self.public_url, t="start-deep-check", repair="true",
2800                       ophandle="124", output="json", followRedirect=True)
2801         d.addCallback(self.wait_for_operation, "124")
2802         def _check_json(data):
2803             self.failUnlessReallyEqual(data["finished"], True)
2804             self.failUnlessReallyEqual(data["count-objects-checked"], 11)
2805             self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 11)
2806             self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2807             self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2808             self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2809             self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2810             self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2811             self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 11)
2812             self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2813             self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2814         d.addCallback(_check_json)
2815         d.addCallback(self.get_operation_results, "124", "html")
2816         def _check_html(res):
2817             self.failUnlessIn("Objects Checked: <span>11</span>", res)
2818
2819             self.failUnlessIn("Objects Healthy (before repair): <span>11</span>", res)
2820             self.failUnlessIn("Objects Unhealthy (before repair): <span>0</span>", res)
2821             self.failUnlessIn("Corrupt Shares (before repair): <span>0</span>", res)
2822
2823             self.failUnlessIn("Repairs Attempted: <span>0</span>", res)
2824             self.failUnlessIn("Repairs Successful: <span>0</span>", res)
2825             self.failUnlessIn("Repairs Unsuccessful: <span>0</span>", res)
2826
2827             self.failUnlessIn("Objects Healthy (after repair): <span>11</span>", res)
2828             self.failUnlessIn("Objects Unhealthy (after repair): <span>0</span>", res)
2829             self.failUnlessIn("Corrupt Shares (after repair): <span>0</span>", res)
2830
2831             self.failUnlessIn(FAVICON_MARKUP, res)
2832         d.addCallback(_check_html)
2833         return d
2834
2835     def test_POST_FILEURL_bad_t(self):
2836         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2837                              "POST to file: bad t=bogus",
2838                              self.POST, self.public_url + "/foo/bar.txt",
2839                              t="bogus")
2840         return d
2841
2842     def test_POST_mkdir(self): # return value?
2843         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2844         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2845         d.addCallback(self.failUnlessNodeKeysAre, [])
2846         return d
2847
2848     def test_POST_mkdir_mdmf(self):
2849         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=mdmf")
2850         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2851         d.addCallback(lambda node:
2852             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2853         return d
2854
2855     def test_POST_mkdir_sdmf(self):
2856         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=sdmf")
2857         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2858         d.addCallback(lambda node:
2859             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2860         return d
2861
2862     def test_POST_mkdir_bad_format(self):
2863         return self.shouldHTTPError("POST_mkdir_bad_format",
2864                                     400, "Bad Request", "Unknown format: foo",
2865                                     self.POST, self.public_url +
2866                                     "/foo?t=mkdir&name=newdir&format=foo")
2867
2868     def test_POST_mkdir_initial_children(self):
2869         (newkids, caps) = self._create_initial_children()
2870         d = self.POST2(self.public_url +
2871                        "/foo?t=mkdir-with-children&name=newdir",
2872                        simplejson.dumps(newkids))
2873         d.addCallback(lambda res:
2874                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2875         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2876         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2877         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2878         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2879         return d
2880
2881     def test_POST_mkdir_initial_children_mdmf(self):
2882         (newkids, caps) = self._create_initial_children()
2883         d = self.POST2(self.public_url +
2884                        "/foo?t=mkdir-with-children&name=newdir&format=mdmf",
2885                        simplejson.dumps(newkids))
2886         d.addCallback(lambda res:
2887                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2888         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2889         d.addCallback(lambda node:
2890             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2891         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2892         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2893                        caps['filecap1'])
2894         return d
2895
2896     # XXX: Duplication.
2897     def test_POST_mkdir_initial_children_sdmf(self):
2898         (newkids, caps) = self._create_initial_children()
2899         d = self.POST2(self.public_url +
2900                        "/foo?t=mkdir-with-children&name=newdir&format=sdmf",
2901                        simplejson.dumps(newkids))
2902         d.addCallback(lambda res:
2903                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2904         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2905         d.addCallback(lambda node:
2906             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2907         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2908         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2909                        caps['filecap1'])
2910         return d
2911
2912     def test_POST_mkdir_initial_children_bad_format(self):
2913         (newkids, caps) = self._create_initial_children()
2914         return self.shouldHTTPError("POST_mkdir_initial_children_bad_format",
2915                                     400, "Bad Request", "Unknown format: foo",
2916                                     self.POST, self.public_url + \
2917                                     "/foo?t=mkdir-with-children&name=newdir&format=foo",
2918                                     simplejson.dumps(newkids))
2919
2920     def test_POST_mkdir_immutable(self):
2921         (newkids, caps) = self._create_immutable_children()
2922         d = self.POST2(self.public_url +
2923                        "/foo?t=mkdir-immutable&name=newdir",
2924                        simplejson.dumps(newkids))
2925         d.addCallback(lambda res:
2926                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2927         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2928         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2929         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2930         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2931         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2932         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2933         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2934         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2935         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2936         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2937         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2938         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2939         return d
2940
2941     def test_POST_mkdir_immutable_bad(self):
2942         (newkids, caps) = self._create_initial_children()
2943         d = self.shouldFail2(error.Error, "POST_mkdir_immutable_bad",
2944                              "400 Bad Request",
2945                              "needed to be immutable but was not",
2946                              self.POST2,
2947                              self.public_url +
2948                              "/foo?t=mkdir-immutable&name=newdir",
2949                              simplejson.dumps(newkids))
2950         return d
2951
2952     def test_POST_mkdir_2(self):
2953         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
2954         d.addCallback(lambda res:
2955                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2956         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2957         d.addCallback(self.failUnlessNodeKeysAre, [])
2958         return d
2959
2960     def test_POST_mkdirs_2(self):
2961         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
2962         d.addCallback(lambda res:
2963                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
2964         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
2965         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
2966         d.addCallback(self.failUnlessNodeKeysAre, [])
2967         return d
2968
2969     def test_POST_mkdir_no_parentdir_noredirect(self):
2970         d = self.POST("/uri?t=mkdir")
2971         def _after_mkdir(res):
2972             uri.DirectoryURI.init_from_string(res)
2973         d.addCallback(_after_mkdir)
2974         return d
2975
2976     def test_POST_mkdir_no_parentdir_noredirect_mdmf(self):
2977         d = self.POST("/uri?t=mkdir&format=mdmf")
2978         def _after_mkdir(res):
2979             u = uri.from_string(res)
2980             # Check that this is an MDMF writecap
2981             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
2982         d.addCallback(_after_mkdir)
2983         return d
2984
2985     def test_POST_mkdir_no_parentdir_noredirect_sdmf(self):
2986         d = self.POST("/uri?t=mkdir&format=sdmf")
2987         def _after_mkdir(res):
2988             u = uri.from_string(res)
2989             self.failUnlessIsInstance(u, uri.DirectoryURI)
2990         d.addCallback(_after_mkdir)
2991         return d
2992
2993     def test_POST_mkdir_no_parentdir_noredirect_bad_format(self):
2994         return self.shouldHTTPError("POST_mkdir_no_parentdir_noredirect_bad_format",
2995                                     400, "Bad Request", "Unknown format: foo",
2996                                     self.POST, self.public_url +
2997                                     "/uri?t=mkdir&format=foo")
2998
2999     def test_POST_mkdir_no_parentdir_noredirect2(self):
3000         # make sure form-based arguments (as on the welcome page) still work
3001         d = self.POST("/uri", t="mkdir")
3002         def _after_mkdir(res):
3003             uri.DirectoryURI.init_from_string(res)
3004         d.addCallback(_after_mkdir)
3005         d.addErrback(self.explain_web_error)
3006         return d
3007
3008     def test_POST_mkdir_no_parentdir_redirect(self):
3009         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
3010         d.addBoth(self.shouldRedirect, None, statuscode='303')
3011         def _check_target(target):
3012             target = urllib.unquote(target)
3013             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
3014         d.addCallback(_check_target)
3015         return d
3016
3017     def test_POST_mkdir_no_parentdir_redirect2(self):
3018         d = self.POST("/uri", t="mkdir", redirect_to_result="true")
3019         d.addBoth(self.shouldRedirect, None, statuscode='303')
3020         def _check_target(target):
3021             target = urllib.unquote(target)
3022             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
3023         d.addCallback(_check_target)
3024         d.addErrback(self.explain_web_error)
3025         return d
3026
3027     def _make_readonly(self, u):
3028         ro_uri = uri.from_string(u).get_readonly()
3029         if ro_uri is None:
3030             return None
3031         return ro_uri.to_string()
3032
3033     def _create_initial_children(self):
3034         contents, n, filecap1 = self.makefile(12)
3035         md1 = {"metakey1": "metavalue1"}
3036         filecap2 = make_mutable_file_uri()
3037         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
3038         filecap3 = node3.get_readonly_uri()
3039         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
3040         dircap = DirectoryNode(node4, None, None).get_uri()
3041         mdmfcap = make_mutable_file_uri(mdmf=True)
3042         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
3043         emptydircap = "URI:DIR2-LIT:"
3044         newkids = {u"child-imm":        ["filenode", {"rw_uri": filecap1,
3045                                                       "ro_uri": self._make_readonly(filecap1),
3046                                                       "metadata": md1, }],
3047                    u"child-mutable":    ["filenode", {"rw_uri": filecap2,
3048                                                       "ro_uri": self._make_readonly(filecap2)}],
3049                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
3050                    u"unknownchild-rw":  ["unknown",  {"rw_uri": unknown_rwcap,
3051                                                       "ro_uri": unknown_rocap}],
3052                    u"unknownchild-ro":  ["unknown",  {"ro_uri": unknown_rocap}],
3053                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
3054                    u"dirchild":         ["dirnode",  {"rw_uri": dircap,
3055                                                       "ro_uri": self._make_readonly(dircap)}],
3056                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
3057                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
3058                    u"child-mutable-mdmf": ["filenode", {"rw_uri": mdmfcap,
3059                                                         "ro_uri": self._make_readonly(mdmfcap)}],
3060                    }
3061         return newkids, {'filecap1': filecap1,
3062                          'filecap2': filecap2,
3063                          'filecap3': filecap3,
3064                          'unknown_rwcap': unknown_rwcap,
3065                          'unknown_rocap': unknown_rocap,
3066                          'unknown_immcap': unknown_immcap,
3067                          'dircap': dircap,
3068                          'litdircap': litdircap,
3069                          'emptydircap': emptydircap,
3070                          'mdmfcap': mdmfcap}
3071
3072     def _create_immutable_children(self):
3073         contents, n, filecap1 = self.makefile(12)
3074         md1 = {"metakey1": "metavalue1"}
3075         tnode = create_chk_filenode("immutable directory contents\n"*10,
3076                                     self.get_all_contents())
3077         dnode = DirectoryNode(tnode, None, None)
3078         assert not dnode.is_mutable()
3079         immdircap = dnode.get_uri()
3080         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
3081         emptydircap = "URI:DIR2-LIT:"
3082         newkids = {u"child-imm":        ["filenode", {"ro_uri": filecap1,
3083                                                       "metadata": md1, }],
3084                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
3085                    u"dirchild-imm":     ["dirnode",  {"ro_uri": immdircap}],
3086                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
3087                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
3088                    }
3089         return newkids, {'filecap1': filecap1,
3090                          'unknown_immcap': unknown_immcap,
3091                          'immdircap': immdircap,
3092                          'litdircap': litdircap,
3093                          'emptydircap': emptydircap}
3094
3095     def test_POST_mkdir_no_parentdir_initial_children(self):
3096         (newkids, caps) = self._create_initial_children()
3097         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
3098         def _after_mkdir(res):
3099             self.failUnless(res.startswith("URI:DIR"), res)
3100             n = self.s.create_node_from_uri(res)
3101             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3102             d2.addCallback(lambda ign:
3103                            self.failUnlessROChildURIIs(n, u"child-imm",
3104                                                        caps['filecap1']))
3105             d2.addCallback(lambda ign:
3106                            self.failUnlessRWChildURIIs(n, u"child-mutable",
3107                                                        caps['filecap2']))
3108             d2.addCallback(lambda ign:
3109                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
3110                                                        caps['filecap3']))
3111             d2.addCallback(lambda ign:
3112                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
3113                                                        caps['unknown_rwcap']))
3114             d2.addCallback(lambda ign:
3115                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
3116                                                        caps['unknown_rocap']))
3117             d2.addCallback(lambda ign:
3118                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3119                                                        caps['unknown_immcap']))
3120             d2.addCallback(lambda ign:
3121                            self.failUnlessRWChildURIIs(n, u"dirchild",
3122                                                        caps['dircap']))
3123             return d2
3124         d.addCallback(_after_mkdir)
3125         return d
3126
3127     def test_POST_mkdir_no_parentdir_unexpected_children(self):
3128         # the regular /uri?t=mkdir operation is specified to ignore its body.
3129         # Only t=mkdir-with-children pays attention to it.
3130         (newkids, caps) = self._create_initial_children()
3131         d = self.shouldHTTPError("POST_mkdir_no_parentdir_unexpected_children",
3132                                  400, "Bad Request",
3133                                  "t=mkdir does not accept children=, "
3134                                  "try t=mkdir-with-children instead",
3135                                  self.POST2, "/uri?t=mkdir", # without children
3136                                  simplejson.dumps(newkids))
3137         return d
3138
3139     def test_POST_noparent_bad(self):
3140         d = self.shouldHTTPError("POST_noparent_bad",
3141                                  400, "Bad Request",
3142                                  "/uri accepts only PUT, PUT?t=mkdir, "
3143                                  "POST?t=upload, and POST?t=mkdir",
3144                                  self.POST, "/uri?t=bogus")
3145         return d
3146
3147     def test_POST_mkdir_no_parentdir_immutable(self):
3148         (newkids, caps) = self._create_immutable_children()
3149         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
3150         def _after_mkdir(res):
3151             self.failUnless(res.startswith("URI:DIR"), res)
3152             n = self.s.create_node_from_uri(res)
3153             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3154             d2.addCallback(lambda ign:
3155                            self.failUnlessROChildURIIs(n, u"child-imm",
3156                                                           caps['filecap1']))
3157             d2.addCallback(lambda ign:
3158                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3159                                                           caps['unknown_immcap']))
3160             d2.addCallback(lambda ign:
3161                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
3162                                                           caps['immdircap']))
3163             d2.addCallback(lambda ign:
3164                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
3165                                                           caps['litdircap']))
3166             d2.addCallback(lambda ign:
3167                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
3168                                                           caps['emptydircap']))
3169             return d2
3170         d.addCallback(_after_mkdir)
3171         return d
3172
3173     def test_POST_mkdir_no_parentdir_immutable_bad(self):
3174         (newkids, caps) = self._create_initial_children()
3175         d = self.shouldFail2(error.Error,
3176                              "test_POST_mkdir_no_parentdir_immutable_bad",
3177                              "400 Bad Request",
3178                              "needed to be immutable but was not",
3179                              self.POST2,
3180                              "/uri?t=mkdir-immutable",
3181                              simplejson.dumps(newkids))
3182         return d
3183
3184     def test_welcome_page_mkdir_button(self):
3185         # Fetch the welcome page.
3186         d = self.GET("/")
3187         def _after_get_welcome_page(res):
3188             MKDIR_BUTTON_RE = re.compile(
3189                 '<form action="([^"]*)" method="post".*?'
3190                 '<input type="hidden" name="t" value="([^"]*)" />'
3191                 '<input type="hidden" name="([^"]*)" value="([^"]*)" />'
3192                 '<input type="submit" value="Create a directory" />',
3193                 re.I)
3194             mo = MKDIR_BUTTON_RE.search(res)
3195             formaction = mo.group(1)
3196             formt = mo.group(2)
3197             formaname = mo.group(3)
3198             formavalue = mo.group(4)
3199             return (formaction, formt, formaname, formavalue)
3200         d.addCallback(_after_get_welcome_page)
3201         def _after_parse_form(res):
3202             (formaction, formt, formaname, formavalue) = res
3203             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
3204         d.addCallback(_after_parse_form)
3205         d.addBoth(self.shouldRedirect, None, statuscode='303')
3206         return d
3207
3208     def test_POST_mkdir_replace(self): # return value?
3209         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
3210         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3211         d.addCallback(self.failUnlessNodeKeysAre, [])
3212         return d
3213
3214     def test_POST_mkdir_no_replace_queryarg(self): # return value?
3215         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
3216         d.addBoth(self.shouldFail, error.Error,
3217                   "POST_mkdir_no_replace_queryarg",
3218                   "409 Conflict",
3219                   "There was already a child by that name, and you asked me "
3220                   "to not replace it")
3221         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3222         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3223         return d
3224
3225     def test_POST_mkdir_no_replace_field(self): # return value?
3226         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
3227                       replace="false")
3228         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
3229                   "409 Conflict",
3230                   "There was already a child by that name, and you asked me "
3231                   "to not replace it")
3232         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3233         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3234         return d
3235
3236     def test_POST_mkdir_whendone_field(self):
3237         d = self.POST(self.public_url + "/foo",
3238                       t="mkdir", name="newdir", when_done="/THERE")
3239         d.addBoth(self.shouldRedirect, "/THERE")
3240         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3241         d.addCallback(self.failUnlessNodeKeysAre, [])
3242         return d
3243
3244     def test_POST_mkdir_whendone_queryarg(self):
3245         d = self.POST(self.public_url + "/foo?when_done=/THERE",
3246                       t="mkdir", name="newdir")
3247         d.addBoth(self.shouldRedirect, "/THERE")
3248         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3249         d.addCallback(self.failUnlessNodeKeysAre, [])
3250         return d
3251
3252     def test_POST_bad_t(self):
3253         d = self.shouldFail2(error.Error, "POST_bad_t",
3254                              "400 Bad Request",
3255                              "POST to a directory with bad t=BOGUS",
3256                              self.POST, self.public_url + "/foo", t="BOGUS")
3257         return d
3258
3259     def test_POST_set_children(self, command_name="set_children"):
3260         contents9, n9, newuri9 = self.makefile(9)
3261         contents10, n10, newuri10 = self.makefile(10)
3262         contents11, n11, newuri11 = self.makefile(11)
3263
3264         reqbody = """{
3265                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
3266                                                 "size": 0,
3267                                                 "metadata": {
3268                                                   "ctime": 1002777696.7564139,
3269                                                   "mtime": 1002777696.7564139
3270                                                  }
3271                                                } ],
3272                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
3273                                                 "size": 1,
3274                                                 "metadata": {
3275                                                   "ctime": 1002777696.7564139,
3276                                                   "mtime": 1002777696.7564139
3277                                                  }
3278                                                } ],
3279                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
3280                                                 "size": 2,
3281                                                 "metadata": {
3282                                                   "ctime": 1002777696.7564139,
3283                                                   "mtime": 1002777696.7564139
3284                                                  }
3285                                                } ]
3286                     }""" % (newuri9, newuri10, newuri11)
3287
3288         url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
3289
3290         d = client.getPage(url, method="POST", postdata=reqbody)
3291         def _then(res):
3292             self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
3293             self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
3294             self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
3295
3296         d.addCallback(_then)
3297         d.addErrback(self.dump_error)
3298         return d
3299
3300     def test_POST_set_children_with_hyphen(self):
3301         return self.test_POST_set_children(command_name="set-children")
3302
3303     def test_POST_link_uri(self):
3304         contents, n, newuri = self.makefile(8)
3305         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
3306         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
3307         d.addCallback(lambda res:
3308                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3309                                                       contents))
3310         return d
3311
3312     def test_POST_link_uri_replace(self):
3313         contents, n, newuri = self.makefile(8)
3314         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
3315         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
3316         d.addCallback(lambda res:
3317                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3318                                                       contents))
3319         return d
3320
3321     def test_POST_link_uri_unknown_bad(self):
3322         d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
3323         d.addBoth(self.shouldFail, error.Error,
3324                   "POST_link_uri_unknown_bad",
3325                   "400 Bad Request",
3326                   "unknown cap in a write slot")
3327         return d
3328
3329     def test_POST_link_uri_unknown_ro_good(self):
3330         d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
3331         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
3332         return d
3333
3334     def test_POST_link_uri_unknown_imm_good(self):
3335         d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
3336         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
3337         return d
3338
3339     def test_POST_link_uri_no_replace_queryarg(self):
3340         contents, n, newuri = self.makefile(8)
3341         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
3342                       name="bar.txt", uri=newuri)
3343         d.addBoth(self.shouldFail, error.Error,
3344                   "POST_link_uri_no_replace_queryarg",
3345                   "409 Conflict",
3346                   "There was already a child by that name, and you asked me "
3347                   "to not replace it")
3348         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3349         d.addCallback(self.failUnlessIsBarDotTxt)
3350         return d
3351
3352     def test_POST_link_uri_no_replace_field(self):
3353         contents, n, newuri = self.makefile(8)
3354         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
3355                       name="bar.txt", uri=newuri)
3356         d.addBoth(self.shouldFail, error.Error,
3357                   "POST_link_uri_no_replace_field",
3358                   "409 Conflict",
3359                   "There was already a child by that name, and you asked me "
3360                   "to not replace it")
3361         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3362         d.addCallback(self.failUnlessIsBarDotTxt)
3363         return d
3364
3365     def test_POST_delete(self, command_name='delete'):
3366         d = self._foo_node.list()
3367         def _check_before(children):
3368             self.failUnlessIn(u"bar.txt", children)
3369         d.addCallback(_check_before)
3370         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t=command_name, name="bar.txt"))
3371         d.addCallback(lambda res: self._foo_node.list())
3372         def _check_after(children):
3373             self.failIfIn(u"bar.txt", children)
3374         d.addCallback(_check_after)
3375         return d
3376
3377     def test_POST_unlink(self):
3378         return self.test_POST_delete(command_name='unlink')
3379
3380     def test_POST_rename_file(self):
3381         d = self.POST(self.public_url + "/foo", t="rename",
3382                       from_name="bar.txt", to_name='wibble.txt')
3383         d.addCallback(lambda res:
3384                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3385         d.addCallback(lambda res:
3386                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
3387         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
3388         d.addCallback(self.failUnlessIsBarDotTxt)
3389         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
3390         d.addCallback(self.failUnlessIsBarJSON)
3391         return d
3392
3393     def test_POST_rename_file_redundant(self):
3394         d = self.POST(self.public_url + "/foo", t="rename",
3395                       from_name="bar.txt", to_name='bar.txt')
3396         d.addCallback(lambda res:
3397                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3398         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3399         d.addCallback(self.failUnlessIsBarDotTxt)
3400         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3401         d.addCallback(self.failUnlessIsBarJSON)
3402         return d
3403
3404     def test_POST_rename_file_replace(self):
3405         # rename a file and replace a directory with it
3406         d = self.POST(self.public_url + "/foo", t="rename",
3407                       from_name="bar.txt", to_name='empty')
3408         d.addCallback(lambda res:
3409                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3410         d.addCallback(lambda res:
3411                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
3412         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
3413         d.addCallback(self.failUnlessIsBarDotTxt)
3414         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3415         d.addCallback(self.failUnlessIsBarJSON)
3416         return d
3417
3418     def test_POST_rename_file_no_replace_queryarg(self):
3419         # rename a file and replace a directory with it
3420         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
3421                       from_name="bar.txt", to_name='empty')
3422         d.addBoth(self.shouldFail, error.Error,
3423                   "POST_rename_file_no_replace_queryarg",
3424                   "409 Conflict",
3425                   "There was already a child by that name, and you asked me "
3426                   "to not replace it")
3427         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3428         d.addCallback(self.failUnlessIsEmptyJSON)
3429         return d
3430
3431     def test_POST_rename_file_no_replace_field(self):
3432         # rename a file and replace a directory with it
3433         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
3434                       from_name="bar.txt", to_name='empty')
3435         d.addBoth(self.shouldFail, error.Error,
3436                   "POST_rename_file_no_replace_field",
3437                   "409 Conflict",
3438                   "There was already a child by that name, and you asked me "
3439                   "to not replace it")
3440         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3441         d.addCallback(self.failUnlessIsEmptyJSON)
3442         return d
3443
3444     def failUnlessIsEmptyJSON(self, res):
3445         data = simplejson.loads(res)
3446         self.failUnlessEqual(data[0], "dirnode", data)
3447         self.failUnlessReallyEqual(len(data[1]["children"]), 0)
3448
3449     def test_POST_rename_file_slash_fail(self):
3450         d = self.POST(self.public_url + "/foo", t="rename",
3451                       from_name="bar.txt", to_name='kirk/spock.txt')
3452         d.addBoth(self.shouldFail, error.Error,
3453                   "test_POST_rename_file_slash_fail",
3454                   "400 Bad Request",
3455                   "to_name= may not contain a slash",
3456                   )
3457         d.addCallback(lambda res:
3458                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3459         return d
3460
3461     def test_POST_rename_dir(self):
3462         d = self.POST(self.public_url, t="rename",
3463                       from_name="foo", to_name='plunk')
3464         d.addCallback(lambda res:
3465                       self.failIfNodeHasChild(self.public_root, u"foo"))
3466         d.addCallback(lambda res:
3467                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
3468         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
3469         d.addCallback(self.failUnlessIsFooJSON)
3470         return d
3471
3472     def test_POST_move_file(self):
3473         d = self.POST(self.public_url + "/foo", t="move",
3474                       from_name="bar.txt", to_dir="sub")
3475         d.addCallback(lambda res:
3476                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3477         d.addCallback(lambda res:
3478                       self.failUnlessNodeHasChild(self._sub_node, u"bar.txt"))
3479         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3480         d.addCallback(self.failUnlessIsBarDotTxt)
3481         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3482         d.addCallback(self.failUnlessIsBarJSON)
3483         return d
3484
3485     def test_POST_move_file_new_name(self):
3486         d = self.POST(self.public_url + "/foo", t="move",
3487                       from_name="bar.txt", to_name="wibble.txt", to_dir="sub")
3488         d.addCallback(lambda res:
3489                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3490         d.addCallback(lambda res:
3491                       self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3492         d.addCallback(lambda res:
3493                       self.failUnlessNodeHasChild(self._sub_node, u"wibble.txt"))
3494         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt"))
3495         d.addCallback(self.failUnlessIsBarDotTxt)
3496         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt?t=json"))
3497         d.addCallback(self.failUnlessIsBarJSON)
3498         return d
3499
3500     def test_POST_move_file_replace(self):
3501         d = self.POST(self.public_url + "/foo", t="move",
3502                       from_name="bar.txt", to_name="baz.txt", to_dir="sub")
3503         d.addCallback(lambda res:
3504                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3505         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3506         d.addCallback(self.failUnlessIsBarDotTxt)
3507         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt?t=json"))
3508         d.addCallback(self.failUnlessIsBarJSON)
3509         return d
3510
3511     def test_POST_move_file_no_replace(self):
3512         d = self.shouldFail2(error.Error, "POST_move_file_no_replace",
3513                              "409 Conflict",
3514                              "There was already a child by that name, and you asked me to not replace it",
3515                              self.POST, self.public_url + "/foo", t="move",
3516                              replace="false", from_name="bar.txt",
3517                              to_name="baz.txt", to_dir="sub")
3518         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3519         d.addCallback(self.failUnlessIsBarDotTxt)
3520         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3521         d.addCallback(self.failUnlessIsBarJSON)
3522         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3523         d.addCallback(self.failUnlessIsSubBazDotTxt)
3524         return d
3525
3526     def test_POST_move_file_slash_fail(self):
3527         d = self.shouldFail2(error.Error, "test_POST_rename_file_slash_fail",
3528                              "400 Bad Request",
3529                              "to_name= may not contain a slash",
3530                              self.POST, self.public_url + "/foo", t="move",
3531                              from_name="bar.txt",
3532                              to_name="slash/fail.txt", to_dir="sub")
3533         d.addCallback(lambda res:
3534                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3535         d.addCallback(lambda res:
3536                       self.failIfNodeHasChild(self._sub_node, u"slash/fail.txt"))
3537         d.addCallback(lambda ign:
3538                       self.shouldFail2(error.Error,
3539                                        "test_POST_rename_file_slash_fail2",
3540                                        "400 Bad Request",
3541                                        "from_name= may not contain a slash",
3542                                        self.POST, self.public_url + "/foo",
3543                                        t="move",
3544                                        from_name="nope/bar.txt",
3545                                        to_name="fail.txt", to_dir="sub"))
3546         return d
3547
3548     def test_POST_move_file_no_target(self):
3549         d = self.shouldFail2(error.Error, "POST_move_file_no_target",
3550                              "400 Bad Request",
3551                              "move requires from_name and to_dir",
3552                              self.POST, self.public_url + "/foo", t="move",
3553                              from_name="bar.txt", to_name="baz.txt")
3554         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3555         d.addCallback(self.failUnlessIsBarDotTxt)
3556         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3557         d.addCallback(self.failUnlessIsBarJSON)
3558         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3559         d.addCallback(self.failUnlessIsBazDotTxt)
3560         return d
3561
3562     def test_POST_move_file_bad_target_type(self):
3563         d = self.shouldFail2(error.Error, "test_POST_move_file_bad_target_type",
3564                              "400 Bad Request", "invalid target_type parameter",
3565                              self.POST,
3566                              self.public_url + "/foo", t="move",
3567                              target_type="*D", from_name="bar.txt",
3568                              to_dir="sub")
3569         return d
3570
3571     def test_POST_move_file_multi_level(self):
3572         d = self.POST(self.public_url + "/foo/sub/level2?t=mkdir", "")
3573         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t="move",
3574                       from_name="bar.txt", to_dir="sub/level2"))
3575         d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3576         d.addCallback(lambda res: self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3577         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt"))
3578         d.addCallback(self.failUnlessIsBarDotTxt)
3579         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt?t=json"))
3580         d.addCallback(self.failUnlessIsBarJSON)
3581         return d
3582
3583     def test_POST_move_file_to_uri(self):
3584         d = self.POST(self.public_url + "/foo", t="move", target_type="uri",
3585                       from_name="bar.txt", to_dir=self._sub_uri)
3586         d.addCallback(lambda res:
3587                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3588         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3589         d.addCallback(self.failUnlessIsBarDotTxt)
3590         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3591         d.addCallback(self.failUnlessIsBarJSON)
3592         return d
3593
3594     def test_POST_move_file_to_nonexist_dir(self):
3595         d = self.shouldFail2(error.Error, "POST_move_file_to_nonexist_dir",
3596                             "404 Not Found", "No such child: nopechucktesta",
3597                             self.POST, self.public_url + "/foo", t="move",
3598                             from_name="bar.txt", to_dir="nopechucktesta")
3599         return d
3600
3601     def test_POST_move_file_into_file(self):
3602         d = self.shouldFail2(error.Error, "POST_move_file_into_file",
3603                              "400 Bad Request", "to_dir is not a directory",
3604                              self.POST, self.public_url + "/foo", t="move",
3605                              from_name="bar.txt", to_dir="baz.txt")
3606         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3607         d.addCallback(self.failUnlessIsBazDotTxt)
3608         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3609         d.addCallback(self.failUnlessIsBarDotTxt)
3610         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3611         d.addCallback(self.failUnlessIsBarJSON)
3612         return d
3613
3614     def test_POST_move_file_to_bad_uri(self):
3615         d =  self.shouldFail2(error.Error, "POST_move_file_to_bad_uri",
3616                               "400 Bad Request", "to_dir is not a directory",
3617                               self.POST, self.public_url + "/foo", t="move",
3618                               from_name="bar.txt", target_type="uri",
3619                               to_dir="URI:DIR2:mn5jlyjnrjeuydyswlzyui72i:rmneifcj6k6sycjljjhj3f6majsq2zqffydnnul5hfa4j577arma")
3620         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3621         d.addCallback(self.failUnlessIsBarDotTxt)
3622         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3623         d.addCallback(self.failUnlessIsBarJSON)
3624         return d
3625
3626     def test_POST_move_dir(self):
3627         d = self.POST(self.public_url + "/foo", t="move",
3628                       from_name="bar.txt", to_dir="empty")
3629         d.addCallback(lambda res: self.POST(self.public_url + "/foo",
3630                       t="move", from_name="empty", to_dir="sub"))
3631         d.addCallback(lambda res:
3632                       self.failIfNodeHasChild(self._foo_node, u"empty"))
3633         d.addCallback(lambda res:
3634                       self.failUnlessNodeHasChild(self._sub_node, u"empty"))
3635         d.addCallback(lambda res:
3636                       self._sub_node.get_child_at_path(u"empty"))
3637         d.addCallback(lambda node:
3638                       self.failUnlessNodeHasChild(node, u"bar.txt"))
3639         d.addCallback(lambda res:
3640                       self.GET(self.public_url + "/foo/sub/empty/bar.txt"))
3641         d.addCallback(self.failUnlessIsBarDotTxt)
3642         return d
3643
3644     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
3645         """ If target is not None then the redirection has to go to target.  If
3646         statuscode is not None then the redirection has to be accomplished with
3647         that HTTP status code."""
3648         if not isinstance(res, failure.Failure):
3649             to_where = (target is None) and "somewhere" or ("to " + target)
3650             self.fail("%s: we were expecting to get redirected %s, not get an"
3651                       " actual page: %s" % (which, to_where, res))
3652         res.trap(error.PageRedirect)
3653         if statuscode is not None:
3654             self.failUnlessReallyEqual(res.value.status, statuscode,
3655                                        "%s: not a redirect" % which)
3656         if target is not None:
3657             # the PageRedirect does not seem to capture the uri= query arg
3658             # properly, so we can't check for it.
3659             realtarget = self.webish_url + target
3660             self.failUnlessReallyEqual(res.value.location, realtarget,
3661                                        "%s: wrong target" % which)
3662         return res.value.location
3663
3664     def test_GET_URI_form(self):
3665         base = "/uri?uri=%s" % self._bar_txt_uri
3666         # this is supposed to give us a redirect to /uri/$URI, plus arguments
3667         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
3668         d = self.GET(base)
3669         d.addBoth(self.shouldRedirect, targetbase)
3670         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
3671         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
3672         d.addCallback(lambda res: self.GET(base+"&t=json"))
3673         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
3674         d.addCallback(self.log, "about to get file by uri")
3675         d.addCallback(lambda res: self.GET(base, followRedirect=True))
3676         d.addCallback(self.failUnlessIsBarDotTxt)
3677         d.addCallback(self.log, "got file by uri, about to get dir by uri")
3678         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
3679                                            followRedirect=True))
3680         d.addCallback(self.failUnlessIsFooJSON)
3681         d.addCallback(self.log, "got dir by uri")
3682
3683         return d
3684
3685     def test_GET_URI_form_bad(self):
3686         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
3687                              "400 Bad Request", "GET /uri requires uri=",
3688                              self.GET, "/uri")
3689         return d
3690
3691     def test_GET_rename_form(self):
3692         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
3693                      followRedirect=True)
3694         def _check(res):
3695             self.failUnlessIn('name="when_done" value="."', res)
3696             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
3697             self.failUnlessIn(FAVICON_MARKUP, res)
3698         d.addCallback(_check)
3699         return d
3700
3701     def log(self, res, msg):
3702         #print "MSG: %s  RES: %s" % (msg, res)
3703         log.msg(msg)
3704         return res
3705
3706     def test_GET_URI_URL(self):
3707         base = "/uri/%s" % self._bar_txt_uri
3708         d = self.GET(base)
3709         d.addCallback(self.failUnlessIsBarDotTxt)
3710         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
3711         d.addCallback(self.failUnlessIsBarDotTxt)
3712         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
3713         d.addCallback(self.failUnlessIsBarDotTxt)
3714         return d
3715
3716     def test_GET_URI_URL_dir(self):
3717         base = "/uri/%s?t=json" % self._foo_uri
3718         d = self.GET(base)
3719         d.addCallback(self.failUnlessIsFooJSON)
3720         return d
3721
3722     def test_GET_URI_URL_missing(self):
3723         base = "/uri/%s" % self._bad_file_uri
3724         d = self.shouldHTTPError("test_GET_URI_URL_missing",
3725                                  http.GONE, None, "NotEnoughSharesError",
3726                                  self.GET, base)
3727         # TODO: how can we exercise both sides of WebDownloadTarget.fail
3728         # here? we must arrange for a download to fail after target.open()
3729         # has been called, and then inspect the response to see that it is
3730         # shorter than we expected.
3731         return d
3732
3733     def test_PUT_DIRURL_uri(self):
3734         d = self.s.create_dirnode()
3735         def _made_dir(dn):
3736             new_uri = dn.get_uri()
3737             # replace /foo with a new (empty) directory
3738             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
3739             d.addCallback(lambda res:
3740                           self.failUnlessReallyEqual(res.strip(), new_uri))
3741             d.addCallback(lambda res:
3742                           self.failUnlessRWChildURIIs(self.public_root,
3743                                                       u"foo",
3744                                                       new_uri))
3745             return d
3746         d.addCallback(_made_dir)
3747         return d
3748
3749     def test_PUT_DIRURL_uri_noreplace(self):
3750         d = self.s.create_dirnode()
3751         def _made_dir(dn):
3752             new_uri = dn.get_uri()
3753             # replace /foo with a new (empty) directory, but ask that
3754             # replace=false, so it should fail
3755             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
3756                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
3757                                  self.PUT,
3758                                  self.public_url + "/foo?t=uri&replace=false",
3759                                  new_uri)
3760             d.addCallback(lambda res:
3761                           self.failUnlessRWChildURIIs(self.public_root,
3762                                                       u"foo",
3763                                                       self._foo_uri))
3764             return d
3765         d.addCallback(_made_dir)
3766         return d
3767
3768     def test_PUT_DIRURL_bad_t(self):
3769         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
3770                              "400 Bad Request", "PUT to a directory",
3771                              self.PUT, self.public_url + "/foo?t=BOGUS", "")
3772         d.addCallback(lambda res:
3773                       self.failUnlessRWChildURIIs(self.public_root,
3774                                                   u"foo",
3775                                                   self._foo_uri))
3776         return d
3777
3778     def test_PUT_NEWFILEURL_uri(self):
3779         contents, n, new_uri = self.makefile(8)
3780         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
3781         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3782         d.addCallback(lambda res:
3783                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3784                                                       contents))
3785         return d
3786
3787     def test_PUT_NEWFILEURL_mdmf(self):
3788         new_contents = self.NEWFILE_CONTENTS * 300000
3789         d = self.PUT(self.public_url + \
3790                      "/foo/mdmf.txt?format=mdmf",
3791                      new_contents)
3792         d.addCallback(lambda ignored:
3793             self.GET(self.public_url + "/foo/mdmf.txt?t=json"))
3794         def _got_json(json):
3795             data = simplejson.loads(json)
3796             data = data[1]
3797             self.failUnlessIn("format", data)
3798             self.failUnlessEqual(data["format"], "MDMF")
3799             self.failUnless(data['rw_uri'].startswith("URI:MDMF"))
3800             self.failUnless(data['ro_uri'].startswith("URI:MDMF"))
3801         d.addCallback(_got_json)
3802         return d
3803
3804     def test_PUT_NEWFILEURL_sdmf(self):
3805         new_contents = self.NEWFILE_CONTENTS * 300000
3806         d = self.PUT(self.public_url + \
3807                      "/foo/sdmf.txt?format=sdmf",
3808                      new_contents)
3809         d.addCallback(lambda ignored:
3810             self.GET(self.public_url + "/foo/sdmf.txt?t=json"))
3811         def _got_json(json):
3812             data = simplejson.loads(json)
3813             data = data[1]
3814             self.failUnlessIn("format", data)
3815             self.failUnlessEqual(data["format"], "SDMF")
3816         d.addCallback(_got_json)
3817         return d
3818
3819     def test_PUT_NEWFILEURL_bad_format(self):
3820         new_contents = self.NEWFILE_CONTENTS * 300000
3821         return self.shouldHTTPError("PUT_NEWFILEURL_bad_format",
3822                                     400, "Bad Request", "Unknown format: foo",
3823                                     self.PUT, self.public_url + \
3824                                     "/foo/foo.txt?format=foo",
3825                                     new_contents)
3826
3827     def test_PUT_NEWFILEURL_uri_replace(self):
3828         contents, n, new_uri = self.makefile(8)
3829         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
3830         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3831         d.addCallback(lambda res:
3832                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3833                                                       contents))
3834         return d
3835
3836     def test_PUT_NEWFILEURL_uri_no_replace(self):
3837         contents, n, new_uri = self.makefile(8)
3838         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
3839         d.addBoth(self.shouldFail, error.Error,
3840                   "PUT_NEWFILEURL_uri_no_replace",
3841                   "409 Conflict",
3842                   "There was already a child by that name, and you asked me "
3843                   "to not replace it")
3844         return d
3845
3846     def test_PUT_NEWFILEURL_uri_unknown_bad(self):
3847         d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
3848         d.addBoth(self.shouldFail, error.Error,
3849                   "POST_put_uri_unknown_bad",
3850                   "400 Bad Request",
3851                   "unknown cap in a write slot")
3852         return d
3853
3854     def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
3855         d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
3856         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3857                       u"put-future-ro.txt")
3858         return d
3859
3860     def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
3861         d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
3862         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3863                       u"put-future-imm.txt")
3864         return d
3865
3866     def test_PUT_NEWFILE_URI(self):
3867         file_contents = "New file contents here\n"
3868         d = self.PUT("/uri", file_contents)
3869         def _check(uri):
3870             assert isinstance(uri, str), uri
3871             self.failUnlessIn(uri, self.get_all_contents())
3872             self.failUnlessReallyEqual(self.get_all_contents()[uri],
3873                                        file_contents)
3874             return self.GET("/uri/%s" % uri)
3875         d.addCallback(_check)
3876         def _check2(res):
3877             self.failUnlessReallyEqual(res, file_contents)
3878         d.addCallback(_check2)
3879         return d
3880
3881     def test_PUT_NEWFILE_URI_not_mutable(self):
3882         file_contents = "New file contents here\n"
3883         d = self.PUT("/uri?mutable=false", file_contents)
3884         def _check(uri):
3885             assert isinstance(uri, str), uri
3886             self.failUnlessIn(uri, self.get_all_contents())
3887             self.failUnlessReallyEqual(self.get_all_contents()[uri],
3888                                        file_contents)
3889             return self.GET("/uri/%s" % uri)
3890         d.addCallback(_check)
3891         def _check2(res):
3892             self.failUnlessReallyEqual(res, file_contents)
3893         d.addCallback(_check2)
3894         return d
3895
3896     def test_PUT_NEWFILE_URI_only_PUT(self):
3897         d = self.PUT("/uri?t=bogus", "")
3898         d.addBoth(self.shouldFail, error.Error,
3899                   "PUT_NEWFILE_URI_only_PUT",
3900                   "400 Bad Request",
3901                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
3902         return d
3903
3904     def test_PUT_NEWFILE_URI_mutable(self):
3905         file_contents = "New file contents here\n"
3906         d = self.PUT("/uri?mutable=true", file_contents)
3907         def _check1(filecap):
3908             filecap = filecap.strip()
3909             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
3910             self.filecap = filecap
3911             u = uri.WriteableSSKFileURI.init_from_string(filecap)
3912             self.failUnlessIn(u.get_storage_index(), self.get_all_contents())
3913             n = self.s.create_node_from_uri(filecap)
3914             return n.download_best_version()
3915         d.addCallback(_check1)
3916         def _check2(data):
3917             self.failUnlessReallyEqual(data, file_contents)
3918             return self.GET("/uri/%s" % urllib.quote(self.filecap))
3919         d.addCallback(_check2)
3920         def _check3(res):
3921             self.failUnlessReallyEqual(res, file_contents)
3922         d.addCallback(_check3)
3923         return d
3924
3925     def test_PUT_mkdir(self):
3926         d = self.PUT("/uri?t=mkdir", "")
3927         def _check(uri):
3928             n = self.s.create_node_from_uri(uri.strip())
3929             d2 = self.failUnlessNodeKeysAre(n, [])
3930             d2.addCallback(lambda res:
3931                            self.GET("/uri/%s?t=json" % uri))
3932             return d2
3933         d.addCallback(_check)
3934         d.addCallback(self.failUnlessIsEmptyJSON)
3935         return d
3936
3937     def test_PUT_mkdir_mdmf(self):
3938         d = self.PUT("/uri?t=mkdir&format=mdmf", "")
3939         def _got(res):
3940             u = uri.from_string(res)
3941             # Check that this is an MDMF writecap
3942             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
3943         d.addCallback(_got)
3944         return d
3945
3946     def test_PUT_mkdir_sdmf(self):
3947         d = self.PUT("/uri?t=mkdir&format=sdmf", "")
3948         def _got(res):
3949             u = uri.from_string(res)
3950             self.failUnlessIsInstance(u, uri.DirectoryURI)
3951         d.addCallback(_got)
3952         return d
3953
3954     def test_PUT_mkdir_bad_format(self):
3955         return self.shouldHTTPError("PUT_mkdir_bad_format",
3956                                     400, "Bad Request", "Unknown format: foo",
3957                                     self.PUT, "/uri?t=mkdir&format=foo",
3958                                     "")
3959
3960     def test_POST_check(self):
3961         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
3962         def _done(res):
3963             # this returns a string form of the results, which are probably
3964             # None since we're using fake filenodes.
3965             # TODO: verify that the check actually happened, by changing
3966             # FakeCHKFileNode to count how many times .check() has been
3967             # called.
3968             pass
3969         d.addCallback(_done)
3970         return d
3971
3972
3973     def test_PUT_update_at_offset(self):
3974         file_contents = "test file" * 100000 # about 900 KiB
3975         d = self.PUT("/uri?mutable=true", file_contents)
3976         def _then(filecap):
3977             self.filecap = filecap
3978             new_data = file_contents[:100]
3979             new = "replaced and so on"
3980             new_data += new
3981             new_data += file_contents[len(new_data):]
3982             assert len(new_data) == len(file_contents)
3983             self.new_data = new_data
3984         d.addCallback(_then)
3985         d.addCallback(lambda ignored:
3986             self.PUT("/uri/%s?replace=True&offset=100" % self.filecap,
3987                      "replaced and so on"))
3988         def _get_data(filecap):
3989             n = self.s.create_node_from_uri(filecap)
3990             return n.download_best_version()
3991         d.addCallback(_get_data)
3992         d.addCallback(lambda results:
3993             self.failUnlessEqual(results, self.new_data))
3994         # Now try appending things to the file
3995         d.addCallback(lambda ignored:
3996             self.PUT("/uri/%s?offset=%d" % (self.filecap, len(self.new_data)),
3997                      "puppies" * 100))
3998         d.addCallback(_get_data)
3999         d.addCallback(lambda results:
4000             self.failUnlessEqual(results, self.new_data + ("puppies" * 100)))
4001         # and try replacing the beginning of the file
4002         d.addCallback(lambda ignored:
4003             self.PUT("/uri/%s?offset=0" % self.filecap, "begin"))
4004         d.addCallback(_get_data)
4005         d.addCallback(lambda results:
4006             self.failUnlessEqual(results, "begin"+self.new_data[len("begin"):]+("puppies"*100)))
4007         return d
4008
4009     def test_PUT_update_at_invalid_offset(self):
4010         file_contents = "test file" * 100000 # about 900 KiB
4011         d = self.PUT("/uri?mutable=true", file_contents)
4012         def _then(filecap):
4013             self.filecap = filecap
4014         d.addCallback(_then)
4015         # Negative offsets should cause an error.
4016         d.addCallback(lambda ignored:
4017             self.shouldHTTPError("PUT_update_at_invalid_offset",
4018                                  400, "Bad Request",
4019                                  "Invalid offset",
4020                                  self.PUT,
4021                                  "/uri/%s?offset=-1" % self.filecap,
4022                                  "foo"))
4023         return d
4024
4025     def test_PUT_update_at_offset_immutable(self):
4026         file_contents = "Test file" * 100000
4027         d = self.PUT("/uri", file_contents)
4028         def _then(filecap):
4029             self.filecap = filecap
4030         d.addCallback(_then)
4031         d.addCallback(lambda ignored:
4032             self.shouldHTTPError("PUT_update_at_offset_immutable",
4033                                  400, "Bad Request",
4034                                  "immutable",
4035                                  self.PUT,
4036                                  "/uri/%s?offset=50" % self.filecap,
4037                                  "foo"))
4038         return d
4039
4040
4041     def test_bad_method(self):
4042         url = self.webish_url + self.public_url + "/foo/bar.txt"
4043         d = self.shouldHTTPError("bad_method",
4044                                  501, "Not Implemented",
4045                                  "I don't know how to treat a BOGUS request.",
4046                                  client.getPage, url, method="BOGUS")
4047         return d
4048
4049     def test_short_url(self):
4050         url = self.webish_url + "/uri"
4051         d = self.shouldHTTPError("short_url", 501, "Not Implemented",
4052                                  "I don't know how to treat a DELETE request.",
4053                                  client.getPage, url, method="DELETE")
4054         return d
4055
4056     def test_ophandle_bad(self):
4057         url = self.webish_url + "/operations/bogus?t=status"
4058         d = self.shouldHTTPError("ophandle_bad", 404, "404 Not Found",
4059                                  "unknown/expired handle 'bogus'",
4060                                  client.getPage, url)
4061         return d
4062
4063     def test_ophandle_cancel(self):
4064         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
4065                       followRedirect=True)
4066         d.addCallback(lambda ignored:
4067                       self.GET("/operations/128?t=status&output=JSON"))
4068         def _check1(res):
4069             data = simplejson.loads(res)
4070             self.failUnless("finished" in data, res)
4071             monitor = self.ws.root.child_operations.handles["128"][0]
4072             d = self.POST("/operations/128?t=cancel&output=JSON")
4073             def _check2(res):
4074                 data = simplejson.loads(res)
4075                 self.failUnless("finished" in data, res)
4076                 # t=cancel causes the handle to be forgotten
4077                 self.failUnless(monitor.is_cancelled())
4078             d.addCallback(_check2)
4079             return d
4080         d.addCallback(_check1)
4081         d.addCallback(lambda ignored:
4082                       self.shouldHTTPError("ophandle_cancel",
4083                                            404, "404 Not Found",
4084                                            "unknown/expired handle '128'",
4085                                            self.GET,
4086                                            "/operations/128?t=status&output=JSON"))
4087         return d
4088
4089     def test_ophandle_retainfor(self):
4090         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
4091                       followRedirect=True)
4092         d.addCallback(lambda ignored:
4093                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
4094         def _check1(res):
4095             data = simplejson.loads(res)
4096             self.failUnless("finished" in data, res)
4097         d.addCallback(_check1)
4098         # the retain-for=0 will cause the handle to be expired very soon
4099         d.addCallback(lambda ign:
4100             self.clock.advance(2.0))
4101         d.addCallback(lambda ignored:
4102                       self.shouldHTTPError("ophandle_retainfor",
4103                                            404, "404 Not Found",
4104                                            "unknown/expired handle '129'",
4105                                            self.GET,
4106                                            "/operations/129?t=status&output=JSON"))
4107         return d
4108
4109     def test_ophandle_release_after_complete(self):
4110         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
4111                       followRedirect=True)
4112         d.addCallback(self.wait_for_operation, "130")
4113         d.addCallback(lambda ignored:
4114                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
4115         # the release-after-complete=true will cause the handle to be expired
4116         d.addCallback(lambda ignored:
4117                       self.shouldHTTPError("ophandle_release_after_complete",
4118                                            404, "404 Not Found",
4119                                            "unknown/expired handle '130'",
4120                                            self.GET,
4121                                            "/operations/130?t=status&output=JSON"))
4122         return d
4123
4124     def test_uncollected_ophandle_expiration(self):
4125         # uncollected ophandles should expire after 4 days
4126         def _make_uncollected_ophandle(ophandle):
4127             d = self.POST(self.public_url +
4128                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
4129                           followRedirect=False)
4130             # When we start the operation, the webapi server will want
4131             # to redirect us to the page for the ophandle, so we get
4132             # confirmation that the operation has started. If the
4133             # manifest operation has finished by the time we get there,
4134             # following that redirect (by setting followRedirect=True
4135             # above) has the side effect of collecting the ophandle that
4136             # we've just created, which means that we can't use the
4137             # ophandle to test the uncollected timeout anymore. So,
4138             # instead, catch the 302 here and don't follow it.
4139             d.addBoth(self.should302, "uncollected_ophandle_creation")
4140             return d
4141         # Create an ophandle, don't collect it, then advance the clock by
4142         # 4 days - 1 second and make sure that the ophandle is still there.
4143         d = _make_uncollected_ophandle(131)
4144         d.addCallback(lambda ign:
4145             self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
4146         d.addCallback(lambda ign:
4147             self.GET("/operations/131?t=status&output=JSON"))
4148         def _check1(res):
4149             data = simplejson.loads(res)
4150             self.failUnless("finished" in data, res)
4151         d.addCallback(_check1)
4152         # Create an ophandle, don't collect it, then try to collect it
4153         # after 4 days. It should be gone.
4154         d.addCallback(lambda ign:
4155             _make_uncollected_ophandle(132))
4156         d.addCallback(lambda ign:
4157             self.clock.advance(96*60*60))
4158         d.addCallback(lambda ign:
4159             self.shouldHTTPError("uncollected_ophandle_expired_after_100_hours",
4160                                  404, "404 Not Found",
4161                                  "unknown/expired handle '132'",
4162                                  self.GET,
4163                                  "/operations/132?t=status&output=JSON"))
4164         return d
4165
4166     def test_collected_ophandle_expiration(self):
4167         # collected ophandles should expire after 1 day
4168         def _make_collected_ophandle(ophandle):
4169             d = self.POST(self.public_url +
4170                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
4171                           followRedirect=True)
4172             # By following the initial redirect, we collect the ophandle
4173             # we've just created.
4174             return d
4175         # Create a collected ophandle, then collect it after 23 hours
4176         # and 59 seconds to make sure that it is still there.
4177         d = _make_collected_ophandle(133)
4178         d.addCallback(lambda ign:
4179             self.clock.advance((24*60*60) - 1))
4180         d.addCallback(lambda ign:
4181             self.GET("/operations/133?t=status&output=JSON"))
4182         def _check1(res):
4183             data = simplejson.loads(res)
4184             self.failUnless("finished" in data, res)
4185         d.addCallback(_check1)
4186         # Create another uncollected ophandle, then try to collect it
4187         # after 24 hours to make sure that it is gone.
4188         d.addCallback(lambda ign:
4189             _make_collected_ophandle(134))
4190         d.addCallback(lambda ign:
4191             self.clock.advance(24*60*60))
4192         d.addCallback(lambda ign:
4193             self.shouldHTTPError("collected_ophandle_expired_after_1_day",
4194                                  404, "404 Not Found",
4195                                  "unknown/expired handle '134'",
4196                                  self.GET,
4197                                  "/operations/134?t=status&output=JSON"))
4198         return d
4199
4200     def test_incident(self):
4201         d = self.POST("/report_incident", details="eek")
4202         def _done(res):
4203             self.failIfIn("<html>", res)
4204             self.failUnlessIn("Thank you for your report!", res)
4205         d.addCallback(_done)
4206         return d
4207
4208     def test_static(self):
4209         webdir = os.path.join(self.staticdir, "subdir")
4210         fileutil.make_dirs(webdir)
4211         f = open(os.path.join(webdir, "hello.txt"), "wb")
4212         f.write("hello")
4213         f.close()
4214
4215         d = self.GET("/static/subdir/hello.txt")
4216         def _check(res):
4217             self.failUnlessReallyEqual(res, "hello")
4218         d.addCallback(_check)
4219         return d
4220
4221
4222 class IntroducerWeb(unittest.TestCase):
4223     def setUp(self):
4224         self.node = None
4225
4226     def tearDown(self):
4227         d = defer.succeed(None)
4228         if self.node:
4229             d.addCallback(lambda ign: self.node.stopService())
4230         d.addCallback(flushEventualQueue)
4231         return d
4232
4233     def test_welcome(self):
4234         basedir = "web.IntroducerWeb.test_welcome"
4235         os.mkdir(basedir)
4236         fileutil.write(os.path.join(basedir, "tahoe.cfg"), "[node]\nweb.port = tcp:0\n")
4237         self.node = IntroducerNode(basedir)
4238         self.ws = self.node.getServiceNamed("webish")
4239
4240         d = fireEventually(None)
4241         d.addCallback(lambda ign: self.node.startService())
4242         d.addCallback(lambda ign: self.node.when_tub_ready())
4243
4244         d.addCallback(lambda ign: self.GET("/"))
4245         def _check(res):
4246             self.failUnlessIn('Welcome to the Tahoe-LAFS Introducer', res)
4247             self.failUnlessIn(FAVICON_MARKUP, res)
4248         d.addCallback(_check)
4249         return d
4250
4251     def GET(self, urlpath, followRedirect=False, return_response=False,
4252             **kwargs):
4253         # if return_response=True, this fires with (data, statuscode,
4254         # respheaders) instead of just data.
4255         assert not isinstance(urlpath, unicode)
4256         url = self.ws.getURL().rstrip('/') + urlpath
4257         factory = HTTPClientGETFactory(url, method="GET",
4258                                        followRedirect=followRedirect, **kwargs)
4259         reactor.connectTCP("localhost", self.ws.getPortnum(), factory)
4260         d = factory.deferred
4261         def _got_data(data):
4262             return (data, factory.status, factory.response_headers)
4263         if return_response:
4264             d.addCallback(_got_data)
4265         return factory.deferred
4266
4267
4268 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4269     def test_load_file(self):
4270         # This will raise an exception unless a well-formed XML file is found under that name.
4271         common.getxmlfile('directory.xhtml').load()
4272
4273     def test_parse_replace_arg(self):
4274         self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
4275         self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
4276         self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
4277                                    "only-files")
4278         self.shouldFail(AssertionError, "test_parse_replace_arg", "",
4279                         common.parse_replace_arg, "only_fles")
4280
4281     def test_abbreviate_time(self):
4282         self.failUnlessReallyEqual(common.abbreviate_time(None), "")
4283         self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
4284         self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
4285         self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
4286         self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
4287         self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
4288
4289     def test_compute_rate(self):
4290         self.failUnlessReallyEqual(common.compute_rate(None, None), None)
4291         self.failUnlessReallyEqual(common.compute_rate(None, 1), None)
4292         self.failUnlessReallyEqual(common.compute_rate(250000, None), None)
4293         self.failUnlessReallyEqual(common.compute_rate(250000, 0), None)
4294         self.failUnlessReallyEqual(common.compute_rate(250000, 10), 25000.0)
4295         self.failUnlessReallyEqual(common.compute_rate(0, 10), 0.0)
4296         self.shouldFail(AssertionError, "test_compute_rate", "",
4297                         common.compute_rate, -100, 10)
4298         self.shouldFail(AssertionError, "test_compute_rate", "",
4299                         common.compute_rate, 100, -10)
4300
4301         # Sanity check
4302         rate = common.compute_rate(10*1000*1000, 1)
4303         self.failUnlessReallyEqual(common.abbreviate_rate(rate), "10.00MBps")
4304
4305     def test_abbreviate_rate(self):
4306         self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
4307         self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
4308         self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
4309         self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
4310
4311     def test_abbreviate_size(self):
4312         self.failUnlessReallyEqual(common.abbreviate_size(None), "")
4313         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
4314         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
4315         self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
4316         self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
4317
4318     def test_plural(self):
4319         def convert(s):
4320             return "%d second%s" % (s, status.plural(s))
4321         self.failUnlessReallyEqual(convert(0), "0 seconds")
4322         self.failUnlessReallyEqual(convert(1), "1 second")
4323         self.failUnlessReallyEqual(convert(2), "2 seconds")
4324         def convert2(s):
4325             return "has share%s: %s" % (status.plural(s), ",".join(s))
4326         self.failUnlessReallyEqual(convert2([]), "has shares: ")
4327         self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
4328         self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
4329
4330
4331 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4332
4333     def CHECK(self, ign, which, args, clientnum=0):
4334         fileurl = self.fileurls[which]
4335         url = fileurl + "?" + args
4336         return self.GET(url, method="POST", clientnum=clientnum)
4337
4338     def test_filecheck(self):
4339         self.basedir = "web/Grid/filecheck"
4340         self.set_up_grid()
4341         c0 = self.g.clients[0]
4342         self.uris = {}
4343         DATA = "data" * 100
4344         d = c0.upload(upload.Data(DATA, convergence=""))
4345         def _stash_uri(ur, which):
4346             self.uris[which] = ur.get_uri()
4347         d.addCallback(_stash_uri, "good")
4348         d.addCallback(lambda ign:
4349                       c0.upload(upload.Data(DATA+"1", convergence="")))
4350         d.addCallback(_stash_uri, "sick")
4351         d.addCallback(lambda ign:
4352                       c0.upload(upload.Data(DATA+"2", convergence="")))
4353         d.addCallback(_stash_uri, "dead")
4354         def _stash_mutable_uri(n, which):
4355             self.uris[which] = n.get_uri()
4356             assert isinstance(self.uris[which], str)
4357         d.addCallback(lambda ign:
4358             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4359         d.addCallback(_stash_mutable_uri, "corrupt")
4360         d.addCallback(lambda ign:
4361                       c0.upload(upload.Data("literal", convergence="")))
4362         d.addCallback(_stash_uri, "small")
4363         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
4364         d.addCallback(_stash_mutable_uri, "smalldir")
4365
4366         def _compute_fileurls(ignored):
4367             self.fileurls = {}
4368             for which in self.uris:
4369                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4370         d.addCallback(_compute_fileurls)
4371
4372         def _clobber_shares(ignored):
4373             good_shares = self.find_uri_shares(self.uris["good"])
4374             self.failUnlessReallyEqual(len(good_shares), 10)
4375             sick_shares = self.find_uri_shares(self.uris["sick"])
4376             os.unlink(sick_shares[0][2])
4377             dead_shares = self.find_uri_shares(self.uris["dead"])
4378             for i in range(1, 10):
4379                 os.unlink(dead_shares[i][2])
4380             c_shares = self.find_uri_shares(self.uris["corrupt"])
4381             cso = CorruptShareOptions()
4382             cso.stdout = StringIO()
4383             cso.parseOptions([c_shares[0][2]])
4384             corrupt_share(cso)
4385         d.addCallback(_clobber_shares)
4386
4387         d.addCallback(self.CHECK, "good", "t=check")
4388         def _got_html_good(res):
4389             self.failUnlessIn("Healthy", res)
4390             self.failIfIn("Not Healthy", res)
4391             self.failUnlessIn(FAVICON_MARKUP, res)
4392         d.addCallback(_got_html_good)
4393         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
4394         def _got_html_good_return_to(res):
4395             self.failUnlessIn("Healthy", res)
4396             self.failIfIn("Not Healthy", res)
4397             self.failUnlessIn('<a href="somewhere">Return to file', res)
4398         d.addCallback(_got_html_good_return_to)
4399         d.addCallback(self.CHECK, "good", "t=check&output=json")
4400         def _got_json_good(res):
4401             r = simplejson.loads(res)
4402             self.failUnlessEqual(r["summary"], "Healthy")
4403             self.failUnless(r["results"]["healthy"])
4404             self.failIf(r["results"]["needs-rebalancing"])
4405             self.failUnless(r["results"]["recoverable"])
4406         d.addCallback(_got_json_good)
4407
4408         d.addCallback(self.CHECK, "small", "t=check")
4409         def _got_html_small(res):
4410             self.failUnlessIn("Literal files are always healthy", res)
4411             self.failIfIn("Not Healthy", res)
4412         d.addCallback(_got_html_small)
4413         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
4414         def _got_html_small_return_to(res):
4415             self.failUnlessIn("Literal files are always healthy", res)
4416             self.failIfIn("Not Healthy", res)
4417             self.failUnlessIn('<a href="somewhere">Return to file', res)
4418         d.addCallback(_got_html_small_return_to)
4419         d.addCallback(self.CHECK, "small", "t=check&output=json")
4420         def _got_json_small(res):
4421             r = simplejson.loads(res)
4422             self.failUnlessEqual(r["storage-index"], "")
4423             self.failUnless(r["results"]["healthy"])
4424         d.addCallback(_got_json_small)
4425
4426         d.addCallback(self.CHECK, "smalldir", "t=check")
4427         def _got_html_smalldir(res):
4428             self.failUnlessIn("Literal files are always healthy", res)
4429             self.failIfIn("Not Healthy", res)
4430         d.addCallback(_got_html_smalldir)
4431         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
4432         def _got_json_smalldir(res):
4433             r = simplejson.loads(res)
4434             self.failUnlessEqual(r["storage-index"], "")
4435             self.failUnless(r["results"]["healthy"])
4436         d.addCallback(_got_json_smalldir)
4437
4438         d.addCallback(self.CHECK, "sick", "t=check")
4439         def _got_html_sick(res):
4440             self.failUnlessIn("Not Healthy", res)
4441         d.addCallback(_got_html_sick)
4442         d.addCallback(self.CHECK, "sick", "t=check&output=json")
4443         def _got_json_sick(res):
4444             r = simplejson.loads(res)
4445             self.failUnlessEqual(r["summary"],
4446                                  "Not Healthy: 9 shares (enc 3-of-10)")
4447             self.failIf(r["results"]["healthy"])
4448             self.failIf(r["results"]["needs-rebalancing"])
4449             self.failUnless(r["results"]["recoverable"])
4450         d.addCallback(_got_json_sick)
4451
4452         d.addCallback(self.CHECK, "dead", "t=check")
4453         def _got_html_dead(res):
4454             self.failUnlessIn("Not Healthy", res)
4455         d.addCallback(_got_html_dead)
4456         d.addCallback(self.CHECK, "dead", "t=check&output=json")
4457         def _got_json_dead(res):
4458             r = simplejson.loads(res)
4459             self.failUnlessEqual(r["summary"],
4460                                  "Not Healthy: 1 shares (enc 3-of-10)")
4461             self.failIf(r["results"]["healthy"])
4462             self.failIf(r["results"]["needs-rebalancing"])
4463             self.failIf(r["results"]["recoverable"])
4464         d.addCallback(_got_json_dead)
4465
4466         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
4467         def _got_html_corrupt(res):
4468             self.failUnlessIn("Not Healthy! : Unhealthy", res)
4469         d.addCallback(_got_html_corrupt)
4470         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
4471         def _got_json_corrupt(res):
4472             r = simplejson.loads(res)
4473             self.failUnlessIn("Unhealthy: 9 shares (enc 3-of-10)", r["summary"])
4474             self.failIf(r["results"]["healthy"])
4475             self.failUnless(r["results"]["recoverable"])
4476             self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
4477             self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
4478         d.addCallback(_got_json_corrupt)
4479
4480         d.addErrback(self.explain_web_error)
4481         return d
4482
4483     def test_repair_html(self):
4484         self.basedir = "web/Grid/repair_html"
4485         self.set_up_grid()
4486         c0 = self.g.clients[0]
4487         self.uris = {}
4488         DATA = "data" * 100
4489         d = c0.upload(upload.Data(DATA, convergence=""))
4490         def _stash_uri(ur, which):
4491             self.uris[which] = ur.get_uri()
4492         d.addCallback(_stash_uri, "good")
4493         d.addCallback(lambda ign:
4494                       c0.upload(upload.Data(DATA+"1", convergence="")))
4495         d.addCallback(_stash_uri, "sick")
4496         d.addCallback(lambda ign:
4497                       c0.upload(upload.Data(DATA+"2", convergence="")))
4498         d.addCallback(_stash_uri, "dead")
4499         def _stash_mutable_uri(n, which):
4500             self.uris[which] = n.get_uri()
4501             assert isinstance(self.uris[which], str)
4502         d.addCallback(lambda ign:
4503             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4504         d.addCallback(_stash_mutable_uri, "corrupt")
4505
4506         def _compute_fileurls(ignored):
4507             self.fileurls = {}
4508             for which in self.uris:
4509                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4510         d.addCallback(_compute_fileurls)
4511
4512         def _clobber_shares(ignored):
4513             good_shares = self.find_uri_shares(self.uris["good"])
4514             self.failUnlessReallyEqual(len(good_shares), 10)
4515             sick_shares = self.find_uri_shares(self.uris["sick"])
4516             os.unlink(sick_shares[0][2])
4517             dead_shares = self.find_uri_shares(self.uris["dead"])
4518             for i in range(1, 10):
4519                 os.unlink(dead_shares[i][2])
4520             c_shares = self.find_uri_shares(self.uris["corrupt"])
4521             cso = CorruptShareOptions()
4522             cso.stdout = StringIO()
4523             cso.parseOptions([c_shares[0][2]])
4524             corrupt_share(cso)
4525         d.addCallback(_clobber_shares)
4526
4527         d.addCallback(self.CHECK, "good", "t=check&repair=true")
4528         def _got_html_good(res):
4529             self.failUnlessIn("Healthy", res)
4530             self.failIfIn("Not Healthy", res)
4531             self.failUnlessIn("No repair necessary", res)
4532             self.failUnlessIn(FAVICON_MARKUP, res)
4533         d.addCallback(_got_html_good)
4534
4535         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
4536         def _got_html_sick(res):
4537             self.failUnlessIn("Healthy : healthy", res)
4538             self.failIfIn("Not Healthy", res)
4539             self.failUnlessIn("Repair successful", res)
4540         d.addCallback(_got_html_sick)
4541
4542         # repair of a dead file will fail, of course, but it isn't yet
4543         # clear how this should be reported. Right now it shows up as
4544         # a "410 Gone".
4545         #
4546         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
4547         #def _got_html_dead(res):
4548         #    print res
4549         #    self.failUnlessIn("Healthy : healthy", res)
4550         #    self.failIfIn("Not Healthy", res)
4551         #    self.failUnlessIn("No repair necessary", res)
4552         #d.addCallback(_got_html_dead)
4553
4554         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
4555         def _got_html_corrupt(res):
4556             self.failUnlessIn("Healthy : Healthy", res)
4557             self.failIfIn("Not Healthy", res)
4558             self.failUnlessIn("Repair successful", res)
4559         d.addCallback(_got_html_corrupt)
4560
4561         d.addErrback(self.explain_web_error)
4562         return d
4563
4564     def test_repair_json(self):
4565         self.basedir = "web/Grid/repair_json"
4566         self.set_up_grid()
4567         c0 = self.g.clients[0]
4568         self.uris = {}
4569         DATA = "data" * 100
4570         d = c0.upload(upload.Data(DATA+"1", convergence=""))
4571         def _stash_uri(ur, which):
4572             self.uris[which] = ur.get_uri()
4573         d.addCallback(_stash_uri, "sick")
4574
4575         def _compute_fileurls(ignored):
4576             self.fileurls = {}
4577             for which in self.uris:
4578                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4579         d.addCallback(_compute_fileurls)
4580
4581         def _clobber_shares(ignored):
4582             sick_shares = self.find_uri_shares(self.uris["sick"])
4583             os.unlink(sick_shares[0][2])
4584         d.addCallback(_clobber_shares)
4585
4586         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
4587         def _got_json_sick(res):
4588             r = simplejson.loads(res)
4589             self.failUnlessReallyEqual(r["repair-attempted"], True)
4590             self.failUnlessReallyEqual(r["repair-successful"], True)
4591             self.failUnlessEqual(r["pre-repair-results"]["summary"],
4592                                  "Not Healthy: 9 shares (enc 3-of-10)")
4593             self.failIf(r["pre-repair-results"]["results"]["healthy"])
4594             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
4595             self.failUnless(r["post-repair-results"]["results"]["healthy"])
4596         d.addCallback(_got_json_sick)
4597
4598         d.addErrback(self.explain_web_error)
4599         return d
4600
4601     def test_unknown(self, immutable=False):
4602         self.basedir = "web/Grid/unknown"
4603         if immutable:
4604             self.basedir = "web/Grid/unknown-immutable"
4605
4606         self.set_up_grid()
4607         c0 = self.g.clients[0]
4608         self.uris = {}
4609         self.fileurls = {}
4610
4611         # the future cap format may contain slashes, which must be tolerated
4612         expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
4613                                                            safe="")
4614
4615         if immutable:
4616             name = u"future-imm"
4617             future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
4618             d = c0.create_immutable_dirnode({name: (future_node, {})})
4619         else:
4620             name = u"future"
4621             future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4622             d = c0.create_dirnode()
4623
4624         def _stash_root_and_create_file(n):
4625             self.rootnode = n
4626             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
4627             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
4628             if not immutable:
4629                 return self.rootnode.set_node(name, future_node)
4630         d.addCallback(_stash_root_and_create_file)
4631
4632         # make sure directory listing tolerates unknown nodes
4633         d.addCallback(lambda ign: self.GET(self.rooturl))
4634         def _check_directory_html(res, expected_type_suffix):
4635             pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
4636                                   '<td>%s</td>' % (expected_type_suffix, str(name)),
4637                                  re.DOTALL)
4638             self.failUnless(re.search(pattern, res), res)
4639             # find the More Info link for name, should be relative
4640             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4641             info_url = mo.group(1)
4642             self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
4643         if immutable:
4644             d.addCallback(_check_directory_html, "-IMM")
4645         else:
4646             d.addCallback(_check_directory_html, "")
4647
4648         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4649         def _check_directory_json(res, expect_rw_uri):
4650             data = simplejson.loads(res)
4651             self.failUnlessEqual(data[0], "dirnode")
4652             f = data[1]["children"][name]
4653             self.failUnlessEqual(f[0], "unknown")
4654             if expect_rw_uri:
4655                 self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data)
4656             else:
4657                 self.failIfIn("rw_uri", f[1])
4658             if immutable:
4659                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
4660             else:
4661                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
4662             self.failUnlessIn("metadata", f[1])
4663         d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
4664
4665         def _check_info(res, expect_rw_uri, expect_ro_uri):
4666             self.failUnlessIn("Object Type: <span>unknown</span>", res)
4667             if expect_rw_uri:
4668                 self.failUnlessIn(unknown_rwcap, res)
4669             if expect_ro_uri:
4670                 if immutable:
4671                     self.failUnlessIn(unknown_immcap, res)
4672                 else:
4673                     self.failUnlessIn(unknown_rocap, res)
4674             else:
4675                 self.failIfIn(unknown_rocap, res)
4676             self.failIfIn("Raw data as", res)
4677             self.failIfIn("Directory writecap", res)
4678             self.failIfIn("Checker Operations", res)
4679             self.failIfIn("Mutable File Operations", res)
4680             self.failIfIn("Directory Operations", res)
4681
4682         # FIXME: these should have expect_rw_uri=not immutable; I don't know
4683         # why they fail. Possibly related to ticket #922.
4684
4685         d.addCallback(lambda ign: self.GET(expected_info_url))
4686         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
4687         d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
4688         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
4689
4690         def _check_json(res, expect_rw_uri):
4691             data = simplejson.loads(res)
4692             self.failUnlessEqual(data[0], "unknown")
4693             if expect_rw_uri:
4694                 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data)
4695             else:
4696                 self.failIfIn("rw_uri", data[1])
4697
4698             if immutable:
4699                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data)
4700                 self.failUnlessReallyEqual(data[1]["mutable"], False)
4701             elif expect_rw_uri:
4702                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4703                 self.failUnlessReallyEqual(data[1]["mutable"], True)
4704             else:
4705                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4706                 self.failIfIn("mutable", data[1])
4707
4708             # TODO: check metadata contents
4709             self.failUnlessIn("metadata", data[1])
4710
4711         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
4712         d.addCallback(_check_json, expect_rw_uri=not immutable)
4713
4714         # and make sure that a read-only version of the directory can be
4715         # rendered too. This version will not have unknown_rwcap, whether
4716         # or not future_node was immutable.
4717         d.addCallback(lambda ign: self.GET(self.rourl))
4718         if immutable:
4719             d.addCallback(_check_directory_html, "-IMM")
4720         else:
4721             d.addCallback(_check_directory_html, "-RO")
4722
4723         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
4724         d.addCallback(_check_directory_json, expect_rw_uri=False)
4725
4726         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
4727         d.addCallback(_check_json, expect_rw_uri=False)
4728
4729         # TODO: check that getting t=info from the Info link in the ro directory
4730         # works, and does not include the writecap URI.
4731         return d
4732
4733     def test_immutable_unknown(self):
4734         return self.test_unknown(immutable=True)
4735
4736     def test_mutant_dirnodes_are_omitted(self):
4737         self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
4738
4739         self.set_up_grid()
4740         c = self.g.clients[0]
4741         nm = c.nodemaker
4742         self.uris = {}
4743         self.fileurls = {}
4744
4745         lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
4746         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
4747         mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
4748
4749         # This method tests mainly dirnode, but we'd have to duplicate code in order to
4750         # test the dirnode and web layers separately.
4751
4752         # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
4753         # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
4754         # When the directory is read, the mutants should be silently disposed of, leaving
4755         # their lonely sibling.
4756         # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
4757         # because immutable directories don't have a writecap and therefore that field
4758         # isn't (and can't be) decrypted.
4759         # TODO: The field still exists in the netstring. Technically we should check what
4760         # happens if something is put there (_unpack_contents should raise ValueError),
4761         # but that can wait.
4762
4763         lonely_child = nm.create_from_cap(lonely_uri)
4764         mutant_ro_child = nm.create_from_cap(mut_read_uri)
4765         mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
4766
4767         def _by_hook_or_by_crook():
4768             return True
4769         for n in [mutant_ro_child, mutant_write_in_ro_child]:
4770             n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
4771
4772         mutant_write_in_ro_child.get_write_uri    = lambda: None
4773         mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
4774
4775         kids = {u"lonely":      (lonely_child, {}),
4776                 u"ro":          (mutant_ro_child, {}),
4777                 u"write-in-ro": (mutant_write_in_ro_child, {}),
4778                 }
4779         d = c.create_immutable_dirnode(kids)
4780
4781         def _created(dn):
4782             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
4783             self.failIf(dn.is_mutable())
4784             self.failUnless(dn.is_readonly())
4785             # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
4786             self.failIf(hasattr(dn._node, 'get_writekey'))
4787             rep = str(dn)
4788             self.failUnlessIn("RO-IMM", rep)
4789             cap = dn.get_cap()
4790             self.failUnlessIn("CHK", cap.to_string())
4791             self.cap = cap
4792             self.rootnode = dn
4793             self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
4794             return download_to_data(dn._node)
4795         d.addCallback(_created)
4796
4797         def _check_data(data):
4798             # Decode the netstring representation of the directory to check that all children
4799             # are present. This is a bit of an abstraction violation, but there's not really
4800             # any other way to do it given that the real DirectoryNode._unpack_contents would
4801             # strip the mutant children out (which is what we're trying to test, later).
4802             position = 0
4803             numkids = 0
4804             while position < len(data):
4805                 entries, position = split_netstring(data, 1, position)
4806                 entry = entries[0]
4807                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
4808                 name = name_utf8.decode("utf-8")
4809                 self.failUnlessEqual(rwcapdata, "")
4810                 self.failUnlessIn(name, kids)
4811                 (expected_child, ign) = kids[name]
4812                 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
4813                 numkids += 1
4814
4815             self.failUnlessReallyEqual(numkids, 3)
4816             return self.rootnode.list()
4817         d.addCallback(_check_data)
4818
4819         # Now when we use the real directory listing code, the mutants should be absent.
4820         def _check_kids(children):
4821             self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
4822             lonely_node, lonely_metadata = children[u"lonely"]
4823
4824             self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
4825             self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
4826         d.addCallback(_check_kids)
4827
4828         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
4829         d.addCallback(lambda n: n.list())
4830         d.addCallback(_check_kids)  # again with dirnode recreated from cap
4831
4832         # Make sure the lonely child can be listed in HTML...
4833         d.addCallback(lambda ign: self.GET(self.rooturl))
4834         def _check_html(res):
4835             self.failIfIn("URI:SSK", res)
4836             get_lonely = "".join([r'<td>FILE</td>',
4837                                   r'\s+<td>',
4838                                   r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
4839                                   r'</td>',
4840                                   r'\s+<td align="right">%d</td>' % len("one"),
4841                                   ])
4842             self.failUnless(re.search(get_lonely, res), res)
4843
4844             # find the More Info link for name, should be relative
4845             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4846             info_url = mo.group(1)
4847             self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
4848         d.addCallback(_check_html)
4849
4850         # ... and in JSON.
4851         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4852         def _check_json(res):
4853             data = simplejson.loads(res)
4854             self.failUnlessEqual(data[0], "dirnode")
4855             listed_children = data[1]["children"]
4856             self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
4857             ll_type, ll_data = listed_children[u"lonely"]
4858             self.failUnlessEqual(ll_type, "filenode")
4859             self.failIfIn("rw_uri", ll_data)
4860             self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
4861         d.addCallback(_check_json)
4862         return d
4863
4864     def test_deep_check(self):
4865         self.basedir = "web/Grid/deep_check"
4866         self.set_up_grid()
4867         c0 = self.g.clients[0]
4868         self.uris = {}
4869         self.fileurls = {}
4870         DATA = "data" * 100
4871         d = c0.create_dirnode()
4872         def _stash_root_and_create_file(n):
4873             self.rootnode = n
4874             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4875             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4876         d.addCallback(_stash_root_and_create_file)
4877         def _stash_uri(fn, which):
4878             self.uris[which] = fn.get_uri()
4879             return fn
4880         d.addCallback(_stash_uri, "good")
4881         d.addCallback(lambda ign:
4882                       self.rootnode.add_file(u"small",
4883                                              upload.Data("literal",
4884                                                         convergence="")))
4885         d.addCallback(_stash_uri, "small")
4886         d.addCallback(lambda ign:
4887                       self.rootnode.add_file(u"sick",
4888                                              upload.Data(DATA+"1",
4889                                                         convergence="")))
4890         d.addCallback(_stash_uri, "sick")
4891
4892         # this tests that deep-check and stream-manifest will ignore
4893         # UnknownNode instances. Hopefully this will also cover deep-stats.
4894         future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4895         d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
4896
4897         def _clobber_shares(ignored):
4898             self.delete_shares_numbered(self.uris["sick"], [0,1])
4899         d.addCallback(_clobber_shares)
4900
4901         # root
4902         # root/good
4903         # root/small
4904         # root/sick
4905         # root/future
4906
4907         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4908         def _done(res):
4909             try:
4910                 units = [simplejson.loads(line)
4911                          for line in res.splitlines()
4912                          if line]
4913             except ValueError:
4914                 print "response is:", res
4915                 print "undecodeable line was '%s'" % line
4916                 raise
4917             self.failUnlessReallyEqual(len(units), 5+1)
4918             # should be parent-first
4919             u0 = units[0]
4920             self.failUnlessEqual(u0["path"], [])
4921             self.failUnlessEqual(u0["type"], "directory")
4922             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4923             u0cr = u0["check-results"]
4924             self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
4925
4926             ugood = [u for u in units
4927                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4928             self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
4929             ugoodcr = ugood["check-results"]
4930             self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
4931
4932             stats = units[-1]
4933             self.failUnlessEqual(stats["type"], "stats")
4934             s = stats["stats"]
4935             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4936             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4937             self.failUnlessReallyEqual(s["count-directories"], 1)
4938             self.failUnlessReallyEqual(s["count-unknown"], 1)
4939         d.addCallback(_done)
4940
4941         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4942         def _check_manifest(res):
4943             self.failUnless(res.endswith("\n"))
4944             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
4945             self.failUnlessReallyEqual(len(units), 5+1)
4946             self.failUnlessEqual(units[-1]["type"], "stats")
4947             first = units[0]
4948             self.failUnlessEqual(first["path"], [])
4949             self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri())
4950             self.failUnlessEqual(first["type"], "directory")
4951             stats = units[-1]["stats"]
4952             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4953             self.failUnlessReallyEqual(stats["count-literal-files"], 1)
4954             self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
4955             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4956             self.failUnlessReallyEqual(stats["count-unknown"], 1)
4957         d.addCallback(_check_manifest)
4958
4959         # now add root/subdir and root/subdir/grandchild, then make subdir
4960         # unrecoverable, then see what happens
4961
4962         d.addCallback(lambda ign:
4963                       self.rootnode.create_subdirectory(u"subdir"))
4964         d.addCallback(_stash_uri, "subdir")
4965         d.addCallback(lambda subdir_node:
4966                       subdir_node.add_file(u"grandchild",
4967                                            upload.Data(DATA+"2",
4968                                                        convergence="")))
4969         d.addCallback(_stash_uri, "grandchild")
4970
4971         d.addCallback(lambda ign:
4972                       self.delete_shares_numbered(self.uris["subdir"],
4973                                                   range(1, 10)))
4974
4975         # root
4976         # root/good
4977         # root/small
4978         # root/sick
4979         # root/future
4980         # root/subdir [unrecoverable]
4981         # root/subdir/grandchild
4982
4983         # how should a streaming-JSON API indicate fatal error?
4984         # answer: emit ERROR: instead of a JSON string
4985
4986         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4987         def _check_broken_manifest(res):
4988             lines = res.splitlines()
4989             error_lines = [i
4990                            for (i,line) in enumerate(lines)
4991                            if line.startswith("ERROR:")]
4992             if not error_lines:
4993                 self.fail("no ERROR: in output: %s" % (res,))
4994             first_error = error_lines[0]
4995             error_line = lines[first_error]
4996             error_msg = lines[first_error+1:]
4997             error_msg_s = "\n".join(error_msg) + "\n"
4998             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4999                               error_line)
5000             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
5001             units = [simplejson.loads(line) for line in lines[:first_error]]
5002             self.failUnlessReallyEqual(len(units), 6) # includes subdir
5003             last_unit = units[-1]
5004             self.failUnlessEqual(last_unit["path"], ["subdir"])
5005         d.addCallback(_check_broken_manifest)
5006
5007         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
5008         def _check_broken_deepcheck(res):
5009             lines = res.splitlines()
5010             error_lines = [i
5011                            for (i,line) in enumerate(lines)
5012                            if line.startswith("ERROR:")]
5013             if not error_lines:
5014                 self.fail("no ERROR: in output: %s" % (res,))
5015             first_error = error_lines[0]
5016             error_line = lines[first_error]
5017             error_msg = lines[first_error+1:]
5018             error_msg_s = "\n".join(error_msg) + "\n"
5019             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
5020                               error_line)
5021             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
5022             units = [simplejson.loads(line) for line in lines[:first_error]]
5023             self.failUnlessReallyEqual(len(units), 6) # includes subdir
5024             last_unit = units[-1]
5025             self.failUnlessEqual(last_unit["path"], ["subdir"])
5026             r = last_unit["check-results"]["results"]
5027             self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
5028             self.failUnlessReallyEqual(r["count-shares-good"], 1)
5029             self.failUnlessReallyEqual(r["recoverable"], False)
5030         d.addCallback(_check_broken_deepcheck)
5031
5032         d.addErrback(self.explain_web_error)
5033         return d
5034
5035     def test_deep_check_and_repair(self):
5036         self.basedir = "web/Grid/deep_check_and_repair"
5037         self.set_up_grid()
5038         c0 = self.g.clients[0]
5039         self.uris = {}
5040         self.fileurls = {}
5041         DATA = "data" * 100
5042         d = c0.create_dirnode()
5043         def _stash_root_and_create_file(n):
5044             self.rootnode = n
5045             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5046             return n.add_file(u"good", upload.Data(DATA, convergence=""))
5047         d.addCallback(_stash_root_and_create_file)
5048         def _stash_uri(fn, which):
5049             self.uris[which] = fn.get_uri()
5050         d.addCallback(_stash_uri, "good")
5051         d.addCallback(lambda ign:
5052                       self.rootnode.add_file(u"small",
5053                                              upload.Data("literal",
5054                                                         convergence="")))
5055         d.addCallback(_stash_uri, "small")
5056         d.addCallback(lambda ign:
5057                       self.rootnode.add_file(u"sick",
5058                                              upload.Data(DATA+"1",
5059                                                         convergence="")))
5060         d.addCallback(_stash_uri, "sick")
5061         #d.addCallback(lambda ign:
5062         #              self.rootnode.add_file(u"dead",
5063         #                                     upload.Data(DATA+"2",
5064         #                                                convergence="")))
5065         #d.addCallback(_stash_uri, "dead")
5066
5067         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
5068         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
5069         #d.addCallback(_stash_uri, "corrupt")
5070
5071         def _clobber_shares(ignored):
5072             good_shares = self.find_uri_shares(self.uris["good"])
5073             self.failUnlessReallyEqual(len(good_shares), 10)
5074             sick_shares = self.find_uri_shares(self.uris["sick"])
5075             os.unlink(sick_shares[0][2])
5076             #dead_shares = self.find_uri_shares(self.uris["dead"])
5077             #for i in range(1, 10):
5078             #    os.unlink(dead_shares[i][2])
5079
5080             #c_shares = self.find_uri_shares(self.uris["corrupt"])
5081             #cso = CorruptShareOptions()
5082             #cso.stdout = StringIO()
5083             #cso.parseOptions([c_shares[0][2]])
5084             #corrupt_share(cso)
5085         d.addCallback(_clobber_shares)
5086
5087         # root
5088         # root/good   CHK, 10 shares
5089         # root/small  LIT
5090         # root/sick   CHK, 9 shares
5091
5092         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
5093         def _done(res):
5094             units = [simplejson.loads(line)
5095                      for line in res.splitlines()
5096                      if line]
5097             self.failUnlessReallyEqual(len(units), 4+1)
5098             # should be parent-first
5099             u0 = units[0]
5100             self.failUnlessEqual(u0["path"], [])
5101             self.failUnlessEqual(u0["type"], "directory")
5102             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
5103             u0crr = u0["check-and-repair-results"]
5104             self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
5105             self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
5106
5107             ugood = [u for u in units
5108                      if u["type"] == "file" and u["path"] == [u"good"]][0]
5109             self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
5110             ugoodcrr = ugood["check-and-repair-results"]
5111             self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
5112             self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
5113
5114             usick = [u for u in units
5115                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
5116             self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"])
5117             usickcrr = usick["check-and-repair-results"]
5118             self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
5119             self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
5120             self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
5121             self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
5122
5123             stats = units[-1]
5124             self.failUnlessEqual(stats["type"], "stats")
5125             s = stats["stats"]
5126             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
5127             self.failUnlessReallyEqual(s["count-literal-files"], 1)
5128             self.failUnlessReallyEqual(s["count-directories"], 1)
5129         d.addCallback(_done)
5130
5131         d.addErrback(self.explain_web_error)
5132         return d
5133
5134     def _count_leases(self, ignored, which):
5135         u = self.uris[which]
5136         shares = self.find_uri_shares(u)
5137         lease_counts = []
5138         for shnum, serverid, fn in shares:
5139             sf = get_share_file(fn)
5140             num_leases = len(list(sf.get_leases()))
5141             lease_counts.append( (fn, num_leases) )
5142         return lease_counts
5143
5144     def _assert_leasecount(self, lease_counts, expected):
5145         for (fn, num_leases) in lease_counts:
5146             if num_leases != expected:
5147                 self.fail("expected %d leases, have %d, on %s" %
5148                           (expected, num_leases, fn))
5149
5150     def test_add_lease(self):
5151         self.basedir = "web/Grid/add_lease"
5152         self.set_up_grid(num_clients=2)
5153         c0 = self.g.clients[0]
5154         self.uris = {}
5155         DATA = "data" * 100
5156         d = c0.upload(upload.Data(DATA, convergence=""))
5157         def _stash_uri(ur, which):
5158             self.uris[which] = ur.get_uri()
5159         d.addCallback(_stash_uri, "one")
5160         d.addCallback(lambda ign:
5161                       c0.upload(upload.Data(DATA+"1", convergence="")))
5162         d.addCallback(_stash_uri, "two")
5163         def _stash_mutable_uri(n, which):
5164             self.uris[which] = n.get_uri()
5165             assert isinstance(self.uris[which], str)
5166         d.addCallback(lambda ign:
5167             c0.create_mutable_file(publish.MutableData(DATA+"2")))
5168         d.addCallback(_stash_mutable_uri, "mutable")
5169
5170         def _compute_fileurls(ignored):
5171             self.fileurls = {}
5172             for which in self.uris:
5173                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
5174         d.addCallback(_compute_fileurls)
5175
5176         d.addCallback(self._count_leases, "one")
5177         d.addCallback(self._assert_leasecount, 1)
5178         d.addCallback(self._count_leases, "two")
5179         d.addCallback(self._assert_leasecount, 1)
5180         d.addCallback(self._count_leases, "mutable")
5181         d.addCallback(self._assert_leasecount, 1)
5182
5183         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
5184         def _got_html_good(res):
5185             self.failUnlessIn("Healthy", res)
5186             self.failIfIn("Not Healthy", res)
5187         d.addCallback(_got_html_good)
5188
5189         d.addCallback(self._count_leases, "one")
5190         d.addCallback(self._assert_leasecount, 1)
5191         d.addCallback(self._count_leases, "two")
5192         d.addCallback(self._assert_leasecount, 1)
5193         d.addCallback(self._count_leases, "mutable")
5194         d.addCallback(self._assert_leasecount, 1)
5195
5196         # this CHECK uses the original client, which uses the same
5197         # lease-secrets, so it will just renew the original lease
5198         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
5199         d.addCallback(_got_html_good)
5200
5201         d.addCallback(self._count_leases, "one")
5202         d.addCallback(self._assert_leasecount, 1)
5203         d.addCallback(self._count_leases, "two")
5204         d.addCallback(self._assert_leasecount, 1)
5205         d.addCallback(self._count_leases, "mutable")
5206         d.addCallback(self._assert_leasecount, 1)
5207
5208         # this CHECK uses an alternate client, which adds a second lease
5209         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
5210         d.addCallback(_got_html_good)
5211
5212         d.addCallback(self._count_leases, "one")
5213         d.addCallback(self._assert_leasecount, 2)
5214         d.addCallback(self._count_leases, "two")
5215         d.addCallback(self._assert_leasecount, 1)
5216         d.addCallback(self._count_leases, "mutable")
5217         d.addCallback(self._assert_leasecount, 1)
5218
5219         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
5220         d.addCallback(_got_html_good)
5221
5222         d.addCallback(self._count_leases, "one")
5223         d.addCallback(self._assert_leasecount, 2)
5224         d.addCallback(self._count_leases, "two")
5225         d.addCallback(self._assert_leasecount, 1)
5226         d.addCallback(self._count_leases, "mutable")
5227         d.addCallback(self._assert_leasecount, 1)
5228
5229         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
5230                       clientnum=1)
5231         d.addCallback(_got_html_good)
5232
5233         d.addCallback(self._count_leases, "one")
5234         d.addCallback(self._assert_leasecount, 2)
5235         d.addCallback(self._count_leases, "two")
5236         d.addCallback(self._assert_leasecount, 1)
5237         d.addCallback(self._count_leases, "mutable")
5238         d.addCallback(self._assert_leasecount, 2)
5239
5240         d.addErrback(self.explain_web_error)
5241         return d
5242
5243     def test_deep_add_lease(self):
5244         self.basedir = "web/Grid/deep_add_lease"
5245         self.set_up_grid(num_clients=2)
5246         c0 = self.g.clients[0]
5247         self.uris = {}
5248         self.fileurls = {}
5249         DATA = "data" * 100
5250         d = c0.create_dirnode()
5251         def _stash_root_and_create_file(n):
5252             self.rootnode = n
5253             self.uris["root"] = n.get_uri()
5254             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5255             return n.add_file(u"one", upload.Data(DATA, convergence=""))
5256         d.addCallback(_stash_root_and_create_file)
5257         def _stash_uri(fn, which):
5258             self.uris[which] = fn.get_uri()
5259         d.addCallback(_stash_uri, "one")
5260         d.addCallback(lambda ign:
5261                       self.rootnode.add_file(u"small",
5262                                              upload.Data("literal",
5263                                                         convergence="")))
5264         d.addCallback(_stash_uri, "small")
5265
5266         d.addCallback(lambda ign:
5267             c0.create_mutable_file(publish.MutableData("mutable")))
5268         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
5269         d.addCallback(_stash_uri, "mutable")
5270
5271         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
5272         def _done(res):
5273             units = [simplejson.loads(line)
5274                      for line in res.splitlines()
5275                      if line]
5276             # root, one, small, mutable,   stats
5277             self.failUnlessReallyEqual(len(units), 4+1)
5278         d.addCallback(_done)
5279
5280         d.addCallback(self._count_leases, "root")
5281         d.addCallback(self._assert_leasecount, 1)
5282         d.addCallback(self._count_leases, "one")
5283         d.addCallback(self._assert_leasecount, 1)
5284         d.addCallback(self._count_leases, "mutable")
5285         d.addCallback(self._assert_leasecount, 1)
5286
5287         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
5288         d.addCallback(_done)
5289
5290         d.addCallback(self._count_leases, "root")
5291         d.addCallback(self._assert_leasecount, 1)
5292         d.addCallback(self._count_leases, "one")
5293         d.addCallback(self._assert_leasecount, 1)
5294         d.addCallback(self._count_leases, "mutable")
5295         d.addCallback(self._assert_leasecount, 1)
5296
5297         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
5298                       clientnum=1)
5299         d.addCallback(_done)
5300
5301         d.addCallback(self._count_leases, "root")
5302         d.addCallback(self._assert_leasecount, 2)
5303         d.addCallback(self._count_leases, "one")
5304         d.addCallback(self._assert_leasecount, 2)
5305         d.addCallback(self._count_leases, "mutable")
5306         d.addCallback(self._assert_leasecount, 2)
5307
5308         d.addErrback(self.explain_web_error)
5309         return d
5310
5311
5312     def test_exceptions(self):
5313         self.basedir = "web/Grid/exceptions"
5314         self.set_up_grid(num_clients=1, num_servers=2)
5315         c0 = self.g.clients[0]
5316         c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 2
5317         self.fileurls = {}
5318         DATA = "data" * 100
5319         d = c0.create_dirnode()
5320         def _stash_root(n):
5321             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5322             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
5323             return n
5324         d.addCallback(_stash_root)
5325         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
5326         def _stash_bad(ur):
5327             self.fileurls["1share"] = "uri/" + urllib.quote(ur.get_uri())
5328             self.delete_shares_numbered(ur.get_uri(), range(1,10))
5329
5330             u = uri.from_string(ur.get_uri())
5331             u.key = testutil.flip_bit(u.key, 0)
5332             baduri = u.to_string()
5333             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
5334         d.addCallback(_stash_bad)
5335         d.addCallback(lambda ign: c0.create_dirnode())
5336         def _mangle_dirnode_1share(n):
5337             u = n.get_uri()
5338             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
5339             self.fileurls["dir-1share-json"] = url + "?t=json"
5340             self.delete_shares_numbered(u, range(1,10))
5341         d.addCallback(_mangle_dirnode_1share)
5342         d.addCallback(lambda ign: c0.create_dirnode())
5343         def _mangle_dirnode_0share(n):
5344             u = n.get_uri()
5345             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
5346             self.fileurls["dir-0share-json"] = url + "?t=json"
5347             self.delete_shares_numbered(u, range(0,10))
5348         d.addCallback(_mangle_dirnode_0share)
5349
5350         # NotEnoughSharesError should be reported sensibly, with a
5351         # text/plain explanation of the problem, and perhaps some
5352         # information on which shares *could* be found.
5353
5354         d.addCallback(lambda ignored:
5355                       self.shouldHTTPError("GET unrecoverable",
5356                                            410, "Gone", "NoSharesError",
5357                                            self.GET, self.fileurls["0shares"]))
5358         def _check_zero_shares(body):
5359             self.failIfIn("<html>", body)
5360             body = " ".join(body.strip().split())
5361             exp = ("NoSharesError: no shares could be found. "
5362                    "Zero shares usually indicates a corrupt URI, or that "
5363                    "no servers were connected, but it might also indicate "
5364                    "severe corruption. You should perform a filecheck on "
5365                    "this object to learn more. The full error message is: "
5366                    "no shares (need 3). Last failure: None")
5367             self.failUnlessReallyEqual(exp, body)
5368         d.addCallback(_check_zero_shares)
5369
5370
5371         d.addCallback(lambda ignored:
5372                       self.shouldHTTPError("GET 1share",
5373                                            410, "Gone", "NotEnoughSharesError",
5374                                            self.GET, self.fileurls["1share"]))
5375         def _check_one_share(body):
5376             self.failIfIn("<html>", body)
5377             body = " ".join(body.strip().split())
5378             msgbase = ("NotEnoughSharesError: This indicates that some "
5379                        "servers were unavailable, or that shares have been "
5380                        "lost to server departure, hard drive failure, or disk "
5381                        "corruption. You should perform a filecheck on "
5382                        "this object to learn more. The full error message is:"
5383                        )
5384             msg1 = msgbase + (" ran out of shares:"
5385                               " complete=sh0"
5386                               " pending="
5387                               " overdue= unused= need 3. Last failure: None")
5388             msg2 = msgbase + (" ran out of shares:"
5389                               " complete="
5390                               " pending=Share(sh0-on-xgru5)"
5391                               " overdue= unused= need 3. Last failure: None")
5392             self.failUnless(body == msg1 or body == msg2, body)
5393         d.addCallback(_check_one_share)
5394
5395         d.addCallback(lambda ignored:
5396                       self.shouldHTTPError("GET imaginary",
5397                                            404, "Not Found", None,
5398                                            self.GET, self.fileurls["imaginary"]))
5399         def _missing_child(body):
5400             self.failUnlessIn("No such child: imaginary", body)
5401         d.addCallback(_missing_child)
5402
5403         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
5404         def _check_0shares_dir_html(body):
5405             self.failUnlessIn("<html>", body)
5406             # we should see the regular page, but without the child table or
5407             # the dirops forms
5408             body = " ".join(body.strip().split())
5409             self.failUnlessIn('href="?t=info">More info on this directory',
5410                               body)
5411             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5412                    "could not be retrieved, because there were insufficient "
5413                    "good shares. This might indicate that no servers were "
5414                    "connected, insufficient servers were connected, the URI "
5415                    "was corrupt, or that shares have been lost due to server "
5416                    "departure, hard drive failure, or disk corruption. You "
5417                    "should perform a filecheck on this object to learn more.")
5418             self.failUnlessIn(exp, body)
5419             self.failUnlessIn("No upload forms: directory is unreadable", body)
5420         d.addCallback(_check_0shares_dir_html)
5421
5422         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
5423         def _check_1shares_dir_html(body):
5424             # at some point, we'll split UnrecoverableFileError into 0-shares
5425             # and some-shares like we did for immutable files (since there
5426             # are different sorts of advice to offer in each case). For now,
5427             # they present the same way.
5428             self.failUnlessIn("<html>", body)
5429             body = " ".join(body.strip().split())
5430             self.failUnlessIn('href="?t=info">More info on this directory',
5431                               body)
5432             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5433                    "could not be retrieved, because there were insufficient "
5434                    "good shares. This might indicate that no servers were "
5435                    "connected, insufficient servers were connected, the URI "
5436                    "was corrupt, or that shares have been lost due to server "
5437                    "departure, hard drive failure, or disk corruption. You "
5438                    "should perform a filecheck on this object to learn more.")
5439             self.failUnlessIn(exp, body)
5440             self.failUnlessIn("No upload forms: directory is unreadable", body)
5441         d.addCallback(_check_1shares_dir_html)
5442
5443         d.addCallback(lambda ignored:
5444                       self.shouldHTTPError("GET dir-0share-json",
5445                                            410, "Gone", "UnrecoverableFileError",
5446                                            self.GET,
5447                                            self.fileurls["dir-0share-json"]))
5448         def _check_unrecoverable_file(body):
5449             self.failIfIn("<html>", body)
5450             body = " ".join(body.strip().split())
5451             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5452                    "could not be retrieved, because there were insufficient "
5453                    "good shares. This might indicate that no servers were "
5454                    "connected, insufficient servers were connected, the URI "
5455                    "was corrupt, or that shares have been lost due to server "
5456                    "departure, hard drive failure, or disk corruption. You "
5457                    "should perform a filecheck on this object to learn more.")
5458             self.failUnlessReallyEqual(exp, body)
5459         d.addCallback(_check_unrecoverable_file)
5460
5461         d.addCallback(lambda ignored:
5462                       self.shouldHTTPError("GET dir-1share-json",
5463                                            410, "Gone", "UnrecoverableFileError",
5464                                            self.GET,
5465                                            self.fileurls["dir-1share-json"]))
5466         d.addCallback(_check_unrecoverable_file)
5467
5468         d.addCallback(lambda ignored:
5469                       self.shouldHTTPError("GET imaginary",
5470                                            404, "Not Found", None,
5471                                            self.GET, self.fileurls["imaginary"]))
5472
5473         # attach a webapi child that throws a random error, to test how it
5474         # gets rendered.
5475         w = c0.getServiceNamed("webish")
5476         w.root.putChild("ERRORBOOM", ErrorBoom())
5477
5478         # "Accept: */*" :        should get a text/html stack trace
5479         # "Accept: text/plain" : should get a text/plain stack trace
5480         # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
5481         # no Accept header:      should get a text/html stack trace
5482
5483         d.addCallback(lambda ignored:
5484                       self.shouldHTTPError("GET errorboom_html",
5485                                            500, "Internal Server Error", None,
5486                                            self.GET, "ERRORBOOM",
5487                                            headers={"accept": "*/*"}))
5488         def _internal_error_html1(body):
5489             self.failUnlessIn("<html>", "expected HTML, not '%s'" % body)
5490         d.addCallback(_internal_error_html1)
5491
5492         d.addCallback(lambda ignored:
5493                       self.shouldHTTPError("GET errorboom_text",
5494                                            500, "Internal Server Error", None,
5495                                            self.GET, "ERRORBOOM",
5496                                            headers={"accept": "text/plain"}))
5497         def _internal_error_text2(body):
5498             self.failIfIn("<html>", body)
5499             self.failUnless(body.startswith("Traceback "), body)
5500         d.addCallback(_internal_error_text2)
5501
5502         CLI_accepts = "text/plain, application/octet-stream"
5503         d.addCallback(lambda ignored:
5504                       self.shouldHTTPError("GET errorboom_text",
5505                                            500, "Internal Server Error", None,
5506                                            self.GET, "ERRORBOOM",
5507                                            headers={"accept": CLI_accepts}))
5508         def _internal_error_text3(body):
5509             self.failIfIn("<html>", body)
5510             self.failUnless(body.startswith("Traceback "), body)
5511         d.addCallback(_internal_error_text3)
5512
5513         d.addCallback(lambda ignored:
5514                       self.shouldHTTPError("GET errorboom_text",
5515                                            500, "Internal Server Error", None,
5516                                            self.GET, "ERRORBOOM"))
5517         def _internal_error_html4(body):
5518             self.failUnlessIn("<html>", body)
5519         d.addCallback(_internal_error_html4)
5520
5521         def _flush_errors(res):
5522             # Trial: please ignore the CompletelyUnhandledError in the logs
5523             self.flushLoggedErrors(CompletelyUnhandledError)
5524             return res
5525         d.addBoth(_flush_errors)
5526
5527         return d
5528
5529     def test_blacklist(self):
5530         # download from a blacklisted URI, get an error
5531         self.basedir = "web/Grid/blacklist"
5532         self.set_up_grid()
5533         c0 = self.g.clients[0]
5534         c0_basedir = c0.basedir
5535         fn = os.path.join(c0_basedir, "access.blacklist")
5536         self.uris = {}
5537         DATA = "off-limits " * 50
5538
5539         d = c0.upload(upload.Data(DATA, convergence=""))
5540         def _stash_uri_and_create_dir(ur):
5541             self.uri = ur.get_uri()
5542             self.url = "uri/"+self.uri
5543             u = uri.from_string_filenode(self.uri)
5544             self.si = u.get_storage_index()
5545             childnode = c0.create_node_from_uri(self.uri, None)
5546             return c0.create_dirnode({u"blacklisted.txt": (childnode,{}) })
5547         d.addCallback(_stash_uri_and_create_dir)
5548         def _stash_dir(node):
5549             self.dir_node = node
5550             self.dir_uri = node.get_uri()
5551             self.dir_url = "uri/"+self.dir_uri
5552         d.addCallback(_stash_dir)
5553         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5554         def _check_dir_html(body):
5555             self.failUnlessIn("<html>", body)
5556             self.failUnlessIn("blacklisted.txt</a>", body)
5557         d.addCallback(_check_dir_html)
5558         d.addCallback(lambda ign: self.GET(self.url))
5559         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5560
5561         def _blacklist(ign):
5562             f = open(fn, "w")
5563             f.write(" # this is a comment\n")
5564             f.write(" \n")
5565             f.write("\n") # also exercise blank lines
5566             f.write("%s %s\n" % (base32.b2a(self.si), "off-limits to you"))
5567             f.close()
5568             # clients should be checking the blacklist each time, so we don't
5569             # need to restart the client
5570         d.addCallback(_blacklist)
5571         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_uri",
5572                                                        403, "Forbidden",
5573                                                        "Access Prohibited: off-limits",
5574                                                        self.GET, self.url))
5575
5576         # We should still be able to list the parent directory, in HTML...
5577         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5578         def _check_dir_html2(body):
5579             self.failUnlessIn("<html>", body)
5580             self.failUnlessIn("blacklisted.txt</strike>", body)
5581         d.addCallback(_check_dir_html2)
5582
5583         # ... and in JSON (used by CLI).
5584         d.addCallback(lambda ign: self.GET(self.dir_url+"?t=json", followRedirect=True))
5585         def _check_dir_json(res):
5586             data = simplejson.loads(res)
5587             self.failUnless(isinstance(data, list), data)
5588             self.failUnlessEqual(data[0], "dirnode")
5589             self.failUnless(isinstance(data[1], dict), data)
5590             self.failUnlessIn("children", data[1])
5591             self.failUnlessIn("blacklisted.txt", data[1]["children"])
5592             childdata = data[1]["children"]["blacklisted.txt"]
5593             self.failUnless(isinstance(childdata, list), data)
5594             self.failUnlessEqual(childdata[0], "filenode")
5595             self.failUnless(isinstance(childdata[1], dict), data)
5596         d.addCallback(_check_dir_json)
5597
5598         def _unblacklist(ign):
5599             open(fn, "w").close()
5600             # the Blacklist object watches mtime to tell when the file has
5601             # changed, but on windows this test will run faster than the
5602             # filesystem's mtime resolution. So we edit Blacklist.last_mtime
5603             # to force a reload.
5604             self.g.clients[0].blacklist.last_mtime -= 2.0
5605         d.addCallback(_unblacklist)
5606
5607         # now a read should work
5608         d.addCallback(lambda ign: self.GET(self.url))
5609         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5610
5611         # read again to exercise the blacklist-is-unchanged logic
5612         d.addCallback(lambda ign: self.GET(self.url))
5613         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5614
5615         # now add a blacklisted directory, and make sure files under it are
5616         # refused too
5617         def _add_dir(ign):
5618             childnode = c0.create_node_from_uri(self.uri, None)
5619             return c0.create_dirnode({u"child": (childnode,{}) })
5620         d.addCallback(_add_dir)
5621         def _get_dircap(dn):
5622             self.dir_si_b32 = base32.b2a(dn.get_storage_index())
5623             self.dir_url_base = "uri/"+dn.get_write_uri()
5624             self.dir_url_json1 = "uri/"+dn.get_write_uri()+"?t=json"
5625             self.dir_url_json2 = "uri/"+dn.get_write_uri()+"/?t=json"
5626             self.dir_url_json_ro = "uri/"+dn.get_readonly_uri()+"/?t=json"
5627             self.child_url = "uri/"+dn.get_readonly_uri()+"/child"
5628         d.addCallback(_get_dircap)
5629         d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True))
5630         d.addCallback(lambda body: self.failUnlessIn("<html>", body))
5631         d.addCallback(lambda ign: self.GET(self.dir_url_json1))
5632         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5633         d.addCallback(lambda ign: self.GET(self.dir_url_json2))
5634         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5635         d.addCallback(lambda ign: self.GET(self.dir_url_json_ro))
5636         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5637         d.addCallback(lambda ign: self.GET(self.child_url))
5638         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5639
5640         def _block_dir(ign):
5641             f = open(fn, "w")
5642             f.write("%s %s\n" % (self.dir_si_b32, "dir-off-limits to you"))
5643             f.close()
5644             self.g.clients[0].blacklist.last_mtime -= 2.0
5645         d.addCallback(_block_dir)
5646         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir base",
5647                                                        403, "Forbidden",
5648                                                        "Access Prohibited: dir-off-limits",
5649                                                        self.GET, self.dir_url_base))
5650         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json1",
5651                                                        403, "Forbidden",
5652                                                        "Access Prohibited: dir-off-limits",
5653                                                        self.GET, self.dir_url_json1))
5654         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json2",
5655                                                        403, "Forbidden",
5656                                                        "Access Prohibited: dir-off-limits",
5657                                                        self.GET, self.dir_url_json2))
5658         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json_ro",
5659                                                        403, "Forbidden",
5660                                                        "Access Prohibited: dir-off-limits",
5661                                                        self.GET, self.dir_url_json_ro))
5662         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir child",
5663                                                        403, "Forbidden",
5664                                                        "Access Prohibited: dir-off-limits",
5665                                                        self.GET, self.child_url))
5666         return d
5667
5668
5669 class CompletelyUnhandledError(Exception):
5670     pass
5671 class ErrorBoom(rend.Page):
5672     def beforeRender(self, ctx):
5673         raise CompletelyUnhandledError("whoops")