src/allmydata/test/test_web.py

   1 import os.path, re, urllib, time, cgi
   2 import simplejson
   3 from StringIO import StringIO
   4
   5 from twisted.application import service
   6 from twisted.trial import unittest
   7 from twisted.internet import defer, reactor
   8 from twisted.internet.task import Clock
   9 from twisted.web import client, error, http
  10 from twisted.python import failure, log
  11
  12 from foolscap.api import fireEventually, flushEventualQueue
  13
  14 from nevow.util import escapeToXML
  15 from nevow import rend
  16
  17 from allmydata import interfaces, uri, webish, dirnode
  18 from allmydata.storage.shares import get_share_file
  19 from allmydata.storage_client import StorageFarmBroker, StubServer
  20 from allmydata.immutable import upload
  21 from allmydata.immutable.downloader.status import DownloadStatus
  22 from allmydata.dirnode import DirectoryNode
  23 from allmydata.nodemaker import NodeMaker
  24 from allmydata.unknown import UnknownNode
  25 from allmydata.web import status, common
  26 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  27 from allmydata.util import fileutil, base32, hashutil
  28 from allmydata.util.consumer import download_to_data
  29 from allmydata.util.netstring import split_netstring
  30 from allmydata.util.encodingutil import to_str
  31 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  32      create_chk_filenode, WebErrorMixin, ShouldFailMixin, \
  33      make_mutable_file_uri, create_mutable_filenode
  34 from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
  35 from allmydata.mutable import servermap, publish, retrieve
  36 import allmydata.test.common_util as testutil
  37 from allmydata.test.no_network import GridTestMixin
  38 from allmydata.test.common_web import HTTPClientGETFactory, \
  39      HTTPClientHEADFactory
  40 from allmydata.client import Client, SecretHolder
  41 from allmydata.introducer import IntroducerNode
  42
  43 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  44 # create a webserver that works against them
  45
  46 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  47
  48 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
  49 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
  50 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
  51
  52 FAVICON_MARKUP = '<link href="/icon.png" rel="shortcut icon" />'
  53
  54
  55 class FakeStatsProvider:
  56     def get_stats(self):
  57         stats = {'stats': {}, 'counters': {}}
  58         return stats
  59
  60 class FakeNodeMaker(NodeMaker):
  61     encoding_params = {
  62         'k': 3,
  63         'n': 10,
  64         'happy': 7,
  65         'max_segment_size':128*1024 # 1024=KiB
  66     }
  67     def _create_lit(self, cap):
  68         return FakeCHKFileNode(cap, self.all_contents)
  69     def _create_immutable(self, cap):
  70         return FakeCHKFileNode(cap, self.all_contents)
  71     def _create_mutable(self, cap):
  72         return FakeMutableFileNode(None, None,
  73                                    self.encoding_params, None,
  74                                    self.all_contents).init_from_cap(cap)
  75     def create_mutable_file(self, contents="", keysize=None,
  76                             version=SDMF_VERSION):
  77         n = FakeMutableFileNode(None, None, self.encoding_params, None,
  78                                 self.all_contents)
  79         return n.create(contents, version=version)
  80
  81 class FakeUploader(service.Service):
  82     name = "uploader"
  83     helper_furl = None
  84     helper_connected = False
  85
  86     def upload(self, uploadable):
  87         d = uploadable.get_size()
  88         d.addCallback(lambda size: uploadable.read(size))
  89         def _got_data(datav):
  90             data = "".join(datav)
  91             n = create_chk_filenode(data, self.all_contents)
  92             ur = upload.UploadResults(file_size=len(data),
  93                                       ciphertext_fetched=0,
  94                                       preexisting_shares=0,
  95                                       pushed_shares=10,
  96                                       sharemap={},
  97                                       servermap={},
  98                                       timings={},
  99                                       uri_extension_data={},
 100                                       uri_extension_hash="fake",
 101                                       verifycapstr="fakevcap")
 102             ur.set_uri(n.get_uri())
 103             return ur
 104         d.addCallback(_got_data)
 105         return d
 106
 107     def get_helper_info(self):
 108         return (self.helper_furl, self.helper_connected)
 109
 110
 111 def build_one_ds():
 112     ds = DownloadStatus("storage_index", 1234)
 113     now = time.time()
 114
 115     serverA = StubServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
 116     serverB = StubServer(hashutil.tagged_hash("foo", "serverid_b")[:20])
 117     storage_index = hashutil.storage_index_hash("SI")
 118     e0 = ds.add_segment_request(0, now)
 119     e0.activate(now+0.5)
 120     e0.deliver(now+1, 0, 100, 0.5) # when, start,len, decodetime
 121     e1 = ds.add_segment_request(1, now+2)
 122     e1.error(now+3)
 123     # two outstanding requests
 124     e2 = ds.add_segment_request(2, now+4)
 125     e3 = ds.add_segment_request(3, now+5)
 126     del e2,e3 # hush pyflakes
 127
 128     # simulate a segment which gets delivered faster than a system clock tick (ticket #1166)
 129     e = ds.add_segment_request(4, now)
 130     e.activate(now)
 131     e.deliver(now, 0, 140, 0.5)
 132
 133     e = ds.add_dyhb_request(serverA, now)
 134     e.finished([1,2], now+1)
 135     e = ds.add_dyhb_request(serverB, now+2) # left unfinished
 136
 137     e = ds.add_read_event(0, 120, now)
 138     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 139     e.finished(now+1)
 140     e = ds.add_read_event(120, 30, now+2) # left unfinished
 141
 142     e = ds.add_block_request(serverA, 1, 100, 20, now)
 143     e.finished(20, now+1)
 144     e = ds.add_block_request(serverB, 1, 120, 30, now+1) # left unfinished
 145
 146     # make sure that add_read_event() can come first too
 147     ds1 = DownloadStatus(storage_index, 1234)
 148     e = ds1.add_read_event(0, 120, now)
 149     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 150     e.finished(now+1)
 151
 152     return ds
 153
 154 class FakeHistory:
 155     _all_upload_status = [upload.UploadStatus()]
 156     _all_download_status = [build_one_ds()]
 157     _all_mapupdate_statuses = [servermap.UpdateStatus()]
 158     _all_publish_statuses = [publish.PublishStatus()]
 159     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
 160
 161     def list_all_upload_statuses(self):
 162         return self._all_upload_status
 163     def list_all_download_statuses(self):
 164         return self._all_download_status
 165     def list_all_mapupdate_statuses(self):
 166         return self._all_mapupdate_statuses
 167     def list_all_publish_statuses(self):
 168         return self._all_publish_statuses
 169     def list_all_retrieve_statuses(self):
 170         return self._all_retrieve_statuses
 171     def list_all_helper_statuses(self):
 172         return []
 173
 174 class FakeDisplayableServer(StubServer):
 175     def __init__(self, serverid, nickname):
 176         StubServer.__init__(self, serverid)
 177         self.announcement = {"my-version": "allmydata-tahoe-fake",
 178                              "service-name": "storage",
 179                              "nickname": nickname}
 180     def is_connected(self):
 181         return True
 182     def get_permutation_seed(self):
 183         return ""
 184     def get_remote_host(self):
 185         return ""
 186     def get_last_loss_time(self):
 187         return None
 188     def get_announcement_time(self):
 189         return None
 190     def get_announcement(self):
 191         return self.announcement
 192     def get_nickname(self):
 193         return self.announcement["nickname"]
 194
 195 class FakeBucketCounter(object):
 196     def get_state(self):
 197         return {"last-complete-bucket-count": 0}
 198     def get_progress(self):
 199         return {"estimated-time-per-cycle": 0,
 200                 "cycle-in-progress": False,
 201                 "remaining-wait-time": 0}
 202
 203 class FakeLeaseChecker(object):
 204     def __init__(self):
 205         self.expiration_enabled = False
 206         self.mode = "age"
 207         self.override_lease_duration = None
 208         self.sharetypes_to_expire = {}
 209     def get_state(self):
 210         return {"history": None}
 211     def get_progress(self):
 212         return {"estimated-time-per-cycle": 0,
 213                 "cycle-in-progress": False,
 214                 "remaining-wait-time": 0}
 215
 216 class FakeStorageServer(service.MultiService):
 217     name = 'storage'
 218     def __init__(self, nodeid, nickname):
 219         service.MultiService.__init__(self)
 220         self.my_nodeid = nodeid
 221         self.nickname = nickname
 222         self.bucket_counter = FakeBucketCounter()
 223         self.lease_checker = FakeLeaseChecker()
 224     def get_stats(self):
 225         return {"storage_server.accepting_immutable_shares": False}
 226
 227 class FakeClient(Client):
 228     def __init__(self):
 229         # don't upcall to Client.__init__, since we only want to initialize a
 230         # minimal subset
 231         service.MultiService.__init__(self)
 232         self.all_contents = {}
 233         self.nodeid = "fake_nodeid"
 234         self.nickname = u"fake_nickname \u263A"
 235         self.introducer_furl = "None"
 236         self.stats_provider = FakeStatsProvider()
 237         self._secret_holder = SecretHolder("lease secret", "convergence secret")
 238         self.helper = None
 239         self.convergence = "some random string"
 240         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 241         # fake knowledge of another server
 242         self.storage_broker.test_add_server("other_nodeid",
 243                                             FakeDisplayableServer("other_nodeid", u"other_nickname \u263B"))
 244         self.introducer_client = None
 245         self.history = FakeHistory()
 246         self.uploader = FakeUploader()
 247         self.uploader.all_contents = self.all_contents
 248         self.uploader.setServiceParent(self)
 249         self.blacklist = None
 250         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 251                                        self.uploader, None,
 252                                        None, None, None)
 253         self.nodemaker.all_contents = self.all_contents
 254         self.mutable_file_default = SDMF_VERSION
 255         self.addService(FakeStorageServer(self.nodeid, self.nickname))
 256
 257     def startService(self):
 258         return service.MultiService.startService(self)
 259     def stopService(self):
 260         return service.MultiService.stopService(self)
 261
 262     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 263
 264 class WebMixin(object):
 265     def setUp(self):
 266         self.s = FakeClient()
 267         self.s.startService()
 268         self.staticdir = self.mktemp()
 269         self.clock = Clock()
 270         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
 271                                       clock=self.clock)
 272         self.ws.setServiceParent(self.s)
 273         self.webish_port = self.ws.getPortnum()
 274         self.webish_url = self.ws.getURL()
 275         assert self.webish_url.endswith("/")
 276         self.webish_url = self.webish_url[:-1] # these tests add their own /
 277
 278         l = [ self.s.create_dirnode() for x in range(6) ]
 279         d = defer.DeferredList(l)
 280         def _then(res):
 281             self.public_root = res[0][1]
 282             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 283             self.public_url = "/uri/" + self.public_root.get_uri()
 284             self.private_root = res[1][1]
 285
 286             foo = res[2][1]
 287             self._foo_node = foo
 288             self._foo_uri = foo.get_uri()
 289             self._foo_readonly_uri = foo.get_readonly_uri()
 290             self._foo_verifycap = foo.get_verify_cap().to_string()
 291             # NOTE: we ignore the deferred on all set_uri() calls, because we
 292             # know the fake nodes do these synchronously
 293             self.public_root.set_uri(u"foo", foo.get_uri(),
 294                                      foo.get_readonly_uri())
 295
 296             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 297             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 298             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 299
 300             # sdmf
 301             # XXX: Do we ever use this?
 302             self.BAZ_CONTENTS, n, self._baz_txt_uri, self._baz_txt_readonly_uri = self.makefile_mutable(0)
 303
 304             foo.set_uri(u"baz.txt", self._baz_txt_uri, self._baz_txt_readonly_uri)
 305
 306             # mdmf
 307             self.QUUX_CONTENTS, n, self._quux_txt_uri, self._quux_txt_readonly_uri = self.makefile_mutable(0, mdmf=True)
 308             assert self._quux_txt_uri.startswith("URI:MDMF")
 309             foo.set_uri(u"quux.txt", self._quux_txt_uri, self._quux_txt_readonly_uri)
 310
 311             foo.set_uri(u"empty", res[3][1].get_uri(),
 312                         res[3][1].get_readonly_uri())
 313             sub_uri = res[4][1].get_uri()
 314             self._sub_uri = sub_uri
 315             foo.set_uri(u"sub", sub_uri, sub_uri)
 316             sub = self.s.create_node_from_uri(sub_uri)
 317             self._sub_node = sub
 318
 319             _ign, n, blocking_uri = self.makefile(1)
 320             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 321
 322             # filenode to test for html encoding issues
 323             self._htmlname_unicode = u"<&weirdly'named\"file>>>_<iframe />.txt"
 324             self._htmlname_raw = self._htmlname_unicode.encode('utf-8')
 325             self._htmlname_urlencoded = urllib.quote(self._htmlname_raw, '')
 326             self._htmlname_escaped = escapeToXML(self._htmlname_raw)
 327             self._htmlname_escaped_attr = cgi.escape(self._htmlname_raw, quote=True)
 328             self._htmlname_escaped_double = escapeToXML(cgi.escape(self._htmlname_raw, quote=True))
 329             self.HTMLNAME_CONTENTS, n, self._htmlname_txt_uri = self.makefile(0)
 330             foo.set_uri(self._htmlname_unicode, self._htmlname_txt_uri, self._htmlname_txt_uri)
 331
 332             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 333             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 334             # still think of it as an umlaut
 335             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 336
 337             self.SUBBAZ_CONTENTS, n, baz_file = self.makefile(2)
 338             self._baz_file_uri = baz_file
 339             sub.set_uri(u"baz.txt", baz_file, baz_file)
 340
 341             _ign, n, self._bad_file_uri = self.makefile(3)
 342             # this uri should not be downloadable
 343             del self.s.all_contents[self._bad_file_uri]
 344
 345             rodir = res[5][1]
 346             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 347                                      rodir.get_readonly_uri())
 348             rodir.set_uri(u"nor", baz_file, baz_file)
 349
 350             # public/
 351             # public/foo/
 352             # public/foo/bar.txt
 353             # public/foo/baz.txt
 354             # public/foo/quux.txt
 355             # public/foo/blockingfile
 356             # public/foo/<&weirdly'named\"file>>>_<iframe />.txt
 357             # public/foo/empty/
 358             # public/foo/sub/
 359             # public/foo/sub/baz.txt
 360             # public/reedownlee/
 361             # public/reedownlee/nor
 362             self.NEWFILE_CONTENTS = "newfile contents\n"
 363
 364             return foo.get_metadata_for(u"bar.txt")
 365         d.addCallback(_then)
 366         def _got_metadata(metadata):
 367             self._bar_txt_metadata = metadata
 368         d.addCallback(_got_metadata)
 369         return d
 370
 371     def get_all_contents(self):
 372         return self.s.all_contents
 373
 374     def makefile(self, number):
 375         contents = "contents of file %s\n" % number
 376         n = create_chk_filenode(contents, self.get_all_contents())
 377         return contents, n, n.get_uri()
 378
 379     def makefile_mutable(self, number, mdmf=False):
 380         contents = "contents of mutable file %s\n" % number
 381         n = create_mutable_filenode(contents, mdmf, self.s.all_contents)
 382         return contents, n, n.get_uri(), n.get_readonly_uri()
 383
 384     def tearDown(self):
 385         return self.s.stopService()
 386
 387     def failUnlessIsBarDotTxt(self, res):
 388         self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
 389
 390     def failUnlessIsQuuxDotTxt(self, res):
 391         self.failUnlessReallyEqual(res, self.QUUX_CONTENTS, res)
 392
 393     def failUnlessIsBazDotTxt(self, res):
 394         self.failUnlessReallyEqual(res, self.BAZ_CONTENTS, res)
 395
 396     def failUnlessIsSubBazDotTxt(self, res):
 397         self.failUnlessReallyEqual(res, self.SUBBAZ_CONTENTS, res)
 398
 399     def failUnlessIsBarJSON(self, res):
 400         data = simplejson.loads(res)
 401         self.failUnless(isinstance(data, list))
 402         self.failUnlessEqual(data[0], "filenode")
 403         self.failUnless(isinstance(data[1], dict))
 404         self.failIf(data[1]["mutable"])
 405         self.failIfIn("rw_uri", data[1]) # immutable
 406         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
 407         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
 408         self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
 409
 410     def failUnlessIsQuuxJSON(self, res, readonly=False):
 411         data = simplejson.loads(res)
 412         self.failUnless(isinstance(data, list))
 413         self.failUnlessEqual(data[0], "filenode")
 414         self.failUnless(isinstance(data[1], dict))
 415         metadata = data[1]
 416         return self.failUnlessIsQuuxDotTxtMetadata(metadata, readonly)
 417
 418     def failUnlessIsQuuxDotTxtMetadata(self, metadata, readonly):
 419         self.failUnless(metadata['mutable'])
 420         if readonly:
 421             self.failIfIn("rw_uri", metadata)
 422         else:
 423             self.failUnlessIn("rw_uri", metadata)
 424             self.failUnlessEqual(metadata['rw_uri'], self._quux_txt_uri)
 425         self.failUnlessIn("ro_uri", metadata)
 426         self.failUnlessEqual(metadata['ro_uri'], self._quux_txt_readonly_uri)
 427         self.failUnlessReallyEqual(metadata['size'], len(self.QUUX_CONTENTS))
 428
 429     def failUnlessIsFooJSON(self, res):
 430         data = simplejson.loads(res)
 431         self.failUnless(isinstance(data, list))
 432         self.failUnlessEqual(data[0], "dirnode", res)
 433         self.failUnless(isinstance(data[1], dict))
 434         self.failUnless(data[1]["mutable"])
 435         self.failUnlessIn("rw_uri", data[1]) # mutable
 436         self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
 437         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
 438         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
 439
 440         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 441         self.failUnlessEqual(kidnames,
 442                              [self._htmlname_unicode, u"bar.txt", u"baz.txt",
 443                               u"blockingfile", u"empty", u"n\u00fc.txt", u"quux.txt", u"sub"])
 444         kids = dict( [(unicode(name),value)
 445                       for (name,value)
 446                       in data[1]["children"].iteritems()] )
 447         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 448         self.failUnlessIn("metadata", kids[u"sub"][1])
 449         self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
 450         tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
 451         self.failUnlessIn("linkcrtime", tahoe_md)
 452         self.failUnlessIn("linkmotime", tahoe_md)
 453         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 454         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 455         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri)
 456         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]),
 457                                    self._bar_txt_verifycap)
 458         self.failUnlessIn("metadata", kids[u"bar.txt"][1])
 459         self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
 460         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
 461                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
 462         self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
 463                                    self._bar_txt_uri)
 464         self.failUnlessIn("quux.txt", kids)
 465         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["rw_uri"]),
 466                                    self._quux_txt_uri)
 467         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["ro_uri"]),
 468                                    self._quux_txt_readonly_uri)
 469
 470     def GET(self, urlpath, followRedirect=False, return_response=False,
 471             **kwargs):
 472         # if return_response=True, this fires with (data, statuscode,
 473         # respheaders) instead of just data.
 474         assert not isinstance(urlpath, unicode)
 475         url = self.webish_url + urlpath
 476         factory = HTTPClientGETFactory(url, method="GET",
 477                                        followRedirect=followRedirect, **kwargs)
 478         reactor.connectTCP("localhost", self.webish_port, factory)
 479         d = factory.deferred
 480         def _got_data(data):
 481             return (data, factory.status, factory.response_headers)
 482         if return_response:
 483             d.addCallback(_got_data)
 484         return factory.deferred
 485
 486     def HEAD(self, urlpath, return_response=False, **kwargs):
 487         # this requires some surgery, because twisted.web.client doesn't want
 488         # to give us back the response headers.
 489         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 490         reactor.connectTCP("localhost", self.webish_port, factory)
 491         d = factory.deferred
 492         def _got_data(data):
 493             return (data, factory.status, factory.response_headers)
 494         if return_response:
 495             d.addCallback(_got_data)
 496         return factory.deferred
 497
 498     def PUT(self, urlpath, data, **kwargs):
 499         url = self.webish_url + urlpath
 500         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 501
 502     def DELETE(self, urlpath):
 503         url = self.webish_url + urlpath
 504         return client.getPage(url, method="DELETE")
 505
 506     def POST(self, urlpath, followRedirect=False, **fields):
 507         sepbase = "boogabooga"
 508         sep = "--" + sepbase
 509         form = []
 510         form.append(sep)
 511         form.append('Content-Disposition: form-data; name="_charset"')
 512         form.append('')
 513         form.append('UTF-8')
 514         form.append(sep)
 515         for name, value in fields.iteritems():
 516             if isinstance(value, tuple):
 517                 filename, value = value
 518                 form.append('Content-Disposition: form-data; name="%s"; '
 519                             'filename="%s"' % (name, filename.encode("utf-8")))
 520             else:
 521                 form.append('Content-Disposition: form-data; name="%s"' % name)
 522             form.append('')
 523             if isinstance(value, unicode):
 524                 value = value.encode("utf-8")
 525             else:
 526                 value = str(value)
 527             assert isinstance(value, str)
 528             form.append(value)
 529             form.append(sep)
 530         form[-1] += "--"
 531         body = ""
 532         headers = {}
 533         if fields:
 534             body = "\r\n".join(form) + "\r\n"
 535             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 536         return self.POST2(urlpath, body, headers, followRedirect)
 537
 538     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 539         url = self.webish_url + urlpath
 540         return client.getPage(url, method="POST", postdata=body,
 541                               headers=headers, followRedirect=followRedirect)
 542
 543     def shouldFail(self, res, expected_failure, which,
 544                    substring=None, response_substring=None):
 545         if isinstance(res, failure.Failure):
 546             res.trap(expected_failure)
 547             if substring:
 548                 self.failUnlessIn(substring, str(res), which)
 549             if response_substring:
 550                 self.failUnlessIn(response_substring, res.value.response, which)
 551         else:
 552             self.fail("%s was supposed to raise %s, not get '%s'" %
 553                       (which, expected_failure, res))
 554
 555     def shouldFail2(self, expected_failure, which, substring,
 556                     response_substring,
 557                     callable, *args, **kwargs):
 558         assert substring is None or isinstance(substring, str)
 559         assert response_substring is None or isinstance(response_substring, str)
 560         d = defer.maybeDeferred(callable, *args, **kwargs)
 561         def done(res):
 562             if isinstance(res, failure.Failure):
 563                 res.trap(expected_failure)
 564                 if substring:
 565                     self.failUnlessIn(substring, str(res),
 566                                       "'%s' not in '%s' for test '%s'" % \
 567                                       (substring, str(res), which))
 568                 if response_substring:
 569                     self.failUnlessIn(response_substring, res.value.response,
 570                                       "'%s' not in '%s' for test '%s'" % \
 571                                       (response_substring, res.value.response,
 572                                        which))
 573             else:
 574                 self.fail("%s was supposed to raise %s, not get '%s'" %
 575                           (which, expected_failure, res))
 576         d.addBoth(done)
 577         return d
 578
 579     def should404(self, res, which):
 580         if isinstance(res, failure.Failure):
 581             res.trap(error.Error)
 582             self.failUnlessReallyEqual(res.value.status, "404")
 583         else:
 584             self.fail("%s was supposed to Error(404), not get '%s'" %
 585                       (which, res))
 586
 587     def should302(self, res, which):
 588         if isinstance(res, failure.Failure):
 589             res.trap(error.Error)
 590             self.failUnlessReallyEqual(res.value.status, "302")
 591         else:
 592             self.fail("%s was supposed to Error(302), not get '%s'" %
 593                       (which, res))
 594
 595
 596 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
 597     def test_create(self):
 598         pass
 599
 600     def test_welcome(self):
 601         d = self.GET("/")
 602         def _check(res):
 603             self.failUnlessIn('<title>Tahoe-LAFS - Welcome</title>', res)
 604             self.failUnlessIn(FAVICON_MARKUP, res)
 605             self.failUnlessIn('<a href="status">Recent and Active Operations</a>', res)
 606             self.failUnlessIn('<a href="statistics">Operational Statistics</a>', res)
 607             self.failUnlessIn('<input type="hidden" name="t" value="report-incident" />', res)
 608             res_u = res.decode('utf-8')
 609             self.failUnlessIn(u'<td>fake_nickname \u263A</td>', res_u)
 610             self.failUnlessIn(u'<div class="nickname">other_nickname \u263B</div>', res_u)
 611             self.failUnlessIn(u'\u00A9 <a href="https://tahoe-lafs.org/">Tahoe-LAFS Software Foundation', res_u)
 612
 613             self.s.basedir = 'web/test_welcome'
 614             fileutil.make_dirs("web/test_welcome")
 615             fileutil.make_dirs("web/test_welcome/private")
 616             return self.GET("/")
 617         d.addCallback(_check)
 618         return d
 619
 620     def test_introducer_status(self):
 621         class MockIntroducerClient(object):
 622             def __init__(self, connected):
 623                 self.connected = connected
 624             def connected_to_introducer(self):
 625                 return self.connected
 626
 627         d = defer.succeed(None)
 628
 629         # introducer not connected, unguessable furl
 630         def _set_introducer_not_connected_unguessable(ign):
 631             self.s.introducer_furl = "pb://someIntroducer/secret"
 632             self.s.introducer_client = MockIntroducerClient(False)
 633             return self.GET("/")
 634         d.addCallback(_set_introducer_not_connected_unguessable)
 635         def _check_introducer_not_connected_unguessable(res):
 636             html = res.replace('\n', ' ')
 637             self.failUnlessIn('<div class="furl">pb://someIntroducer/[censored]</div>', html)
 638             self.failIfIn('pb://someIntroducer/secret', html)
 639             self.failUnless(re.search('<div class="status-indicator connected-no"></div>[ ]*<div>Introducer not connected</div>', html), res)
 640         d.addCallback(_check_introducer_not_connected_unguessable)
 641
 642         # introducer connected, unguessable furl
 643         def _set_introducer_connected_unguessable(ign):
 644             self.s.introducer_furl = "pb://someIntroducer/secret"
 645             self.s.introducer_client = MockIntroducerClient(True)
 646             return self.GET("/")
 647         d.addCallback(_set_introducer_connected_unguessable)
 648         def _check_introducer_connected_unguessable(res):
 649             html = res.replace('\n', ' ')
 650             self.failUnlessIn('<div class="furl">pb://someIntroducer/[censored]</div>', html)
 651             self.failIfIn('pb://someIntroducer/secret', html)
 652             self.failUnless(re.search('<div class="status-indicator connected-yes"></div>[ ]*<div>Introducer</div>', html), res)
 653         d.addCallback(_check_introducer_connected_unguessable)
 654
 655         # introducer connected, guessable furl
 656         def _set_introducer_connected_guessable(ign):
 657             self.s.introducer_furl = "pb://someIntroducer/introducer"
 658             self.s.introducer_client = MockIntroducerClient(True)
 659             return self.GET("/")
 660         d.addCallback(_set_introducer_connected_guessable)
 661         def _check_introducer_connected_guessable(res):
 662             html = res.replace('\n', ' ')
 663             self.failUnlessIn('<div class="furl">pb://someIntroducer/introducer</div>', html)
 664             self.failUnless(re.search('<div class="status-indicator connected-yes"></div>[ ]*<div>Introducer</div>', html), res)
 665         d.addCallback(_check_introducer_connected_guessable)
 666         return d
 667
 668     def test_helper_status(self):
 669         d = defer.succeed(None)
 670
 671         # set helper furl to None
 672         def _set_no_helper(ign):
 673             self.s.uploader.helper_furl = None
 674             return self.GET("/")
 675         d.addCallback(_set_no_helper)
 676         def _check_no_helper(res):
 677             html = res.replace('\n', ' ')
 678             self.failUnless(re.search('<div class="status-indicator connected-not-configured"></div>[ ]*<div>Helper</div>', html), res)
 679         d.addCallback(_check_no_helper)
 680
 681         # enable helper, not connected
 682         def _set_helper_not_connected(ign):
 683             self.s.uploader.helper_furl = "pb://someHelper/secret"
 684             self.s.uploader.helper_connected = False
 685             return self.GET("/")
 686         d.addCallback(_set_helper_not_connected)
 687         def _check_helper_not_connected(res):
 688             html = res.replace('\n', ' ')
 689             self.failUnlessIn('<div class="furl">pb://someHelper/[censored]</div>', html)
 690             self.failIfIn('pb://someHelper/secret', html)
 691             self.failUnless(re.search('<div class="status-indicator connected-no"></div>[ ]*<div>Helper not connected</div>', html), res)
 692         d.addCallback(_check_helper_not_connected)
 693
 694         # enable helper, connected
 695         def _set_helper_connected(ign):
 696             self.s.uploader.helper_furl = "pb://someHelper/secret"
 697             self.s.uploader.helper_connected = True
 698             return self.GET("/")
 699         d.addCallback(_set_helper_connected)
 700         def _check_helper_connected(res):
 701             html = res.replace('\n', ' ')
 702             self.failUnlessIn('<div class="furl">pb://someHelper/[censored]</div>', html)
 703             self.failIfIn('pb://someHelper/secret', html)
 704             self.failUnless(re.search('<div class="status-indicator connected-yes"></div>[ ]*<div>Helper</div>', html), res)
 705         d.addCallback(_check_helper_connected)
 706         return d
 707
 708     def test_storage(self):
 709         d = self.GET("/storage")
 710         def _check(res):
 711             self.failUnlessIn('Storage Server Status', res)
 712             self.failUnlessIn(FAVICON_MARKUP, res)
 713             res_u = res.decode('utf-8')
 714             self.failUnlessIn(u'<li>Server Nickname: <span class="nickname mine">fake_nickname \u263A</span></li>', res_u)
 715         d.addCallback(_check)
 716         return d
 717
 718     def test_status(self):
 719         h = self.s.get_history()
 720         dl_num = h.list_all_download_statuses()[0].get_counter()
 721         ul_num = h.list_all_upload_statuses()[0].get_counter()
 722         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 723         pub_num = h.list_all_publish_statuses()[0].get_counter()
 724         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 725         d = self.GET("/status", followRedirect=True)
 726         def _check(res):
 727             self.failUnlessIn('Recent and Active Operations', res)
 728             self.failUnlessIn('"down-%d"' % dl_num, res)
 729             self.failUnlessIn('"up-%d"' % ul_num, res)
 730             self.failUnlessIn('"mapupdate-%d"' % mu_num, res)
 731             self.failUnlessIn('"publish-%d"' % pub_num, res)
 732             self.failUnlessIn('"retrieve-%d"' % ret_num, res)
 733         d.addCallback(_check)
 734         d.addCallback(lambda res: self.GET("/status/?t=json"))
 735         def _check_json(res):
 736             data = simplejson.loads(res)
 737             self.failUnless(isinstance(data, dict))
 738             #active = data["active"]
 739             # TODO: test more. We need a way to fake an active operation
 740             # here.
 741         d.addCallback(_check_json)
 742
 743         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 744         def _check_dl(res):
 745             self.failUnlessIn("File Download Status", res)
 746         d.addCallback(_check_dl)
 747         d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
 748         def _check_dl_json(res):
 749             data = simplejson.loads(res)
 750             self.failUnless(isinstance(data, dict))
 751             self.failUnlessIn("read", data)
 752             self.failUnlessEqual(data["read"][0]["length"], 120)
 753             self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
 754             self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
 755             self.failUnlessEqual(data["segment"][2]["finish_time"], None)
 756             phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
 757             cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
 758             # serverids[] keys are strings, since that's what JSON does, but
 759             # we'd really like them to be ints
 760             self.failUnlessEqual(data["serverids"]["0"], "phwrsjte")
 761             self.failUnless(data["serverids"].has_key("1"),
 762                             str(data["serverids"]))
 763             self.failUnlessEqual(data["serverids"]["1"], "cmpuvkjm",
 764                                  str(data["serverids"]))
 765             self.failUnlessEqual(data["server_info"][phwr_id]["short"],
 766                                  "phwrsjte")
 767             self.failUnlessEqual(data["server_info"][cmpu_id]["short"],
 768                                  "cmpuvkjm")
 769             self.failUnlessIn("dyhb", data)
 770             self.failUnlessIn("misc", data)
 771         d.addCallback(_check_dl_json)
 772         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 773         def _check_ul(res):
 774             self.failUnlessIn("File Upload Status", res)
 775         d.addCallback(_check_ul)
 776         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 777         def _check_mapupdate(res):
 778             self.failUnlessIn("Mutable File Servermap Update Status", res)
 779         d.addCallback(_check_mapupdate)
 780         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 781         def _check_publish(res):
 782             self.failUnlessIn("Mutable File Publish Status", res)
 783         d.addCallback(_check_publish)
 784         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 785         def _check_retrieve(res):
 786             self.failUnlessIn("Mutable File Retrieve Status", res)
 787         d.addCallback(_check_retrieve)
 788
 789         return d
 790
 791     def test_status_numbers(self):
 792         drrm = status.DownloadResultsRendererMixin()
 793         self.failUnlessReallyEqual(drrm.render_time(None, None), "")
 794         self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
 795         self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
 796         self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
 797         self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
 798         self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
 799         self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 800         self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
 801         self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
 802
 803         urrm = status.UploadResultsRendererMixin()
 804         self.failUnlessReallyEqual(urrm.render_time(None, None), "")
 805         self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
 806         self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
 807         self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
 808         self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
 809         self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
 810         self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 811         self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
 812         self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
 813
 814     def test_GET_FILEURL(self):
 815         d = self.GET(self.public_url + "/foo/bar.txt")
 816         d.addCallback(self.failUnlessIsBarDotTxt)
 817         return d
 818
 819     def test_GET_FILEURL_range(self):
 820         headers = {"range": "bytes=1-10"}
 821         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 822                      return_response=True)
 823         def _got((res, status, headers)):
 824             self.failUnlessReallyEqual(int(status), 206)
 825             self.failUnless(headers.has_key("content-range"))
 826             self.failUnlessReallyEqual(headers["content-range"][0],
 827                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 828             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
 829         d.addCallback(_got)
 830         return d
 831
 832     def test_GET_FILEURL_partial_range(self):
 833         headers = {"range": "bytes=5-"}
 834         length  = len(self.BAR_CONTENTS)
 835         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 836                      return_response=True)
 837         def _got((res, status, headers)):
 838             self.failUnlessReallyEqual(int(status), 206)
 839             self.failUnless(headers.has_key("content-range"))
 840             self.failUnlessReallyEqual(headers["content-range"][0],
 841                                        "bytes 5-%d/%d" % (length-1, length))
 842             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
 843         d.addCallback(_got)
 844         return d
 845
 846     def test_GET_FILEURL_partial_end_range(self):
 847         headers = {"range": "bytes=-5"}
 848         length  = len(self.BAR_CONTENTS)
 849         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 850                      return_response=True)
 851         def _got((res, status, headers)):
 852             self.failUnlessReallyEqual(int(status), 206)
 853             self.failUnless(headers.has_key("content-range"))
 854             self.failUnlessReallyEqual(headers["content-range"][0],
 855                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 856             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
 857         d.addCallback(_got)
 858         return d
 859
 860     def test_GET_FILEURL_partial_range_overrun(self):
 861         headers = {"range": "bytes=100-200"}
 862         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
 863                              "416 Requested Range not satisfiable",
 864                              "First beyond end of file",
 865                              self.GET, self.public_url + "/foo/bar.txt",
 866                              headers=headers)
 867         return d
 868
 869     def test_HEAD_FILEURL_range(self):
 870         headers = {"range": "bytes=1-10"}
 871         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 872                      return_response=True)
 873         def _got((res, status, headers)):
 874             self.failUnlessReallyEqual(res, "")
 875             self.failUnlessReallyEqual(int(status), 206)
 876             self.failUnless(headers.has_key("content-range"))
 877             self.failUnlessReallyEqual(headers["content-range"][0],
 878                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 879         d.addCallback(_got)
 880         return d
 881
 882     def test_HEAD_FILEURL_partial_range(self):
 883         headers = {"range": "bytes=5-"}
 884         length  = len(self.BAR_CONTENTS)
 885         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 886                      return_response=True)
 887         def _got((res, status, headers)):
 888             self.failUnlessReallyEqual(int(status), 206)
 889             self.failUnless(headers.has_key("content-range"))
 890             self.failUnlessReallyEqual(headers["content-range"][0],
 891                                        "bytes 5-%d/%d" % (length-1, length))
 892         d.addCallback(_got)
 893         return d
 894
 895     def test_HEAD_FILEURL_partial_end_range(self):
 896         headers = {"range": "bytes=-5"}
 897         length  = len(self.BAR_CONTENTS)
 898         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 899                      return_response=True)
 900         def _got((res, status, headers)):
 901             self.failUnlessReallyEqual(int(status), 206)
 902             self.failUnless(headers.has_key("content-range"))
 903             self.failUnlessReallyEqual(headers["content-range"][0],
 904                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 905         d.addCallback(_got)
 906         return d
 907
 908     def test_HEAD_FILEURL_partial_range_overrun(self):
 909         headers = {"range": "bytes=100-200"}
 910         d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
 911                              "416 Requested Range not satisfiable",
 912                              "",
 913                              self.HEAD, self.public_url + "/foo/bar.txt",
 914                              headers=headers)
 915         return d
 916
 917     def test_GET_FILEURL_range_bad(self):
 918         headers = {"range": "BOGUS=fizbop-quarnak"}
 919         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 920                      return_response=True)
 921         def _got((res, status, headers)):
 922             self.failUnlessReallyEqual(int(status), 200)
 923             self.failUnless(not headers.has_key("content-range"))
 924             self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
 925         d.addCallback(_got)
 926         return d
 927
 928     def test_HEAD_FILEURL(self):
 929         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 930         def _got((res, status, headers)):
 931             self.failUnlessReallyEqual(res, "")
 932             self.failUnlessReallyEqual(headers["content-length"][0],
 933                                        str(len(self.BAR_CONTENTS)))
 934             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
 935         d.addCallback(_got)
 936         return d
 937
 938     def test_GET_FILEURL_named(self):
 939         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 940         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 941         d = self.GET(base + "/@@name=/blah.txt")
 942         d.addCallback(self.failUnlessIsBarDotTxt)
 943         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 944         d.addCallback(self.failUnlessIsBarDotTxt)
 945         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 946         d.addCallback(self.failUnlessIsBarDotTxt)
 947         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 948         d.addCallback(self.failUnlessIsBarDotTxt)
 949         save_url = base + "?save=true&filename=blah.txt"
 950         d.addCallback(lambda res: self.GET(save_url))
 951         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 952         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 953         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 954         u_url = base + "?save=true&filename=" + u_fn_e
 955         d.addCallback(lambda res: self.GET(u_url))
 956         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 957         return d
 958
 959     def test_PUT_FILEURL_named_bad(self):
 960         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 961         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 962                              "400 Bad Request",
 963                              "/file can only be used with GET or HEAD",
 964                              self.PUT, base + "/@@name=/blah.txt", "")
 965         return d
 966
 967
 968     def test_GET_DIRURL_named_bad(self):
 969         base = "/file/%s" % urllib.quote(self._foo_uri)
 970         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 971                              "400 Bad Request",
 972                              "is not a file-cap",
 973                              self.GET, base + "/@@name=/blah.txt")
 974         return d
 975
 976     def test_GET_slash_file_bad(self):
 977         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 978                              "404 Not Found",
 979                              "/file must be followed by a file-cap and a name",
 980                              self.GET, "/file")
 981         return d
 982
 983     def test_GET_unhandled_URI_named(self):
 984         contents, n, newuri = self.makefile(12)
 985         verifier_cap = n.get_verify_cap().to_string()
 986         base = "/file/%s" % urllib.quote(verifier_cap)
 987         # client.create_node_from_uri() can't handle verify-caps
 988         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 989                              "400 Bad Request", "is not a file-cap",
 990                              self.GET, base)
 991         return d
 992
 993     def test_GET_unhandled_URI(self):
 994         contents, n, newuri = self.makefile(12)
 995         verifier_cap = n.get_verify_cap().to_string()
 996         base = "/uri/%s" % urllib.quote(verifier_cap)
 997         # client.create_node_from_uri() can't handle verify-caps
 998         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 999                              "400 Bad Request",
1000                              "GET unknown URI type: can only do t=info",
1001                              self.GET, base)
1002         return d
1003
1004     def test_GET_FILE_URI(self):
1005         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
1006         d = self.GET(base)
1007         d.addCallback(self.failUnlessIsBarDotTxt)
1008         return d
1009
1010     def test_GET_FILE_URI_mdmf(self):
1011         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
1012         d = self.GET(base)
1013         d.addCallback(self.failUnlessIsQuuxDotTxt)
1014         return d
1015
1016     def test_GET_FILE_URI_mdmf_extensions(self):
1017         base = "/uri/%s" % urllib.quote("%s:RANDOMSTUFF" % self._quux_txt_uri)
1018         d = self.GET(base)
1019         d.addCallback(self.failUnlessIsQuuxDotTxt)
1020         return d
1021
1022     def test_GET_FILE_URI_mdmf_readonly(self):
1023         base = "/uri/%s" % urllib.quote(self._quux_txt_readonly_uri)
1024         d = self.GET(base)
1025         d.addCallback(self.failUnlessIsQuuxDotTxt)
1026         return d
1027
1028     def test_GET_FILE_URI_badchild(self):
1029         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
1030         errmsg = "Files have no children, certainly not named 'boguschild'"
1031         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
1032                              "400 Bad Request", errmsg,
1033                              self.GET, base)
1034         return d
1035
1036     def test_PUT_FILE_URI_badchild(self):
1037         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
1038         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
1039         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
1040                              "400 Bad Request", errmsg,
1041                              self.PUT, base, "")
1042         return d
1043
1044     def test_PUT_FILE_URI_mdmf(self):
1045         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
1046         self._quux_new_contents = "new_contents"
1047         d = self.GET(base)
1048         d.addCallback(lambda res:
1049             self.failUnlessIsQuuxDotTxt(res))
1050         d.addCallback(lambda ignored:
1051             self.PUT(base, self._quux_new_contents))
1052         d.addCallback(lambda ignored:
1053             self.GET(base))
1054         d.addCallback(lambda res:
1055             self.failUnlessReallyEqual(res, self._quux_new_contents))
1056         return d
1057
1058     def test_PUT_FILE_URI_mdmf_extensions(self):
1059         base = "/uri/%s" % urllib.quote("%s:EXTENSIONSTUFF" % self._quux_txt_uri)
1060         self._quux_new_contents = "new_contents"
1061         d = self.GET(base)
1062         d.addCallback(lambda res: self.failUnlessIsQuuxDotTxt(res))
1063         d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
1064         d.addCallback(lambda ignored: self.GET(base))
1065         d.addCallback(lambda res: self.failUnlessEqual(self._quux_new_contents,
1066                                                        res))
1067         return d
1068
1069     def test_PUT_FILE_URI_mdmf_readonly(self):
1070         # We're not allowed to PUT things to a readonly cap.
1071         base = "/uri/%s" % self._quux_txt_readonly_uri
1072         d = self.GET(base)
1073         d.addCallback(lambda res:
1074             self.failUnlessIsQuuxDotTxt(res))
1075         # What should we get here? We get a 500 error now; that's not right.
1076         d.addCallback(lambda ignored:
1077             self.shouldFail2(error.Error, "test_PUT_FILE_URI_mdmf_readonly",
1078                              "400 Bad Request", "read-only cap",
1079                              self.PUT, base, "new data"))
1080         return d
1081
1082     def test_PUT_FILE_URI_sdmf_readonly(self):
1083         # We're not allowed to put things to a readonly cap.
1084         base = "/uri/%s" % self._baz_txt_readonly_uri
1085         d = self.GET(base)
1086         d.addCallback(lambda res:
1087             self.failUnlessIsBazDotTxt(res))
1088         d.addCallback(lambda ignored:
1089             self.shouldFail2(error.Error, "test_PUT_FILE_URI_sdmf_readonly",
1090                              "400 Bad Request", "read-only cap",
1091                              self.PUT, base, "new_data"))
1092         return d
1093
1094     def test_GET_etags(self):
1095
1096         def _check_etags(uri):
1097             d1 = _get_etag(uri)
1098             d2 = _get_etag(uri, 'json')
1099             d = defer.DeferredList([d1, d2], consumeErrors=True)
1100             def _check(results):
1101                 # All deferred must succeed
1102                 self.failUnless(all([r[0] for r in results]))
1103                 # the etag for the t=json form should be just like the etag
1104                 # fo the default t='' form, but with a 'json' suffix
1105                 self.failUnlessEqual(results[0][1] + 'json', results[1][1])
1106             d.addCallback(_check)
1107             return d
1108
1109         def _get_etag(uri, t=''):
1110             targetbase = "/uri/%s?t=%s" % (urllib.quote(uri.strip()), t)
1111             d = self.GET(targetbase, return_response=True, followRedirect=True)
1112             def _just_the_etag(result):
1113                 data, response, headers = result
1114                 etag = headers['etag'][0]
1115                 if uri.startswith('URI:DIR'):
1116                     self.failUnless(etag.startswith('DIR:'), etag)
1117                 return etag
1118             return d.addCallback(_just_the_etag)
1119
1120         # Check that etags work with immutable directories
1121         (newkids, caps) = self._create_immutable_children()
1122         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1123                       simplejson.dumps(newkids))
1124         def _stash_immdir_uri(uri):
1125             self._immdir_uri = uri
1126             return uri
1127         d.addCallback(_stash_immdir_uri)
1128         d.addCallback(_check_etags)
1129
1130         # Check that etags work with immutable files
1131         d.addCallback(lambda _: _check_etags(self._bar_txt_uri))
1132
1133         # use the ETag on GET
1134         def _check_match(ign):
1135             uri = "/uri/%s" % self._bar_txt_uri
1136             d = self.GET(uri, return_response=True)
1137             # extract the ETag
1138             d.addCallback(lambda (data, code, headers):
1139                           headers['etag'][0])
1140             # do a GET that's supposed to match the ETag
1141             d.addCallback(lambda etag:
1142                           self.GET(uri, return_response=True,
1143                                    headers={"If-None-Match": etag}))
1144             # make sure it short-circuited (304 instead of 200)
1145             d.addCallback(lambda (data, code, headers):
1146                           self.failUnlessEqual(int(code), http.NOT_MODIFIED))
1147             return d
1148         d.addCallback(_check_match)
1149
1150         def _no_etag(uri, t):
1151             target = "/uri/%s?t=%s" % (uri, t)
1152             d = self.GET(target, return_response=True, followRedirect=True)
1153             d.addCallback(lambda (data, code, headers):
1154                           self.failIf("etag" in headers, target))
1155             return d
1156         def _yes_etag(uri, t):
1157             target = "/uri/%s?t=%s" % (uri, t)
1158             d = self.GET(target, return_response=True, followRedirect=True)
1159             d.addCallback(lambda (data, code, headers):
1160                           self.failUnless("etag" in headers, target))
1161             return d
1162
1163         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, ""))
1164         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "json"))
1165         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "uri"))
1166         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "readonly-uri"))
1167         d.addCallback(lambda ign: _no_etag(self._bar_txt_uri, "info"))
1168
1169         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, ""))
1170         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "json"))
1171         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "uri"))
1172         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "readonly-uri"))
1173         d.addCallback(lambda ign: _no_etag(self._immdir_uri, "info"))
1174         d.addCallback(lambda ign: _no_etag(self._immdir_uri, "rename-form"))
1175
1176         return d
1177
1178     # TODO: version of this with a Unicode filename
1179     def test_GET_FILEURL_save(self):
1180         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
1181                      return_response=True)
1182         def _got((res, statuscode, headers)):
1183             content_disposition = headers["content-disposition"][0]
1184             self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
1185             self.failUnlessIsBarDotTxt(res)
1186         d.addCallback(_got)
1187         return d
1188
1189     def test_GET_FILEURL_missing(self):
1190         d = self.GET(self.public_url + "/foo/missing")
1191         d.addBoth(self.should404, "test_GET_FILEURL_missing")
1192         return d
1193
1194     def test_GET_FILEURL_info_mdmf(self):
1195         d = self.GET("/uri/%s?t=info" % self._quux_txt_uri)
1196         def _got(res):
1197             self.failUnlessIn("mutable file (mdmf)", res)
1198             self.failUnlessIn(self._quux_txt_uri, res)
1199             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1200         d.addCallback(_got)
1201         return d
1202
1203     def test_GET_FILEURL_info_mdmf_readonly(self):
1204         d = self.GET("/uri/%s?t=info" % self._quux_txt_readonly_uri)
1205         def _got(res):
1206             self.failUnlessIn("mutable file (mdmf)", res)
1207             self.failIfIn(self._quux_txt_uri, res)
1208             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1209         d.addCallback(_got)
1210         return d
1211
1212     def test_GET_FILEURL_info_sdmf(self):
1213         d = self.GET("/uri/%s?t=info" % self._baz_txt_uri)
1214         def _got(res):
1215             self.failUnlessIn("mutable file (sdmf)", res)
1216             self.failUnlessIn(self._baz_txt_uri, res)
1217         d.addCallback(_got)
1218         return d
1219
1220     def test_GET_FILEURL_info_mdmf_extensions(self):
1221         d = self.GET("/uri/%s:STUFF?t=info" % self._quux_txt_uri)
1222         def _got(res):
1223             self.failUnlessIn("mutable file (mdmf)", res)
1224             self.failUnlessIn(self._quux_txt_uri, res)
1225             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1226         d.addCallback(_got)
1227         return d
1228
1229     def test_PUT_overwrite_only_files(self):
1230         # create a directory, put a file in that directory.
1231         contents, n, filecap = self.makefile(8)
1232         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
1233         d.addCallback(lambda res:
1234             self.PUT(self.public_url + "/foo/dir/file1.txt",
1235                      self.NEWFILE_CONTENTS))
1236         # try to overwrite the file with replace=only-files
1237         # (this should work)
1238         d.addCallback(lambda res:
1239             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
1240                      filecap))
1241         d.addCallback(lambda res:
1242             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
1243                  "There was already a child by that name, and you asked me "
1244                  "to not replace it",
1245                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
1246                  filecap))
1247         return d
1248
1249     def test_PUT_NEWFILEURL(self):
1250         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
1251         # TODO: we lose the response code, so we can't check this
1252         #self.failUnlessReallyEqual(responsecode, 201)
1253         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1254         d.addCallback(lambda res:
1255                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1256                                                       self.NEWFILE_CONTENTS))
1257         return d
1258
1259     def test_PUT_NEWFILEURL_not_mutable(self):
1260         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
1261                      self.NEWFILE_CONTENTS)
1262         # TODO: we lose the response code, so we can't check this
1263         #self.failUnlessReallyEqual(responsecode, 201)
1264         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1265         d.addCallback(lambda res:
1266                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1267                                                       self.NEWFILE_CONTENTS))
1268         return d
1269
1270     def test_PUT_NEWFILEURL_unlinked_mdmf(self):
1271         # this should get us a few segments of an MDMF mutable file,
1272         # which we can then test for.
1273         contents = self.NEWFILE_CONTENTS * 300000
1274         d = self.PUT("/uri?format=mdmf",
1275                      contents)
1276         def _got_filecap(filecap):
1277             self.failUnless(filecap.startswith("URI:MDMF"))
1278             return filecap
1279         d.addCallback(_got_filecap)
1280         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1281         d.addCallback(lambda json: self.failUnlessIn("MDMF", json))
1282         return d
1283
1284     def test_PUT_NEWFILEURL_unlinked_sdmf(self):
1285         contents = self.NEWFILE_CONTENTS * 300000
1286         d = self.PUT("/uri?format=sdmf",
1287                      contents)
1288         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1289         d.addCallback(lambda json: self.failUnlessIn("SDMF", json))
1290         return d
1291
1292     def test_PUT_NEWFILEURL_unlinked_bad_format(self):
1293         contents = self.NEWFILE_CONTENTS * 300000
1294         return self.shouldHTTPError("PUT_NEWFILEURL_unlinked_bad_format",
1295                                     400, "Bad Request", "Unknown format: foo",
1296                                     self.PUT, "/uri?format=foo",
1297                                     contents)
1298
1299     def test_PUT_NEWFILEURL_range_bad(self):
1300         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
1301         target = self.public_url + "/foo/new.txt"
1302         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
1303                              "501 Not Implemented",
1304                              "Content-Range in PUT not yet supported",
1305                              # (and certainly not for immutable files)
1306                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
1307                              headers=headers)
1308         d.addCallback(lambda res:
1309                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
1310         return d
1311
1312     def test_PUT_NEWFILEURL_mutable(self):
1313         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1314                      self.NEWFILE_CONTENTS)
1315         # TODO: we lose the response code, so we can't check this
1316         #self.failUnlessReallyEqual(responsecode, 201)
1317         def _check_uri(res):
1318             u = uri.from_string_mutable_filenode(res)
1319             self.failUnless(u.is_mutable())
1320             self.failIf(u.is_readonly())
1321             return res
1322         d.addCallback(_check_uri)
1323         d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
1324         d.addCallback(lambda res:
1325                       self.failUnlessMutableChildContentsAre(self._foo_node,
1326                                                              u"new.txt",
1327                                                              self.NEWFILE_CONTENTS))
1328         return d
1329
1330     def test_PUT_NEWFILEURL_mutable_toobig(self):
1331         # It is okay to upload large mutable files, so we should be able
1332         # to do that.
1333         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1334                      "b" * (self.s.MUTABLE_SIZELIMIT + 1))
1335         return d
1336
1337     def test_PUT_NEWFILEURL_replace(self):
1338         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
1339         # TODO: we lose the response code, so we can't check this
1340         #self.failUnlessReallyEqual(responsecode, 200)
1341         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
1342         d.addCallback(lambda res:
1343                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1344                                                       self.NEWFILE_CONTENTS))
1345         return d
1346
1347     def test_PUT_NEWFILEURL_bad_t(self):
1348         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
1349                              "PUT to a file: bad t=bogus",
1350                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
1351                              "contents")
1352         return d
1353
1354     def test_PUT_NEWFILEURL_no_replace(self):
1355         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
1356                      self.NEWFILE_CONTENTS)
1357         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
1358                   "409 Conflict",
1359                   "There was already a child by that name, and you asked me "
1360                   "to not replace it")
1361         return d
1362
1363     def test_PUT_NEWFILEURL_mkdirs(self):
1364         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
1365         fn = self._foo_node
1366         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
1367         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
1368         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
1369         d.addCallback(lambda res:
1370                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
1371                                                       self.NEWFILE_CONTENTS))
1372         return d
1373
1374     def test_PUT_NEWFILEURL_blocked(self):
1375         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
1376                      self.NEWFILE_CONTENTS)
1377         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
1378                   "409 Conflict",
1379                   "Unable to create directory 'blockingfile': a file was in the way")
1380         return d
1381
1382     def test_PUT_NEWFILEURL_emptyname(self):
1383         # an empty pathname component (i.e. a double-slash) is disallowed
1384         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
1385                              "400 Bad Request",
1386                              "The webapi does not allow empty pathname components",
1387                              self.PUT, self.public_url + "/foo//new.txt", "")
1388         return d
1389
1390     def test_DELETE_FILEURL(self):
1391         d = self.DELETE(self.public_url + "/foo/bar.txt")
1392         d.addCallback(lambda res:
1393                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1394         return d
1395
1396     def test_DELETE_FILEURL_missing(self):
1397         d = self.DELETE(self.public_url + "/foo/missing")
1398         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
1399         return d
1400
1401     def test_DELETE_FILEURL_missing2(self):
1402         d = self.DELETE(self.public_url + "/missing/missing")
1403         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
1404         return d
1405
1406     def failUnlessHasBarDotTxtMetadata(self, res):
1407         data = simplejson.loads(res)
1408         self.failUnless(isinstance(data, list))
1409         self.failUnlessIn("metadata", data[1])
1410         self.failUnlessIn("tahoe", data[1]["metadata"])
1411         self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
1412         self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
1413         self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
1414                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
1415
1416     def test_GET_FILEURL_json(self):
1417         # twisted.web.http.parse_qs ignores any query args without an '=', so
1418         # I can't do "GET /path?json", I have to do "GET /path/t=json"
1419         # instead. This may make it tricky to emulate the S3 interface
1420         # completely.
1421         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
1422         def _check1(data):
1423             self.failUnlessIsBarJSON(data)
1424             self.failUnlessHasBarDotTxtMetadata(data)
1425             return
1426         d.addCallback(_check1)
1427         return d
1428
1429     def test_GET_FILEURL_json_mutable_type(self):
1430         # The JSON should include format, which says whether the
1431         # file is SDMF or MDMF
1432         d = self.PUT("/uri?format=mdmf",
1433                      self.NEWFILE_CONTENTS * 300000)
1434         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1435         def _got_json(json, version):
1436             data = simplejson.loads(json)
1437             assert "filenode" == data[0]
1438             data = data[1]
1439             assert isinstance(data, dict)
1440
1441             self.failUnlessIn("format", data)
1442             self.failUnlessEqual(data["format"], version)
1443
1444         d.addCallback(_got_json, "MDMF")
1445         # Now make an SDMF file and check that it is reported correctly.
1446         d.addCallback(lambda ignored:
1447             self.PUT("/uri?format=sdmf",
1448                       self.NEWFILE_CONTENTS * 300000))
1449         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1450         d.addCallback(_got_json, "SDMF")
1451         return d
1452
1453     def test_GET_FILEURL_json_mdmf(self):
1454         d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
1455         d.addCallback(self.failUnlessIsQuuxJSON)
1456         return d
1457
1458     def test_GET_FILEURL_json_missing(self):
1459         d = self.GET(self.public_url + "/foo/missing?json")
1460         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
1461         return d
1462
1463     def test_GET_FILEURL_uri(self):
1464         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
1465         def _check(res):
1466             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1467         d.addCallback(_check)
1468         d.addCallback(lambda res:
1469                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
1470         def _check2(res):
1471             # for now, for files, uris and readonly-uris are the same
1472             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1473         d.addCallback(_check2)
1474         return d
1475
1476     def test_GET_FILEURL_badtype(self):
1477         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
1478                                  "bad t=bogus",
1479                                  self.GET,
1480                                  self.public_url + "/foo/bar.txt?t=bogus")
1481         return d
1482
1483     def test_CSS_FILE(self):
1484         d = self.GET("/tahoe.css", followRedirect=True)
1485         def _check(res):
1486             CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
1487             self.failUnless(CSS_STYLE.search(res), res)
1488         d.addCallback(_check)
1489         return d
1490
1491     def test_GET_FILEURL_uri_missing(self):
1492         d = self.GET(self.public_url + "/foo/missing?t=uri")
1493         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
1494         return d
1495
1496     def _check_upload_and_mkdir_forms(self, html):
1497         # We should have a form to create a file, with radio buttons that allow
1498         # the user to toggle whether it is a CHK/LIT (default), SDMF, or MDMF file.
1499         self.failUnlessIn('name="t" value="upload"', html)
1500         self.failUnless(re.search('<input [^/]*id="upload-chk"', html), html)
1501         self.failUnless(re.search('<input [^/]*id="upload-sdmf"', html), html)
1502         self.failUnless(re.search('<input [^/]*id="upload-mdmf"', html), html)
1503
1504         # We should also have the ability to create a mutable directory, with
1505         # radio buttons that allow the user to toggle whether it is an SDMF (default)
1506         # or MDMF directory.
1507         self.failUnlessIn('name="t" value="mkdir"', html)
1508         self.failUnless(re.search('<input [^/]*id="mkdir-sdmf"', html), html)
1509         self.failUnless(re.search('<input [^/]*id="mkdir-mdmf"', html), html)
1510
1511         self.failUnlessIn(FAVICON_MARKUP, html)
1512
1513     def test_GET_DIRECTORY_html(self):
1514         d = self.GET(self.public_url + "/foo", followRedirect=True)
1515         def _check(html):
1516             self.failUnlessIn('<div class="toolbar-item"><a href="../../..">Return to Welcome page</a></div>', html)
1517             self._check_upload_and_mkdir_forms(html)
1518             self.failUnlessIn("quux", html)
1519         d.addCallback(_check)
1520         return d
1521
1522     def test_GET_DIRECTORY_html_filenode_encoding(self):
1523         d = self.GET(self.public_url + "/foo", followRedirect=True)
1524         def _check(html):
1525             # Check if encoded entries are there
1526             self.failUnlessIn('@@named=/' + self._htmlname_urlencoded + '">'
1527                               + self._htmlname_escaped + '</a>', html)
1528             self.failUnlessIn('value="' + self._htmlname_escaped_attr + '"', html)
1529             self.failIfIn(self._htmlname_escaped_double, html)
1530             # Make sure that Nevow escaping actually works by checking for unsafe characters
1531             # and that '&' is escaped.
1532             for entity in '<>':
1533                 self.failUnlessIn(entity, self._htmlname_raw)
1534                 self.failIfIn(entity, self._htmlname_escaped)
1535             self.failUnlessIn('&', re.sub(r'&(amp|lt|gt|quot|apos);', '', self._htmlname_raw))
1536             self.failIfIn('&', re.sub(r'&(amp|lt|gt|quot|apos);', '', self._htmlname_escaped))
1537         d.addCallback(_check)
1538         return d
1539
1540     def test_GET_root_html(self):
1541         d = self.GET("/")
1542         d.addCallback(self._check_upload_and_mkdir_forms)
1543         return d
1544
1545     def test_GET_DIRURL(self):
1546         # the addSlash means we get a redirect here
1547         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1548         ROOT = "../../.."
1549         d = self.GET(self.public_url + "/foo", followRedirect=True)
1550         def _check(res):
1551             self.failUnlessIn('<a href="%s">Return to Welcome page' % ROOT, res)
1552
1553             # the FILE reference points to a URI, but it should end in bar.txt
1554             bar_url = ("%s/file/%s/@@named=/bar.txt" %
1555                        (ROOT, urllib.quote(self._bar_txt_uri)))
1556             get_bar = "".join([r'<td>FILE</td>',
1557                                r'\s+<td>',
1558                                r'<a href="%s">bar.txt</a>' % bar_url,
1559                                r'</td>',
1560                                r'\s+<td align="right">%d</td>' % len(self.BAR_CONTENTS),
1561                                ])
1562             self.failUnless(re.search(get_bar, res), res)
1563             for label in ['unlink', 'rename/move']:
1564                 for line in res.split("\n"):
1565                     # find the line that contains the relevant button for bar.txt
1566                     if ("form action" in line and
1567                         ('value="%s"' % (label,)) in line and
1568                         'value="bar.txt"' in line):
1569                         # the form target should use a relative URL
1570                         foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1571                         self.failUnlessIn('action="%s"' % foo_url, line)
1572                         # and the when_done= should too
1573                         #done_url = urllib.quote(???)
1574                         #self.failUnlessIn('name="when_done" value="%s"' % done_url, line)
1575
1576                         # 'unlink' needs to use POST because it directly has a side effect
1577                         if label == 'unlink':
1578                             self.failUnlessIn('method="post"', line)
1579                         break
1580                 else:
1581                     self.fail("unable to find '%s bar.txt' line" % (label,))
1582
1583             # the DIR reference just points to a URI
1584             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1585             get_sub = ((r'<td>DIR</td>')
1586                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1587             self.failUnless(re.search(get_sub, res), res)
1588         d.addCallback(_check)
1589
1590         # look at a readonly directory
1591         d.addCallback(lambda res:
1592                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
1593         def _check2(res):
1594             self.failUnlessIn("(read-only)", res)
1595             self.failIfIn("Upload a file", res)
1596         d.addCallback(_check2)
1597
1598         # and at a directory that contains a readonly directory
1599         d.addCallback(lambda res:
1600                       self.GET(self.public_url, followRedirect=True))
1601         def _check3(res):
1602             self.failUnless(re.search('<td>DIR-RO</td>'
1603                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1604         d.addCallback(_check3)
1605
1606         # and an empty directory
1607         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1608         def _check4(res):
1609             self.failUnlessIn("directory is empty", res)
1610             MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
1611             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1612         d.addCallback(_check4)
1613
1614         # and at a literal directory
1615         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1616         d.addCallback(lambda res:
1617                       self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1618         def _check5(res):
1619             self.failUnlessIn('(immutable)', res)
1620             self.failUnless(re.search('<td>FILE</td>'
1621                                       r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1622         d.addCallback(_check5)
1623         return d
1624
1625     def test_GET_DIRURL_badtype(self):
1626         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1627                                  400, "Bad Request",
1628                                  "bad t=bogus",
1629                                  self.GET,
1630                                  self.public_url + "/foo?t=bogus")
1631         return d
1632
1633     def test_GET_DIRURL_json(self):
1634         d = self.GET(self.public_url + "/foo?t=json")
1635         d.addCallback(self.failUnlessIsFooJSON)
1636         return d
1637
1638     def test_GET_DIRURL_json_format(self):
1639         d = self.PUT(self.public_url + \
1640                      "/foo/sdmf.txt?format=sdmf",
1641                      self.NEWFILE_CONTENTS * 300000)
1642         d.addCallback(lambda ignored:
1643             self.PUT(self.public_url + \
1644                      "/foo/mdmf.txt?format=mdmf",
1645                      self.NEWFILE_CONTENTS * 300000))
1646         # Now we have an MDMF and SDMF file in the directory. If we GET
1647         # its JSON, we should see their encodings.
1648         d.addCallback(lambda ignored:
1649             self.GET(self.public_url + "/foo?t=json"))
1650         def _got_json(json):
1651             data = simplejson.loads(json)
1652             assert data[0] == "dirnode"
1653
1654             data = data[1]
1655             kids = data['children']
1656
1657             mdmf_data = kids['mdmf.txt'][1]
1658             self.failUnlessIn("format", mdmf_data)
1659             self.failUnlessEqual(mdmf_data["format"], "MDMF")
1660
1661             sdmf_data = kids['sdmf.txt'][1]
1662             self.failUnlessIn("format", sdmf_data)
1663             self.failUnlessEqual(sdmf_data["format"], "SDMF")
1664         d.addCallback(_got_json)
1665         return d
1666
1667
1668     def test_POST_DIRURL_manifest_no_ophandle(self):
1669         d = self.shouldFail2(error.Error,
1670                              "test_POST_DIRURL_manifest_no_ophandle",
1671                              "400 Bad Request",
1672                              "slow operation requires ophandle=",
1673                              self.POST, self.public_url, t="start-manifest")
1674         return d
1675
1676     def test_POST_DIRURL_manifest(self):
1677         d = defer.succeed(None)
1678         def getman(ignored, output):
1679             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1680                           followRedirect=True)
1681             d.addCallback(self.wait_for_operation, "125")
1682             d.addCallback(self.get_operation_results, "125", output)
1683             return d
1684         d.addCallback(getman, None)
1685         def _got_html(manifest):
1686             self.failUnlessIn("Manifest of SI=", manifest)
1687             self.failUnlessIn("<td>sub</td>", manifest)
1688             self.failUnlessIn(self._sub_uri, manifest)
1689             self.failUnlessIn("<td>sub/baz.txt</td>", manifest)
1690             self.failUnlessIn(FAVICON_MARKUP, manifest)
1691         d.addCallback(_got_html)
1692
1693         # both t=status and unadorned GET should be identical
1694         d.addCallback(lambda res: self.GET("/operations/125"))
1695         d.addCallback(_got_html)
1696
1697         d.addCallback(getman, "html")
1698         d.addCallback(_got_html)
1699         d.addCallback(getman, "text")
1700         def _got_text(manifest):
1701             self.failUnlessIn("\nsub " + self._sub_uri + "\n", manifest)
1702             self.failUnlessIn("\nsub/baz.txt URI:CHK:", manifest)
1703         d.addCallback(_got_text)
1704         d.addCallback(getman, "JSON")
1705         def _got_json(res):
1706             data = res["manifest"]
1707             got = {}
1708             for (path_list, cap) in data:
1709                 got[tuple(path_list)] = cap
1710             self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
1711             self.failUnlessIn((u"sub", u"baz.txt"), got)
1712             self.failUnlessIn("finished", res)
1713             self.failUnlessIn("origin", res)
1714             self.failUnlessIn("storage-index", res)
1715             self.failUnlessIn("verifycaps", res)
1716             self.failUnlessIn("stats", res)
1717         d.addCallback(_got_json)
1718         return d
1719
1720     def test_POST_DIRURL_deepsize_no_ophandle(self):
1721         d = self.shouldFail2(error.Error,
1722                              "test_POST_DIRURL_deepsize_no_ophandle",
1723                              "400 Bad Request",
1724                              "slow operation requires ophandle=",
1725                              self.POST, self.public_url, t="start-deep-size")
1726         return d
1727
1728     def test_POST_DIRURL_deepsize(self):
1729         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1730                       followRedirect=True)
1731         d.addCallback(self.wait_for_operation, "126")
1732         d.addCallback(self.get_operation_results, "126", "json")
1733         def _got_json(data):
1734             self.failUnlessReallyEqual(data["finished"], True)
1735             size = data["size"]
1736             self.failUnless(size > 1000)
1737         d.addCallback(_got_json)
1738         d.addCallback(self.get_operation_results, "126", "text")
1739         def _got_text(res):
1740             mo = re.search(r'^size: (\d+)$', res, re.M)
1741             self.failUnless(mo, res)
1742             size = int(mo.group(1))
1743             # with directories, the size varies.
1744             self.failUnless(size > 1000)
1745         d.addCallback(_got_text)
1746         return d
1747
1748     def test_POST_DIRURL_deepstats_no_ophandle(self):
1749         d = self.shouldFail2(error.Error,
1750                              "test_POST_DIRURL_deepstats_no_ophandle",
1751                              "400 Bad Request",
1752                              "slow operation requires ophandle=",
1753                              self.POST, self.public_url, t="start-deep-stats")
1754         return d
1755
1756     def test_POST_DIRURL_deepstats(self):
1757         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1758                       followRedirect=True)
1759         d.addCallback(self.wait_for_operation, "127")
1760         d.addCallback(self.get_operation_results, "127", "json")
1761         def _got_json(stats):
1762             expected = {"count-immutable-files": 4,
1763                         "count-mutable-files": 2,
1764                         "count-literal-files": 0,
1765                         "count-files": 6,
1766                         "count-directories": 3,
1767                         "size-immutable-files": 76,
1768                         "size-literal-files": 0,
1769                         #"size-directories": 1912, # varies
1770                         #"largest-directory": 1590,
1771                         "largest-directory-children": 8,
1772                         "largest-immutable-file": 19,
1773                         }
1774             for k,v in expected.iteritems():
1775                 self.failUnlessReallyEqual(stats[k], v,
1776                                            "stats[%s] was %s, not %s" %
1777                                            (k, stats[k], v))
1778             self.failUnlessReallyEqual(stats["size-files-histogram"],
1779                                        [ [11, 31, 4] ])
1780         d.addCallback(_got_json)
1781         return d
1782
1783     def test_POST_DIRURL_stream_manifest(self):
1784         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1785         def _check(res):
1786             self.failUnless(res.endswith("\n"))
1787             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1788             self.failUnlessReallyEqual(len(units), 10)
1789             self.failUnlessEqual(units[-1]["type"], "stats")
1790             first = units[0]
1791             self.failUnlessEqual(first["path"], [])
1792             self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri)
1793             self.failUnlessEqual(first["type"], "directory")
1794             baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0]
1795             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1796             self.failIfEqual(baz["storage-index"], None)
1797             self.failIfEqual(baz["verifycap"], None)
1798             self.failIfEqual(baz["repaircap"], None)
1799             # XXX: Add quux and baz to this test.
1800             return
1801         d.addCallback(_check)
1802         return d
1803
1804     def test_GET_DIRURL_uri(self):
1805         d = self.GET(self.public_url + "/foo?t=uri")
1806         def _check(res):
1807             self.failUnlessReallyEqual(to_str(res), self._foo_uri)
1808         d.addCallback(_check)
1809         return d
1810
1811     def test_GET_DIRURL_readonly_uri(self):
1812         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1813         def _check(res):
1814             self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri)
1815         d.addCallback(_check)
1816         return d
1817
1818     def test_PUT_NEWDIRURL(self):
1819         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1820         d.addCallback(lambda res:
1821                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1822         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1823         d.addCallback(self.failUnlessNodeKeysAre, [])
1824         return d
1825
1826     def test_PUT_NEWDIRURL_mdmf(self):
1827         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1828         d.addCallback(lambda res:
1829                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1830         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1831         d.addCallback(lambda node:
1832             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1833         return d
1834
1835     def test_PUT_NEWDIRURL_sdmf(self):
1836         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=sdmf",
1837                      "")
1838         d.addCallback(lambda res:
1839                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1840         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1841         d.addCallback(lambda node:
1842             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1843         return d
1844
1845     def test_PUT_NEWDIRURL_bad_format(self):
1846         return self.shouldHTTPError("PUT_NEWDIRURL_bad_format",
1847                                     400, "Bad Request", "Unknown format: foo",
1848                                     self.PUT, self.public_url +
1849                                     "/foo/newdir=?t=mkdir&format=foo", "")
1850
1851     def test_POST_NEWDIRURL(self):
1852         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1853         d.addCallback(lambda res:
1854                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1855         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1856         d.addCallback(self.failUnlessNodeKeysAre, [])
1857         return d
1858
1859     def test_POST_NEWDIRURL_mdmf(self):
1860         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1861         d.addCallback(lambda res:
1862                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1863         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1864         d.addCallback(lambda node:
1865             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1866         return d
1867
1868     def test_POST_NEWDIRURL_sdmf(self):
1869         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=sdmf", "")
1870         d.addCallback(lambda res:
1871             self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1872         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1873         d.addCallback(lambda node:
1874             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1875         return d
1876
1877     def test_POST_NEWDIRURL_bad_format(self):
1878         return self.shouldHTTPError("POST_NEWDIRURL_bad_format",
1879                                     400, "Bad Request", "Unknown format: foo",
1880                                     self.POST2, self.public_url + \
1881                                     "/foo/newdir?t=mkdir&format=foo", "")
1882
1883     def test_POST_NEWDIRURL_emptyname(self):
1884         # an empty pathname component (i.e. a double-slash) is disallowed
1885         d = self.shouldFail2(error.Error, "POST_NEWDIRURL_emptyname",
1886                              "400 Bad Request",
1887                              "The webapi does not allow empty pathname components, i.e. a double slash",
1888                              self.POST, self.public_url + "//?t=mkdir")
1889         return d
1890
1891     def _do_POST_NEWDIRURL_initial_children_test(self, version=None):
1892         (newkids, caps) = self._create_initial_children()
1893         query = "/foo/newdir?t=mkdir-with-children"
1894         if version == MDMF_VERSION:
1895             query += "&format=mdmf"
1896         elif version == SDMF_VERSION:
1897             query += "&format=sdmf"
1898         else:
1899             version = SDMF_VERSION # for later
1900         d = self.POST2(self.public_url + query,
1901                        simplejson.dumps(newkids))
1902         def _check(uri):
1903             n = self.s.create_node_from_uri(uri.strip())
1904             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1905             self.failUnlessEqual(n._node.get_version(), version)
1906             d2.addCallback(lambda ign:
1907                            self.failUnlessROChildURIIs(n, u"child-imm",
1908                                                        caps['filecap1']))
1909             d2.addCallback(lambda ign:
1910                            self.failUnlessRWChildURIIs(n, u"child-mutable",
1911                                                        caps['filecap2']))
1912             d2.addCallback(lambda ign:
1913                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1914                                                        caps['filecap3']))
1915             d2.addCallback(lambda ign:
1916                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1917                                                        caps['unknown_rocap']))
1918             d2.addCallback(lambda ign:
1919                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1920                                                        caps['unknown_rwcap']))
1921             d2.addCallback(lambda ign:
1922                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1923                                                        caps['unknown_immcap']))
1924             d2.addCallback(lambda ign:
1925                            self.failUnlessRWChildURIIs(n, u"dirchild",
1926                                                        caps['dircap']))
1927             d2.addCallback(lambda ign:
1928                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1929                                                        caps['litdircap']))
1930             d2.addCallback(lambda ign:
1931                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1932                                                        caps['emptydircap']))
1933             return d2
1934         d.addCallback(_check)
1935         d.addCallback(lambda res:
1936                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1937         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1938         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1939         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1940         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1941         return d
1942
1943     def test_POST_NEWDIRURL_initial_children(self):
1944         return self._do_POST_NEWDIRURL_initial_children_test()
1945
1946     def test_POST_NEWDIRURL_initial_children_mdmf(self):
1947         return self._do_POST_NEWDIRURL_initial_children_test(MDMF_VERSION)
1948
1949     def test_POST_NEWDIRURL_initial_children_sdmf(self):
1950         return self._do_POST_NEWDIRURL_initial_children_test(SDMF_VERSION)
1951
1952     def test_POST_NEWDIRURL_initial_children_bad_format(self):
1953         (newkids, caps) = self._create_initial_children()
1954         return self.shouldHTTPError("POST_NEWDIRURL_initial_children_bad_format",
1955                                     400, "Bad Request", "Unknown format: foo",
1956                                     self.POST2, self.public_url + \
1957                                     "/foo/newdir?t=mkdir-with-children&format=foo",
1958                                     simplejson.dumps(newkids))
1959
1960     def test_POST_NEWDIRURL_immutable(self):
1961         (newkids, caps) = self._create_immutable_children()
1962         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1963                        simplejson.dumps(newkids))
1964         def _check(uri):
1965             n = self.s.create_node_from_uri(uri.strip())
1966             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1967             d2.addCallback(lambda ign:
1968                            self.failUnlessROChildURIIs(n, u"child-imm",
1969                                                        caps['filecap1']))
1970             d2.addCallback(lambda ign:
1971                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1972                                                        caps['unknown_immcap']))
1973             d2.addCallback(lambda ign:
1974                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
1975                                                        caps['immdircap']))
1976             d2.addCallback(lambda ign:
1977                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1978                                                        caps['litdircap']))
1979             d2.addCallback(lambda ign:
1980                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1981                                                        caps['emptydircap']))
1982             return d2
1983         d.addCallback(_check)
1984         d.addCallback(lambda res:
1985                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1986         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1987         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1988         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1989         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1990         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1991         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1992         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1993         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
1994         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1995         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
1996         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1997         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
1998         d.addErrback(self.explain_web_error)
1999         return d
2000
2001     def test_POST_NEWDIRURL_immutable_bad(self):
2002         (newkids, caps) = self._create_initial_children()
2003         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
2004                              "400 Bad Request",
2005                              "needed to be immutable but was not",
2006                              self.POST2,
2007                              self.public_url + "/foo/newdir?t=mkdir-immutable",
2008                              simplejson.dumps(newkids))
2009         return d
2010
2011     def test_PUT_NEWDIRURL_exists(self):
2012         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
2013         d.addCallback(lambda res:
2014                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
2015         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2016         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2017         return d
2018
2019     def test_PUT_NEWDIRURL_blocked(self):
2020         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
2021                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
2022                              self.PUT,
2023                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
2024         d.addCallback(lambda res:
2025                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
2026         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2027         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2028         return d
2029
2030     def test_PUT_NEWDIRURL_mkdirs(self):
2031         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
2032         d.addCallback(lambda res:
2033                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
2034         d.addCallback(lambda res:
2035                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
2036         d.addCallback(lambda res:
2037                       self._foo_node.get_child_at_path(u"subdir/newdir"))
2038         d.addCallback(self.failUnlessNodeKeysAre, [])
2039         return d
2040
2041     def test_PUT_NEWDIRURL_mkdirs_mdmf(self):
2042         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=mdmf", "")
2043         d.addCallback(lambda ignored:
2044             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
2045         d.addCallback(lambda ignored:
2046             self.failIfNodeHasChild(self._foo_node, u"newdir"))
2047         d.addCallback(lambda ignored:
2048             self._foo_node.get_child_at_path(u"subdir"))
2049         def _got_subdir(subdir):
2050             # XXX: What we want?
2051             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
2052             self.failUnlessNodeHasChild(subdir, u"newdir")
2053             return subdir.get_child_at_path(u"newdir")
2054         d.addCallback(_got_subdir)
2055         d.addCallback(lambda newdir:
2056             self.failUnlessEqual(newdir._node.get_version(), MDMF_VERSION))
2057         return d
2058
2059     def test_PUT_NEWDIRURL_mkdirs_sdmf(self):
2060         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=sdmf", "")
2061         d.addCallback(lambda ignored:
2062             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
2063         d.addCallback(lambda ignored:
2064             self.failIfNodeHasChild(self._foo_node, u"newdir"))
2065         d.addCallback(lambda ignored:
2066             self._foo_node.get_child_at_path(u"subdir"))
2067         def _got_subdir(subdir):
2068             # XXX: What we want?
2069             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
2070             self.failUnlessNodeHasChild(subdir, u"newdir")
2071             return subdir.get_child_at_path(u"newdir")
2072         d.addCallback(_got_subdir)
2073         d.addCallback(lambda newdir:
2074             self.failUnlessEqual(newdir._node.get_version(), SDMF_VERSION))
2075         return d
2076
2077     def test_PUT_NEWDIRURL_mkdirs_bad_format(self):
2078         return self.shouldHTTPError("PUT_NEWDIRURL_mkdirs_bad_format",
2079                                     400, "Bad Request", "Unknown format: foo",
2080                                     self.PUT, self.public_url + \
2081                                     "/foo/subdir/newdir?t=mkdir&format=foo",
2082                                     "")
2083
2084     def test_DELETE_DIRURL(self):
2085         d = self.DELETE(self.public_url + "/foo")
2086         d.addCallback(lambda res:
2087                       self.failIfNodeHasChild(self.public_root, u"foo"))
2088         return d
2089
2090     def test_DELETE_DIRURL_missing(self):
2091         d = self.DELETE(self.public_url + "/foo/missing")
2092         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
2093         d.addCallback(lambda res:
2094                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
2095         return d
2096
2097     def test_DELETE_DIRURL_missing2(self):
2098         d = self.DELETE(self.public_url + "/missing")
2099         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
2100         return d
2101
2102     def dump_root(self):
2103         print "NODEWALK"
2104         w = webish.DirnodeWalkerMixin()
2105         def visitor(childpath, childnode, metadata):
2106             print childpath
2107         d = w.walk(self.public_root, visitor)
2108         return d
2109
2110     def failUnlessNodeKeysAre(self, node, expected_keys):
2111         for k in expected_keys:
2112             assert isinstance(k, unicode)
2113         d = node.list()
2114         def _check(children):
2115             self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
2116         d.addCallback(_check)
2117         return d
2118     def failUnlessNodeHasChild(self, node, name):
2119         assert isinstance(name, unicode)
2120         d = node.list()
2121         def _check(children):
2122             self.failUnlessIn(name, children)
2123         d.addCallback(_check)
2124         return d
2125     def failIfNodeHasChild(self, node, name):
2126         assert isinstance(name, unicode)
2127         d = node.list()
2128         def _check(children):
2129             self.failIfIn(name, children)
2130         d.addCallback(_check)
2131         return d
2132
2133     def failUnlessChildContentsAre(self, node, name, expected_contents):
2134         assert isinstance(name, unicode)
2135         d = node.get_child_at_path(name)
2136         d.addCallback(lambda node: download_to_data(node))
2137         def _check(contents):
2138             self.failUnlessReallyEqual(contents, expected_contents)
2139         d.addCallback(_check)
2140         return d
2141
2142     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
2143         assert isinstance(name, unicode)
2144         d = node.get_child_at_path(name)
2145         d.addCallback(lambda node: node.download_best_version())
2146         def _check(contents):
2147             self.failUnlessReallyEqual(contents, expected_contents)
2148         d.addCallback(_check)
2149         return d
2150
2151     def failUnlessRWChildURIIs(self, node, name, expected_uri):
2152         assert isinstance(name, unicode)
2153         d = node.get_child_at_path(name)
2154         def _check(child):
2155             self.failUnless(child.is_unknown() or not child.is_readonly())
2156             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
2157             self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
2158             expected_ro_uri = self._make_readonly(expected_uri)
2159             if expected_ro_uri:
2160                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
2161         d.addCallback(_check)
2162         return d
2163
2164     def failUnlessROChildURIIs(self, node, name, expected_uri):
2165         assert isinstance(name, unicode)
2166         d = node.get_child_at_path(name)
2167         def _check(child):
2168             self.failUnless(child.is_unknown() or child.is_readonly())
2169             self.failUnlessReallyEqual(child.get_write_uri(), None)
2170             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
2171             self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
2172         d.addCallback(_check)
2173         return d
2174
2175     def failUnlessURIMatchesRWChild(self, got_uri, node, name):
2176         assert isinstance(name, unicode)
2177         d = node.get_child_at_path(name)
2178         def _check(child):
2179             self.failUnless(child.is_unknown() or not child.is_readonly())
2180             self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
2181             self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
2182             expected_ro_uri = self._make_readonly(got_uri)
2183             if expected_ro_uri:
2184                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
2185         d.addCallback(_check)
2186         return d
2187
2188     def failUnlessURIMatchesROChild(self, got_uri, node, name):
2189         assert isinstance(name, unicode)
2190         d = node.get_child_at_path(name)
2191         def _check(child):
2192             self.failUnless(child.is_unknown() or child.is_readonly())
2193             self.failUnlessReallyEqual(child.get_write_uri(), None)
2194             self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
2195             self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
2196         d.addCallback(_check)
2197         return d
2198
2199     def failUnlessCHKURIHasContents(self, got_uri, contents):
2200         self.failUnless(self.get_all_contents()[got_uri] == contents)
2201
2202     def test_POST_upload(self):
2203         d = self.POST(self.public_url + "/foo", t="upload",
2204                       file=("new.txt", self.NEWFILE_CONTENTS))
2205         fn = self._foo_node
2206         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2207         d.addCallback(lambda res:
2208                       self.failUnlessChildContentsAre(fn, u"new.txt",
2209                                                       self.NEWFILE_CONTENTS))
2210         return d
2211
2212     def test_POST_upload_unicode(self):
2213         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2214         d = self.POST(self.public_url + "/foo", t="upload",
2215                       file=(filename, self.NEWFILE_CONTENTS))
2216         fn = self._foo_node
2217         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2218         d.addCallback(lambda res:
2219                       self.failUnlessChildContentsAre(fn, filename,
2220                                                       self.NEWFILE_CONTENTS))
2221         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2222         d.addCallback(lambda res: self.GET(target_url))
2223         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2224                                                                   self.NEWFILE_CONTENTS,
2225                                                                   contents))
2226         return d
2227
2228     def test_POST_upload_unicode_named(self):
2229         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2230         d = self.POST(self.public_url + "/foo", t="upload",
2231                       name=filename,
2232                       file=("overridden", self.NEWFILE_CONTENTS))
2233         fn = self._foo_node
2234         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2235         d.addCallback(lambda res:
2236                       self.failUnlessChildContentsAre(fn, filename,
2237                                                       self.NEWFILE_CONTENTS))
2238         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2239         d.addCallback(lambda res: self.GET(target_url))
2240         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2241                                                                   self.NEWFILE_CONTENTS,
2242                                                                   contents))
2243         return d
2244
2245     def test_POST_upload_no_link(self):
2246         d = self.POST("/uri", t="upload",
2247                       file=("new.txt", self.NEWFILE_CONTENTS))
2248         def _check_upload_results(page):
2249             # this should be a page which describes the results of the upload
2250             # that just finished.
2251             self.failUnlessIn("Upload Results:", page)
2252             self.failUnlessIn("URI:", page)
2253             uri_re = re.compile("URI: <tt><span>(.*)</span>")
2254             mo = uri_re.search(page)
2255             self.failUnless(mo, page)
2256             new_uri = mo.group(1)
2257             return new_uri
2258         d.addCallback(_check_upload_results)
2259         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
2260         return d
2261
2262     def test_POST_upload_no_link_whendone(self):
2263         d = self.POST("/uri", t="upload", when_done="/",
2264                       file=("new.txt", self.NEWFILE_CONTENTS))
2265         d.addBoth(self.shouldRedirect, "/")
2266         return d
2267
2268     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
2269         d = defer.maybeDeferred(callable, *args, **kwargs)
2270         def done(res):
2271             if isinstance(res, failure.Failure):
2272                 res.trap(error.PageRedirect)
2273                 statuscode = res.value.status
2274                 target = res.value.location
2275                 return checker(statuscode, target)
2276             self.fail("%s: callable was supposed to redirect, not return '%s'"
2277                       % (which, res))
2278         d.addBoth(done)
2279         return d
2280
2281     def test_POST_upload_no_link_whendone_results(self):
2282         def check(statuscode, target):
2283             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2284             self.failUnless(target.startswith(self.webish_url), target)
2285             return client.getPage(target, method="GET")
2286         # We encode "uri" as "%75ri" to exercise a case affected by ticket #1860.
2287         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
2288                                  check,
2289                                  self.POST, "/uri", t="upload",
2290                                  when_done="/%75ri/%(uri)s",
2291                                  file=("new.txt", self.NEWFILE_CONTENTS))
2292         d.addCallback(lambda res:
2293                       self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
2294         return d
2295
2296     def test_POST_upload_no_link_mutable(self):
2297         d = self.POST("/uri", t="upload", mutable="true",
2298                       file=("new.txt", self.NEWFILE_CONTENTS))
2299         def _check(filecap):
2300             filecap = filecap.strip()
2301             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2302             self.filecap = filecap
2303             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2304             self.failUnlessIn(u.get_storage_index(), self.get_all_contents())
2305             n = self.s.create_node_from_uri(filecap)
2306             return n.download_best_version()
2307         d.addCallback(_check)
2308         def _check2(data):
2309             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2310             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2311         d.addCallback(_check2)
2312         def _check3(data):
2313             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2314             return self.GET("/file/%s" % urllib.quote(self.filecap))
2315         d.addCallback(_check3)
2316         def _check4(data):
2317             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2318         d.addCallback(_check4)
2319         return d
2320
2321     def test_POST_upload_no_link_mutable_toobig(self):
2322         # The SDMF size limit is no longer in place, so we should be
2323         # able to upload mutable files that are as large as we want them
2324         # to be.
2325         d = self.POST("/uri", t="upload", mutable="true",
2326                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2327         return d
2328
2329
2330     def test_POST_upload_format_unlinked(self):
2331         def _check_upload_unlinked(ign, format, uri_prefix):
2332             filename = format + ".txt"
2333             d = self.POST("/uri?t=upload&format=" + format,
2334                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2335             def _got_results(results):
2336                 if format.upper() in ("SDMF", "MDMF"):
2337                     # webapi.rst says this returns a filecap
2338                     filecap = results
2339                 else:
2340                     # for immutable, it returns an "upload results page", and
2341                     # the filecap is buried inside
2342                     line = [l for l in results.split("\n") if "URI: " in l][0]
2343                     mo = re.search(r'<span>([^<]+)</span>', line)
2344                     filecap = mo.group(1)
2345                 self.failUnless(filecap.startswith(uri_prefix),
2346                                 (uri_prefix, filecap))
2347                 return self.GET("/uri/%s?t=json" % filecap)
2348             d.addCallback(_got_results)
2349             def _got_json(json):
2350                 data = simplejson.loads(json)
2351                 data = data[1]
2352                 self.failUnlessIn("format", data)
2353                 self.failUnlessEqual(data["format"], format.upper())
2354             d.addCallback(_got_json)
2355             return d
2356         d = defer.succeed(None)
2357         d.addCallback(_check_upload_unlinked, "chk", "URI:CHK")
2358         d.addCallback(_check_upload_unlinked, "CHK", "URI:CHK")
2359         d.addCallback(_check_upload_unlinked, "sdmf", "URI:SSK")
2360         d.addCallback(_check_upload_unlinked, "mdmf", "URI:MDMF")
2361         return d
2362
2363     def test_POST_upload_bad_format_unlinked(self):
2364         return self.shouldHTTPError("POST_upload_bad_format_unlinked",
2365                                     400, "Bad Request", "Unknown format: foo",
2366                                     self.POST,
2367                                     "/uri?t=upload&format=foo",
2368                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2369
2370     def test_POST_upload_format(self):
2371         def _check_upload(ign, format, uri_prefix, fn=None):
2372             filename = format + ".txt"
2373             d = self.POST(self.public_url +
2374                           "/foo?t=upload&format=" + format,
2375                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2376             def _got_filecap(filecap):
2377                 if fn is not None:
2378                     filenameu = unicode(filename)
2379                     self.failUnlessURIMatchesRWChild(filecap, fn, filenameu)
2380                 self.failUnless(filecap.startswith(uri_prefix))
2381                 return self.GET(self.public_url + "/foo/%s?t=json" % filename)
2382             d.addCallback(_got_filecap)
2383             def _got_json(json):
2384                 data = simplejson.loads(json)
2385                 data = data[1]
2386                 self.failUnlessIn("format", data)
2387                 self.failUnlessEqual(data["format"], format.upper())
2388             d.addCallback(_got_json)
2389             return d
2390
2391         d = defer.succeed(None)
2392         d.addCallback(_check_upload, "chk", "URI:CHK")
2393         d.addCallback(_check_upload, "sdmf", "URI:SSK", self._foo_node)
2394         d.addCallback(_check_upload, "mdmf", "URI:MDMF")
2395         d.addCallback(_check_upload, "MDMF", "URI:MDMF")
2396         return d
2397
2398     def test_POST_upload_bad_format(self):
2399         return self.shouldHTTPError("POST_upload_bad_format",
2400                                     400, "Bad Request", "Unknown format: foo",
2401                                     self.POST, self.public_url + \
2402                                     "/foo?t=upload&format=foo",
2403                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2404
2405     def test_POST_upload_mutable(self):
2406         # this creates a mutable file
2407         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
2408                       file=("new.txt", self.NEWFILE_CONTENTS))
2409         fn = self._foo_node
2410         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2411         d.addCallback(lambda res:
2412                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2413                                                              self.NEWFILE_CONTENTS))
2414         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2415         def _got(newnode):
2416             self.failUnless(IMutableFileNode.providedBy(newnode))
2417             self.failUnless(newnode.is_mutable())
2418             self.failIf(newnode.is_readonly())
2419             self._mutable_node = newnode
2420             self._mutable_uri = newnode.get_uri()
2421         d.addCallback(_got)
2422
2423         # now upload it again and make sure that the URI doesn't change
2424         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
2425         d.addCallback(lambda res:
2426                       self.POST(self.public_url + "/foo", t="upload",
2427                                 mutable="true",
2428                                 file=("new.txt", NEWER_CONTENTS)))
2429         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2430         d.addCallback(lambda res:
2431                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2432                                                              NEWER_CONTENTS))
2433         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2434         def _got2(newnode):
2435             self.failUnless(IMutableFileNode.providedBy(newnode))
2436             self.failUnless(newnode.is_mutable())
2437             self.failIf(newnode.is_readonly())
2438             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2439         d.addCallback(_got2)
2440
2441         # upload a second time, using PUT instead of POST
2442         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
2443         d.addCallback(lambda res:
2444                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
2445         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2446         d.addCallback(lambda res:
2447                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2448                                                              NEW2_CONTENTS))
2449
2450         # finally list the directory, since mutable files are displayed
2451         # slightly differently
2452
2453         d.addCallback(lambda res:
2454                       self.GET(self.public_url + "/foo/",
2455                                followRedirect=True))
2456         def _check_page(res):
2457             # TODO: assert more about the contents
2458             self.failUnlessIn("SSK", res)
2459             return res
2460         d.addCallback(_check_page)
2461
2462         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2463         def _got3(newnode):
2464             self.failUnless(IMutableFileNode.providedBy(newnode))
2465             self.failUnless(newnode.is_mutable())
2466             self.failIf(newnode.is_readonly())
2467             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2468         d.addCallback(_got3)
2469
2470         # look at the JSON form of the enclosing directory
2471         d.addCallback(lambda res:
2472                       self.GET(self.public_url + "/foo/?t=json",
2473                                followRedirect=True))
2474         def _check_page_json(res):
2475             parsed = simplejson.loads(res)
2476             self.failUnlessEqual(parsed[0], "dirnode")
2477             children = dict( [(unicode(name),value)
2478                               for (name,value)
2479                               in parsed[1]["children"].iteritems()] )
2480             self.failUnlessIn(u"new.txt", children)
2481             new_json = children[u"new.txt"]
2482             self.failUnlessEqual(new_json[0], "filenode")
2483             self.failUnless(new_json[1]["mutable"])
2484             self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri)
2485             ro_uri = self._mutable_node.get_readonly().to_string()
2486             self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri)
2487         d.addCallback(_check_page_json)
2488
2489         # and the JSON form of the file
2490         d.addCallback(lambda res:
2491                       self.GET(self.public_url + "/foo/new.txt?t=json"))
2492         def _check_file_json(res):
2493             parsed = simplejson.loads(res)
2494             self.failUnlessEqual(parsed[0], "filenode")
2495             self.failUnless(parsed[1]["mutable"])
2496             self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri)
2497             ro_uri = self._mutable_node.get_readonly().to_string()
2498             self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri)
2499         d.addCallback(_check_file_json)
2500
2501         # and look at t=uri and t=readonly-uri
2502         d.addCallback(lambda res:
2503                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
2504         d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
2505         d.addCallback(lambda res:
2506                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
2507         def _check_ro_uri(res):
2508             ro_uri = self._mutable_node.get_readonly().to_string()
2509             self.failUnlessReallyEqual(res, ro_uri)
2510         d.addCallback(_check_ro_uri)
2511
2512         # make sure we can get to it from /uri/URI
2513         d.addCallback(lambda res:
2514                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
2515         d.addCallback(lambda res:
2516                       self.failUnlessReallyEqual(res, NEW2_CONTENTS))
2517
2518         # and that HEAD computes the size correctly
2519         d.addCallback(lambda res:
2520                       self.HEAD(self.public_url + "/foo/new.txt",
2521                                 return_response=True))
2522         def _got_headers((res, status, headers)):
2523             self.failUnlessReallyEqual(res, "")
2524             self.failUnlessReallyEqual(headers["content-length"][0],
2525                                        str(len(NEW2_CONTENTS)))
2526             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
2527         d.addCallback(_got_headers)
2528
2529         # make sure that outdated size limits aren't enforced anymore.
2530         d.addCallback(lambda ignored:
2531             self.POST(self.public_url + "/foo", t="upload",
2532                       mutable="true",
2533                       file=("new.txt",
2534                             "b" * (self.s.MUTABLE_SIZELIMIT+1))))
2535         d.addErrback(self.dump_error)
2536         return d
2537
2538     def test_POST_upload_mutable_toobig(self):
2539         # SDMF had a size limti that was removed a while ago. MDMF has
2540         # never had a size limit. Test to make sure that we do not
2541         # encounter errors when trying to upload large mutable files,
2542         # since there should be no coded prohibitions regarding large
2543         # mutable files.
2544         d = self.POST(self.public_url + "/foo",
2545                       t="upload", mutable="true",
2546                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2547         return d
2548
2549     def dump_error(self, f):
2550         # if the web server returns an error code (like 400 Bad Request),
2551         # web.client.getPage puts the HTTP response body into the .response
2552         # attribute of the exception object that it gives back. It does not
2553         # appear in the Failure's repr(), so the ERROR that trial displays
2554         # will be rather terse and unhelpful. addErrback this method to the
2555         # end of your chain to get more information out of these errors.
2556         if f.check(error.Error):
2557             print "web.error.Error:"
2558             print f
2559             print f.value.response
2560         return f
2561
2562     def test_POST_upload_replace(self):
2563         d = self.POST(self.public_url + "/foo", t="upload",
2564                       file=("bar.txt", self.NEWFILE_CONTENTS))
2565         fn = self._foo_node
2566         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
2567         d.addCallback(lambda res:
2568                       self.failUnlessChildContentsAre(fn, u"bar.txt",
2569                                                       self.NEWFILE_CONTENTS))
2570         return d
2571
2572     def test_POST_upload_no_replace_ok(self):
2573         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2574                       file=("new.txt", self.NEWFILE_CONTENTS))
2575         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
2576         d.addCallback(lambda res: self.failUnlessReallyEqual(res,
2577                                                              self.NEWFILE_CONTENTS))
2578         return d
2579
2580     def test_POST_upload_no_replace_queryarg(self):
2581         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2582                       file=("bar.txt", self.NEWFILE_CONTENTS))
2583         d.addBoth(self.shouldFail, error.Error,
2584                   "POST_upload_no_replace_queryarg",
2585                   "409 Conflict",
2586                   "There was already a child by that name, and you asked me "
2587                   "to not replace it")
2588         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2589         d.addCallback(self.failUnlessIsBarDotTxt)
2590         return d
2591
2592     def test_POST_upload_no_replace_field(self):
2593         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
2594                       file=("bar.txt", self.NEWFILE_CONTENTS))
2595         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
2596                   "409 Conflict",
2597                   "There was already a child by that name, and you asked me "
2598                   "to not replace it")
2599         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2600         d.addCallback(self.failUnlessIsBarDotTxt)
2601         return d
2602
2603     def test_POST_upload_whendone(self):
2604         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
2605                       file=("new.txt", self.NEWFILE_CONTENTS))
2606         d.addBoth(self.shouldRedirect, "/THERE")
2607         fn = self._foo_node
2608         d.addCallback(lambda res:
2609                       self.failUnlessChildContentsAre(fn, u"new.txt",
2610                                                       self.NEWFILE_CONTENTS))
2611         return d
2612
2613     def test_POST_upload_named(self):
2614         fn = self._foo_node
2615         d = self.POST(self.public_url + "/foo", t="upload",
2616                       name="new.txt", file=self.NEWFILE_CONTENTS)
2617         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2618         d.addCallback(lambda res:
2619                       self.failUnlessChildContentsAre(fn, u"new.txt",
2620                                                       self.NEWFILE_CONTENTS))
2621         return d
2622
2623     def test_POST_upload_named_badfilename(self):
2624         d = self.POST(self.public_url + "/foo", t="upload",
2625                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
2626         d.addBoth(self.shouldFail, error.Error,
2627                   "test_POST_upload_named_badfilename",
2628                   "400 Bad Request",
2629                   "name= may not contain a slash",
2630                   )
2631         # make sure that nothing was added
2632         d.addCallback(lambda res:
2633                       self.failUnlessNodeKeysAre(self._foo_node,
2634                                                  [self._htmlname_unicode,
2635                                                   u"bar.txt", u"baz.txt", u"blockingfile",
2636                                                   u"empty", u"n\u00fc.txt", u"quux.txt",
2637                                                   u"sub"]))
2638         return d
2639
2640     def test_POST_FILEURL_check(self):
2641         bar_url = self.public_url + "/foo/bar.txt"
2642         d = self.POST(bar_url, t="check")
2643         def _check(res):
2644             self.failUnlessIn("Healthy :", res)
2645         d.addCallback(_check)
2646         redir_url = "http://allmydata.org/TARGET"
2647         def _check2(statuscode, target):
2648             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2649             self.failUnlessReallyEqual(target, redir_url)
2650         d.addCallback(lambda res:
2651                       self.shouldRedirect2("test_POST_FILEURL_check",
2652                                            _check2,
2653                                            self.POST, bar_url,
2654                                            t="check",
2655                                            when_done=redir_url))
2656         d.addCallback(lambda res:
2657                       self.POST(bar_url, t="check", return_to=redir_url))
2658         def _check3(res):
2659             self.failUnlessIn("Healthy :", res)
2660             self.failUnlessIn("Return to file", res)
2661             self.failUnlessIn(redir_url, res)
2662         d.addCallback(_check3)
2663
2664         d.addCallback(lambda res:
2665                       self.POST(bar_url, t="check", output="JSON"))
2666         def _check_json(res):
2667             data = simplejson.loads(res)
2668             self.failUnlessIn("storage-index", data)
2669             self.failUnless(data["results"]["healthy"])
2670         d.addCallback(_check_json)
2671
2672         return d
2673
2674     def test_POST_FILEURL_check_and_repair(self):
2675         bar_url = self.public_url + "/foo/bar.txt"
2676         d = self.POST(bar_url, t="check", repair="true")
2677         def _check(res):
2678             self.failUnlessIn("Healthy :", res)
2679         d.addCallback(_check)
2680         redir_url = "http://allmydata.org/TARGET"
2681         def _check2(statuscode, target):
2682             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2683             self.failUnlessReallyEqual(target, redir_url)
2684         d.addCallback(lambda res:
2685                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
2686                                            _check2,
2687                                            self.POST, bar_url,
2688                                            t="check", repair="true",
2689                                            when_done=redir_url))
2690         d.addCallback(lambda res:
2691                       self.POST(bar_url, t="check", return_to=redir_url))
2692         def _check3(res):
2693             self.failUnlessIn("Healthy :", res)
2694             self.failUnlessIn("Return to file", res)
2695             self.failUnlessIn(redir_url, res)
2696         d.addCallback(_check3)
2697         return d
2698
2699     def test_POST_DIRURL_check(self):
2700         foo_url = self.public_url + "/foo/"
2701         d = self.POST(foo_url, t="check")
2702         def _check(res):
2703             self.failUnlessIn("Healthy :", res)
2704         d.addCallback(_check)
2705         redir_url = "http://allmydata.org/TARGET"
2706         def _check2(statuscode, target):
2707             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2708             self.failUnlessReallyEqual(target, redir_url)
2709         d.addCallback(lambda res:
2710                       self.shouldRedirect2("test_POST_DIRURL_check",
2711                                            _check2,
2712                                            self.POST, foo_url,
2713                                            t="check",
2714                                            when_done=redir_url))
2715         d.addCallback(lambda res:
2716                       self.POST(foo_url, t="check", return_to=redir_url))
2717         def _check3(res):
2718             self.failUnlessIn("Healthy :", res)
2719             self.failUnlessIn("Return to file/directory", res)
2720             self.failUnlessIn(redir_url, res)
2721         d.addCallback(_check3)
2722
2723         d.addCallback(lambda res:
2724                       self.POST(foo_url, t="check", output="JSON"))
2725         def _check_json(res):
2726             data = simplejson.loads(res)
2727             self.failUnlessIn("storage-index", data)
2728             self.failUnless(data["results"]["healthy"])
2729         d.addCallback(_check_json)
2730
2731         return d
2732
2733     def test_POST_DIRURL_check_and_repair(self):
2734         foo_url = self.public_url + "/foo/"
2735         d = self.POST(foo_url, t="check", repair="true")
2736         def _check(res):
2737             self.failUnlessIn("Healthy :", res)
2738         d.addCallback(_check)
2739         redir_url = "http://allmydata.org/TARGET"
2740         def _check2(statuscode, target):
2741             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2742             self.failUnlessReallyEqual(target, redir_url)
2743         d.addCallback(lambda res:
2744                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2745                                            _check2,
2746                                            self.POST, foo_url,
2747                                            t="check", repair="true",
2748                                            when_done=redir_url))
2749         d.addCallback(lambda res:
2750                       self.POST(foo_url, t="check", return_to=redir_url))
2751         def _check3(res):
2752             self.failUnlessIn("Healthy :", res)
2753             self.failUnlessIn("Return to file/directory", res)
2754             self.failUnlessIn(redir_url, res)
2755         d.addCallback(_check3)
2756         return d
2757
2758     def test_POST_FILEURL_mdmf_check(self):
2759         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2760         d = self.POST(quux_url, t="check")
2761         def _check(res):
2762             self.failUnlessIn("Healthy", res)
2763         d.addCallback(_check)
2764         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2765         d.addCallback(lambda ignored:
2766                       self.POST(quux_extension_url, t="check"))
2767         d.addCallback(_check)
2768         return d
2769
2770     def test_POST_FILEURL_mdmf_check_and_repair(self):
2771         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2772         d = self.POST(quux_url, t="check", repair="true")
2773         def _check(res):
2774             self.failUnlessIn("Healthy", res)
2775         d.addCallback(_check)
2776         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2777         d.addCallback(lambda ignored:
2778                       self.POST(quux_extension_url, t="check", repair="true"))
2779         d.addCallback(_check)
2780         return d
2781
2782     def wait_for_operation(self, ignored, ophandle):
2783         url = "/operations/" + ophandle
2784         url += "?t=status&output=JSON"
2785         d = self.GET(url)
2786         def _got(res):
2787             data = simplejson.loads(res)
2788             if not data["finished"]:
2789                 d = self.stall(delay=1.0)
2790                 d.addCallback(self.wait_for_operation, ophandle)
2791                 return d
2792             return data
2793         d.addCallback(_got)
2794         return d
2795
2796     def get_operation_results(self, ignored, ophandle, output=None):
2797         url = "/operations/" + ophandle
2798         url += "?t=status"
2799         if output:
2800             url += "&output=" + output
2801         d = self.GET(url)
2802         def _got(res):
2803             if output and output.lower() == "json":
2804                 return simplejson.loads(res)
2805             return res
2806         d.addCallback(_got)
2807         return d
2808
2809     def test_POST_DIRURL_deepcheck_no_ophandle(self):
2810         d = self.shouldFail2(error.Error,
2811                              "test_POST_DIRURL_deepcheck_no_ophandle",
2812                              "400 Bad Request",
2813                              "slow operation requires ophandle=",
2814                              self.POST, self.public_url, t="start-deep-check")
2815         return d
2816
2817     def test_POST_DIRURL_deepcheck(self):
2818         def _check_redirect(statuscode, target):
2819             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2820             self.failUnless(target.endswith("/operations/123"))
2821         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2822                                  self.POST, self.public_url,
2823                                  t="start-deep-check", ophandle="123")
2824         d.addCallback(self.wait_for_operation, "123")
2825         def _check_json(data):
2826             self.failUnlessReallyEqual(data["finished"], True)
2827             self.failUnlessReallyEqual(data["count-objects-checked"], 11)
2828             self.failUnlessReallyEqual(data["count-objects-healthy"], 11)
2829         d.addCallback(_check_json)
2830         d.addCallback(self.get_operation_results, "123", "html")
2831         def _check_html(res):
2832             self.failUnlessIn("Objects Checked: <span>11</span>", res)
2833             self.failUnlessIn("Objects Healthy: <span>11</span>", res)
2834             self.failUnlessIn(FAVICON_MARKUP, res)
2835         d.addCallback(_check_html)
2836
2837         d.addCallback(lambda res:
2838                       self.GET("/operations/123/"))
2839         d.addCallback(_check_html) # should be the same as without the slash
2840
2841         d.addCallback(lambda res:
2842                       self.shouldFail2(error.Error, "one", "404 Not Found",
2843                                        "No detailed results for SI bogus",
2844                                        self.GET, "/operations/123/bogus"))
2845
2846         foo_si = self._foo_node.get_storage_index()
2847         foo_si_s = base32.b2a(foo_si)
2848         d.addCallback(lambda res:
2849                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2850         def _check_foo_json(res):
2851             data = simplejson.loads(res)
2852             self.failUnlessEqual(data["storage-index"], foo_si_s)
2853             self.failUnless(data["results"]["healthy"])
2854         d.addCallback(_check_foo_json)
2855         return d
2856
2857     def test_POST_DIRURL_deepcheck_and_repair(self):
2858         d = self.POST(self.public_url, t="start-deep-check", repair="true",
2859                       ophandle="124", output="json", followRedirect=True)
2860         d.addCallback(self.wait_for_operation, "124")
2861         def _check_json(data):
2862             self.failUnlessReallyEqual(data["finished"], True)
2863             self.failUnlessReallyEqual(data["count-objects-checked"], 11)
2864             self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 11)
2865             self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2866             self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2867             self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2868             self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2869             self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2870             self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 11)
2871             self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2872             self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2873         d.addCallback(_check_json)
2874         d.addCallback(self.get_operation_results, "124", "html")
2875         def _check_html(res):
2876             self.failUnlessIn("Objects Checked: <span>11</span>", res)
2877
2878             self.failUnlessIn("Objects Healthy (before repair): <span>11</span>", res)
2879             self.failUnlessIn("Objects Unhealthy (before repair): <span>0</span>", res)
2880             self.failUnlessIn("Corrupt Shares (before repair): <span>0</span>", res)
2881
2882             self.failUnlessIn("Repairs Attempted: <span>0</span>", res)
2883             self.failUnlessIn("Repairs Successful: <span>0</span>", res)
2884             self.failUnlessIn("Repairs Unsuccessful: <span>0</span>", res)
2885
2886             self.failUnlessIn("Objects Healthy (after repair): <span>11</span>", res)
2887             self.failUnlessIn("Objects Unhealthy (after repair): <span>0</span>", res)
2888             self.failUnlessIn("Corrupt Shares (after repair): <span>0</span>", res)
2889
2890             self.failUnlessIn(FAVICON_MARKUP, res)
2891         d.addCallback(_check_html)
2892         return d
2893
2894     def test_POST_FILEURL_bad_t(self):
2895         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2896                              "POST to file: bad t=bogus",
2897                              self.POST, self.public_url + "/foo/bar.txt",
2898                              t="bogus")
2899         return d
2900
2901     def test_POST_mkdir(self): # return value?
2902         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2903         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2904         d.addCallback(self.failUnlessNodeKeysAre, [])
2905         return d
2906
2907     def test_POST_mkdir_mdmf(self):
2908         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=mdmf")
2909         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2910         d.addCallback(lambda node:
2911             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2912         return d
2913
2914     def test_POST_mkdir_sdmf(self):
2915         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=sdmf")
2916         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2917         d.addCallback(lambda node:
2918             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2919         return d
2920
2921     def test_POST_mkdir_bad_format(self):
2922         return self.shouldHTTPError("POST_mkdir_bad_format",
2923                                     400, "Bad Request", "Unknown format: foo",
2924                                     self.POST, self.public_url +
2925                                     "/foo?t=mkdir&name=newdir&format=foo")
2926
2927     def test_POST_mkdir_initial_children(self):
2928         (newkids, caps) = self._create_initial_children()
2929         d = self.POST2(self.public_url +
2930                        "/foo?t=mkdir-with-children&name=newdir",
2931                        simplejson.dumps(newkids))
2932         d.addCallback(lambda res:
2933                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2934         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2935         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2936         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2937         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2938         return d
2939
2940     def test_POST_mkdir_initial_children_mdmf(self):
2941         (newkids, caps) = self._create_initial_children()
2942         d = self.POST2(self.public_url +
2943                        "/foo?t=mkdir-with-children&name=newdir&format=mdmf",
2944                        simplejson.dumps(newkids))
2945         d.addCallback(lambda res:
2946                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2947         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2948         d.addCallback(lambda node:
2949             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2950         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2951         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2952                        caps['filecap1'])
2953         return d
2954
2955     # XXX: Duplication.
2956     def test_POST_mkdir_initial_children_sdmf(self):
2957         (newkids, caps) = self._create_initial_children()
2958         d = self.POST2(self.public_url +
2959                        "/foo?t=mkdir-with-children&name=newdir&format=sdmf",
2960                        simplejson.dumps(newkids))
2961         d.addCallback(lambda res:
2962                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2963         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2964         d.addCallback(lambda node:
2965             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2966         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2967         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2968                        caps['filecap1'])
2969         return d
2970
2971     def test_POST_mkdir_initial_children_bad_format(self):
2972         (newkids, caps) = self._create_initial_children()
2973         return self.shouldHTTPError("POST_mkdir_initial_children_bad_format",
2974                                     400, "Bad Request", "Unknown format: foo",
2975                                     self.POST, self.public_url + \
2976                                     "/foo?t=mkdir-with-children&name=newdir&format=foo",
2977                                     simplejson.dumps(newkids))
2978
2979     def test_POST_mkdir_immutable(self):
2980         (newkids, caps) = self._create_immutable_children()
2981         d = self.POST2(self.public_url +
2982                        "/foo?t=mkdir-immutable&name=newdir",
2983                        simplejson.dumps(newkids))
2984         d.addCallback(lambda res:
2985                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2986         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2987         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2988         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2989         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2990         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2991         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2992         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2993         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2994         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2995         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2996         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2997         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2998         return d
2999
3000     def test_POST_mkdir_immutable_bad(self):
3001         (newkids, caps) = self._create_initial_children()
3002         d = self.shouldFail2(error.Error, "POST_mkdir_immutable_bad",
3003                              "400 Bad Request",
3004                              "needed to be immutable but was not",
3005                              self.POST2,
3006                              self.public_url +
3007                              "/foo?t=mkdir-immutable&name=newdir",
3008                              simplejson.dumps(newkids))
3009         return d
3010
3011     def test_POST_mkdir_2(self):
3012         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
3013         d.addCallback(lambda res:
3014                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
3015         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3016         d.addCallback(self.failUnlessNodeKeysAre, [])
3017         return d
3018
3019     def test_POST_mkdirs_2(self):
3020         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
3021         d.addCallback(lambda res:
3022                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
3023         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
3024         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
3025         d.addCallback(self.failUnlessNodeKeysAre, [])
3026         return d
3027
3028     def test_POST_mkdir_no_parentdir_noredirect(self):
3029         d = self.POST("/uri?t=mkdir")
3030         def _after_mkdir(res):
3031             uri.DirectoryURI.init_from_string(res)
3032         d.addCallback(_after_mkdir)
3033         return d
3034
3035     def test_POST_mkdir_no_parentdir_noredirect_mdmf(self):
3036         d = self.POST("/uri?t=mkdir&format=mdmf")
3037         def _after_mkdir(res):
3038             u = uri.from_string(res)
3039             # Check that this is an MDMF writecap
3040             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
3041         d.addCallback(_after_mkdir)
3042         return d
3043
3044     def test_POST_mkdir_no_parentdir_noredirect_sdmf(self):
3045         d = self.POST("/uri?t=mkdir&format=sdmf")
3046         def _after_mkdir(res):
3047             u = uri.from_string(res)
3048             self.failUnlessIsInstance(u, uri.DirectoryURI)
3049         d.addCallback(_after_mkdir)
3050         return d
3051
3052     def test_POST_mkdir_no_parentdir_noredirect_bad_format(self):
3053         return self.shouldHTTPError("POST_mkdir_no_parentdir_noredirect_bad_format",
3054                                     400, "Bad Request", "Unknown format: foo",
3055                                     self.POST, self.public_url +
3056                                     "/uri?t=mkdir&format=foo")
3057
3058     def test_POST_mkdir_no_parentdir_noredirect2(self):
3059         # make sure form-based arguments (as on the welcome page) still work
3060         d = self.POST("/uri", t="mkdir")
3061         def _after_mkdir(res):
3062             uri.DirectoryURI.init_from_string(res)
3063         d.addCallback(_after_mkdir)
3064         d.addErrback(self.explain_web_error)
3065         return d
3066
3067     def test_POST_mkdir_no_parentdir_redirect(self):
3068         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
3069         d.addBoth(self.shouldRedirect, None, statuscode='303')
3070         def _check_target(target):
3071             target = urllib.unquote(target)
3072             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
3073         d.addCallback(_check_target)
3074         return d
3075
3076     def test_POST_mkdir_no_parentdir_redirect2(self):
3077         d = self.POST("/uri", t="mkdir", redirect_to_result="true")
3078         d.addBoth(self.shouldRedirect, None, statuscode='303')
3079         def _check_target(target):
3080             target = urllib.unquote(target)
3081             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
3082         d.addCallback(_check_target)
3083         d.addErrback(self.explain_web_error)
3084         return d
3085
3086     def _make_readonly(self, u):
3087         ro_uri = uri.from_string(u).get_readonly()
3088         if ro_uri is None:
3089             return None
3090         return ro_uri.to_string()
3091
3092     def _create_initial_children(self):
3093         contents, n, filecap1 = self.makefile(12)
3094         md1 = {"metakey1": "metavalue1"}
3095         filecap2 = make_mutable_file_uri()
3096         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
3097         filecap3 = node3.get_readonly_uri()
3098         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
3099         dircap = DirectoryNode(node4, None, None).get_uri()
3100         mdmfcap = make_mutable_file_uri(mdmf=True)
3101         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
3102         emptydircap = "URI:DIR2-LIT:"
3103         newkids = {u"child-imm":        ["filenode", {"rw_uri": filecap1,
3104                                                       "ro_uri": self._make_readonly(filecap1),
3105                                                       "metadata": md1, }],
3106                    u"child-mutable":    ["filenode", {"rw_uri": filecap2,
3107                                                       "ro_uri": self._make_readonly(filecap2)}],
3108                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
3109                    u"unknownchild-rw":  ["unknown",  {"rw_uri": unknown_rwcap,
3110                                                       "ro_uri": unknown_rocap}],
3111                    u"unknownchild-ro":  ["unknown",  {"ro_uri": unknown_rocap}],
3112                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
3113                    u"dirchild":         ["dirnode",  {"rw_uri": dircap,
3114                                                       "ro_uri": self._make_readonly(dircap)}],
3115                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
3116                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
3117                    u"child-mutable-mdmf": ["filenode", {"rw_uri": mdmfcap,
3118                                                         "ro_uri": self._make_readonly(mdmfcap)}],
3119                    }
3120         return newkids, {'filecap1': filecap1,
3121                          'filecap2': filecap2,
3122                          'filecap3': filecap3,
3123                          'unknown_rwcap': unknown_rwcap,
3124                          'unknown_rocap': unknown_rocap,
3125                          'unknown_immcap': unknown_immcap,
3126                          'dircap': dircap,
3127                          'litdircap': litdircap,
3128                          'emptydircap': emptydircap,
3129                          'mdmfcap': mdmfcap}
3130
3131     def _create_immutable_children(self):
3132         contents, n, filecap1 = self.makefile(12)
3133         md1 = {"metakey1": "metavalue1"}
3134         tnode = create_chk_filenode("immutable directory contents\n"*10,
3135                                     self.get_all_contents())
3136         dnode = DirectoryNode(tnode, None, None)
3137         assert not dnode.is_mutable()
3138         immdircap = dnode.get_uri()
3139         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
3140         emptydircap = "URI:DIR2-LIT:"
3141         newkids = {u"child-imm":        ["filenode", {"ro_uri": filecap1,
3142                                                       "metadata": md1, }],
3143                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
3144                    u"dirchild-imm":     ["dirnode",  {"ro_uri": immdircap}],
3145                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
3146                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
3147                    }
3148         return newkids, {'filecap1': filecap1,
3149                          'unknown_immcap': unknown_immcap,
3150                          'immdircap': immdircap,
3151                          'litdircap': litdircap,
3152                          'emptydircap': emptydircap}
3153
3154     def test_POST_mkdir_no_parentdir_initial_children(self):
3155         (newkids, caps) = self._create_initial_children()
3156         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
3157         def _after_mkdir(res):
3158             self.failUnless(res.startswith("URI:DIR"), res)
3159             n = self.s.create_node_from_uri(res)
3160             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3161             d2.addCallback(lambda ign:
3162                            self.failUnlessROChildURIIs(n, u"child-imm",
3163                                                        caps['filecap1']))
3164             d2.addCallback(lambda ign:
3165                            self.failUnlessRWChildURIIs(n, u"child-mutable",
3166                                                        caps['filecap2']))
3167             d2.addCallback(lambda ign:
3168                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
3169                                                        caps['filecap3']))
3170             d2.addCallback(lambda ign:
3171                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
3172                                                        caps['unknown_rwcap']))
3173             d2.addCallback(lambda ign:
3174                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
3175                                                        caps['unknown_rocap']))
3176             d2.addCallback(lambda ign:
3177                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3178                                                        caps['unknown_immcap']))
3179             d2.addCallback(lambda ign:
3180                            self.failUnlessRWChildURIIs(n, u"dirchild",
3181                                                        caps['dircap']))
3182             return d2
3183         d.addCallback(_after_mkdir)
3184         return d
3185
3186     def test_POST_mkdir_no_parentdir_unexpected_children(self):
3187         # the regular /uri?t=mkdir operation is specified to ignore its body.
3188         # Only t=mkdir-with-children pays attention to it.
3189         (newkids, caps) = self._create_initial_children()
3190         d = self.shouldHTTPError("POST_mkdir_no_parentdir_unexpected_children",
3191                                  400, "Bad Request",
3192                                  "t=mkdir does not accept children=, "
3193                                  "try t=mkdir-with-children instead",
3194                                  self.POST2, "/uri?t=mkdir", # without children
3195                                  simplejson.dumps(newkids))
3196         return d
3197
3198     def test_POST_noparent_bad(self):
3199         d = self.shouldHTTPError("POST_noparent_bad",
3200                                  400, "Bad Request",
3201                                  "/uri accepts only PUT, PUT?t=mkdir, "
3202                                  "POST?t=upload, and POST?t=mkdir",
3203                                  self.POST, "/uri?t=bogus")
3204         return d
3205
3206     def test_POST_mkdir_no_parentdir_immutable(self):
3207         (newkids, caps) = self._create_immutable_children()
3208         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
3209         def _after_mkdir(res):
3210             self.failUnless(res.startswith("URI:DIR"), res)
3211             n = self.s.create_node_from_uri(res)
3212             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3213             d2.addCallback(lambda ign:
3214                            self.failUnlessROChildURIIs(n, u"child-imm",
3215                                                           caps['filecap1']))
3216             d2.addCallback(lambda ign:
3217                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3218                                                           caps['unknown_immcap']))
3219             d2.addCallback(lambda ign:
3220                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
3221                                                           caps['immdircap']))
3222             d2.addCallback(lambda ign:
3223                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
3224                                                           caps['litdircap']))
3225             d2.addCallback(lambda ign:
3226                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
3227                                                           caps['emptydircap']))
3228             return d2
3229         d.addCallback(_after_mkdir)
3230         return d
3231
3232     def test_POST_mkdir_no_parentdir_immutable_bad(self):
3233         (newkids, caps) = self._create_initial_children()
3234         d = self.shouldFail2(error.Error,
3235                              "test_POST_mkdir_no_parentdir_immutable_bad",
3236                              "400 Bad Request",
3237                              "needed to be immutable but was not",
3238                              self.POST2,
3239                              "/uri?t=mkdir-immutable",
3240                              simplejson.dumps(newkids))
3241         return d
3242
3243     def test_welcome_page_mkdir_button(self):
3244         # Fetch the welcome page.
3245         d = self.GET("/")
3246         def _after_get_welcome_page(res):
3247             MKDIR_BUTTON_RE = re.compile(
3248                 '<form action="([^"]*)" method="post".*'
3249                 '<input type="hidden" name="t" value="([^"]*)" />[ ]*'
3250                 '<input type="hidden" name="([^"]*)" value="([^"]*)" />[ ]*'
3251                 '<input type="submit" class="btn" value="Create a directory[^"]*" />')
3252             html = res.replace('\n', ' ')
3253             mo = MKDIR_BUTTON_RE.search(html)
3254             self.failUnless(mo, html)
3255             formaction = mo.group(1)
3256             formt = mo.group(2)
3257             formaname = mo.group(3)
3258             formavalue = mo.group(4)
3259             return (formaction, formt, formaname, formavalue)
3260         d.addCallback(_after_get_welcome_page)
3261         def _after_parse_form(res):
3262             (formaction, formt, formaname, formavalue) = res
3263             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
3264         d.addCallback(_after_parse_form)
3265         d.addBoth(self.shouldRedirect, None, statuscode='303')
3266         return d
3267
3268     def test_POST_mkdir_replace(self): # return value?
3269         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
3270         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3271         d.addCallback(self.failUnlessNodeKeysAre, [])
3272         return d
3273
3274     def test_POST_mkdir_no_replace_queryarg(self): # return value?
3275         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
3276         d.addBoth(self.shouldFail, error.Error,
3277                   "POST_mkdir_no_replace_queryarg",
3278                   "409 Conflict",
3279                   "There was already a child by that name, and you asked me "
3280                   "to not replace it")
3281         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3282         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3283         return d
3284
3285     def test_POST_mkdir_no_replace_field(self): # return value?
3286         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
3287                       replace="false")
3288         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
3289                   "409 Conflict",
3290                   "There was already a child by that name, and you asked me "
3291                   "to not replace it")
3292         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3293         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3294         return d
3295
3296     def test_POST_mkdir_whendone_field(self):
3297         d = self.POST(self.public_url + "/foo",
3298                       t="mkdir", name="newdir", when_done="/THERE")
3299         d.addBoth(self.shouldRedirect, "/THERE")
3300         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3301         d.addCallback(self.failUnlessNodeKeysAre, [])
3302         return d
3303
3304     def test_POST_mkdir_whendone_queryarg(self):
3305         d = self.POST(self.public_url + "/foo?when_done=/THERE",
3306                       t="mkdir", name="newdir")
3307         d.addBoth(self.shouldRedirect, "/THERE")
3308         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3309         d.addCallback(self.failUnlessNodeKeysAre, [])
3310         return d
3311
3312     def test_POST_bad_t(self):
3313         d = self.shouldFail2(error.Error, "POST_bad_t",
3314                              "400 Bad Request",
3315                              "POST to a directory with bad t=BOGUS",
3316                              self.POST, self.public_url + "/foo", t="BOGUS")
3317         return d
3318
3319     def test_POST_set_children(self, command_name="set_children"):
3320         contents9, n9, newuri9 = self.makefile(9)
3321         contents10, n10, newuri10 = self.makefile(10)
3322         contents11, n11, newuri11 = self.makefile(11)
3323
3324         reqbody = """{
3325                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
3326                                                 "size": 0,
3327                                                 "metadata": {
3328                                                   "ctime": 1002777696.7564139,
3329                                                   "mtime": 1002777696.7564139
3330                                                  }
3331                                                } ],
3332                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
3333                                                 "size": 1,
3334                                                 "metadata": {
3335                                                   "ctime": 1002777696.7564139,
3336                                                   "mtime": 1002777696.7564139
3337                                                  }
3338                                                } ],
3339                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
3340                                                 "size": 2,
3341                                                 "metadata": {
3342                                                   "ctime": 1002777696.7564139,
3343                                                   "mtime": 1002777696.7564139
3344                                                  }
3345                                                } ]
3346                     }""" % (newuri9, newuri10, newuri11)
3347
3348         url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
3349
3350         d = client.getPage(url, method="POST", postdata=reqbody)
3351         def _then(res):
3352             self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
3353             self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
3354             self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
3355
3356         d.addCallback(_then)
3357         d.addErrback(self.dump_error)
3358         return d
3359
3360     def test_POST_set_children_with_hyphen(self):
3361         return self.test_POST_set_children(command_name="set-children")
3362
3363     def test_POST_link_uri(self):
3364         contents, n, newuri = self.makefile(8)
3365         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
3366         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
3367         d.addCallback(lambda res:
3368                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3369                                                       contents))
3370         return d
3371
3372     def test_POST_link_uri_replace(self):
3373         contents, n, newuri = self.makefile(8)
3374         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
3375         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
3376         d.addCallback(lambda res:
3377                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3378                                                       contents))
3379         return d
3380
3381     def test_POST_link_uri_unknown_bad(self):
3382         d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
3383         d.addBoth(self.shouldFail, error.Error,
3384                   "POST_link_uri_unknown_bad",
3385                   "400 Bad Request",
3386                   "unknown cap in a write slot")
3387         return d
3388
3389     def test_POST_link_uri_unknown_ro_good(self):
3390         d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
3391         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
3392         return d
3393
3394     def test_POST_link_uri_unknown_imm_good(self):
3395         d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
3396         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
3397         return d
3398
3399     def test_POST_link_uri_no_replace_queryarg(self):
3400         contents, n, newuri = self.makefile(8)
3401         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
3402                       name="bar.txt", uri=newuri)
3403         d.addBoth(self.shouldFail, error.Error,
3404                   "POST_link_uri_no_replace_queryarg",
3405                   "409 Conflict",
3406                   "There was already a child by that name, and you asked me "
3407                   "to not replace it")
3408         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3409         d.addCallback(self.failUnlessIsBarDotTxt)
3410         return d
3411
3412     def test_POST_link_uri_no_replace_field(self):
3413         contents, n, newuri = self.makefile(8)
3414         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
3415                       name="bar.txt", uri=newuri)
3416         d.addBoth(self.shouldFail, error.Error,
3417                   "POST_link_uri_no_replace_field",
3418                   "409 Conflict",
3419                   "There was already a child by that name, and you asked me "
3420                   "to not replace it")
3421         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3422         d.addCallback(self.failUnlessIsBarDotTxt)
3423         return d
3424
3425     def test_POST_delete(self, command_name='delete'):
3426         d = self._foo_node.list()
3427         def _check_before(children):
3428             self.failUnlessIn(u"bar.txt", children)
3429         d.addCallback(_check_before)
3430         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t=command_name, name="bar.txt"))
3431         d.addCallback(lambda res: self._foo_node.list())
3432         def _check_after(children):
3433             self.failIfIn(u"bar.txt", children)
3434         d.addCallback(_check_after)
3435         return d
3436
3437     def test_POST_unlink(self):
3438         return self.test_POST_delete(command_name='unlink')
3439
3440     def test_POST_rename_file(self):
3441         d = self.POST(self.public_url + "/foo", t="rename",
3442                       from_name="bar.txt", to_name='wibble.txt')
3443         d.addCallback(lambda res:
3444                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3445         d.addCallback(lambda res:
3446                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
3447         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
3448         d.addCallback(self.failUnlessIsBarDotTxt)
3449         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
3450         d.addCallback(self.failUnlessIsBarJSON)
3451         return d
3452
3453     def test_POST_rename_file_redundant(self):
3454         d = self.POST(self.public_url + "/foo", t="rename",
3455                       from_name="bar.txt", to_name='bar.txt')
3456         d.addCallback(lambda res:
3457                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3458         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3459         d.addCallback(self.failUnlessIsBarDotTxt)
3460         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3461         d.addCallback(self.failUnlessIsBarJSON)
3462         return d
3463
3464     def test_POST_rename_file_replace(self):
3465         # rename a file and replace a directory with it
3466         d = self.POST(self.public_url + "/foo", t="rename",
3467                       from_name="bar.txt", to_name='empty')
3468         d.addCallback(lambda res:
3469                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3470         d.addCallback(lambda res:
3471                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
3472         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
3473         d.addCallback(self.failUnlessIsBarDotTxt)
3474         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3475         d.addCallback(self.failUnlessIsBarJSON)
3476         return d
3477
3478     def test_POST_rename_file_no_replace_queryarg(self):
3479         # rename a file and replace a directory with it
3480         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
3481                       from_name="bar.txt", to_name='empty')
3482         d.addBoth(self.shouldFail, error.Error,
3483                   "POST_rename_file_no_replace_queryarg",
3484                   "409 Conflict",
3485                   "There was already a child by that name, and you asked me "
3486                   "to not replace it")
3487         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3488         d.addCallback(self.failUnlessIsEmptyJSON)
3489         return d
3490
3491     def test_POST_rename_file_no_replace_field(self):
3492         # rename a file and replace a directory with it
3493         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
3494                       from_name="bar.txt", to_name='empty')
3495         d.addBoth(self.shouldFail, error.Error,
3496                   "POST_rename_file_no_replace_field",
3497                   "409 Conflict",
3498                   "There was already a child by that name, and you asked me "
3499                   "to not replace it")
3500         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3501         d.addCallback(self.failUnlessIsEmptyJSON)
3502         return d
3503
3504     def failUnlessIsEmptyJSON(self, res):
3505         data = simplejson.loads(res)
3506         self.failUnlessEqual(data[0], "dirnode", data)
3507         self.failUnlessReallyEqual(len(data[1]["children"]), 0)
3508
3509     def test_POST_rename_file_slash_fail(self):
3510         d = self.POST(self.public_url + "/foo", t="rename",
3511                       from_name="bar.txt", to_name='kirk/spock.txt')
3512         d.addBoth(self.shouldFail, error.Error,
3513                   "test_POST_rename_file_slash_fail",
3514                   "400 Bad Request",
3515                   "to_name= may not contain a slash",
3516                   )
3517         d.addCallback(lambda res:
3518                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3519         return d
3520
3521     def test_POST_rename_dir(self):
3522         d = self.POST(self.public_url, t="rename",
3523                       from_name="foo", to_name='plunk')
3524         d.addCallback(lambda res:
3525                       self.failIfNodeHasChild(self.public_root, u"foo"))
3526         d.addCallback(lambda res:
3527                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
3528         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
3529         d.addCallback(self.failUnlessIsFooJSON)
3530         return d
3531
3532     def test_POST_move_file(self):
3533         d = self.POST(self.public_url + "/foo", t="move",
3534                       from_name="bar.txt", to_dir="sub")
3535         d.addCallback(lambda res:
3536                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3537         d.addCallback(lambda res:
3538                       self.failUnlessNodeHasChild(self._sub_node, u"bar.txt"))
3539         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3540         d.addCallback(self.failUnlessIsBarDotTxt)
3541         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3542         d.addCallback(self.failUnlessIsBarJSON)
3543         return d
3544
3545     def test_POST_move_file_new_name(self):
3546         d = self.POST(self.public_url + "/foo", t="move",
3547                       from_name="bar.txt", to_name="wibble.txt", to_dir="sub")
3548         d.addCallback(lambda res:
3549                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3550         d.addCallback(lambda res:
3551                       self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3552         d.addCallback(lambda res:
3553                       self.failUnlessNodeHasChild(self._sub_node, u"wibble.txt"))
3554         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt"))
3555         d.addCallback(self.failUnlessIsBarDotTxt)
3556         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt?t=json"))
3557         d.addCallback(self.failUnlessIsBarJSON)
3558         return d
3559
3560     def test_POST_move_file_replace(self):
3561         d = self.POST(self.public_url + "/foo", t="move",
3562                       from_name="bar.txt", to_name="baz.txt", to_dir="sub")
3563         d.addCallback(lambda res:
3564                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3565         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3566         d.addCallback(self.failUnlessIsBarDotTxt)
3567         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt?t=json"))
3568         d.addCallback(self.failUnlessIsBarJSON)
3569         return d
3570
3571     def test_POST_move_file_no_replace(self):
3572         d = self.shouldFail2(error.Error, "POST_move_file_no_replace",
3573                              "409 Conflict",
3574                              "There was already a child by that name, and you asked me to not replace it",
3575                              self.POST, self.public_url + "/foo", t="move",
3576                              replace="false", from_name="bar.txt",
3577                              to_name="baz.txt", to_dir="sub")
3578         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3579         d.addCallback(self.failUnlessIsBarDotTxt)
3580         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3581         d.addCallback(self.failUnlessIsBarJSON)
3582         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3583         d.addCallback(self.failUnlessIsSubBazDotTxt)
3584         return d
3585
3586     def test_POST_move_file_slash_fail(self):
3587         d = self.shouldFail2(error.Error, "test_POST_rename_file_slash_fail",
3588                              "400 Bad Request",
3589                              "to_name= may not contain a slash",
3590                              self.POST, self.public_url + "/foo", t="move",
3591                              from_name="bar.txt",
3592                              to_name="slash/fail.txt", to_dir="sub")
3593         d.addCallback(lambda res:
3594                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3595         d.addCallback(lambda res:
3596                       self.failIfNodeHasChild(self._sub_node, u"slash/fail.txt"))
3597         d.addCallback(lambda ign:
3598                       self.shouldFail2(error.Error,
3599                                        "test_POST_rename_file_slash_fail2",
3600                                        "400 Bad Request",
3601                                        "from_name= may not contain a slash",
3602                                        self.POST, self.public_url + "/foo",
3603                                        t="move",
3604                                        from_name="nope/bar.txt",
3605                                        to_name="fail.txt", to_dir="sub"))
3606         return d
3607
3608     def test_POST_move_file_no_target(self):
3609         d = self.shouldFail2(error.Error, "POST_move_file_no_target",
3610                              "400 Bad Request",
3611                              "move requires from_name and to_dir",
3612                              self.POST, self.public_url + "/foo", t="move",
3613                              from_name="bar.txt", to_name="baz.txt")
3614         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3615         d.addCallback(self.failUnlessIsBarDotTxt)
3616         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3617         d.addCallback(self.failUnlessIsBarJSON)
3618         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3619         d.addCallback(self.failUnlessIsBazDotTxt)
3620         return d
3621
3622     def test_POST_move_file_bad_target_type(self):
3623         d = self.shouldFail2(error.Error, "test_POST_move_file_bad_target_type",
3624                              "400 Bad Request", "invalid target_type parameter",
3625                              self.POST,
3626                              self.public_url + "/foo", t="move",
3627                              target_type="*D", from_name="bar.txt",
3628                              to_dir="sub")
3629         return d
3630
3631     def test_POST_move_file_multi_level(self):
3632         d = self.POST(self.public_url + "/foo/sub/level2?t=mkdir", "")
3633         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t="move",
3634                       from_name="bar.txt", to_dir="sub/level2"))
3635         d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3636         d.addCallback(lambda res: self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3637         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt"))
3638         d.addCallback(self.failUnlessIsBarDotTxt)
3639         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt?t=json"))
3640         d.addCallback(self.failUnlessIsBarJSON)
3641         return d
3642
3643     def test_POST_move_file_to_uri(self):
3644         d = self.POST(self.public_url + "/foo", t="move", target_type="uri",
3645                       from_name="bar.txt", to_dir=self._sub_uri)
3646         d.addCallback(lambda res:
3647                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3648         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3649         d.addCallback(self.failUnlessIsBarDotTxt)
3650         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3651         d.addCallback(self.failUnlessIsBarJSON)
3652         return d
3653
3654     def test_POST_move_file_to_nonexist_dir(self):
3655         d = self.shouldFail2(error.Error, "POST_move_file_to_nonexist_dir",
3656                             "404 Not Found", "No such child: nopechucktesta",
3657                             self.POST, self.public_url + "/foo", t="move",
3658                             from_name="bar.txt", to_dir="nopechucktesta")
3659         return d
3660
3661     def test_POST_move_file_into_file(self):
3662         d = self.shouldFail2(error.Error, "POST_move_file_into_file",
3663                              "400 Bad Request", "to_dir is not a directory",
3664                              self.POST, self.public_url + "/foo", t="move",
3665                              from_name="bar.txt", to_dir="baz.txt")
3666         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3667         d.addCallback(self.failUnlessIsBazDotTxt)
3668         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3669         d.addCallback(self.failUnlessIsBarDotTxt)
3670         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3671         d.addCallback(self.failUnlessIsBarJSON)
3672         return d
3673
3674     def test_POST_move_file_to_bad_uri(self):
3675         d =  self.shouldFail2(error.Error, "POST_move_file_to_bad_uri",
3676                               "400 Bad Request", "to_dir is not a directory",
3677                               self.POST, self.public_url + "/foo", t="move",
3678                               from_name="bar.txt", target_type="uri",
3679                               to_dir="URI:DIR2:mn5jlyjnrjeuydyswlzyui72i:rmneifcj6k6sycjljjhj3f6majsq2zqffydnnul5hfa4j577arma")
3680         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3681         d.addCallback(self.failUnlessIsBarDotTxt)
3682         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3683         d.addCallback(self.failUnlessIsBarJSON)
3684         return d
3685
3686     def test_POST_move_dir(self):
3687         d = self.POST(self.public_url + "/foo", t="move",
3688                       from_name="bar.txt", to_dir="empty")
3689         d.addCallback(lambda res: self.POST(self.public_url + "/foo",
3690                       t="move", from_name="empty", to_dir="sub"))
3691         d.addCallback(lambda res:
3692                       self.failIfNodeHasChild(self._foo_node, u"empty"))
3693         d.addCallback(lambda res:
3694                       self.failUnlessNodeHasChild(self._sub_node, u"empty"))
3695         d.addCallback(lambda res:
3696                       self._sub_node.get_child_at_path(u"empty"))
3697         d.addCallback(lambda node:
3698                       self.failUnlessNodeHasChild(node, u"bar.txt"))
3699         d.addCallback(lambda res:
3700                       self.GET(self.public_url + "/foo/sub/empty/bar.txt"))
3701         d.addCallback(self.failUnlessIsBarDotTxt)
3702         return d
3703
3704     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
3705         """ If target is not None then the redirection has to go to target.  If
3706         statuscode is not None then the redirection has to be accomplished with
3707         that HTTP status code."""
3708         if not isinstance(res, failure.Failure):
3709             to_where = (target is None) and "somewhere" or ("to " + target)
3710             self.fail("%s: we were expecting to get redirected %s, not get an"
3711                       " actual page: %s" % (which, to_where, res))
3712         res.trap(error.PageRedirect)
3713         if statuscode is not None:
3714             self.failUnlessReallyEqual(res.value.status, statuscode,
3715                                        "%s: not a redirect" % which)
3716         if target is not None:
3717             # the PageRedirect does not seem to capture the uri= query arg
3718             # properly, so we can't check for it.
3719             realtarget = self.webish_url + target
3720             self.failUnlessReallyEqual(res.value.location, realtarget,
3721                                        "%s: wrong target" % which)
3722         return res.value.location
3723
3724     def test_GET_URI_form(self):
3725         base = "/uri?uri=%s" % self._bar_txt_uri
3726         # this is supposed to give us a redirect to /uri/$URI, plus arguments
3727         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
3728         d = self.GET(base)
3729         d.addBoth(self.shouldRedirect, targetbase)
3730         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
3731         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
3732         d.addCallback(lambda res: self.GET(base+"&t=json"))
3733         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
3734         d.addCallback(self.log, "about to get file by uri")
3735         d.addCallback(lambda res: self.GET(base, followRedirect=True))
3736         d.addCallback(self.failUnlessIsBarDotTxt)
3737         d.addCallback(self.log, "got file by uri, about to get dir by uri")
3738         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
3739                                            followRedirect=True))
3740         d.addCallback(self.failUnlessIsFooJSON)
3741         d.addCallback(self.log, "got dir by uri")
3742
3743         return d
3744
3745     def test_GET_URI_form_bad(self):
3746         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
3747                              "400 Bad Request", "GET /uri requires uri=",
3748                              self.GET, "/uri")
3749         return d
3750
3751     def test_GET_rename_form(self):
3752         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
3753                      followRedirect=True)
3754         def _check(res):
3755             self.failUnlessIn('name="when_done" value="."', res)
3756             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
3757             self.failUnlessIn(FAVICON_MARKUP, res)
3758         d.addCallback(_check)
3759         return d
3760
3761     def log(self, res, msg):
3762         #print "MSG: %s  RES: %s" % (msg, res)
3763         log.msg(msg)
3764         return res
3765
3766     def test_GET_URI_URL(self):
3767         base = "/uri/%s" % self._bar_txt_uri
3768         d = self.GET(base)
3769         d.addCallback(self.failUnlessIsBarDotTxt)
3770         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
3771         d.addCallback(self.failUnlessIsBarDotTxt)
3772         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
3773         d.addCallback(self.failUnlessIsBarDotTxt)
3774         return d
3775
3776     def test_GET_URI_URL_dir(self):
3777         base = "/uri/%s?t=json" % self._foo_uri
3778         d = self.GET(base)
3779         d.addCallback(self.failUnlessIsFooJSON)
3780         return d
3781
3782     def test_GET_URI_URL_missing(self):
3783         base = "/uri/%s" % self._bad_file_uri
3784         d = self.shouldHTTPError("test_GET_URI_URL_missing",
3785                                  http.GONE, None, "NotEnoughSharesError",
3786                                  self.GET, base)
3787         # TODO: how can we exercise both sides of WebDownloadTarget.fail
3788         # here? we must arrange for a download to fail after target.open()
3789         # has been called, and then inspect the response to see that it is
3790         # shorter than we expected.
3791         return d
3792
3793     def test_PUT_DIRURL_uri(self):
3794         d = self.s.create_dirnode()
3795         def _made_dir(dn):
3796             new_uri = dn.get_uri()
3797             # replace /foo with a new (empty) directory
3798             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
3799             d.addCallback(lambda res:
3800                           self.failUnlessReallyEqual(res.strip(), new_uri))
3801             d.addCallback(lambda res:
3802                           self.failUnlessRWChildURIIs(self.public_root,
3803                                                       u"foo",
3804                                                       new_uri))
3805             return d
3806         d.addCallback(_made_dir)
3807         return d
3808
3809     def test_PUT_DIRURL_uri_noreplace(self):
3810         d = self.s.create_dirnode()
3811         def _made_dir(dn):
3812             new_uri = dn.get_uri()
3813             # replace /foo with a new (empty) directory, but ask that
3814             # replace=false, so it should fail
3815             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
3816                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
3817                                  self.PUT,
3818                                  self.public_url + "/foo?t=uri&replace=false",
3819                                  new_uri)
3820             d.addCallback(lambda res:
3821                           self.failUnlessRWChildURIIs(self.public_root,
3822                                                       u"foo",
3823                                                       self._foo_uri))
3824             return d
3825         d.addCallback(_made_dir)
3826         return d
3827
3828     def test_PUT_DIRURL_bad_t(self):
3829         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
3830                              "400 Bad Request", "PUT to a directory",
3831                              self.PUT, self.public_url + "/foo?t=BOGUS", "")
3832         d.addCallback(lambda res:
3833                       self.failUnlessRWChildURIIs(self.public_root,
3834                                                   u"foo",
3835                                                   self._foo_uri))
3836         return d
3837
3838     def test_PUT_NEWFILEURL_uri(self):
3839         contents, n, new_uri = self.makefile(8)
3840         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
3841         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3842         d.addCallback(lambda res:
3843                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3844                                                       contents))
3845         return d
3846
3847     def test_PUT_NEWFILEURL_mdmf(self):
3848         new_contents = self.NEWFILE_CONTENTS * 300000
3849         d = self.PUT(self.public_url + \
3850                      "/foo/mdmf.txt?format=mdmf",
3851                      new_contents)
3852         d.addCallback(lambda ignored:
3853             self.GET(self.public_url + "/foo/mdmf.txt?t=json"))
3854         def _got_json(json):
3855             data = simplejson.loads(json)
3856             data = data[1]
3857             self.failUnlessIn("format", data)
3858             self.failUnlessEqual(data["format"], "MDMF")
3859             self.failUnless(data['rw_uri'].startswith("URI:MDMF"))
3860             self.failUnless(data['ro_uri'].startswith("URI:MDMF"))
3861         d.addCallback(_got_json)
3862         return d
3863
3864     def test_PUT_NEWFILEURL_sdmf(self):
3865         new_contents = self.NEWFILE_CONTENTS * 300000
3866         d = self.PUT(self.public_url + \
3867                      "/foo/sdmf.txt?format=sdmf",
3868                      new_contents)
3869         d.addCallback(lambda ignored:
3870             self.GET(self.public_url + "/foo/sdmf.txt?t=json"))
3871         def _got_json(json):
3872             data = simplejson.loads(json)
3873             data = data[1]
3874             self.failUnlessIn("format", data)
3875             self.failUnlessEqual(data["format"], "SDMF")
3876         d.addCallback(_got_json)
3877         return d
3878
3879     def test_PUT_NEWFILEURL_bad_format(self):
3880         new_contents = self.NEWFILE_CONTENTS * 300000
3881         return self.shouldHTTPError("PUT_NEWFILEURL_bad_format",
3882                                     400, "Bad Request", "Unknown format: foo",
3883                                     self.PUT, self.public_url + \
3884                                     "/foo/foo.txt?format=foo",
3885                                     new_contents)
3886
3887     def test_PUT_NEWFILEURL_uri_replace(self):
3888         contents, n, new_uri = self.makefile(8)
3889         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
3890         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3891         d.addCallback(lambda res:
3892                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3893                                                       contents))
3894         return d
3895
3896     def test_PUT_NEWFILEURL_uri_no_replace(self):
3897         contents, n, new_uri = self.makefile(8)
3898         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
3899         d.addBoth(self.shouldFail, error.Error,
3900                   "PUT_NEWFILEURL_uri_no_replace",
3901                   "409 Conflict",
3902                   "There was already a child by that name, and you asked me "
3903                   "to not replace it")
3904         return d
3905
3906     def test_PUT_NEWFILEURL_uri_unknown_bad(self):
3907         d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
3908         d.addBoth(self.shouldFail, error.Error,
3909                   "POST_put_uri_unknown_bad",
3910                   "400 Bad Request",
3911                   "unknown cap in a write slot")
3912         return d
3913
3914     def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
3915         d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
3916         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3917                       u"put-future-ro.txt")
3918         return d
3919
3920     def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
3921         d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
3922         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3923                       u"put-future-imm.txt")
3924         return d
3925
3926     def test_PUT_NEWFILE_URI(self):
3927         file_contents = "New file contents here\n"
3928         d = self.PUT("/uri", file_contents)
3929         def _check(uri):
3930             assert isinstance(uri, str), uri
3931             self.failUnlessIn(uri, self.get_all_contents())
3932             self.failUnlessReallyEqual(self.get_all_contents()[uri],
3933                                        file_contents)
3934             return self.GET("/uri/%s" % uri)
3935         d.addCallback(_check)
3936         def _check2(res):
3937             self.failUnlessReallyEqual(res, file_contents)
3938         d.addCallback(_check2)
3939         return d
3940
3941     def test_PUT_NEWFILE_URI_not_mutable(self):
3942         file_contents = "New file contents here\n"
3943         d = self.PUT("/uri?mutable=false", file_contents)
3944         def _check(uri):
3945             assert isinstance(uri, str), uri
3946             self.failUnlessIn(uri, self.get_all_contents())
3947             self.failUnlessReallyEqual(self.get_all_contents()[uri],
3948                                        file_contents)
3949             return self.GET("/uri/%s" % uri)
3950         d.addCallback(_check)
3951         def _check2(res):
3952             self.failUnlessReallyEqual(res, file_contents)
3953         d.addCallback(_check2)
3954         return d
3955
3956     def test_PUT_NEWFILE_URI_only_PUT(self):
3957         d = self.PUT("/uri?t=bogus", "")
3958         d.addBoth(self.shouldFail, error.Error,
3959                   "PUT_NEWFILE_URI_only_PUT",
3960                   "400 Bad Request",
3961                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
3962         return d
3963
3964     def test_PUT_NEWFILE_URI_mutable(self):
3965         file_contents = "New file contents here\n"
3966         d = self.PUT("/uri?mutable=true", file_contents)
3967         def _check1(filecap):
3968             filecap = filecap.strip()
3969             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
3970             self.filecap = filecap
3971             u = uri.WriteableSSKFileURI.init_from_string(filecap)
3972             self.failUnlessIn(u.get_storage_index(), self.get_all_contents())
3973             n = self.s.create_node_from_uri(filecap)
3974             return n.download_best_version()
3975         d.addCallback(_check1)
3976         def _check2(data):
3977             self.failUnlessReallyEqual(data, file_contents)
3978             return self.GET("/uri/%s" % urllib.quote(self.filecap))
3979         d.addCallback(_check2)
3980         def _check3(res):
3981             self.failUnlessReallyEqual(res, file_contents)
3982         d.addCallback(_check3)
3983         return d
3984
3985     def test_PUT_mkdir(self):
3986         d = self.PUT("/uri?t=mkdir", "")
3987         def _check(uri):
3988             n = self.s.create_node_from_uri(uri.strip())
3989             d2 = self.failUnlessNodeKeysAre(n, [])
3990             d2.addCallback(lambda res:
3991                            self.GET("/uri/%s?t=json" % uri))
3992             return d2
3993         d.addCallback(_check)
3994         d.addCallback(self.failUnlessIsEmptyJSON)
3995         return d
3996
3997     def test_PUT_mkdir_mdmf(self):
3998         d = self.PUT("/uri?t=mkdir&format=mdmf", "")
3999         def _got(res):
4000             u = uri.from_string(res)
4001             # Check that this is an MDMF writecap
4002             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
4003         d.addCallback(_got)
4004         return d
4005
4006     def test_PUT_mkdir_sdmf(self):
4007         d = self.PUT("/uri?t=mkdir&format=sdmf", "")
4008         def _got(res):
4009             u = uri.from_string(res)
4010             self.failUnlessIsInstance(u, uri.DirectoryURI)
4011         d.addCallback(_got)
4012         return d
4013
4014     def test_PUT_mkdir_bad_format(self):
4015         return self.shouldHTTPError("PUT_mkdir_bad_format",
4016                                     400, "Bad Request", "Unknown format: foo",
4017                                     self.PUT, "/uri?t=mkdir&format=foo",
4018                                     "")
4019
4020     def test_POST_check(self):
4021         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
4022         def _done(res):
4023             # this returns a string form of the results, which are probably
4024             # None since we're using fake filenodes.
4025             # TODO: verify that the check actually happened, by changing
4026             # FakeCHKFileNode to count how many times .check() has been
4027             # called.
4028             pass
4029         d.addCallback(_done)
4030         return d
4031
4032
4033     def test_PUT_update_at_offset(self):
4034         file_contents = "test file" * 100000 # about 900 KiB
4035         d = self.PUT("/uri?mutable=true", file_contents)
4036         def _then(filecap):
4037             self.filecap = filecap
4038             new_data = file_contents[:100]
4039             new = "replaced and so on"
4040             new_data += new
4041             new_data += file_contents[len(new_data):]
4042             assert len(new_data) == len(file_contents)
4043             self.new_data = new_data
4044         d.addCallback(_then)
4045         d.addCallback(lambda ignored:
4046             self.PUT("/uri/%s?replace=True&offset=100" % self.filecap,
4047                      "replaced and so on"))
4048         def _get_data(filecap):
4049             n = self.s.create_node_from_uri(filecap)
4050             return n.download_best_version()
4051         d.addCallback(_get_data)
4052         d.addCallback(lambda results:
4053             self.failUnlessEqual(results, self.new_data))
4054         # Now try appending things to the file
4055         d.addCallback(lambda ignored:
4056             self.PUT("/uri/%s?offset=%d" % (self.filecap, len(self.new_data)),
4057                      "puppies" * 100))
4058         d.addCallback(_get_data)
4059         d.addCallback(lambda results:
4060             self.failUnlessEqual(results, self.new_data + ("puppies" * 100)))
4061         # and try replacing the beginning of the file
4062         d.addCallback(lambda ignored:
4063             self.PUT("/uri/%s?offset=0" % self.filecap, "begin"))
4064         d.addCallback(_get_data)
4065         d.addCallback(lambda results:
4066             self.failUnlessEqual(results, "begin"+self.new_data[len("begin"):]+("puppies"*100)))
4067         return d
4068
4069     def test_PUT_update_at_invalid_offset(self):
4070         file_contents = "test file" * 100000 # about 900 KiB
4071         d = self.PUT("/uri?mutable=true", file_contents)
4072         def _then(filecap):
4073             self.filecap = filecap
4074         d.addCallback(_then)
4075         # Negative offsets should cause an error.
4076         d.addCallback(lambda ignored:
4077             self.shouldHTTPError("PUT_update_at_invalid_offset",
4078                                  400, "Bad Request",
4079                                  "Invalid offset",
4080                                  self.PUT,
4081                                  "/uri/%s?offset=-1" % self.filecap,
4082                                  "foo"))
4083         return d
4084
4085     def test_PUT_update_at_offset_immutable(self):
4086         file_contents = "Test file" * 100000
4087         d = self.PUT("/uri", file_contents)
4088         def _then(filecap):
4089             self.filecap = filecap
4090         d.addCallback(_then)
4091         d.addCallback(lambda ignored:
4092             self.shouldHTTPError("PUT_update_at_offset_immutable",
4093                                  400, "Bad Request",
4094                                  "immutable",
4095                                  self.PUT,
4096                                  "/uri/%s?offset=50" % self.filecap,
4097                                  "foo"))
4098         return d
4099
4100
4101     def test_bad_method(self):
4102         url = self.webish_url + self.public_url + "/foo/bar.txt"
4103         d = self.shouldHTTPError("bad_method",
4104                                  501, "Not Implemented",
4105                                  "I don't know how to treat a BOGUS request.",
4106                                  client.getPage, url, method="BOGUS")
4107         return d
4108
4109     def test_short_url(self):
4110         url = self.webish_url + "/uri"
4111         d = self.shouldHTTPError("short_url", 501, "Not Implemented",
4112                                  "I don't know how to treat a DELETE request.",
4113                                  client.getPage, url, method="DELETE")
4114         return d
4115
4116     def test_ophandle_bad(self):
4117         url = self.webish_url + "/operations/bogus?t=status"
4118         d = self.shouldHTTPError("ophandle_bad", 404, "404 Not Found",
4119                                  "unknown/expired handle 'bogus'",
4120                                  client.getPage, url)
4121         return d
4122
4123     def test_ophandle_cancel(self):
4124         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
4125                       followRedirect=True)
4126         d.addCallback(lambda ignored:
4127                       self.GET("/operations/128?t=status&output=JSON"))
4128         def _check1(res):
4129             data = simplejson.loads(res)
4130             self.failUnless("finished" in data, res)
4131             monitor = self.ws.root.child_operations.handles["128"][0]
4132             d = self.POST("/operations/128?t=cancel&output=JSON")
4133             def _check2(res):
4134                 data = simplejson.loads(res)
4135                 self.failUnless("finished" in data, res)
4136                 # t=cancel causes the handle to be forgotten
4137                 self.failUnless(monitor.is_cancelled())
4138             d.addCallback(_check2)
4139             return d
4140         d.addCallback(_check1)
4141         d.addCallback(lambda ignored:
4142                       self.shouldHTTPError("ophandle_cancel",
4143                                            404, "404 Not Found",
4144                                            "unknown/expired handle '128'",
4145                                            self.GET,
4146                                            "/operations/128?t=status&output=JSON"))
4147         return d
4148
4149     def test_ophandle_retainfor(self):
4150         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
4151                       followRedirect=True)
4152         d.addCallback(lambda ignored:
4153                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
4154         def _check1(res):
4155             data = simplejson.loads(res)
4156             self.failUnless("finished" in data, res)
4157         d.addCallback(_check1)
4158         # the retain-for=0 will cause the handle to be expired very soon
4159         d.addCallback(lambda ign:
4160             self.clock.advance(2.0))
4161         d.addCallback(lambda ignored:
4162                       self.shouldHTTPError("ophandle_retainfor",
4163                                            404, "404 Not Found",
4164                                            "unknown/expired handle '129'",
4165                                            self.GET,
4166                                            "/operations/129?t=status&output=JSON"))
4167         return d
4168
4169     def test_ophandle_release_after_complete(self):
4170         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
4171                       followRedirect=True)
4172         d.addCallback(self.wait_for_operation, "130")
4173         d.addCallback(lambda ignored:
4174                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
4175         # the release-after-complete=true will cause the handle to be expired
4176         d.addCallback(lambda ignored:
4177                       self.shouldHTTPError("ophandle_release_after_complete",
4178                                            404, "404 Not Found",
4179                                            "unknown/expired handle '130'",
4180                                            self.GET,
4181                                            "/operations/130?t=status&output=JSON"))
4182         return d
4183
4184     def test_uncollected_ophandle_expiration(self):
4185         # uncollected ophandles should expire after 4 days
4186         def _make_uncollected_ophandle(ophandle):
4187             d = self.POST(self.public_url +
4188                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
4189                           followRedirect=False)
4190             # When we start the operation, the webapi server will want
4191             # to redirect us to the page for the ophandle, so we get
4192             # confirmation that the operation has started. If the
4193             # manifest operation has finished by the time we get there,
4194             # following that redirect (by setting followRedirect=True
4195             # above) has the side effect of collecting the ophandle that
4196             # we've just created, which means that we can't use the
4197             # ophandle to test the uncollected timeout anymore. So,
4198             # instead, catch the 302 here and don't follow it.
4199             d.addBoth(self.should302, "uncollected_ophandle_creation")
4200             return d
4201         # Create an ophandle, don't collect it, then advance the clock by
4202         # 4 days - 1 second and make sure that the ophandle is still there.
4203         d = _make_uncollected_ophandle(131)
4204         d.addCallback(lambda ign:
4205             self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
4206         d.addCallback(lambda ign:
4207             self.GET("/operations/131?t=status&output=JSON"))
4208         def _check1(res):
4209             data = simplejson.loads(res)
4210             self.failUnless("finished" in data, res)
4211         d.addCallback(_check1)
4212         # Create an ophandle, don't collect it, then try to collect it
4213         # after 4 days. It should be gone.
4214         d.addCallback(lambda ign:
4215             _make_uncollected_ophandle(132))
4216         d.addCallback(lambda ign:
4217             self.clock.advance(96*60*60))
4218         d.addCallback(lambda ign:
4219             self.shouldHTTPError("uncollected_ophandle_expired_after_100_hours",
4220                                  404, "404 Not Found",
4221                                  "unknown/expired handle '132'",
4222                                  self.GET,
4223                                  "/operations/132?t=status&output=JSON"))
4224         return d
4225
4226     def test_collected_ophandle_expiration(self):
4227         # collected ophandles should expire after 1 day
4228         def _make_collected_ophandle(ophandle):
4229             d = self.POST(self.public_url +
4230                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
4231                           followRedirect=True)
4232             # By following the initial redirect, we collect the ophandle
4233             # we've just created.
4234             return d
4235         # Create a collected ophandle, then collect it after 23 hours
4236         # and 59 seconds to make sure that it is still there.
4237         d = _make_collected_ophandle(133)
4238         d.addCallback(lambda ign:
4239             self.clock.advance((24*60*60) - 1))
4240         d.addCallback(lambda ign:
4241             self.GET("/operations/133?t=status&output=JSON"))
4242         def _check1(res):
4243             data = simplejson.loads(res)
4244             self.failUnless("finished" in data, res)
4245         d.addCallback(_check1)
4246         # Create another uncollected ophandle, then try to collect it
4247         # after 24 hours to make sure that it is gone.
4248         d.addCallback(lambda ign:
4249             _make_collected_ophandle(134))
4250         d.addCallback(lambda ign:
4251             self.clock.advance(24*60*60))
4252         d.addCallback(lambda ign:
4253             self.shouldHTTPError("collected_ophandle_expired_after_1_day",
4254                                  404, "404 Not Found",
4255                                  "unknown/expired handle '134'",
4256                                  self.GET,
4257                                  "/operations/134?t=status&output=JSON"))
4258         return d
4259
4260     def test_incident(self):
4261         d = self.POST("/report_incident", details="eek")
4262         def _done(res):
4263             self.failIfIn("<html>", res)
4264             self.failUnlessIn("Thank you for your report!", res)
4265         d.addCallback(_done)
4266         return d
4267
4268     def test_static(self):
4269         webdir = os.path.join(self.staticdir, "subdir")
4270         fileutil.make_dirs(webdir)
4271         f = open(os.path.join(webdir, "hello.txt"), "wb")
4272         f.write("hello")
4273         f.close()
4274
4275         d = self.GET("/static/subdir/hello.txt")
4276         def _check(res):
4277             self.failUnlessReallyEqual(res, "hello")
4278         d.addCallback(_check)
4279         return d
4280
4281
4282 class IntroducerWeb(unittest.TestCase):
4283     def setUp(self):
4284         self.node = None
4285
4286     def tearDown(self):
4287         d = defer.succeed(None)
4288         if self.node:
4289             d.addCallback(lambda ign: self.node.stopService())
4290         d.addCallback(flushEventualQueue)
4291         return d
4292
4293     def test_welcome(self):
4294         basedir = "web.IntroducerWeb.test_welcome"
4295         os.mkdir(basedir)
4296         fileutil.write(os.path.join(basedir, "tahoe.cfg"), "[node]\nweb.port = tcp:0\n")
4297         self.node = IntroducerNode(basedir)
4298         self.ws = self.node.getServiceNamed("webish")
4299
4300         d = fireEventually(None)
4301         d.addCallback(lambda ign: self.node.startService())
4302         d.addCallback(lambda ign: self.node.when_tub_ready())
4303
4304         d.addCallback(lambda ign: self.GET("/"))
4305         def _check(res):
4306             self.failUnlessIn('Welcome to the Tahoe-LAFS Introducer', res)
4307             self.failUnlessIn(FAVICON_MARKUP, res)
4308         d.addCallback(_check)
4309         return d
4310
4311     def GET(self, urlpath, followRedirect=False, return_response=False,
4312             **kwargs):
4313         # if return_response=True, this fires with (data, statuscode,
4314         # respheaders) instead of just data.
4315         assert not isinstance(urlpath, unicode)
4316         url = self.ws.getURL().rstrip('/') + urlpath
4317         factory = HTTPClientGETFactory(url, method="GET",
4318                                        followRedirect=followRedirect, **kwargs)
4319         reactor.connectTCP("localhost", self.ws.getPortnum(), factory)
4320         d = factory.deferred
4321         def _got_data(data):
4322             return (data, factory.status, factory.response_headers)
4323         if return_response:
4324             d.addCallback(_got_data)
4325         return factory.deferred
4326
4327
4328 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4329     def test_load_file(self):
4330         # This will raise an exception unless a well-formed XML file is found under that name.
4331         common.getxmlfile('directory.xhtml').load()
4332
4333     def test_parse_replace_arg(self):
4334         self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
4335         self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
4336         self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
4337                                    "only-files")
4338         self.shouldFail(AssertionError, "test_parse_replace_arg", "",
4339                         common.parse_replace_arg, "only_fles")
4340
4341     def test_abbreviate_time(self):
4342         self.failUnlessReallyEqual(common.abbreviate_time(None), "")
4343         self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
4344         self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
4345         self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
4346         self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
4347         self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
4348
4349     def test_compute_rate(self):
4350         self.failUnlessReallyEqual(common.compute_rate(None, None), None)
4351         self.failUnlessReallyEqual(common.compute_rate(None, 1), None)
4352         self.failUnlessReallyEqual(common.compute_rate(250000, None), None)
4353         self.failUnlessReallyEqual(common.compute_rate(250000, 0), None)
4354         self.failUnlessReallyEqual(common.compute_rate(250000, 10), 25000.0)
4355         self.failUnlessReallyEqual(common.compute_rate(0, 10), 0.0)
4356         self.shouldFail(AssertionError, "test_compute_rate", "",
4357                         common.compute_rate, -100, 10)
4358         self.shouldFail(AssertionError, "test_compute_rate", "",
4359                         common.compute_rate, 100, -10)
4360
4361         # Sanity check
4362         rate = common.compute_rate(10*1000*1000, 1)
4363         self.failUnlessReallyEqual(common.abbreviate_rate(rate), "10.00MBps")
4364
4365     def test_abbreviate_rate(self):
4366         self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
4367         self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
4368         self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
4369         self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
4370
4371     def test_abbreviate_size(self):
4372         self.failUnlessReallyEqual(common.abbreviate_size(None), "")
4373         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
4374         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
4375         self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
4376         self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
4377
4378     def test_plural(self):
4379         def convert(s):
4380             return "%d second%s" % (s, status.plural(s))
4381         self.failUnlessReallyEqual(convert(0), "0 seconds")
4382         self.failUnlessReallyEqual(convert(1), "1 second")
4383         self.failUnlessReallyEqual(convert(2), "2 seconds")
4384         def convert2(s):
4385             return "has share%s: %s" % (status.plural(s), ",".join(s))
4386         self.failUnlessReallyEqual(convert2([]), "has shares: ")
4387         self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
4388         self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
4389
4390
4391 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4392
4393     def CHECK(self, ign, which, args, clientnum=0):
4394         fileurl = self.fileurls[which]
4395         url = fileurl + "?" + args
4396         return self.GET(url, method="POST", clientnum=clientnum)
4397
4398     def test_filecheck(self):
4399         self.basedir = "web/Grid/filecheck"
4400         self.set_up_grid()
4401         c0 = self.g.clients[0]
4402         self.uris = {}
4403         DATA = "data" * 100
4404         d = c0.upload(upload.Data(DATA, convergence=""))
4405         def _stash_uri(ur, which):
4406             self.uris[which] = ur.get_uri()
4407         d.addCallback(_stash_uri, "good")
4408         d.addCallback(lambda ign:
4409                       c0.upload(upload.Data(DATA+"1", convergence="")))
4410         d.addCallback(_stash_uri, "sick")
4411         d.addCallback(lambda ign:
4412                       c0.upload(upload.Data(DATA+"2", convergence="")))
4413         d.addCallback(_stash_uri, "dead")
4414         def _stash_mutable_uri(n, which):
4415             self.uris[which] = n.get_uri()
4416             assert isinstance(self.uris[which], str)
4417         d.addCallback(lambda ign:
4418             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4419         d.addCallback(_stash_mutable_uri, "corrupt")
4420         d.addCallback(lambda ign:
4421                       c0.upload(upload.Data("literal", convergence="")))
4422         d.addCallback(_stash_uri, "small")
4423         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
4424         d.addCallback(_stash_mutable_uri, "smalldir")
4425
4426         def _compute_fileurls(ignored):
4427             self.fileurls = {}
4428             for which in self.uris:
4429                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4430         d.addCallback(_compute_fileurls)
4431
4432         def _clobber_shares(ignored):
4433             good_shares = self.find_uri_shares(self.uris["good"])
4434             self.failUnlessReallyEqual(len(good_shares), 10)
4435             sick_shares = self.find_uri_shares(self.uris["sick"])
4436             os.unlink(sick_shares[0][2])
4437             dead_shares = self.find_uri_shares(self.uris["dead"])
4438             for i in range(1, 10):
4439                 os.unlink(dead_shares[i][2])
4440             c_shares = self.find_uri_shares(self.uris["corrupt"])
4441             cso = CorruptShareOptions()
4442             cso.stdout = StringIO()
4443             cso.parseOptions([c_shares[0][2]])
4444             corrupt_share(cso)
4445         d.addCallback(_clobber_shares)
4446
4447         d.addCallback(self.CHECK, "good", "t=check")
4448         def _got_html_good(res):
4449             self.failUnlessIn("Healthy", res)
4450             self.failIfIn("Not Healthy", res)
4451             self.failUnlessIn(FAVICON_MARKUP, res)
4452         d.addCallback(_got_html_good)
4453         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
4454         def _got_html_good_return_to(res):
4455             self.failUnlessIn("Healthy", res)
4456             self.failIfIn("Not Healthy", res)
4457             self.failUnlessIn('<a href="somewhere">Return to file', res)
4458         d.addCallback(_got_html_good_return_to)
4459         d.addCallback(self.CHECK, "good", "t=check&output=json")
4460         def _got_json_good(res):
4461             r = simplejson.loads(res)
4462             self.failUnlessEqual(r["summary"], "Healthy")
4463             self.failUnless(r["results"]["healthy"])
4464             self.failIf(r["results"]["needs-rebalancing"])
4465             self.failUnless(r["results"]["recoverable"])
4466         d.addCallback(_got_json_good)
4467
4468         d.addCallback(self.CHECK, "small", "t=check")
4469         def _got_html_small(res):
4470             self.failUnlessIn("Literal files are always healthy", res)
4471             self.failIfIn("Not Healthy", res)
4472         d.addCallback(_got_html_small)
4473         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
4474         def _got_html_small_return_to(res):
4475             self.failUnlessIn("Literal files are always healthy", res)
4476             self.failIfIn("Not Healthy", res)
4477             self.failUnlessIn('<a href="somewhere">Return to file', res)
4478         d.addCallback(_got_html_small_return_to)
4479         d.addCallback(self.CHECK, "small", "t=check&output=json")
4480         def _got_json_small(res):
4481             r = simplejson.loads(res)
4482             self.failUnlessEqual(r["storage-index"], "")
4483             self.failUnless(r["results"]["healthy"])
4484         d.addCallback(_got_json_small)
4485
4486         d.addCallback(self.CHECK, "smalldir", "t=check")
4487         def _got_html_smalldir(res):
4488             self.failUnlessIn("Literal files are always healthy", res)
4489             self.failIfIn("Not Healthy", res)
4490         d.addCallback(_got_html_smalldir)
4491         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
4492         def _got_json_smalldir(res):
4493             r = simplejson.loads(res)
4494             self.failUnlessEqual(r["storage-index"], "")
4495             self.failUnless(r["results"]["healthy"])
4496         d.addCallback(_got_json_smalldir)
4497
4498         d.addCallback(self.CHECK, "sick", "t=check")
4499         def _got_html_sick(res):
4500             self.failUnlessIn("Not Healthy", res)
4501         d.addCallback(_got_html_sick)
4502         d.addCallback(self.CHECK, "sick", "t=check&output=json")
4503         def _got_json_sick(res):
4504             r = simplejson.loads(res)
4505             self.failUnlessEqual(r["summary"],
4506                                  "Not Healthy: 9 shares (enc 3-of-10)")
4507             self.failIf(r["results"]["healthy"])
4508             self.failIf(r["results"]["needs-rebalancing"])
4509             self.failUnless(r["results"]["recoverable"])
4510         d.addCallback(_got_json_sick)
4511
4512         d.addCallback(self.CHECK, "dead", "t=check")
4513         def _got_html_dead(res):
4514             self.failUnlessIn("Not Healthy", res)
4515         d.addCallback(_got_html_dead)
4516         d.addCallback(self.CHECK, "dead", "t=check&output=json")
4517         def _got_json_dead(res):
4518             r = simplejson.loads(res)
4519             self.failUnlessEqual(r["summary"],
4520                                  "Not Healthy: 1 shares (enc 3-of-10)")
4521             self.failIf(r["results"]["healthy"])
4522             self.failIf(r["results"]["needs-rebalancing"])
4523             self.failIf(r["results"]["recoverable"])
4524         d.addCallback(_got_json_dead)
4525
4526         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
4527         def _got_html_corrupt(res):
4528             self.failUnlessIn("Not Healthy! : Unhealthy", res)
4529         d.addCallback(_got_html_corrupt)
4530         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
4531         def _got_json_corrupt(res):
4532             r = simplejson.loads(res)
4533             self.failUnlessIn("Unhealthy: 9 shares (enc 3-of-10)", r["summary"])
4534             self.failIf(r["results"]["healthy"])
4535             self.failUnless(r["results"]["recoverable"])
4536             self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
4537             self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
4538         d.addCallback(_got_json_corrupt)
4539
4540         d.addErrback(self.explain_web_error)
4541         return d
4542
4543     def test_repair_html(self):
4544         self.basedir = "web/Grid/repair_html"
4545         self.set_up_grid()
4546         c0 = self.g.clients[0]
4547         self.uris = {}
4548         DATA = "data" * 100
4549         d = c0.upload(upload.Data(DATA, convergence=""))
4550         def _stash_uri(ur, which):
4551             self.uris[which] = ur.get_uri()
4552         d.addCallback(_stash_uri, "good")
4553         d.addCallback(lambda ign:
4554                       c0.upload(upload.Data(DATA+"1", convergence="")))
4555         d.addCallback(_stash_uri, "sick")
4556         d.addCallback(lambda ign:
4557                       c0.upload(upload.Data(DATA+"2", convergence="")))
4558         d.addCallback(_stash_uri, "dead")
4559         def _stash_mutable_uri(n, which):
4560             self.uris[which] = n.get_uri()
4561             assert isinstance(self.uris[which], str)
4562         d.addCallback(lambda ign:
4563             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4564         d.addCallback(_stash_mutable_uri, "corrupt")
4565
4566         def _compute_fileurls(ignored):
4567             self.fileurls = {}
4568             for which in self.uris:
4569                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4570         d.addCallback(_compute_fileurls)
4571
4572         def _clobber_shares(ignored):
4573             good_shares = self.find_uri_shares(self.uris["good"])
4574             self.failUnlessReallyEqual(len(good_shares), 10)
4575             sick_shares = self.find_uri_shares(self.uris["sick"])
4576             os.unlink(sick_shares[0][2])
4577             dead_shares = self.find_uri_shares(self.uris["dead"])
4578             for i in range(1, 10):
4579                 os.unlink(dead_shares[i][2])
4580             c_shares = self.find_uri_shares(self.uris["corrupt"])
4581             cso = CorruptShareOptions()
4582             cso.stdout = StringIO()
4583             cso.parseOptions([c_shares[0][2]])
4584             corrupt_share(cso)
4585         d.addCallback(_clobber_shares)
4586
4587         d.addCallback(self.CHECK, "good", "t=check&repair=true")
4588         def _got_html_good(res):
4589             self.failUnlessIn("Healthy", res)
4590             self.failIfIn("Not Healthy", res)
4591             self.failUnlessIn("No repair necessary", res)
4592             self.failUnlessIn(FAVICON_MARKUP, res)
4593         d.addCallback(_got_html_good)
4594
4595         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
4596         def _got_html_sick(res):
4597             self.failUnlessIn("Healthy : healthy", res)
4598             self.failIfIn("Not Healthy", res)
4599             self.failUnlessIn("Repair successful", res)
4600         d.addCallback(_got_html_sick)
4601
4602         # repair of a dead file will fail, of course, but it isn't yet
4603         # clear how this should be reported. Right now it shows up as
4604         # a "410 Gone".
4605         #
4606         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
4607         #def _got_html_dead(res):
4608         #    print res
4609         #    self.failUnlessIn("Healthy : healthy", res)
4610         #    self.failIfIn("Not Healthy", res)
4611         #    self.failUnlessIn("No repair necessary", res)
4612         #d.addCallback(_got_html_dead)
4613
4614         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
4615         def _got_html_corrupt(res):
4616             self.failUnlessIn("Healthy : Healthy", res)
4617             self.failIfIn("Not Healthy", res)
4618             self.failUnlessIn("Repair successful", res)
4619         d.addCallback(_got_html_corrupt)
4620
4621         d.addErrback(self.explain_web_error)
4622         return d
4623
4624     def test_repair_json(self):
4625         self.basedir = "web/Grid/repair_json"
4626         self.set_up_grid()
4627         c0 = self.g.clients[0]
4628         self.uris = {}
4629         DATA = "data" * 100
4630         d = c0.upload(upload.Data(DATA+"1", convergence=""))
4631         def _stash_uri(ur, which):
4632             self.uris[which] = ur.get_uri()
4633         d.addCallback(_stash_uri, "sick")
4634
4635         def _compute_fileurls(ignored):
4636             self.fileurls = {}
4637             for which in self.uris:
4638                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4639         d.addCallback(_compute_fileurls)
4640
4641         def _clobber_shares(ignored):
4642             sick_shares = self.find_uri_shares(self.uris["sick"])
4643             os.unlink(sick_shares[0][2])
4644         d.addCallback(_clobber_shares)
4645
4646         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
4647         def _got_json_sick(res):
4648             r = simplejson.loads(res)
4649             self.failUnlessReallyEqual(r["repair-attempted"], True)
4650             self.failUnlessReallyEqual(r["repair-successful"], True)
4651             self.failUnlessEqual(r["pre-repair-results"]["summary"],
4652                                  "Not Healthy: 9 shares (enc 3-of-10)")
4653             self.failIf(r["pre-repair-results"]["results"]["healthy"])
4654             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
4655             self.failUnless(r["post-repair-results"]["results"]["healthy"])
4656         d.addCallback(_got_json_sick)
4657
4658         d.addErrback(self.explain_web_error)
4659         return d
4660
4661     def test_unknown(self, immutable=False):
4662         self.basedir = "web/Grid/unknown"
4663         if immutable:
4664             self.basedir = "web/Grid/unknown-immutable"
4665
4666         self.set_up_grid()
4667         c0 = self.g.clients[0]
4668         self.uris = {}
4669         self.fileurls = {}
4670
4671         # the future cap format may contain slashes, which must be tolerated
4672         expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
4673                                                            safe="")
4674
4675         if immutable:
4676             name = u"future-imm"
4677             future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
4678             d = c0.create_immutable_dirnode({name: (future_node, {})})
4679         else:
4680             name = u"future"
4681             future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4682             d = c0.create_dirnode()
4683
4684         def _stash_root_and_create_file(n):
4685             self.rootnode = n
4686             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
4687             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
4688             if not immutable:
4689                 return self.rootnode.set_node(name, future_node)
4690         d.addCallback(_stash_root_and_create_file)
4691
4692         # make sure directory listing tolerates unknown nodes
4693         d.addCallback(lambda ign: self.GET(self.rooturl))
4694         def _check_directory_html(res, expected_type_suffix):
4695             pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
4696                                   '<td>%s</td>' % (expected_type_suffix, str(name)),
4697                                  re.DOTALL)
4698             self.failUnless(re.search(pattern, res), res)
4699             # find the More Info link for name, should be relative
4700             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4701             info_url = mo.group(1)
4702             self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
4703         if immutable:
4704             d.addCallback(_check_directory_html, "-IMM")
4705         else:
4706             d.addCallback(_check_directory_html, "")
4707
4708         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4709         def _check_directory_json(res, expect_rw_uri):
4710             data = simplejson.loads(res)
4711             self.failUnlessEqual(data[0], "dirnode")
4712             f = data[1]["children"][name]
4713             self.failUnlessEqual(f[0], "unknown")
4714             if expect_rw_uri:
4715                 self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data)
4716             else:
4717                 self.failIfIn("rw_uri", f[1])
4718             if immutable:
4719                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
4720             else:
4721                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
4722             self.failUnlessIn("metadata", f[1])
4723         d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
4724
4725         def _check_info(res, expect_rw_uri, expect_ro_uri):
4726             self.failUnlessIn("Object Type: <span>unknown</span>", res)
4727             if expect_rw_uri:
4728                 self.failUnlessIn(unknown_rwcap, res)
4729             if expect_ro_uri:
4730                 if immutable:
4731                     self.failUnlessIn(unknown_immcap, res)
4732                 else:
4733                     self.failUnlessIn(unknown_rocap, res)
4734             else:
4735                 self.failIfIn(unknown_rocap, res)
4736             self.failIfIn("Raw data as", res)
4737             self.failIfIn("Directory writecap", res)
4738             self.failIfIn("Checker Operations", res)
4739             self.failIfIn("Mutable File Operations", res)
4740             self.failIfIn("Directory Operations", res)
4741
4742         # FIXME: these should have expect_rw_uri=not immutable; I don't know
4743         # why they fail. Possibly related to ticket #922.
4744
4745         d.addCallback(lambda ign: self.GET(expected_info_url))
4746         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
4747         d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
4748         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
4749
4750         def _check_json(res, expect_rw_uri):
4751             data = simplejson.loads(res)
4752             self.failUnlessEqual(data[0], "unknown")
4753             if expect_rw_uri:
4754                 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data)
4755             else:
4756                 self.failIfIn("rw_uri", data[1])
4757
4758             if immutable:
4759                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data)
4760                 self.failUnlessReallyEqual(data[1]["mutable"], False)
4761             elif expect_rw_uri:
4762                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4763                 self.failUnlessReallyEqual(data[1]["mutable"], True)
4764             else:
4765                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4766                 self.failIfIn("mutable", data[1])
4767
4768             # TODO: check metadata contents
4769             self.failUnlessIn("metadata", data[1])
4770
4771         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
4772         d.addCallback(_check_json, expect_rw_uri=not immutable)
4773
4774         # and make sure that a read-only version of the directory can be
4775         # rendered too. This version will not have unknown_rwcap, whether
4776         # or not future_node was immutable.
4777         d.addCallback(lambda ign: self.GET(self.rourl))
4778         if immutable:
4779             d.addCallback(_check_directory_html, "-IMM")
4780         else:
4781             d.addCallback(_check_directory_html, "-RO")
4782
4783         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
4784         d.addCallback(_check_directory_json, expect_rw_uri=False)
4785
4786         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
4787         d.addCallback(_check_json, expect_rw_uri=False)
4788
4789         # TODO: check that getting t=info from the Info link in the ro directory
4790         # works, and does not include the writecap URI.
4791         return d
4792
4793     def test_immutable_unknown(self):
4794         return self.test_unknown(immutable=True)
4795
4796     def test_mutant_dirnodes_are_omitted(self):
4797         self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
4798
4799         self.set_up_grid()
4800         c = self.g.clients[0]
4801         nm = c.nodemaker
4802         self.uris = {}
4803         self.fileurls = {}
4804
4805         lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
4806         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
4807         mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
4808
4809         # This method tests mainly dirnode, but we'd have to duplicate code in order to
4810         # test the dirnode and web layers separately.
4811
4812         # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
4813         # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
4814         # When the directory is read, the mutants should be silently disposed of, leaving
4815         # their lonely sibling.
4816         # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
4817         # because immutable directories don't have a writecap and therefore that field
4818         # isn't (and can't be) decrypted.
4819         # TODO: The field still exists in the netstring. Technically we should check what
4820         # happens if something is put there (_unpack_contents should raise ValueError),
4821         # but that can wait.
4822
4823         lonely_child = nm.create_from_cap(lonely_uri)
4824         mutant_ro_child = nm.create_from_cap(mut_read_uri)
4825         mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
4826
4827         def _by_hook_or_by_crook():
4828             return True
4829         for n in [mutant_ro_child, mutant_write_in_ro_child]:
4830             n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
4831
4832         mutant_write_in_ro_child.get_write_uri    = lambda: None
4833         mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
4834
4835         kids = {u"lonely":      (lonely_child, {}),
4836                 u"ro":          (mutant_ro_child, {}),
4837                 u"write-in-ro": (mutant_write_in_ro_child, {}),
4838                 }
4839         d = c.create_immutable_dirnode(kids)
4840
4841         def _created(dn):
4842             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
4843             self.failIf(dn.is_mutable())
4844             self.failUnless(dn.is_readonly())
4845             # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
4846             self.failIf(hasattr(dn._node, 'get_writekey'))
4847             rep = str(dn)
4848             self.failUnlessIn("RO-IMM", rep)
4849             cap = dn.get_cap()
4850             self.failUnlessIn("CHK", cap.to_string())
4851             self.cap = cap
4852             self.rootnode = dn
4853             self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
4854             return download_to_data(dn._node)
4855         d.addCallback(_created)
4856
4857         def _check_data(data):
4858             # Decode the netstring representation of the directory to check that all children
4859             # are present. This is a bit of an abstraction violation, but there's not really
4860             # any other way to do it given that the real DirectoryNode._unpack_contents would
4861             # strip the mutant children out (which is what we're trying to test, later).
4862             position = 0
4863             numkids = 0
4864             while position < len(data):
4865                 entries, position = split_netstring(data, 1, position)
4866                 entry = entries[0]
4867                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
4868                 name = name_utf8.decode("utf-8")
4869                 self.failUnlessEqual(rwcapdata, "")
4870                 self.failUnlessIn(name, kids)
4871                 (expected_child, ign) = kids[name]
4872                 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
4873                 numkids += 1
4874
4875             self.failUnlessReallyEqual(numkids, 3)
4876             return self.rootnode.list()
4877         d.addCallback(_check_data)
4878
4879         # Now when we use the real directory listing code, the mutants should be absent.
4880         def _check_kids(children):
4881             self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
4882             lonely_node, lonely_metadata = children[u"lonely"]
4883
4884             self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
4885             self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
4886         d.addCallback(_check_kids)
4887
4888         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
4889         d.addCallback(lambda n: n.list())
4890         d.addCallback(_check_kids)  # again with dirnode recreated from cap
4891
4892         # Make sure the lonely child can be listed in HTML...
4893         d.addCallback(lambda ign: self.GET(self.rooturl))
4894         def _check_html(res):
4895             self.failIfIn("URI:SSK", res)
4896             get_lonely = "".join([r'<td>FILE</td>',
4897                                   r'\s+<td>',
4898                                   r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
4899                                   r'</td>',
4900                                   r'\s+<td align="right">%d</td>' % len("one"),
4901                                   ])
4902             self.failUnless(re.search(get_lonely, res), res)
4903
4904             # find the More Info link for name, should be relative
4905             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4906             info_url = mo.group(1)
4907             self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
4908         d.addCallback(_check_html)
4909
4910         # ... and in JSON.
4911         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4912         def _check_json(res):
4913             data = simplejson.loads(res)
4914             self.failUnlessEqual(data[0], "dirnode")
4915             listed_children = data[1]["children"]
4916             self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
4917             ll_type, ll_data = listed_children[u"lonely"]
4918             self.failUnlessEqual(ll_type, "filenode")
4919             self.failIfIn("rw_uri", ll_data)
4920             self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
4921         d.addCallback(_check_json)
4922         return d
4923
4924     def test_deep_check(self):
4925         self.basedir = "web/Grid/deep_check"
4926         self.set_up_grid()
4927         c0 = self.g.clients[0]
4928         self.uris = {}
4929         self.fileurls = {}
4930         DATA = "data" * 100
4931         d = c0.create_dirnode()
4932         def _stash_root_and_create_file(n):
4933             self.rootnode = n
4934             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4935             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4936         d.addCallback(_stash_root_and_create_file)
4937         def _stash_uri(fn, which):
4938             self.uris[which] = fn.get_uri()
4939             return fn
4940         d.addCallback(_stash_uri, "good")
4941         d.addCallback(lambda ign:
4942                       self.rootnode.add_file(u"small",
4943                                              upload.Data("literal",
4944                                                         convergence="")))
4945         d.addCallback(_stash_uri, "small")
4946         d.addCallback(lambda ign:
4947                       self.rootnode.add_file(u"sick",
4948                                              upload.Data(DATA+"1",
4949                                                         convergence="")))
4950         d.addCallback(_stash_uri, "sick")
4951
4952         # this tests that deep-check and stream-manifest will ignore
4953         # UnknownNode instances. Hopefully this will also cover deep-stats.
4954         future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4955         d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
4956
4957         def _clobber_shares(ignored):
4958             self.delete_shares_numbered(self.uris["sick"], [0,1])
4959         d.addCallback(_clobber_shares)
4960
4961         # root
4962         # root/good
4963         # root/small
4964         # root/sick
4965         # root/future
4966
4967         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4968         def _done(res):
4969             try:
4970                 units = [simplejson.loads(line)
4971                          for line in res.splitlines()
4972                          if line]
4973             except ValueError:
4974                 print "response is:", res
4975                 print "undecodeable line was '%s'" % line
4976                 raise
4977             self.failUnlessReallyEqual(len(units), 5+1)
4978             # should be parent-first
4979             u0 = units[0]
4980             self.failUnlessEqual(u0["path"], [])
4981             self.failUnlessEqual(u0["type"], "directory")
4982             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4983             u0cr = u0["check-results"]
4984             self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
4985
4986             ugood = [u for u in units
4987                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4988             self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
4989             ugoodcr = ugood["check-results"]
4990             self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
4991
4992             stats = units[-1]
4993             self.failUnlessEqual(stats["type"], "stats")
4994             s = stats["stats"]
4995             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4996             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4997             self.failUnlessReallyEqual(s["count-directories"], 1)
4998             self.failUnlessReallyEqual(s["count-unknown"], 1)
4999         d.addCallback(_done)
5000
5001         d.addCallback(self.CHECK, "root", "t=stream-manifest")
5002         def _check_manifest(res):
5003             self.failUnless(res.endswith("\n"))
5004             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
5005             self.failUnlessReallyEqual(len(units), 5+1)
5006             self.failUnlessEqual(units[-1]["type"], "stats")
5007             first = units[0]
5008             self.failUnlessEqual(first["path"], [])
5009             self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri())
5010             self.failUnlessEqual(first["type"], "directory")
5011             stats = units[-1]["stats"]
5012             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
5013             self.failUnlessReallyEqual(stats["count-literal-files"], 1)
5014             self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
5015             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
5016             self.failUnlessReallyEqual(stats["count-unknown"], 1)
5017         d.addCallback(_check_manifest)
5018
5019         # now add root/subdir and root/subdir/grandchild, then make subdir
5020         # unrecoverable, then see what happens
5021
5022         d.addCallback(lambda ign:
5023                       self.rootnode.create_subdirectory(u"subdir"))
5024         d.addCallback(_stash_uri, "subdir")
5025         d.addCallback(lambda subdir_node:
5026                       subdir_node.add_file(u"grandchild",
5027                                            upload.Data(DATA+"2",
5028                                                        convergence="")))
5029         d.addCallback(_stash_uri, "grandchild")
5030
5031         d.addCallback(lambda ign:
5032                       self.delete_shares_numbered(self.uris["subdir"],
5033                                                   range(1, 10)))
5034
5035         # root
5036         # root/good
5037         # root/small
5038         # root/sick
5039         # root/future
5040         # root/subdir [unrecoverable]
5041         # root/subdir/grandchild
5042
5043         # how should a streaming-JSON API indicate fatal error?
5044         # answer: emit ERROR: instead of a JSON string
5045
5046         d.addCallback(self.CHECK, "root", "t=stream-manifest")
5047         def _check_broken_manifest(res):
5048             lines = res.splitlines()
5049             error_lines = [i
5050                            for (i,line) in enumerate(lines)
5051                            if line.startswith("ERROR:")]
5052             if not error_lines:
5053                 self.fail("no ERROR: in output: %s" % (res,))
5054             first_error = error_lines[0]
5055             error_line = lines[first_error]
5056             error_msg = lines[first_error+1:]
5057             error_msg_s = "\n".join(error_msg) + "\n"
5058             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
5059                               error_line)
5060             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
5061             units = [simplejson.loads(line) for line in lines[:first_error]]
5062             self.failUnlessReallyEqual(len(units), 6) # includes subdir
5063             last_unit = units[-1]
5064             self.failUnlessEqual(last_unit["path"], ["subdir"])
5065         d.addCallback(_check_broken_manifest)
5066
5067         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
5068         def _check_broken_deepcheck(res):
5069             lines = res.splitlines()
5070             error_lines = [i
5071                            for (i,line) in enumerate(lines)
5072                            if line.startswith("ERROR:")]
5073             if not error_lines:
5074                 self.fail("no ERROR: in output: %s" % (res,))
5075             first_error = error_lines[0]
5076             error_line = lines[first_error]
5077             error_msg = lines[first_error+1:]
5078             error_msg_s = "\n".join(error_msg) + "\n"
5079             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
5080                               error_line)
5081             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
5082             units = [simplejson.loads(line) for line in lines[:first_error]]
5083             self.failUnlessReallyEqual(len(units), 6) # includes subdir
5084             last_unit = units[-1]
5085             self.failUnlessEqual(last_unit["path"], ["subdir"])
5086             r = last_unit["check-results"]["results"]
5087             self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
5088             self.failUnlessReallyEqual(r["count-shares-good"], 1)
5089             self.failUnlessReallyEqual(r["recoverable"], False)
5090         d.addCallback(_check_broken_deepcheck)
5091
5092         d.addErrback(self.explain_web_error)
5093         return d
5094
5095     def test_deep_check_and_repair(self):
5096         self.basedir = "web/Grid/deep_check_and_repair"
5097         self.set_up_grid()
5098         c0 = self.g.clients[0]
5099         self.uris = {}
5100         self.fileurls = {}
5101         DATA = "data" * 100
5102         d = c0.create_dirnode()
5103         def _stash_root_and_create_file(n):
5104             self.rootnode = n
5105             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5106             return n.add_file(u"good", upload.Data(DATA, convergence=""))
5107         d.addCallback(_stash_root_and_create_file)
5108         def _stash_uri(fn, which):
5109             self.uris[which] = fn.get_uri()
5110         d.addCallback(_stash_uri, "good")
5111         d.addCallback(lambda ign:
5112                       self.rootnode.add_file(u"small",
5113                                              upload.Data("literal",
5114                                                         convergence="")))
5115         d.addCallback(_stash_uri, "small")
5116         d.addCallback(lambda ign:
5117                       self.rootnode.add_file(u"sick",
5118                                              upload.Data(DATA+"1",
5119                                                         convergence="")))
5120         d.addCallback(_stash_uri, "sick")
5121         #d.addCallback(lambda ign:
5122         #              self.rootnode.add_file(u"dead",
5123         #                                     upload.Data(DATA+"2",
5124         #                                                convergence="")))
5125         #d.addCallback(_stash_uri, "dead")
5126
5127         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
5128         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
5129         #d.addCallback(_stash_uri, "corrupt")
5130
5131         def _clobber_shares(ignored):
5132             good_shares = self.find_uri_shares(self.uris["good"])
5133             self.failUnlessReallyEqual(len(good_shares), 10)
5134             sick_shares = self.find_uri_shares(self.uris["sick"])
5135             os.unlink(sick_shares[0][2])
5136             #dead_shares = self.find_uri_shares(self.uris["dead"])
5137             #for i in range(1, 10):
5138             #    os.unlink(dead_shares[i][2])
5139
5140             #c_shares = self.find_uri_shares(self.uris["corrupt"])
5141             #cso = CorruptShareOptions()
5142             #cso.stdout = StringIO()
5143             #cso.parseOptions([c_shares[0][2]])
5144             #corrupt_share(cso)
5145         d.addCallback(_clobber_shares)
5146
5147         # root
5148         # root/good   CHK, 10 shares
5149         # root/small  LIT
5150         # root/sick   CHK, 9 shares
5151
5152         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
5153         def _done(res):
5154             units = [simplejson.loads(line)
5155                      for line in res.splitlines()
5156                      if line]
5157             self.failUnlessReallyEqual(len(units), 4+1)
5158             # should be parent-first
5159             u0 = units[0]
5160             self.failUnlessEqual(u0["path"], [])
5161             self.failUnlessEqual(u0["type"], "directory")
5162             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
5163             u0crr = u0["check-and-repair-results"]
5164             self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
5165             self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
5166
5167             ugood = [u for u in units
5168                      if u["type"] == "file" and u["path"] == [u"good"]][0]
5169             self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
5170             ugoodcrr = ugood["check-and-repair-results"]
5171             self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
5172             self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
5173
5174             usick = [u for u in units
5175                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
5176             self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"])
5177             usickcrr = usick["check-and-repair-results"]
5178             self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
5179             self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
5180             self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
5181             self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
5182
5183             stats = units[-1]
5184             self.failUnlessEqual(stats["type"], "stats")
5185             s = stats["stats"]
5186             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
5187             self.failUnlessReallyEqual(s["count-literal-files"], 1)
5188             self.failUnlessReallyEqual(s["count-directories"], 1)
5189         d.addCallback(_done)
5190
5191         d.addErrback(self.explain_web_error)
5192         return d
5193
5194     def _count_leases(self, ignored, which):
5195         u = self.uris[which]
5196         shares = self.find_uri_shares(u)
5197         lease_counts = []
5198         for shnum, serverid, fn in shares:
5199             sf = get_share_file(fn)
5200             num_leases = len(list(sf.get_leases()))
5201             lease_counts.append( (fn, num_leases) )
5202         return lease_counts
5203
5204     def _assert_leasecount(self, lease_counts, expected):
5205         for (fn, num_leases) in lease_counts:
5206             if num_leases != expected:
5207                 self.fail("expected %d leases, have %d, on %s" %
5208                           (expected, num_leases, fn))
5209
5210     def test_add_lease(self):
5211         self.basedir = "web/Grid/add_lease"
5212         self.set_up_grid(num_clients=2)
5213         c0 = self.g.clients[0]
5214         self.uris = {}
5215         DATA = "data" * 100
5216         d = c0.upload(upload.Data(DATA, convergence=""))
5217         def _stash_uri(ur, which):
5218             self.uris[which] = ur.get_uri()
5219         d.addCallback(_stash_uri, "one")
5220         d.addCallback(lambda ign:
5221                       c0.upload(upload.Data(DATA+"1", convergence="")))
5222         d.addCallback(_stash_uri, "two")
5223         def _stash_mutable_uri(n, which):
5224             self.uris[which] = n.get_uri()
5225             assert isinstance(self.uris[which], str)
5226         d.addCallback(lambda ign:
5227             c0.create_mutable_file(publish.MutableData(DATA+"2")))
5228         d.addCallback(_stash_mutable_uri, "mutable")
5229
5230         def _compute_fileurls(ignored):
5231             self.fileurls = {}
5232             for which in self.uris:
5233                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
5234         d.addCallback(_compute_fileurls)
5235
5236         d.addCallback(self._count_leases, "one")
5237         d.addCallback(self._assert_leasecount, 1)
5238         d.addCallback(self._count_leases, "two")
5239         d.addCallback(self._assert_leasecount, 1)
5240         d.addCallback(self._count_leases, "mutable")
5241         d.addCallback(self._assert_leasecount, 1)
5242
5243         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
5244         def _got_html_good(res):
5245             self.failUnlessIn("Healthy", res)
5246             self.failIfIn("Not Healthy", res)
5247         d.addCallback(_got_html_good)
5248
5249         d.addCallback(self._count_leases, "one")
5250         d.addCallback(self._assert_leasecount, 1)
5251         d.addCallback(self._count_leases, "two")
5252         d.addCallback(self._assert_leasecount, 1)
5253         d.addCallback(self._count_leases, "mutable")
5254         d.addCallback(self._assert_leasecount, 1)
5255
5256         # this CHECK uses the original client, which uses the same
5257         # lease-secrets, so it will just renew the original lease
5258         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
5259         d.addCallback(_got_html_good)
5260
5261         d.addCallback(self._count_leases, "one")
5262         d.addCallback(self._assert_leasecount, 1)
5263         d.addCallback(self._count_leases, "two")
5264         d.addCallback(self._assert_leasecount, 1)
5265         d.addCallback(self._count_leases, "mutable")
5266         d.addCallback(self._assert_leasecount, 1)
5267
5268         # this CHECK uses an alternate client, which adds a second lease
5269         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
5270         d.addCallback(_got_html_good)
5271
5272         d.addCallback(self._count_leases, "one")
5273         d.addCallback(self._assert_leasecount, 2)
5274         d.addCallback(self._count_leases, "two")
5275         d.addCallback(self._assert_leasecount, 1)
5276         d.addCallback(self._count_leases, "mutable")
5277         d.addCallback(self._assert_leasecount, 1)
5278
5279         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
5280         d.addCallback(_got_html_good)
5281
5282         d.addCallback(self._count_leases, "one")
5283         d.addCallback(self._assert_leasecount, 2)
5284         d.addCallback(self._count_leases, "two")
5285         d.addCallback(self._assert_leasecount, 1)
5286         d.addCallback(self._count_leases, "mutable")
5287         d.addCallback(self._assert_leasecount, 1)
5288
5289         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
5290                       clientnum=1)
5291         d.addCallback(_got_html_good)
5292
5293         d.addCallback(self._count_leases, "one")
5294         d.addCallback(self._assert_leasecount, 2)
5295         d.addCallback(self._count_leases, "two")
5296         d.addCallback(self._assert_leasecount, 1)
5297         d.addCallback(self._count_leases, "mutable")
5298         d.addCallback(self._assert_leasecount, 2)
5299
5300         d.addErrback(self.explain_web_error)
5301         return d
5302
5303     def test_deep_add_lease(self):
5304         self.basedir = "web/Grid/deep_add_lease"
5305         self.set_up_grid(num_clients=2)
5306         c0 = self.g.clients[0]
5307         self.uris = {}
5308         self.fileurls = {}
5309         DATA = "data" * 100
5310         d = c0.create_dirnode()
5311         def _stash_root_and_create_file(n):
5312             self.rootnode = n
5313             self.uris["root"] = n.get_uri()
5314             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5315             return n.add_file(u"one", upload.Data(DATA, convergence=""))
5316         d.addCallback(_stash_root_and_create_file)
5317         def _stash_uri(fn, which):
5318             self.uris[which] = fn.get_uri()
5319         d.addCallback(_stash_uri, "one")
5320         d.addCallback(lambda ign:
5321                       self.rootnode.add_file(u"small",
5322                                              upload.Data("literal",
5323                                                         convergence="")))
5324         d.addCallback(_stash_uri, "small")
5325
5326         d.addCallback(lambda ign:
5327             c0.create_mutable_file(publish.MutableData("mutable")))
5328         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
5329         d.addCallback(_stash_uri, "mutable")
5330
5331         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
5332         def _done(res):
5333             units = [simplejson.loads(line)
5334                      for line in res.splitlines()
5335                      if line]
5336             # root, one, small, mutable,   stats
5337             self.failUnlessReallyEqual(len(units), 4+1)
5338         d.addCallback(_done)
5339
5340         d.addCallback(self._count_leases, "root")
5341         d.addCallback(self._assert_leasecount, 1)
5342         d.addCallback(self._count_leases, "one")
5343         d.addCallback(self._assert_leasecount, 1)
5344         d.addCallback(self._count_leases, "mutable")
5345         d.addCallback(self._assert_leasecount, 1)
5346
5347         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
5348         d.addCallback(_done)
5349
5350         d.addCallback(self._count_leases, "root")
5351         d.addCallback(self._assert_leasecount, 1)
5352         d.addCallback(self._count_leases, "one")
5353         d.addCallback(self._assert_leasecount, 1)
5354         d.addCallback(self._count_leases, "mutable")
5355         d.addCallback(self._assert_leasecount, 1)
5356
5357         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
5358                       clientnum=1)
5359         d.addCallback(_done)
5360
5361         d.addCallback(self._count_leases, "root")
5362         d.addCallback(self._assert_leasecount, 2)
5363         d.addCallback(self._count_leases, "one")
5364         d.addCallback(self._assert_leasecount, 2)
5365         d.addCallback(self._count_leases, "mutable")
5366         d.addCallback(self._assert_leasecount, 2)
5367
5368         d.addErrback(self.explain_web_error)
5369         return d
5370
5371
5372     def test_exceptions(self):
5373         self.basedir = "web/Grid/exceptions"
5374         self.set_up_grid(num_clients=1, num_servers=2)
5375         c0 = self.g.clients[0]
5376         c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 2
5377         self.fileurls = {}
5378         DATA = "data" * 100
5379         d = c0.create_dirnode()
5380         def _stash_root(n):
5381             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5382             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
5383             return n
5384         d.addCallback(_stash_root)
5385         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
5386         def _stash_bad(ur):
5387             self.fileurls["1share"] = "uri/" + urllib.quote(ur.get_uri())
5388             self.delete_shares_numbered(ur.get_uri(), range(1,10))
5389
5390             u = uri.from_string(ur.get_uri())
5391             u.key = testutil.flip_bit(u.key, 0)
5392             baduri = u.to_string()
5393             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
5394         d.addCallback(_stash_bad)
5395         d.addCallback(lambda ign: c0.create_dirnode())
5396         def _mangle_dirnode_1share(n):
5397             u = n.get_uri()
5398             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
5399             self.fileurls["dir-1share-json"] = url + "?t=json"
5400             self.delete_shares_numbered(u, range(1,10))
5401         d.addCallback(_mangle_dirnode_1share)
5402         d.addCallback(lambda ign: c0.create_dirnode())
5403         def _mangle_dirnode_0share(n):
5404             u = n.get_uri()
5405             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
5406             self.fileurls["dir-0share-json"] = url + "?t=json"
5407             self.delete_shares_numbered(u, range(0,10))
5408         d.addCallback(_mangle_dirnode_0share)
5409
5410         # NotEnoughSharesError should be reported sensibly, with a
5411         # text/plain explanation of the problem, and perhaps some
5412         # information on which shares *could* be found.
5413
5414         d.addCallback(lambda ignored:
5415                       self.shouldHTTPError("GET unrecoverable",
5416                                            410, "Gone", "NoSharesError",
5417                                            self.GET, self.fileurls["0shares"]))
5418         def _check_zero_shares(body):
5419             self.failIfIn("<html>", body)
5420             body = " ".join(body.strip().split())
5421             exp = ("NoSharesError: no shares could be found. "
5422                    "Zero shares usually indicates a corrupt URI, or that "
5423                    "no servers were connected, but it might also indicate "
5424                    "severe corruption. You should perform a filecheck on "
5425                    "this object to learn more. The full error message is: "
5426                    "no shares (need 3). Last failure: None")
5427             self.failUnlessReallyEqual(exp, body)
5428         d.addCallback(_check_zero_shares)
5429
5430
5431         d.addCallback(lambda ignored:
5432                       self.shouldHTTPError("GET 1share",
5433                                            410, "Gone", "NotEnoughSharesError",
5434                                            self.GET, self.fileurls["1share"]))
5435         def _check_one_share(body):
5436             self.failIfIn("<html>", body)
5437             body = " ".join(body.strip().split())
5438             msgbase = ("NotEnoughSharesError: This indicates that some "
5439                        "servers were unavailable, or that shares have been "
5440                        "lost to server departure, hard drive failure, or disk "
5441                        "corruption. You should perform a filecheck on "
5442                        "this object to learn more. The full error message is:"
5443                        )
5444             msg1 = msgbase + (" ran out of shares:"
5445                               " complete=sh0"
5446                               " pending="
5447                               " overdue= unused= need 3. Last failure: None")
5448             msg2 = msgbase + (" ran out of shares:"
5449                               " complete="
5450                               " pending=Share(sh0-on-xgru5)"
5451                               " overdue= unused= need 3. Last failure: None")
5452             self.failUnless(body == msg1 or body == msg2, body)
5453         d.addCallback(_check_one_share)
5454
5455         d.addCallback(lambda ignored:
5456                       self.shouldHTTPError("GET imaginary",
5457                                            404, "Not Found", None,
5458                                            self.GET, self.fileurls["imaginary"]))
5459         def _missing_child(body):
5460             self.failUnlessIn("No such child: imaginary", body)
5461         d.addCallback(_missing_child)
5462
5463         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
5464         def _check_0shares_dir_html(body):
5465             self.failUnlessIn("<html>", body)
5466             # we should see the regular page, but without the child table or
5467             # the dirops forms
5468             body = " ".join(body.strip().split())
5469             self.failUnlessIn('href="?t=info">More info on this directory',
5470                               body)
5471             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5472                    "could not be retrieved, because there were insufficient "
5473                    "good shares. This might indicate that no servers were "
5474                    "connected, insufficient servers were connected, the URI "
5475                    "was corrupt, or that shares have been lost due to server "
5476                    "departure, hard drive failure, or disk corruption. You "
5477                    "should perform a filecheck on this object to learn more.")
5478             self.failUnlessIn(exp, body)
5479             self.failUnlessIn("No upload forms: directory is unreadable", body)
5480         d.addCallback(_check_0shares_dir_html)
5481
5482         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
5483         def _check_1shares_dir_html(body):
5484             # at some point, we'll split UnrecoverableFileError into 0-shares
5485             # and some-shares like we did for immutable files (since there
5486             # are different sorts of advice to offer in each case). For now,
5487             # they present the same way.
5488             self.failUnlessIn("<html>", body)
5489             body = " ".join(body.strip().split())
5490             self.failUnlessIn('href="?t=info">More info on this directory',
5491                               body)
5492             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5493                    "could not be retrieved, because there were insufficient "
5494                    "good shares. This might indicate that no servers were "
5495                    "connected, insufficient servers were connected, the URI "
5496                    "was corrupt, or that shares have been lost due to server "
5497                    "departure, hard drive failure, or disk corruption. You "
5498                    "should perform a filecheck on this object to learn more.")
5499             self.failUnlessIn(exp, body)
5500             self.failUnlessIn("No upload forms: directory is unreadable", body)
5501         d.addCallback(_check_1shares_dir_html)
5502
5503         d.addCallback(lambda ignored:
5504                       self.shouldHTTPError("GET dir-0share-json",
5505                                            410, "Gone", "UnrecoverableFileError",
5506                                            self.GET,
5507                                            self.fileurls["dir-0share-json"]))
5508         def _check_unrecoverable_file(body):
5509             self.failIfIn("<html>", body)
5510             body = " ".join(body.strip().split())
5511             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5512                    "could not be retrieved, because there were insufficient "
5513                    "good shares. This might indicate that no servers were "
5514                    "connected, insufficient servers were connected, the URI "
5515                    "was corrupt, or that shares have been lost due to server "
5516                    "departure, hard drive failure, or disk corruption. You "
5517                    "should perform a filecheck on this object to learn more.")
5518             self.failUnlessReallyEqual(exp, body)
5519         d.addCallback(_check_unrecoverable_file)
5520
5521         d.addCallback(lambda ignored:
5522                       self.shouldHTTPError("GET dir-1share-json",
5523                                            410, "Gone", "UnrecoverableFileError",
5524                                            self.GET,
5525                                            self.fileurls["dir-1share-json"]))
5526         d.addCallback(_check_unrecoverable_file)
5527
5528         d.addCallback(lambda ignored:
5529                       self.shouldHTTPError("GET imaginary",
5530                                            404, "Not Found", None,
5531                                            self.GET, self.fileurls["imaginary"]))
5532
5533         # attach a webapi child that throws a random error, to test how it
5534         # gets rendered.
5535         w = c0.getServiceNamed("webish")
5536         w.root.putChild("ERRORBOOM", ErrorBoom())
5537
5538         # "Accept: */*" :        should get a text/html stack trace
5539         # "Accept: text/plain" : should get a text/plain stack trace
5540         # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
5541         # no Accept header:      should get a text/html stack trace
5542
5543         d.addCallback(lambda ignored:
5544                       self.shouldHTTPError("GET errorboom_html",
5545                                            500, "Internal Server Error", None,
5546                                            self.GET, "ERRORBOOM",
5547                                            headers={"accept": "*/*"}))
5548         def _internal_error_html1(body):
5549             self.failUnlessIn("<html>", "expected HTML, not '%s'" % body)
5550         d.addCallback(_internal_error_html1)
5551
5552         d.addCallback(lambda ignored:
5553                       self.shouldHTTPError("GET errorboom_text",
5554                                            500, "Internal Server Error", None,
5555                                            self.GET, "ERRORBOOM",
5556                                            headers={"accept": "text/plain"}))
5557         def _internal_error_text2(body):
5558             self.failIfIn("<html>", body)
5559             self.failUnless(body.startswith("Traceback "), body)
5560         d.addCallback(_internal_error_text2)
5561
5562         CLI_accepts = "text/plain, application/octet-stream"
5563         d.addCallback(lambda ignored:
5564                       self.shouldHTTPError("GET errorboom_text",
5565                                            500, "Internal Server Error", None,
5566                                            self.GET, "ERRORBOOM",
5567                                            headers={"accept": CLI_accepts}))
5568         def _internal_error_text3(body):
5569             self.failIfIn("<html>", body)
5570             self.failUnless(body.startswith("Traceback "), body)
5571         d.addCallback(_internal_error_text3)
5572
5573         d.addCallback(lambda ignored:
5574                       self.shouldHTTPError("GET errorboom_text",
5575                                            500, "Internal Server Error", None,
5576                                            self.GET, "ERRORBOOM"))
5577         def _internal_error_html4(body):
5578             self.failUnlessIn("<html>", body)
5579         d.addCallback(_internal_error_html4)
5580
5581         def _flush_errors(res):
5582             # Trial: please ignore the CompletelyUnhandledError in the logs
5583             self.flushLoggedErrors(CompletelyUnhandledError)
5584             return res
5585         d.addBoth(_flush_errors)
5586
5587         return d
5588
5589     def test_blacklist(self):
5590         # download from a blacklisted URI, get an error
5591         self.basedir = "web/Grid/blacklist"
5592         self.set_up_grid()
5593         c0 = self.g.clients[0]
5594         c0_basedir = c0.basedir
5595         fn = os.path.join(c0_basedir, "access.blacklist")
5596         self.uris = {}
5597         DATA = "off-limits " * 50
5598
5599         d = c0.upload(upload.Data(DATA, convergence=""))
5600         def _stash_uri_and_create_dir(ur):
5601             self.uri = ur.get_uri()
5602             self.url = "uri/"+self.uri
5603             u = uri.from_string_filenode(self.uri)
5604             self.si = u.get_storage_index()
5605             childnode = c0.create_node_from_uri(self.uri, None)
5606             return c0.create_dirnode({u"blacklisted.txt": (childnode,{}) })
5607         d.addCallback(_stash_uri_and_create_dir)
5608         def _stash_dir(node):
5609             self.dir_node = node
5610             self.dir_uri = node.get_uri()
5611             self.dir_url = "uri/"+self.dir_uri
5612         d.addCallback(_stash_dir)
5613         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5614         def _check_dir_html(body):
5615             self.failUnlessIn("<html>", body)
5616             self.failUnlessIn("blacklisted.txt</a>", body)
5617         d.addCallback(_check_dir_html)
5618         d.addCallback(lambda ign: self.GET(self.url))
5619         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5620
5621         def _blacklist(ign):
5622             f = open(fn, "w")
5623             f.write(" # this is a comment\n")
5624             f.write(" \n")
5625             f.write("\n") # also exercise blank lines
5626             f.write("%s %s\n" % (base32.b2a(self.si), "off-limits to you"))
5627             f.close()
5628             # clients should be checking the blacklist each time, so we don't
5629             # need to restart the client
5630         d.addCallback(_blacklist)
5631         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_uri",
5632                                                        403, "Forbidden",
5633                                                        "Access Prohibited: off-limits",
5634                                                        self.GET, self.url))
5635
5636         # We should still be able to list the parent directory, in HTML...
5637         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5638         def _check_dir_html2(body):
5639             self.failUnlessIn("<html>", body)
5640             self.failUnlessIn("blacklisted.txt</strike>", body)
5641         d.addCallback(_check_dir_html2)
5642
5643         # ... and in JSON (used by CLI).
5644         d.addCallback(lambda ign: self.GET(self.dir_url+"?t=json", followRedirect=True))
5645         def _check_dir_json(res):
5646             data = simplejson.loads(res)
5647             self.failUnless(isinstance(data, list), data)
5648             self.failUnlessEqual(data[0], "dirnode")
5649             self.failUnless(isinstance(data[1], dict), data)
5650             self.failUnlessIn("children", data[1])
5651             self.failUnlessIn("blacklisted.txt", data[1]["children"])
5652             childdata = data[1]["children"]["blacklisted.txt"]
5653             self.failUnless(isinstance(childdata, list), data)
5654             self.failUnlessEqual(childdata[0], "filenode")
5655             self.failUnless(isinstance(childdata[1], dict), data)
5656         d.addCallback(_check_dir_json)
5657
5658         def _unblacklist(ign):
5659             open(fn, "w").close()
5660             # the Blacklist object watches mtime to tell when the file has
5661             # changed, but on windows this test will run faster than the
5662             # filesystem's mtime resolution. So we edit Blacklist.last_mtime
5663             # to force a reload.
5664             self.g.clients[0].blacklist.last_mtime -= 2.0
5665         d.addCallback(_unblacklist)
5666
5667         # now a read should work
5668         d.addCallback(lambda ign: self.GET(self.url))
5669         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5670
5671         # read again to exercise the blacklist-is-unchanged logic
5672         d.addCallback(lambda ign: self.GET(self.url))
5673         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5674
5675         # now add a blacklisted directory, and make sure files under it are
5676         # refused too
5677         def _add_dir(ign):
5678             childnode = c0.create_node_from_uri(self.uri, None)
5679             return c0.create_dirnode({u"child": (childnode,{}) })
5680         d.addCallback(_add_dir)
5681         def _get_dircap(dn):
5682             self.dir_si_b32 = base32.b2a(dn.get_storage_index())
5683             self.dir_url_base = "uri/"+dn.get_write_uri()
5684             self.dir_url_json1 = "uri/"+dn.get_write_uri()+"?t=json"
5685             self.dir_url_json2 = "uri/"+dn.get_write_uri()+"/?t=json"
5686             self.dir_url_json_ro = "uri/"+dn.get_readonly_uri()+"/?t=json"
5687             self.child_url = "uri/"+dn.get_readonly_uri()+"/child"
5688         d.addCallback(_get_dircap)
5689         d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True))
5690         d.addCallback(lambda body: self.failUnlessIn("<html>", body))
5691         d.addCallback(lambda ign: self.GET(self.dir_url_json1))
5692         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5693         d.addCallback(lambda ign: self.GET(self.dir_url_json2))
5694         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5695         d.addCallback(lambda ign: self.GET(self.dir_url_json_ro))
5696         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5697         d.addCallback(lambda ign: self.GET(self.child_url))
5698         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5699
5700         def _block_dir(ign):
5701             f = open(fn, "w")
5702             f.write("%s %s\n" % (self.dir_si_b32, "dir-off-limits to you"))
5703             f.close()
5704             self.g.clients[0].blacklist.last_mtime -= 2.0
5705         d.addCallback(_block_dir)
5706         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir base",
5707                                                        403, "Forbidden",
5708                                                        "Access Prohibited: dir-off-limits",
5709                                                        self.GET, self.dir_url_base))
5710         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json1",
5711                                                        403, "Forbidden",
5712                                                        "Access Prohibited: dir-off-limits",
5713                                                        self.GET, self.dir_url_json1))
5714         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json2",
5715                                                        403, "Forbidden",
5716                                                        "Access Prohibited: dir-off-limits",
5717                                                        self.GET, self.dir_url_json2))
5718         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json_ro",
5719                                                        403, "Forbidden",
5720                                                        "Access Prohibited: dir-off-limits",
5721                                                        self.GET, self.dir_url_json_ro))
5722         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir child",
5723                                                        403, "Forbidden",
5724                                                        "Access Prohibited: dir-off-limits",
5725                                                        self.GET, self.child_url))
5726         return d
5727
5728
5729 class CompletelyUnhandledError(Exception):
5730     pass
5731 class ErrorBoom(rend.Page):
5732     def beforeRender(self, ctx):
5733         raise CompletelyUnhandledError("whoops")