src/allmydata/test/test_web.py

   1 import os.path, re, urllib, time, cgi
   2 import simplejson
   3 from StringIO import StringIO
   4
   5 from twisted.application import service
   6 from twisted.trial import unittest
   7 from twisted.internet import defer, reactor
   8 from twisted.internet.task import Clock
   9 from twisted.web import client, error, http
  10 from twisted.python import failure, log
  11
  12 from foolscap.api import fireEventually, flushEventualQueue
  13
  14 from nevow.util import escapeToXML
  15 from nevow import rend
  16
  17 from allmydata import interfaces, uri, webish, dirnode
  18 from allmydata.storage.shares import get_share_file
  19 from allmydata.storage_client import StorageFarmBroker, StubServer
  20 from allmydata.immutable import upload
  21 from allmydata.immutable.downloader.status import DownloadStatus
  22 from allmydata.dirnode import DirectoryNode
  23 from allmydata.nodemaker import NodeMaker
  24 from allmydata.unknown import UnknownNode
  25 from allmydata.web import status, common
  26 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  27 from allmydata.util import fileutil, base32, hashutil
  28 from allmydata.util.consumer import download_to_data
  29 from allmydata.util.netstring import split_netstring
  30 from allmydata.util.encodingutil import to_str
  31 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  32      create_chk_filenode, WebErrorMixin, ShouldFailMixin, \
  33      make_mutable_file_uri, create_mutable_filenode
  34 from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
  35 from allmydata.mutable import servermap, publish, retrieve
  36 import allmydata.test.common_util as testutil
  37 from allmydata.test.no_network import GridTestMixin
  38 from allmydata.test.common_web import HTTPClientGETFactory, \
  39      HTTPClientHEADFactory
  40 from allmydata.client import Client, SecretHolder
  41 from allmydata.introducer import IntroducerNode
  42
  43 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  44 # create a webserver that works against them
  45
  46 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  47
  48 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
  49 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
  50 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
  51
  52 FAVICON_MARKUP = '<link href="/icon.png" rel="shortcut icon" />'
  53
  54
  55 class FakeStatsProvider:
  56     def get_stats(self):
  57         stats = {'stats': {}, 'counters': {}}
  58         return stats
  59
  60 class FakeNodeMaker(NodeMaker):
  61     encoding_params = {
  62         'k': 3,
  63         'n': 10,
  64         'happy': 7,
  65         'max_segment_size':128*1024 # 1024=KiB
  66     }
  67     def _create_lit(self, cap):
  68         return FakeCHKFileNode(cap, self.all_contents)
  69     def _create_immutable(self, cap):
  70         return FakeCHKFileNode(cap, self.all_contents)
  71     def _create_mutable(self, cap):
  72         return FakeMutableFileNode(None, None,
  73                                    self.encoding_params, None,
  74                                    self.all_contents).init_from_cap(cap)
  75     def create_mutable_file(self, contents="", keysize=None,
  76                             version=SDMF_VERSION):
  77         n = FakeMutableFileNode(None, None, self.encoding_params, None,
  78                                 self.all_contents)
  79         return n.create(contents, version=version)
  80
  81 class FakeUploader(service.Service):
  82     name = "uploader"
  83     helper_furl = None
  84     helper_connected = False
  85
  86     def upload(self, uploadable):
  87         d = uploadable.get_size()
  88         d.addCallback(lambda size: uploadable.read(size))
  89         def _got_data(datav):
  90             data = "".join(datav)
  91             n = create_chk_filenode(data, self.all_contents)
  92             ur = upload.UploadResults(file_size=len(data),
  93                                       ciphertext_fetched=0,
  94                                       preexisting_shares=0,
  95                                       pushed_shares=10,
  96                                       sharemap={},
  97                                       servermap={},
  98                                       timings={},
  99                                       uri_extension_data={},
 100                                       uri_extension_hash="fake",
 101                                       verifycapstr="fakevcap")
 102             ur.set_uri(n.get_uri())
 103             return ur
 104         d.addCallback(_got_data)
 105         return d
 106
 107     def get_helper_info(self):
 108         return (self.helper_furl, self.helper_connected)
 109
 110
 111 def build_one_ds():
 112     ds = DownloadStatus("storage_index", 1234)
 113     now = time.time()
 114
 115     serverA = StubServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
 116     serverB = StubServer(hashutil.tagged_hash("foo", "serverid_b")[:20])
 117     storage_index = hashutil.storage_index_hash("SI")
 118     e0 = ds.add_segment_request(0, now)
 119     e0.activate(now+0.5)
 120     e0.deliver(now+1, 0, 100, 0.5) # when, start,len, decodetime
 121     e1 = ds.add_segment_request(1, now+2)
 122     e1.error(now+3)
 123     # two outstanding requests
 124     e2 = ds.add_segment_request(2, now+4)
 125     e3 = ds.add_segment_request(3, now+5)
 126     del e2,e3 # hush pyflakes
 127
 128     # simulate a segment which gets delivered faster than a system clock tick (ticket #1166)
 129     e = ds.add_segment_request(4, now)
 130     e.activate(now)
 131     e.deliver(now, 0, 140, 0.5)
 132
 133     e = ds.add_dyhb_request(serverA, now)
 134     e.finished([1,2], now+1)
 135     e = ds.add_dyhb_request(serverB, now+2) # left unfinished
 136
 137     e = ds.add_read_event(0, 120, now)
 138     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 139     e.finished(now+1)
 140     e = ds.add_read_event(120, 30, now+2) # left unfinished
 141
 142     e = ds.add_block_request(serverA, 1, 100, 20, now)
 143     e.finished(20, now+1)
 144     e = ds.add_block_request(serverB, 1, 120, 30, now+1) # left unfinished
 145
 146     # make sure that add_read_event() can come first too
 147     ds1 = DownloadStatus(storage_index, 1234)
 148     e = ds1.add_read_event(0, 120, now)
 149     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 150     e.finished(now+1)
 151
 152     return ds
 153
 154 class FakeHistory:
 155     _all_upload_status = [upload.UploadStatus()]
 156     _all_download_status = [build_one_ds()]
 157     _all_mapupdate_statuses = [servermap.UpdateStatus()]
 158     _all_publish_statuses = [publish.PublishStatus()]
 159     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
 160
 161     def list_all_upload_statuses(self):
 162         return self._all_upload_status
 163     def list_all_download_statuses(self):
 164         return self._all_download_status
 165     def list_all_mapupdate_statuses(self):
 166         return self._all_mapupdate_statuses
 167     def list_all_publish_statuses(self):
 168         return self._all_publish_statuses
 169     def list_all_retrieve_statuses(self):
 170         return self._all_retrieve_statuses
 171     def list_all_helper_statuses(self):
 172         return []
 173
 174 class FakeDisplayableServer(StubServer):
 175     def __init__(self, serverid, nickname):
 176         StubServer.__init__(self, serverid)
 177         self.announcement = {"my-version": "allmydata-tahoe-fake",
 178                              "service-name": "storage",
 179                              "nickname": nickname}
 180     def is_connected(self):
 181         return True
 182     def get_permutation_seed(self):
 183         return ""
 184     def get_remote_host(self):
 185         return ""
 186     def get_last_loss_time(self):
 187         return None
 188     def get_announcement_time(self):
 189         return None
 190     def get_announcement(self):
 191         return self.announcement
 192     def get_nickname(self):
 193         return self.announcement["nickname"]
 194
 195 class FakeBucketCounter(object):
 196     def get_state(self):
 197         return {"last-complete-bucket-count": 0}
 198     def get_progress(self):
 199         return {"estimated-time-per-cycle": 0,
 200                 "cycle-in-progress": False,
 201                 "remaining-wait-time": 0}
 202
 203 class FakeLeaseChecker(object):
 204     def __init__(self):
 205         self.expiration_enabled = False
 206         self.mode = "age"
 207         self.override_lease_duration = None
 208         self.sharetypes_to_expire = {}
 209     def get_state(self):
 210         return {"history": None}
 211     def get_progress(self):
 212         return {"estimated-time-per-cycle": 0,
 213                 "cycle-in-progress": False,
 214                 "remaining-wait-time": 0}
 215
 216 class FakeStorageServer(service.MultiService):
 217     name = 'storage'
 218     def __init__(self, nodeid, nickname):
 219         service.MultiService.__init__(self)
 220         self.my_nodeid = nodeid
 221         self.nickname = nickname
 222         self.bucket_counter = FakeBucketCounter()
 223         self.lease_checker = FakeLeaseChecker()
 224     def get_stats(self):
 225         return {"storage_server.accepting_immutable_shares": False}
 226
 227 class FakeClient(Client):
 228     def __init__(self):
 229         # don't upcall to Client.__init__, since we only want to initialize a
 230         # minimal subset
 231         service.MultiService.__init__(self)
 232         self.all_contents = {}
 233         self.nodeid = "fake_nodeid"
 234         self.nickname = u"fake_nickname \u263A"
 235         self.introducer_furl = "None"
 236         self.stats_provider = FakeStatsProvider()
 237         self._secret_holder = SecretHolder("lease secret", "convergence secret")
 238         self.helper = None
 239         self.convergence = "some random string"
 240         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 241         # fake knowledge of another server
 242         self.storage_broker.test_add_server("other_nodeid",
 243                                             FakeDisplayableServer("other_nodeid", u"other_nickname \u263B"))
 244         self.introducer_client = None
 245         self.history = FakeHistory()
 246         self.uploader = FakeUploader()
 247         self.uploader.all_contents = self.all_contents
 248         self.uploader.setServiceParent(self)
 249         self.blacklist = None
 250         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 251                                        self.uploader, None,
 252                                        None, None, None)
 253         self.nodemaker.all_contents = self.all_contents
 254         self.mutable_file_default = SDMF_VERSION
 255         self.addService(FakeStorageServer(self.nodeid, self.nickname))
 256
 257     def startService(self):
 258         return service.MultiService.startService(self)
 259     def stopService(self):
 260         return service.MultiService.stopService(self)
 261
 262     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 263
 264 class WebMixin(object):
 265     def setUp(self):
 266         self.s = FakeClient()
 267         self.s.startService()
 268         self.staticdir = self.mktemp()
 269         self.clock = Clock()
 270         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
 271                                       clock=self.clock)
 272         self.ws.setServiceParent(self.s)
 273         self.webish_port = self.ws.getPortnum()
 274         self.webish_url = self.ws.getURL()
 275         assert self.webish_url.endswith("/")
 276         self.webish_url = self.webish_url[:-1] # these tests add their own /
 277
 278         l = [ self.s.create_dirnode() for x in range(6) ]
 279         d = defer.DeferredList(l)
 280         def _then(res):
 281             self.public_root = res[0][1]
 282             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 283             self.public_url = "/uri/" + self.public_root.get_uri()
 284             self.private_root = res[1][1]
 285
 286             foo = res[2][1]
 287             self._foo_node = foo
 288             self._foo_uri = foo.get_uri()
 289             self._foo_readonly_uri = foo.get_readonly_uri()
 290             self._foo_verifycap = foo.get_verify_cap().to_string()
 291             # NOTE: we ignore the deferred on all set_uri() calls, because we
 292             # know the fake nodes do these synchronously
 293             self.public_root.set_uri(u"foo", foo.get_uri(),
 294                                      foo.get_readonly_uri())
 295
 296             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 297             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 298             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 299
 300             # sdmf
 301             # XXX: Do we ever use this?
 302             self.BAZ_CONTENTS, n, self._baz_txt_uri, self._baz_txt_readonly_uri = self.makefile_mutable(0)
 303
 304             foo.set_uri(u"baz.txt", self._baz_txt_uri, self._baz_txt_readonly_uri)
 305
 306             # mdmf
 307             self.QUUX_CONTENTS, n, self._quux_txt_uri, self._quux_txt_readonly_uri = self.makefile_mutable(0, mdmf=True)
 308             assert self._quux_txt_uri.startswith("URI:MDMF")
 309             foo.set_uri(u"quux.txt", self._quux_txt_uri, self._quux_txt_readonly_uri)
 310
 311             foo.set_uri(u"empty", res[3][1].get_uri(),
 312                         res[3][1].get_readonly_uri())
 313             sub_uri = res[4][1].get_uri()
 314             self._sub_uri = sub_uri
 315             foo.set_uri(u"sub", sub_uri, sub_uri)
 316             sub = self.s.create_node_from_uri(sub_uri)
 317             self._sub_node = sub
 318
 319             _ign, n, blocking_uri = self.makefile(1)
 320             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 321
 322             # filenode to test for html encoding issues
 323             self._htmlname_unicode = u"<&weirdly'named\"file>>>_<iframe />.txt"
 324             self._htmlname_raw = self._htmlname_unicode.encode('utf-8')
 325             self._htmlname_urlencoded = urllib.quote(self._htmlname_raw, '')
 326             self._htmlname_escaped = escapeToXML(self._htmlname_raw)
 327             self._htmlname_escaped_attr = cgi.escape(self._htmlname_raw, quote=True)
 328             self._htmlname_escaped_double = escapeToXML(cgi.escape(self._htmlname_raw, quote=True))
 329             self.HTMLNAME_CONTENTS, n, self._htmlname_txt_uri = self.makefile(0)
 330             foo.set_uri(self._htmlname_unicode, self._htmlname_txt_uri, self._htmlname_txt_uri)
 331
 332             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 333             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 334             # still think of it as an umlaut
 335             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 336
 337             self.SUBBAZ_CONTENTS, n, baz_file = self.makefile(2)
 338             self._baz_file_uri = baz_file
 339             sub.set_uri(u"baz.txt", baz_file, baz_file)
 340
 341             _ign, n, self._bad_file_uri = self.makefile(3)
 342             # this uri should not be downloadable
 343             del self.s.all_contents[self._bad_file_uri]
 344
 345             rodir = res[5][1]
 346             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 347                                      rodir.get_readonly_uri())
 348             rodir.set_uri(u"nor", baz_file, baz_file)
 349
 350             # public/
 351             # public/foo/
 352             # public/foo/bar.txt
 353             # public/foo/baz.txt
 354             # public/foo/quux.txt
 355             # public/foo/blockingfile
 356             # public/foo/<&weirdly'named\"file>>>_<iframe />.txt
 357             # public/foo/empty/
 358             # public/foo/sub/
 359             # public/foo/sub/baz.txt
 360             # public/reedownlee/
 361             # public/reedownlee/nor
 362             self.NEWFILE_CONTENTS = "newfile contents\n"
 363
 364             return foo.get_metadata_for(u"bar.txt")
 365         d.addCallback(_then)
 366         def _got_metadata(metadata):
 367             self._bar_txt_metadata = metadata
 368         d.addCallback(_got_metadata)
 369         return d
 370
 371     def get_all_contents(self):
 372         return self.s.all_contents
 373
 374     def makefile(self, number):
 375         contents = "contents of file %s\n" % number
 376         n = create_chk_filenode(contents, self.get_all_contents())
 377         return contents, n, n.get_uri()
 378
 379     def makefile_mutable(self, number, mdmf=False):
 380         contents = "contents of mutable file %s\n" % number
 381         n = create_mutable_filenode(contents, mdmf, self.s.all_contents)
 382         return contents, n, n.get_uri(), n.get_readonly_uri()
 383
 384     def tearDown(self):
 385         return self.s.stopService()
 386
 387     def failUnlessIsBarDotTxt(self, res):
 388         self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
 389
 390     def failUnlessIsQuuxDotTxt(self, res):
 391         self.failUnlessReallyEqual(res, self.QUUX_CONTENTS, res)
 392
 393     def failUnlessIsBazDotTxt(self, res):
 394         self.failUnlessReallyEqual(res, self.BAZ_CONTENTS, res)
 395
 396     def failUnlessIsSubBazDotTxt(self, res):
 397         self.failUnlessReallyEqual(res, self.SUBBAZ_CONTENTS, res)
 398
 399     def failUnlessIsBarJSON(self, res):
 400         data = simplejson.loads(res)
 401         self.failUnless(isinstance(data, list))
 402         self.failUnlessEqual(data[0], "filenode")
 403         self.failUnless(isinstance(data[1], dict))
 404         self.failIf(data[1]["mutable"])
 405         self.failIfIn("rw_uri", data[1]) # immutable
 406         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
 407         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
 408         self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
 409
 410     def failUnlessIsQuuxJSON(self, res, readonly=False):
 411         data = simplejson.loads(res)
 412         self.failUnless(isinstance(data, list))
 413         self.failUnlessEqual(data[0], "filenode")
 414         self.failUnless(isinstance(data[1], dict))
 415         metadata = data[1]
 416         return self.failUnlessIsQuuxDotTxtMetadata(metadata, readonly)
 417
 418     def failUnlessIsQuuxDotTxtMetadata(self, metadata, readonly):
 419         self.failUnless(metadata['mutable'])
 420         if readonly:
 421             self.failIfIn("rw_uri", metadata)
 422         else:
 423             self.failUnlessIn("rw_uri", metadata)
 424             self.failUnlessEqual(metadata['rw_uri'], self._quux_txt_uri)
 425         self.failUnlessIn("ro_uri", metadata)
 426         self.failUnlessEqual(metadata['ro_uri'], self._quux_txt_readonly_uri)
 427         self.failUnlessReallyEqual(metadata['size'], len(self.QUUX_CONTENTS))
 428
 429     def failUnlessIsFooJSON(self, res):
 430         data = simplejson.loads(res)
 431         self.failUnless(isinstance(data, list))
 432         self.failUnlessEqual(data[0], "dirnode", res)
 433         self.failUnless(isinstance(data[1], dict))
 434         self.failUnless(data[1]["mutable"])
 435         self.failUnlessIn("rw_uri", data[1]) # mutable
 436         self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
 437         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
 438         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
 439
 440         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 441         self.failUnlessEqual(kidnames,
 442                              [self._htmlname_unicode, u"bar.txt", u"baz.txt",
 443                               u"blockingfile", u"empty", u"n\u00fc.txt", u"quux.txt", u"sub"])
 444         kids = dict( [(unicode(name),value)
 445                       for (name,value)
 446                       in data[1]["children"].iteritems()] )
 447         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 448         self.failUnlessIn("metadata", kids[u"sub"][1])
 449         self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
 450         tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
 451         self.failUnlessIn("linkcrtime", tahoe_md)
 452         self.failUnlessIn("linkmotime", tahoe_md)
 453         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 454         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 455         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri)
 456         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]),
 457                                    self._bar_txt_verifycap)
 458         self.failUnlessIn("metadata", kids[u"bar.txt"][1])
 459         self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
 460         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
 461                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
 462         self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
 463                                    self._bar_txt_uri)
 464         self.failUnlessIn("quux.txt", kids)
 465         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["rw_uri"]),
 466                                    self._quux_txt_uri)
 467         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["ro_uri"]),
 468                                    self._quux_txt_readonly_uri)
 469
 470     def GET(self, urlpath, followRedirect=False, return_response=False,
 471             **kwargs):
 472         # if return_response=True, this fires with (data, statuscode,
 473         # respheaders) instead of just data.
 474         assert not isinstance(urlpath, unicode)
 475         url = self.webish_url + urlpath
 476         factory = HTTPClientGETFactory(url, method="GET",
 477                                        followRedirect=followRedirect, **kwargs)
 478         reactor.connectTCP("localhost", self.webish_port, factory)
 479         d = factory.deferred
 480         def _got_data(data):
 481             return (data, factory.status, factory.response_headers)
 482         if return_response:
 483             d.addCallback(_got_data)
 484         return factory.deferred
 485
 486     def HEAD(self, urlpath, return_response=False, **kwargs):
 487         # this requires some surgery, because twisted.web.client doesn't want
 488         # to give us back the response headers.
 489         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 490         reactor.connectTCP("localhost", self.webish_port, factory)
 491         d = factory.deferred
 492         def _got_data(data):
 493             return (data, factory.status, factory.response_headers)
 494         if return_response:
 495             d.addCallback(_got_data)
 496         return factory.deferred
 497
 498     def PUT(self, urlpath, data, **kwargs):
 499         url = self.webish_url + urlpath
 500         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 501
 502     def DELETE(self, urlpath):
 503         url = self.webish_url + urlpath
 504         return client.getPage(url, method="DELETE")
 505
 506     def POST(self, urlpath, followRedirect=False, **fields):
 507         sepbase = "boogabooga"
 508         sep = "--" + sepbase
 509         form = []
 510         form.append(sep)
 511         form.append('Content-Disposition: form-data; name="_charset"')
 512         form.append('')
 513         form.append('UTF-8')
 514         form.append(sep)
 515         for name, value in fields.iteritems():
 516             if isinstance(value, tuple):
 517                 filename, value = value
 518                 form.append('Content-Disposition: form-data; name="%s"; '
 519                             'filename="%s"' % (name, filename.encode("utf-8")))
 520             else:
 521                 form.append('Content-Disposition: form-data; name="%s"' % name)
 522             form.append('')
 523             if isinstance(value, unicode):
 524                 value = value.encode("utf-8")
 525             else:
 526                 value = str(value)
 527             assert isinstance(value, str)
 528             form.append(value)
 529             form.append(sep)
 530         form[-1] += "--"
 531         body = ""
 532         headers = {}
 533         if fields:
 534             body = "\r\n".join(form) + "\r\n"
 535             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 536         return self.POST2(urlpath, body, headers, followRedirect)
 537
 538     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 539         url = self.webish_url + urlpath
 540         return client.getPage(url, method="POST", postdata=body,
 541                               headers=headers, followRedirect=followRedirect)
 542
 543     def shouldFail(self, res, expected_failure, which,
 544                    substring=None, response_substring=None):
 545         if isinstance(res, failure.Failure):
 546             res.trap(expected_failure)
 547             if substring:
 548                 self.failUnlessIn(substring, str(res), which)
 549             if response_substring:
 550                 self.failUnlessIn(response_substring, res.value.response, which)
 551         else:
 552             self.fail("%s was supposed to raise %s, not get '%s'" %
 553                       (which, expected_failure, res))
 554
 555     def shouldFail2(self, expected_failure, which, substring,
 556                     response_substring,
 557                     callable, *args, **kwargs):
 558         assert substring is None or isinstance(substring, str)
 559         assert response_substring is None or isinstance(response_substring, str)
 560         d = defer.maybeDeferred(callable, *args, **kwargs)
 561         def done(res):
 562             if isinstance(res, failure.Failure):
 563                 res.trap(expected_failure)
 564                 if substring:
 565                     self.failUnlessIn(substring, str(res),
 566                                       "'%s' not in '%s' (response is '%s') for test '%s'" % \
 567                                       (substring, str(res),
 568                                        getattr(res.value, "response", ""),
 569                                        which))
 570                 if response_substring:
 571                     self.failUnlessIn(response_substring, res.value.response,
 572                                       "'%s' not in '%s' for test '%s'" % \
 573                                       (response_substring, res.value.response,
 574                                        which))
 575             else:
 576                 self.fail("%s was supposed to raise %s, not get '%s'" %
 577                           (which, expected_failure, res))
 578         d.addBoth(done)
 579         return d
 580
 581     def should404(self, res, which):
 582         if isinstance(res, failure.Failure):
 583             res.trap(error.Error)
 584             self.failUnlessReallyEqual(res.value.status, "404")
 585         else:
 586             self.fail("%s was supposed to Error(404), not get '%s'" %
 587                       (which, res))
 588
 589     def should302(self, res, which):
 590         if isinstance(res, failure.Failure):
 591             res.trap(error.Error)
 592             self.failUnlessReallyEqual(res.value.status, "302")
 593         else:
 594             self.fail("%s was supposed to Error(302), not get '%s'" %
 595                       (which, res))
 596
 597
 598 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
 599     def test_create(self):
 600         pass
 601
 602     def test_welcome(self):
 603         d = self.GET("/")
 604         def _check(res):
 605             self.failUnlessIn('<title>Tahoe-LAFS - Welcome</title>', res)
 606             self.failUnlessIn(FAVICON_MARKUP, res)
 607             self.failUnlessIn('<a href="status">Recent and Active Operations</a>', res)
 608             self.failUnlessIn('<a href="statistics">Operational Statistics</a>', res)
 609             self.failUnlessIn('<input type="hidden" name="t" value="report-incident" />', res)
 610             res_u = res.decode('utf-8')
 611             self.failUnlessIn(u'<td>fake_nickname \u263A</td>', res_u)
 612             self.failUnlessIn(u'<div class="nickname">other_nickname \u263B</div>', res_u)
 613             self.failUnlessIn(u'\u00A9 <a href="https://tahoe-lafs.org/">Tahoe-LAFS Software Foundation', res_u)
 614
 615             self.s.basedir = 'web/test_welcome'
 616             fileutil.make_dirs("web/test_welcome")
 617             fileutil.make_dirs("web/test_welcome/private")
 618             return self.GET("/")
 619         d.addCallback(_check)
 620         return d
 621
 622     def test_introducer_status(self):
 623         class MockIntroducerClient(object):
 624             def __init__(self, connected):
 625                 self.connected = connected
 626             def connected_to_introducer(self):
 627                 return self.connected
 628
 629         d = defer.succeed(None)
 630
 631         # introducer not connected, unguessable furl
 632         def _set_introducer_not_connected_unguessable(ign):
 633             self.s.introducer_furl = "pb://someIntroducer/secret"
 634             self.s.introducer_client = MockIntroducerClient(False)
 635             return self.GET("/")
 636         d.addCallback(_set_introducer_not_connected_unguessable)
 637         def _check_introducer_not_connected_unguessable(res):
 638             html = res.replace('\n', ' ')
 639             self.failUnlessIn('<div class="furl">pb://someIntroducer/[censored]</div>', html)
 640             self.failIfIn('pb://someIntroducer/secret', html)
 641             self.failUnless(re.search('<div class="status-indicator connected-no"></div>[ ]*<div>Introducer not connected</div>', html), res)
 642         d.addCallback(_check_introducer_not_connected_unguessable)
 643
 644         # introducer connected, unguessable furl
 645         def _set_introducer_connected_unguessable(ign):
 646             self.s.introducer_furl = "pb://someIntroducer/secret"
 647             self.s.introducer_client = MockIntroducerClient(True)
 648             return self.GET("/")
 649         d.addCallback(_set_introducer_connected_unguessable)
 650         def _check_introducer_connected_unguessable(res):
 651             html = res.replace('\n', ' ')
 652             self.failUnlessIn('<div class="furl">pb://someIntroducer/[censored]</div>', html)
 653             self.failIfIn('pb://someIntroducer/secret', html)
 654             self.failUnless(re.search('<div class="status-indicator connected-yes"></div>[ ]*<div>Introducer</div>', html), res)
 655         d.addCallback(_check_introducer_connected_unguessable)
 656
 657         # introducer connected, guessable furl
 658         def _set_introducer_connected_guessable(ign):
 659             self.s.introducer_furl = "pb://someIntroducer/introducer"
 660             self.s.introducer_client = MockIntroducerClient(True)
 661             return self.GET("/")
 662         d.addCallback(_set_introducer_connected_guessable)
 663         def _check_introducer_connected_guessable(res):
 664             html = res.replace('\n', ' ')
 665             self.failUnlessIn('<div class="furl">pb://someIntroducer/introducer</div>', html)
 666             self.failUnless(re.search('<div class="status-indicator connected-yes"></div>[ ]*<div>Introducer</div>', html), res)
 667         d.addCallback(_check_introducer_connected_guessable)
 668         return d
 669
 670     def test_helper_status(self):
 671         d = defer.succeed(None)
 672
 673         # set helper furl to None
 674         def _set_no_helper(ign):
 675             self.s.uploader.helper_furl = None
 676             return self.GET("/")
 677         d.addCallback(_set_no_helper)
 678         def _check_no_helper(res):
 679             html = res.replace('\n', ' ')
 680             self.failUnless(re.search('<div class="status-indicator connected-not-configured"></div>[ ]*<div>Helper</div>', html), res)
 681         d.addCallback(_check_no_helper)
 682
 683         # enable helper, not connected
 684         def _set_helper_not_connected(ign):
 685             self.s.uploader.helper_furl = "pb://someHelper/secret"
 686             self.s.uploader.helper_connected = False
 687             return self.GET("/")
 688         d.addCallback(_set_helper_not_connected)
 689         def _check_helper_not_connected(res):
 690             html = res.replace('\n', ' ')
 691             self.failUnlessIn('<div class="furl">pb://someHelper/[censored]</div>', html)
 692             self.failIfIn('pb://someHelper/secret', html)
 693             self.failUnless(re.search('<div class="status-indicator connected-no"></div>[ ]*<div>Helper not connected</div>', html), res)
 694         d.addCallback(_check_helper_not_connected)
 695
 696         # enable helper, connected
 697         def _set_helper_connected(ign):
 698             self.s.uploader.helper_furl = "pb://someHelper/secret"
 699             self.s.uploader.helper_connected = True
 700             return self.GET("/")
 701         d.addCallback(_set_helper_connected)
 702         def _check_helper_connected(res):
 703             html = res.replace('\n', ' ')
 704             self.failUnlessIn('<div class="furl">pb://someHelper/[censored]</div>', html)
 705             self.failIfIn('pb://someHelper/secret', html)
 706             self.failUnless(re.search('<div class="status-indicator connected-yes"></div>[ ]*<div>Helper</div>', html), res)
 707         d.addCallback(_check_helper_connected)
 708         return d
 709
 710     def test_storage(self):
 711         d = self.GET("/storage")
 712         def _check(res):
 713             self.failUnlessIn('Storage Server Status', res)
 714             self.failUnlessIn(FAVICON_MARKUP, res)
 715             res_u = res.decode('utf-8')
 716             self.failUnlessIn(u'<li>Server Nickname: <span class="nickname mine">fake_nickname \u263A</span></li>', res_u)
 717         d.addCallback(_check)
 718         return d
 719
 720     def test_status(self):
 721         h = self.s.get_history()
 722         dl_num = h.list_all_download_statuses()[0].get_counter()
 723         ul_num = h.list_all_upload_statuses()[0].get_counter()
 724         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 725         pub_num = h.list_all_publish_statuses()[0].get_counter()
 726         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 727         d = self.GET("/status", followRedirect=True)
 728         def _check(res):
 729             self.failUnlessIn('Recent and Active Operations', res)
 730             self.failUnlessIn('"down-%d"' % dl_num, res)
 731             self.failUnlessIn('"up-%d"' % ul_num, res)
 732             self.failUnlessIn('"mapupdate-%d"' % mu_num, res)
 733             self.failUnlessIn('"publish-%d"' % pub_num, res)
 734             self.failUnlessIn('"retrieve-%d"' % ret_num, res)
 735         d.addCallback(_check)
 736         d.addCallback(lambda res: self.GET("/status/?t=json"))
 737         def _check_json(res):
 738             data = simplejson.loads(res)
 739             self.failUnless(isinstance(data, dict))
 740             #active = data["active"]
 741             # TODO: test more. We need a way to fake an active operation
 742             # here.
 743         d.addCallback(_check_json)
 744
 745         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 746         def _check_dl(res):
 747             self.failUnlessIn("File Download Status", res)
 748         d.addCallback(_check_dl)
 749         d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
 750         def _check_dl_json(res):
 751             data = simplejson.loads(res)
 752             self.failUnless(isinstance(data, dict))
 753             self.failUnlessIn("read", data)
 754             self.failUnlessEqual(data["read"][0]["length"], 120)
 755             self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
 756             self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
 757             self.failUnlessEqual(data["segment"][2]["finish_time"], None)
 758             phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
 759             cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
 760             # serverids[] keys are strings, since that's what JSON does, but
 761             # we'd really like them to be ints
 762             self.failUnlessEqual(data["serverids"]["0"], "phwrsjte")
 763             self.failUnless(data["serverids"].has_key("1"),
 764                             str(data["serverids"]))
 765             self.failUnlessEqual(data["serverids"]["1"], "cmpuvkjm",
 766                                  str(data["serverids"]))
 767             self.failUnlessEqual(data["server_info"][phwr_id]["short"],
 768                                  "phwrsjte")
 769             self.failUnlessEqual(data["server_info"][cmpu_id]["short"],
 770                                  "cmpuvkjm")
 771             self.failUnlessIn("dyhb", data)
 772             self.failUnlessIn("misc", data)
 773         d.addCallback(_check_dl_json)
 774         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 775         def _check_ul(res):
 776             self.failUnlessIn("File Upload Status", res)
 777         d.addCallback(_check_ul)
 778         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 779         def _check_mapupdate(res):
 780             self.failUnlessIn("Mutable File Servermap Update Status", res)
 781         d.addCallback(_check_mapupdate)
 782         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 783         def _check_publish(res):
 784             self.failUnlessIn("Mutable File Publish Status", res)
 785         d.addCallback(_check_publish)
 786         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 787         def _check_retrieve(res):
 788             self.failUnlessIn("Mutable File Retrieve Status", res)
 789         d.addCallback(_check_retrieve)
 790
 791         return d
 792
 793     def test_status_numbers(self):
 794         drrm = status.DownloadResultsRendererMixin()
 795         self.failUnlessReallyEqual(drrm.render_time(None, None), "")
 796         self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
 797         self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
 798         self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
 799         self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
 800         self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
 801         self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 802         self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
 803         self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
 804
 805         urrm = status.UploadResultsRendererMixin()
 806         self.failUnlessReallyEqual(urrm.render_time(None, None), "")
 807         self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
 808         self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
 809         self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
 810         self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
 811         self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
 812         self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 813         self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
 814         self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
 815
 816     def test_GET_FILEURL(self):
 817         d = self.GET(self.public_url + "/foo/bar.txt")
 818         d.addCallback(self.failUnlessIsBarDotTxt)
 819         return d
 820
 821     def test_GET_FILEURL_range(self):
 822         headers = {"range": "bytes=1-10"}
 823         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 824                      return_response=True)
 825         def _got((res, status, headers)):
 826             self.failUnlessReallyEqual(int(status), 206)
 827             self.failUnless(headers.has_key("content-range"))
 828             self.failUnlessReallyEqual(headers["content-range"][0],
 829                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 830             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
 831         d.addCallback(_got)
 832         return d
 833
 834     def test_GET_FILEURL_partial_range(self):
 835         headers = {"range": "bytes=5-"}
 836         length  = len(self.BAR_CONTENTS)
 837         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 838                      return_response=True)
 839         def _got((res, status, headers)):
 840             self.failUnlessReallyEqual(int(status), 206)
 841             self.failUnless(headers.has_key("content-range"))
 842             self.failUnlessReallyEqual(headers["content-range"][0],
 843                                        "bytes 5-%d/%d" % (length-1, length))
 844             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
 845         d.addCallback(_got)
 846         return d
 847
 848     def test_GET_FILEURL_partial_end_range(self):
 849         headers = {"range": "bytes=-5"}
 850         length  = len(self.BAR_CONTENTS)
 851         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 852                      return_response=True)
 853         def _got((res, status, headers)):
 854             self.failUnlessReallyEqual(int(status), 206)
 855             self.failUnless(headers.has_key("content-range"))
 856             self.failUnlessReallyEqual(headers["content-range"][0],
 857                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 858             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
 859         d.addCallback(_got)
 860         return d
 861
 862     def test_GET_FILEURL_partial_range_overrun(self):
 863         headers = {"range": "bytes=100-200"}
 864         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
 865                              "416 Requested Range not satisfiable",
 866                              "First beyond end of file",
 867                              self.GET, self.public_url + "/foo/bar.txt",
 868                              headers=headers)
 869         return d
 870
 871     def test_HEAD_FILEURL_range(self):
 872         headers = {"range": "bytes=1-10"}
 873         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 874                      return_response=True)
 875         def _got((res, status, headers)):
 876             self.failUnlessReallyEqual(res, "")
 877             self.failUnlessReallyEqual(int(status), 206)
 878             self.failUnless(headers.has_key("content-range"))
 879             self.failUnlessReallyEqual(headers["content-range"][0],
 880                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 881         d.addCallback(_got)
 882         return d
 883
 884     def test_HEAD_FILEURL_partial_range(self):
 885         headers = {"range": "bytes=5-"}
 886         length  = len(self.BAR_CONTENTS)
 887         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 888                      return_response=True)
 889         def _got((res, status, headers)):
 890             self.failUnlessReallyEqual(int(status), 206)
 891             self.failUnless(headers.has_key("content-range"))
 892             self.failUnlessReallyEqual(headers["content-range"][0],
 893                                        "bytes 5-%d/%d" % (length-1, length))
 894         d.addCallback(_got)
 895         return d
 896
 897     def test_HEAD_FILEURL_partial_end_range(self):
 898         headers = {"range": "bytes=-5"}
 899         length  = len(self.BAR_CONTENTS)
 900         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 901                      return_response=True)
 902         def _got((res, status, headers)):
 903             self.failUnlessReallyEqual(int(status), 206)
 904             self.failUnless(headers.has_key("content-range"))
 905             self.failUnlessReallyEqual(headers["content-range"][0],
 906                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 907         d.addCallback(_got)
 908         return d
 909
 910     def test_HEAD_FILEURL_partial_range_overrun(self):
 911         headers = {"range": "bytes=100-200"}
 912         d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
 913                              "416 Requested Range not satisfiable",
 914                              "",
 915                              self.HEAD, self.public_url + "/foo/bar.txt",
 916                              headers=headers)
 917         return d
 918
 919     def test_GET_FILEURL_range_bad(self):
 920         headers = {"range": "BOGUS=fizbop-quarnak"}
 921         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 922                      return_response=True)
 923         def _got((res, status, headers)):
 924             self.failUnlessReallyEqual(int(status), 200)
 925             self.failUnless(not headers.has_key("content-range"))
 926             self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
 927         d.addCallback(_got)
 928         return d
 929
 930     def test_HEAD_FILEURL(self):
 931         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 932         def _got((res, status, headers)):
 933             self.failUnlessReallyEqual(res, "")
 934             self.failUnlessReallyEqual(headers["content-length"][0],
 935                                        str(len(self.BAR_CONTENTS)))
 936             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
 937         d.addCallback(_got)
 938         return d
 939
 940     def test_GET_FILEURL_named(self):
 941         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 942         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 943         d = self.GET(base + "/@@name=/blah.txt")
 944         d.addCallback(self.failUnlessIsBarDotTxt)
 945         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 946         d.addCallback(self.failUnlessIsBarDotTxt)
 947         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 948         d.addCallback(self.failUnlessIsBarDotTxt)
 949         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 950         d.addCallback(self.failUnlessIsBarDotTxt)
 951         save_url = base + "?save=true&filename=blah.txt"
 952         d.addCallback(lambda res: self.GET(save_url))
 953         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 954         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 955         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 956         u_url = base + "?save=true&filename=" + u_fn_e
 957         d.addCallback(lambda res: self.GET(u_url))
 958         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 959         return d
 960
 961     def test_PUT_FILEURL_named_bad(self):
 962         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 963         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 964                              "400 Bad Request",
 965                              "/file can only be used with GET or HEAD",
 966                              self.PUT, base + "/@@name=/blah.txt", "")
 967         return d
 968
 969
 970     def test_GET_DIRURL_named_bad(self):
 971         base = "/file/%s" % urllib.quote(self._foo_uri)
 972         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 973                              "400 Bad Request",
 974                              "is not a file-cap",
 975                              self.GET, base + "/@@name=/blah.txt")
 976         return d
 977
 978     def test_GET_slash_file_bad(self):
 979         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 980                              "404 Not Found",
 981                              "/file must be followed by a file-cap and a name",
 982                              self.GET, "/file")
 983         return d
 984
 985     def test_GET_unhandled_URI_named(self):
 986         contents, n, newuri = self.makefile(12)
 987         verifier_cap = n.get_verify_cap().to_string()
 988         base = "/file/%s" % urllib.quote(verifier_cap)
 989         # client.create_node_from_uri() can't handle verify-caps
 990         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 991                              "400 Bad Request", "is not a file-cap",
 992                              self.GET, base)
 993         return d
 994
 995     def test_GET_unhandled_URI(self):
 996         contents, n, newuri = self.makefile(12)
 997         verifier_cap = n.get_verify_cap().to_string()
 998         base = "/uri/%s" % urllib.quote(verifier_cap)
 999         # client.create_node_from_uri() can't handle verify-caps
1000         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
1001                              "400 Bad Request",
1002                              "GET unknown URI type: can only do t=info",
1003                              self.GET, base)
1004         return d
1005
1006     def test_GET_FILE_URI(self):
1007         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
1008         d = self.GET(base)
1009         d.addCallback(self.failUnlessIsBarDotTxt)
1010         return d
1011
1012     def test_GET_FILE_URI_mdmf(self):
1013         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
1014         d = self.GET(base)
1015         d.addCallback(self.failUnlessIsQuuxDotTxt)
1016         return d
1017
1018     def test_GET_FILE_URI_mdmf_extensions(self):
1019         base = "/uri/%s" % urllib.quote("%s:RANDOMSTUFF" % self._quux_txt_uri)
1020         d = self.GET(base)
1021         d.addCallback(self.failUnlessIsQuuxDotTxt)
1022         return d
1023
1024     def test_GET_FILE_URI_mdmf_readonly(self):
1025         base = "/uri/%s" % urllib.quote(self._quux_txt_readonly_uri)
1026         d = self.GET(base)
1027         d.addCallback(self.failUnlessIsQuuxDotTxt)
1028         return d
1029
1030     def test_GET_FILE_URI_badchild(self):
1031         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
1032         errmsg = "Files have no children, certainly not named 'boguschild'"
1033         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
1034                              "400 Bad Request", errmsg,
1035                              self.GET, base)
1036         return d
1037
1038     def test_PUT_FILE_URI_badchild(self):
1039         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
1040         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
1041         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
1042                              "400 Bad Request", errmsg,
1043                              self.PUT, base, "")
1044         return d
1045
1046     def test_PUT_FILE_URI_mdmf(self):
1047         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
1048         self._quux_new_contents = "new_contents"
1049         d = self.GET(base)
1050         d.addCallback(lambda res:
1051             self.failUnlessIsQuuxDotTxt(res))
1052         d.addCallback(lambda ignored:
1053             self.PUT(base, self._quux_new_contents))
1054         d.addCallback(lambda ignored:
1055             self.GET(base))
1056         d.addCallback(lambda res:
1057             self.failUnlessReallyEqual(res, self._quux_new_contents))
1058         return d
1059
1060     def test_PUT_FILE_URI_mdmf_extensions(self):
1061         base = "/uri/%s" % urllib.quote("%s:EXTENSIONSTUFF" % self._quux_txt_uri)
1062         self._quux_new_contents = "new_contents"
1063         d = self.GET(base)
1064         d.addCallback(lambda res: self.failUnlessIsQuuxDotTxt(res))
1065         d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
1066         d.addCallback(lambda ignored: self.GET(base))
1067         d.addCallback(lambda res: self.failUnlessEqual(self._quux_new_contents,
1068                                                        res))
1069         return d
1070
1071     def test_PUT_FILE_URI_mdmf_readonly(self):
1072         # We're not allowed to PUT things to a readonly cap.
1073         base = "/uri/%s" % self._quux_txt_readonly_uri
1074         d = self.GET(base)
1075         d.addCallback(lambda res:
1076             self.failUnlessIsQuuxDotTxt(res))
1077         # What should we get here? We get a 500 error now; that's not right.
1078         d.addCallback(lambda ignored:
1079             self.shouldFail2(error.Error, "test_PUT_FILE_URI_mdmf_readonly",
1080                              "400 Bad Request", "read-only cap",
1081                              self.PUT, base, "new data"))
1082         return d
1083
1084     def test_PUT_FILE_URI_sdmf_readonly(self):
1085         # We're not allowed to put things to a readonly cap.
1086         base = "/uri/%s" % self._baz_txt_readonly_uri
1087         d = self.GET(base)
1088         d.addCallback(lambda res:
1089             self.failUnlessIsBazDotTxt(res))
1090         d.addCallback(lambda ignored:
1091             self.shouldFail2(error.Error, "test_PUT_FILE_URI_sdmf_readonly",
1092                              "400 Bad Request", "read-only cap",
1093                              self.PUT, base, "new_data"))
1094         return d
1095
1096     def test_GET_etags(self):
1097
1098         def _check_etags(uri):
1099             d1 = _get_etag(uri)
1100             d2 = _get_etag(uri, 'json')
1101             d = defer.DeferredList([d1, d2], consumeErrors=True)
1102             def _check(results):
1103                 # All deferred must succeed
1104                 self.failUnless(all([r[0] for r in results]))
1105                 # the etag for the t=json form should be just like the etag
1106                 # fo the default t='' form, but with a 'json' suffix
1107                 self.failUnlessEqual(results[0][1] + 'json', results[1][1])
1108             d.addCallback(_check)
1109             return d
1110
1111         def _get_etag(uri, t=''):
1112             targetbase = "/uri/%s?t=%s" % (urllib.quote(uri.strip()), t)
1113             d = self.GET(targetbase, return_response=True, followRedirect=True)
1114             def _just_the_etag(result):
1115                 data, response, headers = result
1116                 etag = headers['etag'][0]
1117                 if uri.startswith('URI:DIR'):
1118                     self.failUnless(etag.startswith('DIR:'), etag)
1119                 return etag
1120             return d.addCallback(_just_the_etag)
1121
1122         # Check that etags work with immutable directories
1123         (newkids, caps) = self._create_immutable_children()
1124         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1125                       simplejson.dumps(newkids))
1126         def _stash_immdir_uri(uri):
1127             self._immdir_uri = uri
1128             return uri
1129         d.addCallback(_stash_immdir_uri)
1130         d.addCallback(_check_etags)
1131
1132         # Check that etags work with immutable files
1133         d.addCallback(lambda _: _check_etags(self._bar_txt_uri))
1134
1135         # use the ETag on GET
1136         def _check_match(ign):
1137             uri = "/uri/%s" % self._bar_txt_uri
1138             d = self.GET(uri, return_response=True)
1139             # extract the ETag
1140             d.addCallback(lambda (data, code, headers):
1141                           headers['etag'][0])
1142             # do a GET that's supposed to match the ETag
1143             d.addCallback(lambda etag:
1144                           self.GET(uri, return_response=True,
1145                                    headers={"If-None-Match": etag}))
1146             # make sure it short-circuited (304 instead of 200)
1147             d.addCallback(lambda (data, code, headers):
1148                           self.failUnlessEqual(int(code), http.NOT_MODIFIED))
1149             return d
1150         d.addCallback(_check_match)
1151
1152         def _no_etag(uri, t):
1153             target = "/uri/%s?t=%s" % (uri, t)
1154             d = self.GET(target, return_response=True, followRedirect=True)
1155             d.addCallback(lambda (data, code, headers):
1156                           self.failIf("etag" in headers, target))
1157             return d
1158         def _yes_etag(uri, t):
1159             target = "/uri/%s?t=%s" % (uri, t)
1160             d = self.GET(target, return_response=True, followRedirect=True)
1161             d.addCallback(lambda (data, code, headers):
1162                           self.failUnless("etag" in headers, target))
1163             return d
1164
1165         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, ""))
1166         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "json"))
1167         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "uri"))
1168         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "readonly-uri"))
1169         d.addCallback(lambda ign: _no_etag(self._bar_txt_uri, "info"))
1170
1171         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, ""))
1172         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "json"))
1173         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "uri"))
1174         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "readonly-uri"))
1175         d.addCallback(lambda ign: _no_etag(self._immdir_uri, "info"))
1176         d.addCallback(lambda ign: _no_etag(self._immdir_uri, "rename-form"))
1177
1178         return d
1179
1180     # TODO: version of this with a Unicode filename
1181     def test_GET_FILEURL_save(self):
1182         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
1183                      return_response=True)
1184         def _got((res, statuscode, headers)):
1185             content_disposition = headers["content-disposition"][0]
1186             self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
1187             self.failUnlessIsBarDotTxt(res)
1188         d.addCallback(_got)
1189         return d
1190
1191     def test_GET_FILEURL_missing(self):
1192         d = self.GET(self.public_url + "/foo/missing")
1193         d.addBoth(self.should404, "test_GET_FILEURL_missing")
1194         return d
1195
1196     def test_GET_FILEURL_info_mdmf(self):
1197         d = self.GET("/uri/%s?t=info" % self._quux_txt_uri)
1198         def _got(res):
1199             self.failUnlessIn("mutable file (mdmf)", res)
1200             self.failUnlessIn(self._quux_txt_uri, res)
1201             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1202         d.addCallback(_got)
1203         return d
1204
1205     def test_GET_FILEURL_info_mdmf_readonly(self):
1206         d = self.GET("/uri/%s?t=info" % self._quux_txt_readonly_uri)
1207         def _got(res):
1208             self.failUnlessIn("mutable file (mdmf)", res)
1209             self.failIfIn(self._quux_txt_uri, res)
1210             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1211         d.addCallback(_got)
1212         return d
1213
1214     def test_GET_FILEURL_info_sdmf(self):
1215         d = self.GET("/uri/%s?t=info" % self._baz_txt_uri)
1216         def _got(res):
1217             self.failUnlessIn("mutable file (sdmf)", res)
1218             self.failUnlessIn(self._baz_txt_uri, res)
1219         d.addCallback(_got)
1220         return d
1221
1222     def test_GET_FILEURL_info_mdmf_extensions(self):
1223         d = self.GET("/uri/%s:STUFF?t=info" % self._quux_txt_uri)
1224         def _got(res):
1225             self.failUnlessIn("mutable file (mdmf)", res)
1226             self.failUnlessIn(self._quux_txt_uri, res)
1227             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1228         d.addCallback(_got)
1229         return d
1230
1231     def test_PUT_overwrite_only_files(self):
1232         # create a directory, put a file in that directory.
1233         contents, n, filecap = self.makefile(8)
1234         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
1235         d.addCallback(lambda res:
1236             self.PUT(self.public_url + "/foo/dir/file1.txt",
1237                      self.NEWFILE_CONTENTS))
1238         # try to overwrite the file with replace=only-files
1239         # (this should work)
1240         d.addCallback(lambda res:
1241             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
1242                      filecap))
1243         d.addCallback(lambda res:
1244             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
1245                  "There was already a child by that name, and you asked me "
1246                  "to not replace it",
1247                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
1248                  filecap))
1249         return d
1250
1251     def test_PUT_NEWFILEURL(self):
1252         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
1253         # TODO: we lose the response code, so we can't check this
1254         #self.failUnlessReallyEqual(responsecode, 201)
1255         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1256         d.addCallback(lambda res:
1257                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1258                                                       self.NEWFILE_CONTENTS))
1259         return d
1260
1261     def test_PUT_NEWFILEURL_not_mutable(self):
1262         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
1263                      self.NEWFILE_CONTENTS)
1264         # TODO: we lose the response code, so we can't check this
1265         #self.failUnlessReallyEqual(responsecode, 201)
1266         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1267         d.addCallback(lambda res:
1268                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1269                                                       self.NEWFILE_CONTENTS))
1270         return d
1271
1272     def test_PUT_NEWFILEURL_unlinked_mdmf(self):
1273         # this should get us a few segments of an MDMF mutable file,
1274         # which we can then test for.
1275         contents = self.NEWFILE_CONTENTS * 300000
1276         d = self.PUT("/uri?format=mdmf",
1277                      contents)
1278         def _got_filecap(filecap):
1279             self.failUnless(filecap.startswith("URI:MDMF"))
1280             return filecap
1281         d.addCallback(_got_filecap)
1282         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1283         d.addCallback(lambda json: self.failUnlessIn("MDMF", json))
1284         return d
1285
1286     def test_PUT_NEWFILEURL_unlinked_sdmf(self):
1287         contents = self.NEWFILE_CONTENTS * 300000
1288         d = self.PUT("/uri?format=sdmf",
1289                      contents)
1290         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1291         d.addCallback(lambda json: self.failUnlessIn("SDMF", json))
1292         return d
1293
1294     def test_PUT_NEWFILEURL_unlinked_bad_format(self):
1295         contents = self.NEWFILE_CONTENTS * 300000
1296         return self.shouldHTTPError("PUT_NEWFILEURL_unlinked_bad_format",
1297                                     400, "Bad Request", "Unknown format: foo",
1298                                     self.PUT, "/uri?format=foo",
1299                                     contents)
1300
1301     def test_PUT_NEWFILEURL_range_bad(self):
1302         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
1303         target = self.public_url + "/foo/new.txt"
1304         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
1305                              "501 Not Implemented",
1306                              "Content-Range in PUT not yet supported",
1307                              # (and certainly not for immutable files)
1308                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
1309                              headers=headers)
1310         d.addCallback(lambda res:
1311                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
1312         return d
1313
1314     def test_PUT_NEWFILEURL_mutable(self):
1315         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1316                      self.NEWFILE_CONTENTS)
1317         # TODO: we lose the response code, so we can't check this
1318         #self.failUnlessReallyEqual(responsecode, 201)
1319         def _check_uri(res):
1320             u = uri.from_string_mutable_filenode(res)
1321             self.failUnless(u.is_mutable())
1322             self.failIf(u.is_readonly())
1323             return res
1324         d.addCallback(_check_uri)
1325         d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
1326         d.addCallback(lambda res:
1327                       self.failUnlessMutableChildContentsAre(self._foo_node,
1328                                                              u"new.txt",
1329                                                              self.NEWFILE_CONTENTS))
1330         return d
1331
1332     def test_PUT_NEWFILEURL_mutable_toobig(self):
1333         # It is okay to upload large mutable files, so we should be able
1334         # to do that.
1335         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1336                      "b" * (self.s.MUTABLE_SIZELIMIT + 1))
1337         return d
1338
1339     def test_PUT_NEWFILEURL_replace(self):
1340         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
1341         # TODO: we lose the response code, so we can't check this
1342         #self.failUnlessReallyEqual(responsecode, 200)
1343         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
1344         d.addCallback(lambda res:
1345                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1346                                                       self.NEWFILE_CONTENTS))
1347         return d
1348
1349     def test_PUT_NEWFILEURL_bad_t(self):
1350         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
1351                              "PUT to a file: bad t=bogus",
1352                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
1353                              "contents")
1354         return d
1355
1356     def test_PUT_NEWFILEURL_no_replace(self):
1357         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
1358                      self.NEWFILE_CONTENTS)
1359         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
1360                   "409 Conflict",
1361                   "There was already a child by that name, and you asked me "
1362                   "to not replace it")
1363         return d
1364
1365     def test_PUT_NEWFILEURL_mkdirs(self):
1366         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
1367         fn = self._foo_node
1368         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
1369         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
1370         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
1371         d.addCallback(lambda res:
1372                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
1373                                                       self.NEWFILE_CONTENTS))
1374         return d
1375
1376     def test_PUT_NEWFILEURL_blocked(self):
1377         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
1378                      self.NEWFILE_CONTENTS)
1379         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
1380                   "409 Conflict",
1381                   "Unable to create directory 'blockingfile': a file was in the way")
1382         return d
1383
1384     def test_PUT_NEWFILEURL_emptyname(self):
1385         # an empty pathname component (i.e. a double-slash) is disallowed
1386         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
1387                              "400 Bad Request",
1388                              "The webapi does not allow empty pathname components",
1389                              self.PUT, self.public_url + "/foo//new.txt", "")
1390         return d
1391
1392     def test_DELETE_FILEURL(self):
1393         d = self.DELETE(self.public_url + "/foo/bar.txt")
1394         d.addCallback(lambda res:
1395                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1396         return d
1397
1398     def test_DELETE_FILEURL_missing(self):
1399         d = self.DELETE(self.public_url + "/foo/missing")
1400         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
1401         return d
1402
1403     def test_DELETE_FILEURL_missing2(self):
1404         d = self.DELETE(self.public_url + "/missing/missing")
1405         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
1406         return d
1407
1408     def failUnlessHasBarDotTxtMetadata(self, res):
1409         data = simplejson.loads(res)
1410         self.failUnless(isinstance(data, list))
1411         self.failUnlessIn("metadata", data[1])
1412         self.failUnlessIn("tahoe", data[1]["metadata"])
1413         self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
1414         self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
1415         self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
1416                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
1417
1418     def test_GET_FILEURL_json(self):
1419         # twisted.web.http.parse_qs ignores any query args without an '=', so
1420         # I can't do "GET /path?json", I have to do "GET /path/t=json"
1421         # instead. This may make it tricky to emulate the S3 interface
1422         # completely.
1423         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
1424         def _check1(data):
1425             self.failUnlessIsBarJSON(data)
1426             self.failUnlessHasBarDotTxtMetadata(data)
1427             return
1428         d.addCallback(_check1)
1429         return d
1430
1431     def test_GET_FILEURL_json_mutable_type(self):
1432         # The JSON should include format, which says whether the
1433         # file is SDMF or MDMF
1434         d = self.PUT("/uri?format=mdmf",
1435                      self.NEWFILE_CONTENTS * 300000)
1436         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1437         def _got_json(json, version):
1438             data = simplejson.loads(json)
1439             assert "filenode" == data[0]
1440             data = data[1]
1441             assert isinstance(data, dict)
1442
1443             self.failUnlessIn("format", data)
1444             self.failUnlessEqual(data["format"], version)
1445
1446         d.addCallback(_got_json, "MDMF")
1447         # Now make an SDMF file and check that it is reported correctly.
1448         d.addCallback(lambda ignored:
1449             self.PUT("/uri?format=sdmf",
1450                       self.NEWFILE_CONTENTS * 300000))
1451         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1452         d.addCallback(_got_json, "SDMF")
1453         return d
1454
1455     def test_GET_FILEURL_json_mdmf(self):
1456         d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
1457         d.addCallback(self.failUnlessIsQuuxJSON)
1458         return d
1459
1460     def test_GET_FILEURL_json_missing(self):
1461         d = self.GET(self.public_url + "/foo/missing?json")
1462         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
1463         return d
1464
1465     def test_GET_FILEURL_uri(self):
1466         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
1467         def _check(res):
1468             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1469         d.addCallback(_check)
1470         d.addCallback(lambda res:
1471                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
1472         def _check2(res):
1473             # for now, for files, uris and readonly-uris are the same
1474             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1475         d.addCallback(_check2)
1476         return d
1477
1478     def test_GET_FILEURL_badtype(self):
1479         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
1480                                  "bad t=bogus",
1481                                  self.GET,
1482                                  self.public_url + "/foo/bar.txt?t=bogus")
1483         return d
1484
1485     def test_CSS_FILE(self):
1486         d = self.GET("/tahoe.css", followRedirect=True)
1487         def _check(res):
1488             CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
1489             self.failUnless(CSS_STYLE.search(res), res)
1490         d.addCallback(_check)
1491         return d
1492
1493     def test_GET_FILEURL_uri_missing(self):
1494         d = self.GET(self.public_url + "/foo/missing?t=uri")
1495         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
1496         return d
1497
1498     def _check_upload_and_mkdir_forms(self, html):
1499         # We should have a form to create a file, with radio buttons that allow
1500         # the user to toggle whether it is a CHK/LIT (default), SDMF, or MDMF file.
1501         self.failUnlessIn('name="t" value="upload"', html)
1502         self.failUnless(re.search('<input [^/]*id="upload-chk"', html), html)
1503         self.failUnless(re.search('<input [^/]*id="upload-sdmf"', html), html)
1504         self.failUnless(re.search('<input [^/]*id="upload-mdmf"', html), html)
1505
1506         # We should also have the ability to create a mutable directory, with
1507         # radio buttons that allow the user to toggle whether it is an SDMF (default)
1508         # or MDMF directory.
1509         self.failUnlessIn('name="t" value="mkdir"', html)
1510         self.failUnless(re.search('<input [^/]*id="mkdir-sdmf"', html), html)
1511         self.failUnless(re.search('<input [^/]*id="mkdir-mdmf"', html), html)
1512
1513         self.failUnlessIn(FAVICON_MARKUP, html)
1514
1515     def test_GET_DIRECTORY_html(self):
1516         d = self.GET(self.public_url + "/foo", followRedirect=True)
1517         def _check(html):
1518             self.failUnlessIn('<div class="toolbar-item"><a href="../../..">Return to Welcome page</a></div>', html)
1519             self._check_upload_and_mkdir_forms(html)
1520             self.failUnlessIn("quux", html)
1521         d.addCallback(_check)
1522         return d
1523
1524     def test_GET_DIRECTORY_html_filenode_encoding(self):
1525         d = self.GET(self.public_url + "/foo", followRedirect=True)
1526         def _check(html):
1527             # Check if encoded entries are there
1528             self.failUnlessIn('@@named=/' + self._htmlname_urlencoded + '">'
1529                               + self._htmlname_escaped + '</a>', html)
1530             self.failUnlessIn('value="' + self._htmlname_escaped_attr + '"', html)
1531             self.failIfIn(self._htmlname_escaped_double, html)
1532             # Make sure that Nevow escaping actually works by checking for unsafe characters
1533             # and that '&' is escaped.
1534             for entity in '<>':
1535                 self.failUnlessIn(entity, self._htmlname_raw)
1536                 self.failIfIn(entity, self._htmlname_escaped)
1537             self.failUnlessIn('&', re.sub(r'&(amp|lt|gt|quot|apos);', '', self._htmlname_raw))
1538             self.failIfIn('&', re.sub(r'&(amp|lt|gt|quot|apos);', '', self._htmlname_escaped))
1539         d.addCallback(_check)
1540         return d
1541
1542     def test_GET_root_html(self):
1543         d = self.GET("/")
1544         d.addCallback(self._check_upload_and_mkdir_forms)
1545         return d
1546
1547     def test_GET_DIRURL(self):
1548         # the addSlash means we get a redirect here
1549         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1550         ROOT = "../../.."
1551         d = self.GET(self.public_url + "/foo", followRedirect=True)
1552         def _check(res):
1553             self.failUnlessIn('<a href="%s">Return to Welcome page' % ROOT, res)
1554
1555             # the FILE reference points to a URI, but it should end in bar.txt
1556             bar_url = ("%s/file/%s/@@named=/bar.txt" %
1557                        (ROOT, urllib.quote(self._bar_txt_uri)))
1558             get_bar = "".join([r'<td>FILE</td>',
1559                                r'\s+<td>',
1560                                r'<a href="%s">bar.txt</a>' % bar_url,
1561                                r'</td>',
1562                                r'\s+<td align="right">%d</td>' % len(self.BAR_CONTENTS),
1563                                ])
1564             self.failUnless(re.search(get_bar, res), res)
1565             for label in ['unlink', 'rename/relink']:
1566                 for line in res.split("\n"):
1567                     # find the line that contains the relevant button for bar.txt
1568                     if ("form action" in line and
1569                         ('value="%s"' % (label,)) in line and
1570                         'value="bar.txt"' in line):
1571                         # the form target should use a relative URL
1572                         foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1573                         self.failUnlessIn('action="%s"' % foo_url, line)
1574                         # and the when_done= should too
1575                         #done_url = urllib.quote(???)
1576                         #self.failUnlessIn('name="when_done" value="%s"' % done_url, line)
1577
1578                         # 'unlink' needs to use POST because it directly has a side effect
1579                         if label == 'unlink':
1580                             self.failUnlessIn('method="post"', line)
1581                         break
1582                 else:
1583                     self.fail("unable to find '%s bar.txt' line" % (label,))
1584
1585             # the DIR reference just points to a URI
1586             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1587             get_sub = ((r'<td>DIR</td>')
1588                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1589             self.failUnless(re.search(get_sub, res), res)
1590         d.addCallback(_check)
1591
1592         # look at a readonly directory
1593         d.addCallback(lambda res:
1594                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
1595         def _check2(res):
1596             self.failUnlessIn("(read-only)", res)
1597             self.failIfIn("Upload a file", res)
1598         d.addCallback(_check2)
1599
1600         # and at a directory that contains a readonly directory
1601         d.addCallback(lambda res:
1602                       self.GET(self.public_url, followRedirect=True))
1603         def _check3(res):
1604             self.failUnless(re.search('<td>DIR-RO</td>'
1605                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1606         d.addCallback(_check3)
1607
1608         # and an empty directory
1609         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1610         def _check4(res):
1611             self.failUnlessIn("directory is empty", res)
1612             MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
1613             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1614         d.addCallback(_check4)
1615
1616         # and at a literal directory
1617         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1618         d.addCallback(lambda res:
1619                       self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1620         def _check5(res):
1621             self.failUnlessIn('(immutable)', res)
1622             self.failUnless(re.search('<td>FILE</td>'
1623                                       r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1624         d.addCallback(_check5)
1625         return d
1626
1627     def test_GET_DIRURL_badtype(self):
1628         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1629                                  400, "Bad Request",
1630                                  "bad t=bogus",
1631                                  self.GET,
1632                                  self.public_url + "/foo?t=bogus")
1633         return d
1634
1635     def test_GET_DIRURL_json(self):
1636         d = self.GET(self.public_url + "/foo?t=json")
1637         d.addCallback(self.failUnlessIsFooJSON)
1638         return d
1639
1640     def test_GET_DIRURL_json_format(self):
1641         d = self.PUT(self.public_url + \
1642                      "/foo/sdmf.txt?format=sdmf",
1643                      self.NEWFILE_CONTENTS * 300000)
1644         d.addCallback(lambda ignored:
1645             self.PUT(self.public_url + \
1646                      "/foo/mdmf.txt?format=mdmf",
1647                      self.NEWFILE_CONTENTS * 300000))
1648         # Now we have an MDMF and SDMF file in the directory. If we GET
1649         # its JSON, we should see their encodings.
1650         d.addCallback(lambda ignored:
1651             self.GET(self.public_url + "/foo?t=json"))
1652         def _got_json(json):
1653             data = simplejson.loads(json)
1654             assert data[0] == "dirnode"
1655
1656             data = data[1]
1657             kids = data['children']
1658
1659             mdmf_data = kids['mdmf.txt'][1]
1660             self.failUnlessIn("format", mdmf_data)
1661             self.failUnlessEqual(mdmf_data["format"], "MDMF")
1662
1663             sdmf_data = kids['sdmf.txt'][1]
1664             self.failUnlessIn("format", sdmf_data)
1665             self.failUnlessEqual(sdmf_data["format"], "SDMF")
1666         d.addCallback(_got_json)
1667         return d
1668
1669
1670     def test_POST_DIRURL_manifest_no_ophandle(self):
1671         d = self.shouldFail2(error.Error,
1672                              "test_POST_DIRURL_manifest_no_ophandle",
1673                              "400 Bad Request",
1674                              "slow operation requires ophandle=",
1675                              self.POST, self.public_url, t="start-manifest")
1676         return d
1677
1678     def test_POST_DIRURL_manifest(self):
1679         d = defer.succeed(None)
1680         def getman(ignored, output):
1681             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1682                           followRedirect=True)
1683             d.addCallback(self.wait_for_operation, "125")
1684             d.addCallback(self.get_operation_results, "125", output)
1685             return d
1686         d.addCallback(getman, None)
1687         def _got_html(manifest):
1688             self.failUnlessIn("Manifest of SI=", manifest)
1689             self.failUnlessIn("<td>sub</td>", manifest)
1690             self.failUnlessIn(self._sub_uri, manifest)
1691             self.failUnlessIn("<td>sub/baz.txt</td>", manifest)
1692             self.failUnlessIn(FAVICON_MARKUP, manifest)
1693         d.addCallback(_got_html)
1694
1695         # both t=status and unadorned GET should be identical
1696         d.addCallback(lambda res: self.GET("/operations/125"))
1697         d.addCallback(_got_html)
1698
1699         d.addCallback(getman, "html")
1700         d.addCallback(_got_html)
1701         d.addCallback(getman, "text")
1702         def _got_text(manifest):
1703             self.failUnlessIn("\nsub " + self._sub_uri + "\n", manifest)
1704             self.failUnlessIn("\nsub/baz.txt URI:CHK:", manifest)
1705         d.addCallback(_got_text)
1706         d.addCallback(getman, "JSON")
1707         def _got_json(res):
1708             data = res["manifest"]
1709             got = {}
1710             for (path_list, cap) in data:
1711                 got[tuple(path_list)] = cap
1712             self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
1713             self.failUnlessIn((u"sub", u"baz.txt"), got)
1714             self.failUnlessIn("finished", res)
1715             self.failUnlessIn("origin", res)
1716             self.failUnlessIn("storage-index", res)
1717             self.failUnlessIn("verifycaps", res)
1718             self.failUnlessIn("stats", res)
1719         d.addCallback(_got_json)
1720         return d
1721
1722     def test_POST_DIRURL_deepsize_no_ophandle(self):
1723         d = self.shouldFail2(error.Error,
1724                              "test_POST_DIRURL_deepsize_no_ophandle",
1725                              "400 Bad Request",
1726                              "slow operation requires ophandle=",
1727                              self.POST, self.public_url, t="start-deep-size")
1728         return d
1729
1730     def test_POST_DIRURL_deepsize(self):
1731         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1732                       followRedirect=True)
1733         d.addCallback(self.wait_for_operation, "126")
1734         d.addCallback(self.get_operation_results, "126", "json")
1735         def _got_json(data):
1736             self.failUnlessReallyEqual(data["finished"], True)
1737             size = data["size"]
1738             self.failUnless(size > 1000)
1739         d.addCallback(_got_json)
1740         d.addCallback(self.get_operation_results, "126", "text")
1741         def _got_text(res):
1742             mo = re.search(r'^size: (\d+)$', res, re.M)
1743             self.failUnless(mo, res)
1744             size = int(mo.group(1))
1745             # with directories, the size varies.
1746             self.failUnless(size > 1000)
1747         d.addCallback(_got_text)
1748         return d
1749
1750     def test_POST_DIRURL_deepstats_no_ophandle(self):
1751         d = self.shouldFail2(error.Error,
1752                              "test_POST_DIRURL_deepstats_no_ophandle",
1753                              "400 Bad Request",
1754                              "slow operation requires ophandle=",
1755                              self.POST, self.public_url, t="start-deep-stats")
1756         return d
1757
1758     def test_POST_DIRURL_deepstats(self):
1759         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1760                       followRedirect=True)
1761         d.addCallback(self.wait_for_operation, "127")
1762         d.addCallback(self.get_operation_results, "127", "json")
1763         def _got_json(stats):
1764             expected = {"count-immutable-files": 4,
1765                         "count-mutable-files": 2,
1766                         "count-literal-files": 0,
1767                         "count-files": 6,
1768                         "count-directories": 3,
1769                         "size-immutable-files": 76,
1770                         "size-literal-files": 0,
1771                         #"size-directories": 1912, # varies
1772                         #"largest-directory": 1590,
1773                         "largest-directory-children": 8,
1774                         "largest-immutable-file": 19,
1775                         }
1776             for k,v in expected.iteritems():
1777                 self.failUnlessReallyEqual(stats[k], v,
1778                                            "stats[%s] was %s, not %s" %
1779                                            (k, stats[k], v))
1780             self.failUnlessReallyEqual(stats["size-files-histogram"],
1781                                        [ [11, 31, 4] ])
1782         d.addCallback(_got_json)
1783         return d
1784
1785     def test_POST_DIRURL_stream_manifest(self):
1786         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1787         def _check(res):
1788             self.failUnless(res.endswith("\n"))
1789             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1790             self.failUnlessReallyEqual(len(units), 10)
1791             self.failUnlessEqual(units[-1]["type"], "stats")
1792             first = units[0]
1793             self.failUnlessEqual(first["path"], [])
1794             self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri)
1795             self.failUnlessEqual(first["type"], "directory")
1796             baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0]
1797             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1798             self.failIfEqual(baz["storage-index"], None)
1799             self.failIfEqual(baz["verifycap"], None)
1800             self.failIfEqual(baz["repaircap"], None)
1801             # XXX: Add quux and baz to this test.
1802             return
1803         d.addCallback(_check)
1804         return d
1805
1806     def test_GET_DIRURL_uri(self):
1807         d = self.GET(self.public_url + "/foo?t=uri")
1808         def _check(res):
1809             self.failUnlessReallyEqual(to_str(res), self._foo_uri)
1810         d.addCallback(_check)
1811         return d
1812
1813     def test_GET_DIRURL_readonly_uri(self):
1814         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1815         def _check(res):
1816             self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri)
1817         d.addCallback(_check)
1818         return d
1819
1820     def test_PUT_NEWDIRURL(self):
1821         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1822         d.addCallback(lambda res:
1823                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1824         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1825         d.addCallback(self.failUnlessNodeKeysAre, [])
1826         return d
1827
1828     def test_PUT_NEWDIRURL_mdmf(self):
1829         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1830         d.addCallback(lambda res:
1831                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1832         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1833         d.addCallback(lambda node:
1834             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1835         return d
1836
1837     def test_PUT_NEWDIRURL_sdmf(self):
1838         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=sdmf",
1839                      "")
1840         d.addCallback(lambda res:
1841                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1842         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1843         d.addCallback(lambda node:
1844             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1845         return d
1846
1847     def test_PUT_NEWDIRURL_bad_format(self):
1848         return self.shouldHTTPError("PUT_NEWDIRURL_bad_format",
1849                                     400, "Bad Request", "Unknown format: foo",
1850                                     self.PUT, self.public_url +
1851                                     "/foo/newdir=?t=mkdir&format=foo", "")
1852
1853     def test_POST_NEWDIRURL(self):
1854         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1855         d.addCallback(lambda res:
1856                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1857         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1858         d.addCallback(self.failUnlessNodeKeysAre, [])
1859         return d
1860
1861     def test_POST_NEWDIRURL_mdmf(self):
1862         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1863         d.addCallback(lambda res:
1864                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1865         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1866         d.addCallback(lambda node:
1867             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1868         return d
1869
1870     def test_POST_NEWDIRURL_sdmf(self):
1871         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=sdmf", "")
1872         d.addCallback(lambda res:
1873             self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1874         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1875         d.addCallback(lambda node:
1876             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1877         return d
1878
1879     def test_POST_NEWDIRURL_bad_format(self):
1880         return self.shouldHTTPError("POST_NEWDIRURL_bad_format",
1881                                     400, "Bad Request", "Unknown format: foo",
1882                                     self.POST2, self.public_url + \
1883                                     "/foo/newdir?t=mkdir&format=foo", "")
1884
1885     def test_POST_NEWDIRURL_emptyname(self):
1886         # an empty pathname component (i.e. a double-slash) is disallowed
1887         d = self.shouldFail2(error.Error, "POST_NEWDIRURL_emptyname",
1888                              "400 Bad Request",
1889                              "The webapi does not allow empty pathname components, i.e. a double slash",
1890                              self.POST, self.public_url + "//?t=mkdir")
1891         return d
1892
1893     def _do_POST_NEWDIRURL_initial_children_test(self, version=None):
1894         (newkids, caps) = self._create_initial_children()
1895         query = "/foo/newdir?t=mkdir-with-children"
1896         if version == MDMF_VERSION:
1897             query += "&format=mdmf"
1898         elif version == SDMF_VERSION:
1899             query += "&format=sdmf"
1900         else:
1901             version = SDMF_VERSION # for later
1902         d = self.POST2(self.public_url + query,
1903                        simplejson.dumps(newkids))
1904         def _check(uri):
1905             n = self.s.create_node_from_uri(uri.strip())
1906             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1907             self.failUnlessEqual(n._node.get_version(), version)
1908             d2.addCallback(lambda ign:
1909                            self.failUnlessROChildURIIs(n, u"child-imm",
1910                                                        caps['filecap1']))
1911             d2.addCallback(lambda ign:
1912                            self.failUnlessRWChildURIIs(n, u"child-mutable",
1913                                                        caps['filecap2']))
1914             d2.addCallback(lambda ign:
1915                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1916                                                        caps['filecap3']))
1917             d2.addCallback(lambda ign:
1918                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1919                                                        caps['unknown_rocap']))
1920             d2.addCallback(lambda ign:
1921                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1922                                                        caps['unknown_rwcap']))
1923             d2.addCallback(lambda ign:
1924                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1925                                                        caps['unknown_immcap']))
1926             d2.addCallback(lambda ign:
1927                            self.failUnlessRWChildURIIs(n, u"dirchild",
1928                                                        caps['dircap']))
1929             d2.addCallback(lambda ign:
1930                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1931                                                        caps['litdircap']))
1932             d2.addCallback(lambda ign:
1933                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1934                                                        caps['emptydircap']))
1935             return d2
1936         d.addCallback(_check)
1937         d.addCallback(lambda res:
1938                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1939         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1940         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1941         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1942         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1943         return d
1944
1945     def test_POST_NEWDIRURL_initial_children(self):
1946         return self._do_POST_NEWDIRURL_initial_children_test()
1947
1948     def test_POST_NEWDIRURL_initial_children_mdmf(self):
1949         return self._do_POST_NEWDIRURL_initial_children_test(MDMF_VERSION)
1950
1951     def test_POST_NEWDIRURL_initial_children_sdmf(self):
1952         return self._do_POST_NEWDIRURL_initial_children_test(SDMF_VERSION)
1953
1954     def test_POST_NEWDIRURL_initial_children_bad_format(self):
1955         (newkids, caps) = self._create_initial_children()
1956         return self.shouldHTTPError("POST_NEWDIRURL_initial_children_bad_format",
1957                                     400, "Bad Request", "Unknown format: foo",
1958                                     self.POST2, self.public_url + \
1959                                     "/foo/newdir?t=mkdir-with-children&format=foo",
1960                                     simplejson.dumps(newkids))
1961
1962     def test_POST_NEWDIRURL_immutable(self):
1963         (newkids, caps) = self._create_immutable_children()
1964         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1965                        simplejson.dumps(newkids))
1966         def _check(uri):
1967             n = self.s.create_node_from_uri(uri.strip())
1968             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1969             d2.addCallback(lambda ign:
1970                            self.failUnlessROChildURIIs(n, u"child-imm",
1971                                                        caps['filecap1']))
1972             d2.addCallback(lambda ign:
1973                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1974                                                        caps['unknown_immcap']))
1975             d2.addCallback(lambda ign:
1976                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
1977                                                        caps['immdircap']))
1978             d2.addCallback(lambda ign:
1979                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1980                                                        caps['litdircap']))
1981             d2.addCallback(lambda ign:
1982                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1983                                                        caps['emptydircap']))
1984             return d2
1985         d.addCallback(_check)
1986         d.addCallback(lambda res:
1987                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1988         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1989         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1990         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1991         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1992         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1993         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1994         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1995         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
1996         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1997         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
1998         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1999         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2000         d.addErrback(self.explain_web_error)
2001         return d
2002
2003     def test_POST_NEWDIRURL_immutable_bad(self):
2004         (newkids, caps) = self._create_initial_children()
2005         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
2006                              "400 Bad Request",
2007                              "needed to be immutable but was not",
2008                              self.POST2,
2009                              self.public_url + "/foo/newdir?t=mkdir-immutable",
2010                              simplejson.dumps(newkids))
2011         return d
2012
2013     def test_PUT_NEWDIRURL_exists(self):
2014         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
2015         d.addCallback(lambda res:
2016                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
2017         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2018         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2019         return d
2020
2021     def test_PUT_NEWDIRURL_blocked(self):
2022         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
2023                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
2024                              self.PUT,
2025                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
2026         d.addCallback(lambda res:
2027                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
2028         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2029         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2030         return d
2031
2032     def test_PUT_NEWDIRURL_mkdirs(self):
2033         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
2034         d.addCallback(lambda res:
2035                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
2036         d.addCallback(lambda res:
2037                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
2038         d.addCallback(lambda res:
2039                       self._foo_node.get_child_at_path(u"subdir/newdir"))
2040         d.addCallback(self.failUnlessNodeKeysAre, [])
2041         return d
2042
2043     def test_PUT_NEWDIRURL_mkdirs_mdmf(self):
2044         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=mdmf", "")
2045         d.addCallback(lambda ignored:
2046             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
2047         d.addCallback(lambda ignored:
2048             self.failIfNodeHasChild(self._foo_node, u"newdir"))
2049         d.addCallback(lambda ignored:
2050             self._foo_node.get_child_at_path(u"subdir"))
2051         def _got_subdir(subdir):
2052             # XXX: What we want?
2053             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
2054             self.failUnlessNodeHasChild(subdir, u"newdir")
2055             return subdir.get_child_at_path(u"newdir")
2056         d.addCallback(_got_subdir)
2057         d.addCallback(lambda newdir:
2058             self.failUnlessEqual(newdir._node.get_version(), MDMF_VERSION))
2059         return d
2060
2061     def test_PUT_NEWDIRURL_mkdirs_sdmf(self):
2062         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=sdmf", "")
2063         d.addCallback(lambda ignored:
2064             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
2065         d.addCallback(lambda ignored:
2066             self.failIfNodeHasChild(self._foo_node, u"newdir"))
2067         d.addCallback(lambda ignored:
2068             self._foo_node.get_child_at_path(u"subdir"))
2069         def _got_subdir(subdir):
2070             # XXX: What we want?
2071             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
2072             self.failUnlessNodeHasChild(subdir, u"newdir")
2073             return subdir.get_child_at_path(u"newdir")
2074         d.addCallback(_got_subdir)
2075         d.addCallback(lambda newdir:
2076             self.failUnlessEqual(newdir._node.get_version(), SDMF_VERSION))
2077         return d
2078
2079     def test_PUT_NEWDIRURL_mkdirs_bad_format(self):
2080         return self.shouldHTTPError("PUT_NEWDIRURL_mkdirs_bad_format",
2081                                     400, "Bad Request", "Unknown format: foo",
2082                                     self.PUT, self.public_url + \
2083                                     "/foo/subdir/newdir?t=mkdir&format=foo",
2084                                     "")
2085
2086     def test_DELETE_DIRURL(self):
2087         d = self.DELETE(self.public_url + "/foo")
2088         d.addCallback(lambda res:
2089                       self.failIfNodeHasChild(self.public_root, u"foo"))
2090         return d
2091
2092     def test_DELETE_DIRURL_missing(self):
2093         d = self.DELETE(self.public_url + "/foo/missing")
2094         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
2095         d.addCallback(lambda res:
2096                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
2097         return d
2098
2099     def test_DELETE_DIRURL_missing2(self):
2100         d = self.DELETE(self.public_url + "/missing")
2101         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
2102         return d
2103
2104     def dump_root(self):
2105         print "NODEWALK"
2106         w = webish.DirnodeWalkerMixin()
2107         def visitor(childpath, childnode, metadata):
2108             print childpath
2109         d = w.walk(self.public_root, visitor)
2110         return d
2111
2112     def failUnlessNodeKeysAre(self, node, expected_keys):
2113         for k in expected_keys:
2114             assert isinstance(k, unicode)
2115         d = node.list()
2116         def _check(children):
2117             self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
2118         d.addCallback(_check)
2119         return d
2120     def failUnlessNodeHasChild(self, node, name):
2121         assert isinstance(name, unicode)
2122         d = node.list()
2123         def _check(children):
2124             self.failUnlessIn(name, children)
2125         d.addCallback(_check)
2126         return d
2127     def failIfNodeHasChild(self, node, name):
2128         assert isinstance(name, unicode)
2129         d = node.list()
2130         def _check(children):
2131             self.failIfIn(name, children)
2132         d.addCallback(_check)
2133         return d
2134
2135     def failUnlessChildContentsAre(self, node, name, expected_contents):
2136         assert isinstance(name, unicode)
2137         d = node.get_child_at_path(name)
2138         d.addCallback(lambda node: download_to_data(node))
2139         def _check(contents):
2140             self.failUnlessReallyEqual(contents, expected_contents)
2141         d.addCallback(_check)
2142         return d
2143
2144     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
2145         assert isinstance(name, unicode)
2146         d = node.get_child_at_path(name)
2147         d.addCallback(lambda node: node.download_best_version())
2148         def _check(contents):
2149             self.failUnlessReallyEqual(contents, expected_contents)
2150         d.addCallback(_check)
2151         return d
2152
2153     def failUnlessRWChildURIIs(self, node, name, expected_uri):
2154         assert isinstance(name, unicode)
2155         d = node.get_child_at_path(name)
2156         def _check(child):
2157             self.failUnless(child.is_unknown() or not child.is_readonly())
2158             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
2159             self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
2160             expected_ro_uri = self._make_readonly(expected_uri)
2161             if expected_ro_uri:
2162                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
2163         d.addCallback(_check)
2164         return d
2165
2166     def failUnlessROChildURIIs(self, node, name, expected_uri):
2167         assert isinstance(name, unicode)
2168         d = node.get_child_at_path(name)
2169         def _check(child):
2170             self.failUnless(child.is_unknown() or child.is_readonly())
2171             self.failUnlessReallyEqual(child.get_write_uri(), None)
2172             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
2173             self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
2174         d.addCallback(_check)
2175         return d
2176
2177     def failUnlessURIMatchesRWChild(self, got_uri, node, name):
2178         assert isinstance(name, unicode)
2179         d = node.get_child_at_path(name)
2180         def _check(child):
2181             self.failUnless(child.is_unknown() or not child.is_readonly())
2182             self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
2183             self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
2184             expected_ro_uri = self._make_readonly(got_uri)
2185             if expected_ro_uri:
2186                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
2187         d.addCallback(_check)
2188         return d
2189
2190     def failUnlessURIMatchesROChild(self, got_uri, node, name):
2191         assert isinstance(name, unicode)
2192         d = node.get_child_at_path(name)
2193         def _check(child):
2194             self.failUnless(child.is_unknown() or child.is_readonly())
2195             self.failUnlessReallyEqual(child.get_write_uri(), None)
2196             self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
2197             self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
2198         d.addCallback(_check)
2199         return d
2200
2201     def failUnlessCHKURIHasContents(self, got_uri, contents):
2202         self.failUnless(self.get_all_contents()[got_uri] == contents)
2203
2204     def test_POST_upload(self):
2205         d = self.POST(self.public_url + "/foo", t="upload",
2206                       file=("new.txt", self.NEWFILE_CONTENTS))
2207         fn = self._foo_node
2208         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2209         d.addCallback(lambda res:
2210                       self.failUnlessChildContentsAre(fn, u"new.txt",
2211                                                       self.NEWFILE_CONTENTS))
2212         return d
2213
2214     def test_POST_upload_unicode(self):
2215         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2216         d = self.POST(self.public_url + "/foo", t="upload",
2217                       file=(filename, self.NEWFILE_CONTENTS))
2218         fn = self._foo_node
2219         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2220         d.addCallback(lambda res:
2221                       self.failUnlessChildContentsAre(fn, filename,
2222                                                       self.NEWFILE_CONTENTS))
2223         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2224         d.addCallback(lambda res: self.GET(target_url))
2225         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2226                                                                   self.NEWFILE_CONTENTS,
2227                                                                   contents))
2228         return d
2229
2230     def test_POST_upload_unicode_named(self):
2231         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2232         d = self.POST(self.public_url + "/foo", t="upload",
2233                       name=filename,
2234                       file=("overridden", self.NEWFILE_CONTENTS))
2235         fn = self._foo_node
2236         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2237         d.addCallback(lambda res:
2238                       self.failUnlessChildContentsAre(fn, filename,
2239                                                       self.NEWFILE_CONTENTS))
2240         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2241         d.addCallback(lambda res: self.GET(target_url))
2242         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2243                                                                   self.NEWFILE_CONTENTS,
2244                                                                   contents))
2245         return d
2246
2247     def test_POST_upload_no_link(self):
2248         d = self.POST("/uri", t="upload",
2249                       file=("new.txt", self.NEWFILE_CONTENTS))
2250         def _check_upload_results(page):
2251             # this should be a page which describes the results of the upload
2252             # that just finished.
2253             self.failUnlessIn("Upload Results:", page)
2254             self.failUnlessIn("URI:", page)
2255             uri_re = re.compile("URI: <tt><span>(.*)</span>")
2256             mo = uri_re.search(page)
2257             self.failUnless(mo, page)
2258             new_uri = mo.group(1)
2259             return new_uri
2260         d.addCallback(_check_upload_results)
2261         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
2262         return d
2263
2264     def test_POST_upload_no_link_whendone(self):
2265         d = self.POST("/uri", t="upload", when_done="/",
2266                       file=("new.txt", self.NEWFILE_CONTENTS))
2267         d.addBoth(self.shouldRedirect, "/")
2268         return d
2269
2270     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
2271         d = defer.maybeDeferred(callable, *args, **kwargs)
2272         def done(res):
2273             if isinstance(res, failure.Failure):
2274                 res.trap(error.PageRedirect)
2275                 statuscode = res.value.status
2276                 target = res.value.location
2277                 return checker(statuscode, target)
2278             self.fail("%s: callable was supposed to redirect, not return '%s'"
2279                       % (which, res))
2280         d.addBoth(done)
2281         return d
2282
2283     def test_POST_upload_no_link_whendone_results(self):
2284         def check(statuscode, target):
2285             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2286             self.failUnless(target.startswith(self.webish_url), target)
2287             return client.getPage(target, method="GET")
2288         # We encode "uri" as "%75ri" to exercise a case affected by ticket #1860.
2289         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
2290                                  check,
2291                                  self.POST, "/uri", t="upload",
2292                                  when_done="/%75ri/%(uri)s",
2293                                  file=("new.txt", self.NEWFILE_CONTENTS))
2294         d.addCallback(lambda res:
2295                       self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
2296         return d
2297
2298     def test_POST_upload_no_link_mutable(self):
2299         d = self.POST("/uri", t="upload", mutable="true",
2300                       file=("new.txt", self.NEWFILE_CONTENTS))
2301         def _check(filecap):
2302             filecap = filecap.strip()
2303             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2304             self.filecap = filecap
2305             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2306             self.failUnlessIn(u.get_storage_index(), self.get_all_contents())
2307             n = self.s.create_node_from_uri(filecap)
2308             return n.download_best_version()
2309         d.addCallback(_check)
2310         def _check2(data):
2311             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2312             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2313         d.addCallback(_check2)
2314         def _check3(data):
2315             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2316             return self.GET("/file/%s" % urllib.quote(self.filecap))
2317         d.addCallback(_check3)
2318         def _check4(data):
2319             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2320         d.addCallback(_check4)
2321         return d
2322
2323     def test_POST_upload_no_link_mutable_toobig(self):
2324         # The SDMF size limit is no longer in place, so we should be
2325         # able to upload mutable files that are as large as we want them
2326         # to be.
2327         d = self.POST("/uri", t="upload", mutable="true",
2328                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2329         return d
2330
2331
2332     def test_POST_upload_format_unlinked(self):
2333         def _check_upload_unlinked(ign, format, uri_prefix):
2334             filename = format + ".txt"
2335             d = self.POST("/uri?t=upload&format=" + format,
2336                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2337             def _got_results(results):
2338                 if format.upper() in ("SDMF", "MDMF"):
2339                     # webapi.rst says this returns a filecap
2340                     filecap = results
2341                 else:
2342                     # for immutable, it returns an "upload results page", and
2343                     # the filecap is buried inside
2344                     line = [l for l in results.split("\n") if "URI: " in l][0]
2345                     mo = re.search(r'<span>([^<]+)</span>', line)
2346                     filecap = mo.group(1)
2347                 self.failUnless(filecap.startswith(uri_prefix),
2348                                 (uri_prefix, filecap))
2349                 return self.GET("/uri/%s?t=json" % filecap)
2350             d.addCallback(_got_results)
2351             def _got_json(json):
2352                 data = simplejson.loads(json)
2353                 data = data[1]
2354                 self.failUnlessIn("format", data)
2355                 self.failUnlessEqual(data["format"], format.upper())
2356             d.addCallback(_got_json)
2357             return d
2358         d = defer.succeed(None)
2359         d.addCallback(_check_upload_unlinked, "chk", "URI:CHK")
2360         d.addCallback(_check_upload_unlinked, "CHK", "URI:CHK")
2361         d.addCallback(_check_upload_unlinked, "sdmf", "URI:SSK")
2362         d.addCallback(_check_upload_unlinked, "mdmf", "URI:MDMF")
2363         return d
2364
2365     def test_POST_upload_bad_format_unlinked(self):
2366         return self.shouldHTTPError("POST_upload_bad_format_unlinked",
2367                                     400, "Bad Request", "Unknown format: foo",
2368                                     self.POST,
2369                                     "/uri?t=upload&format=foo",
2370                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2371
2372     def test_POST_upload_format(self):
2373         def _check_upload(ign, format, uri_prefix, fn=None):
2374             filename = format + ".txt"
2375             d = self.POST(self.public_url +
2376                           "/foo?t=upload&format=" + format,
2377                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2378             def _got_filecap(filecap):
2379                 if fn is not None:
2380                     filenameu = unicode(filename)
2381                     self.failUnlessURIMatchesRWChild(filecap, fn, filenameu)
2382                 self.failUnless(filecap.startswith(uri_prefix))
2383                 return self.GET(self.public_url + "/foo/%s?t=json" % filename)
2384             d.addCallback(_got_filecap)
2385             def _got_json(json):
2386                 data = simplejson.loads(json)
2387                 data = data[1]
2388                 self.failUnlessIn("format", data)
2389                 self.failUnlessEqual(data["format"], format.upper())
2390             d.addCallback(_got_json)
2391             return d
2392
2393         d = defer.succeed(None)
2394         d.addCallback(_check_upload, "chk", "URI:CHK")
2395         d.addCallback(_check_upload, "sdmf", "URI:SSK", self._foo_node)
2396         d.addCallback(_check_upload, "mdmf", "URI:MDMF")
2397         d.addCallback(_check_upload, "MDMF", "URI:MDMF")
2398         return d
2399
2400     def test_POST_upload_bad_format(self):
2401         return self.shouldHTTPError("POST_upload_bad_format",
2402                                     400, "Bad Request", "Unknown format: foo",
2403                                     self.POST, self.public_url + \
2404                                     "/foo?t=upload&format=foo",
2405                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2406
2407     def test_POST_upload_mutable(self):
2408         # this creates a mutable file
2409         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
2410                       file=("new.txt", self.NEWFILE_CONTENTS))
2411         fn = self._foo_node
2412         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2413         d.addCallback(lambda res:
2414                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2415                                                              self.NEWFILE_CONTENTS))
2416         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2417         def _got(newnode):
2418             self.failUnless(IMutableFileNode.providedBy(newnode))
2419             self.failUnless(newnode.is_mutable())
2420             self.failIf(newnode.is_readonly())
2421             self._mutable_node = newnode
2422             self._mutable_uri = newnode.get_uri()
2423         d.addCallback(_got)
2424
2425         # now upload it again and make sure that the URI doesn't change
2426         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
2427         d.addCallback(lambda res:
2428                       self.POST(self.public_url + "/foo", t="upload",
2429                                 mutable="true",
2430                                 file=("new.txt", NEWER_CONTENTS)))
2431         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2432         d.addCallback(lambda res:
2433                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2434                                                              NEWER_CONTENTS))
2435         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2436         def _got2(newnode):
2437             self.failUnless(IMutableFileNode.providedBy(newnode))
2438             self.failUnless(newnode.is_mutable())
2439             self.failIf(newnode.is_readonly())
2440             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2441         d.addCallback(_got2)
2442
2443         # upload a second time, using PUT instead of POST
2444         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
2445         d.addCallback(lambda res:
2446                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
2447         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2448         d.addCallback(lambda res:
2449                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2450                                                              NEW2_CONTENTS))
2451
2452         # finally list the directory, since mutable files are displayed
2453         # slightly differently
2454
2455         d.addCallback(lambda res:
2456                       self.GET(self.public_url + "/foo/",
2457                                followRedirect=True))
2458         def _check_page(res):
2459             # TODO: assert more about the contents
2460             self.failUnlessIn("SSK", res)
2461             return res
2462         d.addCallback(_check_page)
2463
2464         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2465         def _got3(newnode):
2466             self.failUnless(IMutableFileNode.providedBy(newnode))
2467             self.failUnless(newnode.is_mutable())
2468             self.failIf(newnode.is_readonly())
2469             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2470         d.addCallback(_got3)
2471
2472         # look at the JSON form of the enclosing directory
2473         d.addCallback(lambda res:
2474                       self.GET(self.public_url + "/foo/?t=json",
2475                                followRedirect=True))
2476         def _check_page_json(res):
2477             parsed = simplejson.loads(res)
2478             self.failUnlessEqual(parsed[0], "dirnode")
2479             children = dict( [(unicode(name),value)
2480                               for (name,value)
2481                               in parsed[1]["children"].iteritems()] )
2482             self.failUnlessIn(u"new.txt", children)
2483             new_json = children[u"new.txt"]
2484             self.failUnlessEqual(new_json[0], "filenode")
2485             self.failUnless(new_json[1]["mutable"])
2486             self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri)
2487             ro_uri = self._mutable_node.get_readonly().to_string()
2488             self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri)
2489         d.addCallback(_check_page_json)
2490
2491         # and the JSON form of the file
2492         d.addCallback(lambda res:
2493                       self.GET(self.public_url + "/foo/new.txt?t=json"))
2494         def _check_file_json(res):
2495             parsed = simplejson.loads(res)
2496             self.failUnlessEqual(parsed[0], "filenode")
2497             self.failUnless(parsed[1]["mutable"])
2498             self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri)
2499             ro_uri = self._mutable_node.get_readonly().to_string()
2500             self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri)
2501         d.addCallback(_check_file_json)
2502
2503         # and look at t=uri and t=readonly-uri
2504         d.addCallback(lambda res:
2505                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
2506         d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
2507         d.addCallback(lambda res:
2508                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
2509         def _check_ro_uri(res):
2510             ro_uri = self._mutable_node.get_readonly().to_string()
2511             self.failUnlessReallyEqual(res, ro_uri)
2512         d.addCallback(_check_ro_uri)
2513
2514         # make sure we can get to it from /uri/URI
2515         d.addCallback(lambda res:
2516                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
2517         d.addCallback(lambda res:
2518                       self.failUnlessReallyEqual(res, NEW2_CONTENTS))
2519
2520         # and that HEAD computes the size correctly
2521         d.addCallback(lambda res:
2522                       self.HEAD(self.public_url + "/foo/new.txt",
2523                                 return_response=True))
2524         def _got_headers((res, status, headers)):
2525             self.failUnlessReallyEqual(res, "")
2526             self.failUnlessReallyEqual(headers["content-length"][0],
2527                                        str(len(NEW2_CONTENTS)))
2528             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
2529         d.addCallback(_got_headers)
2530
2531         # make sure that outdated size limits aren't enforced anymore.
2532         d.addCallback(lambda ignored:
2533             self.POST(self.public_url + "/foo", t="upload",
2534                       mutable="true",
2535                       file=("new.txt",
2536                             "b" * (self.s.MUTABLE_SIZELIMIT+1))))
2537         d.addErrback(self.dump_error)
2538         return d
2539
2540     def test_POST_upload_mutable_toobig(self):
2541         # SDMF had a size limti that was removed a while ago. MDMF has
2542         # never had a size limit. Test to make sure that we do not
2543         # encounter errors when trying to upload large mutable files,
2544         # since there should be no coded prohibitions regarding large
2545         # mutable files.
2546         d = self.POST(self.public_url + "/foo",
2547                       t="upload", mutable="true",
2548                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2549         return d
2550
2551     def dump_error(self, f):
2552         # if the web server returns an error code (like 400 Bad Request),
2553         # web.client.getPage puts the HTTP response body into the .response
2554         # attribute of the exception object that it gives back. It does not
2555         # appear in the Failure's repr(), so the ERROR that trial displays
2556         # will be rather terse and unhelpful. addErrback this method to the
2557         # end of your chain to get more information out of these errors.
2558         if f.check(error.Error):
2559             print "web.error.Error:"
2560             print f
2561             print f.value.response
2562         return f
2563
2564     def test_POST_upload_replace(self):
2565         d = self.POST(self.public_url + "/foo", t="upload",
2566                       file=("bar.txt", self.NEWFILE_CONTENTS))
2567         fn = self._foo_node
2568         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
2569         d.addCallback(lambda res:
2570                       self.failUnlessChildContentsAre(fn, u"bar.txt",
2571                                                       self.NEWFILE_CONTENTS))
2572         return d
2573
2574     def test_POST_upload_no_replace_ok(self):
2575         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2576                       file=("new.txt", self.NEWFILE_CONTENTS))
2577         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
2578         d.addCallback(lambda res: self.failUnlessReallyEqual(res,
2579                                                              self.NEWFILE_CONTENTS))
2580         return d
2581
2582     def test_POST_upload_no_replace_queryarg(self):
2583         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2584                       file=("bar.txt", self.NEWFILE_CONTENTS))
2585         d.addBoth(self.shouldFail, error.Error,
2586                   "POST_upload_no_replace_queryarg",
2587                   "409 Conflict",
2588                   "There was already a child by that name, and you asked me "
2589                   "to not replace it")
2590         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2591         d.addCallback(self.failUnlessIsBarDotTxt)
2592         return d
2593
2594     def test_POST_upload_no_replace_field(self):
2595         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
2596                       file=("bar.txt", self.NEWFILE_CONTENTS))
2597         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
2598                   "409 Conflict",
2599                   "There was already a child by that name, and you asked me "
2600                   "to not replace it")
2601         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2602         d.addCallback(self.failUnlessIsBarDotTxt)
2603         return d
2604
2605     def test_POST_upload_whendone(self):
2606         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
2607                       file=("new.txt", self.NEWFILE_CONTENTS))
2608         d.addBoth(self.shouldRedirect, "/THERE")
2609         fn = self._foo_node
2610         d.addCallback(lambda res:
2611                       self.failUnlessChildContentsAre(fn, u"new.txt",
2612                                                       self.NEWFILE_CONTENTS))
2613         return d
2614
2615     def test_POST_upload_named(self):
2616         fn = self._foo_node
2617         d = self.POST(self.public_url + "/foo", t="upload",
2618                       name="new.txt", file=self.NEWFILE_CONTENTS)
2619         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2620         d.addCallback(lambda res:
2621                       self.failUnlessChildContentsAre(fn, u"new.txt",
2622                                                       self.NEWFILE_CONTENTS))
2623         return d
2624
2625     def test_POST_upload_named_badfilename(self):
2626         d = self.POST(self.public_url + "/foo", t="upload",
2627                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
2628         d.addBoth(self.shouldFail, error.Error,
2629                   "test_POST_upload_named_badfilename",
2630                   "400 Bad Request",
2631                   "name= may not contain a slash",
2632                   )
2633         # make sure that nothing was added
2634         d.addCallback(lambda res:
2635                       self.failUnlessNodeKeysAre(self._foo_node,
2636                                                  [self._htmlname_unicode,
2637                                                   u"bar.txt", u"baz.txt", u"blockingfile",
2638                                                   u"empty", u"n\u00fc.txt", u"quux.txt",
2639                                                   u"sub"]))
2640         return d
2641
2642     def test_POST_FILEURL_check(self):
2643         bar_url = self.public_url + "/foo/bar.txt"
2644         d = self.POST(bar_url, t="check")
2645         def _check(res):
2646             self.failUnlessIn("Healthy :", res)
2647         d.addCallback(_check)
2648         redir_url = "http://allmydata.org/TARGET"
2649         def _check2(statuscode, target):
2650             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2651             self.failUnlessReallyEqual(target, redir_url)
2652         d.addCallback(lambda res:
2653                       self.shouldRedirect2("test_POST_FILEURL_check",
2654                                            _check2,
2655                                            self.POST, bar_url,
2656                                            t="check",
2657                                            when_done=redir_url))
2658         d.addCallback(lambda res:
2659                       self.POST(bar_url, t="check", return_to=redir_url))
2660         def _check3(res):
2661             self.failUnlessIn("Healthy :", res)
2662             self.failUnlessIn("Return to file", res)
2663             self.failUnlessIn(redir_url, res)
2664         d.addCallback(_check3)
2665
2666         d.addCallback(lambda res:
2667                       self.POST(bar_url, t="check", output="JSON"))
2668         def _check_json(res):
2669             data = simplejson.loads(res)
2670             self.failUnlessIn("storage-index", data)
2671             self.failUnless(data["results"]["healthy"])
2672         d.addCallback(_check_json)
2673
2674         return d
2675
2676     def test_POST_FILEURL_check_and_repair(self):
2677         bar_url = self.public_url + "/foo/bar.txt"
2678         d = self.POST(bar_url, t="check", repair="true")
2679         def _check(res):
2680             self.failUnlessIn("Healthy :", res)
2681         d.addCallback(_check)
2682         redir_url = "http://allmydata.org/TARGET"
2683         def _check2(statuscode, target):
2684             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2685             self.failUnlessReallyEqual(target, redir_url)
2686         d.addCallback(lambda res:
2687                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
2688                                            _check2,
2689                                            self.POST, bar_url,
2690                                            t="check", repair="true",
2691                                            when_done=redir_url))
2692         d.addCallback(lambda res:
2693                       self.POST(bar_url, t="check", return_to=redir_url))
2694         def _check3(res):
2695             self.failUnlessIn("Healthy :", res)
2696             self.failUnlessIn("Return to file", res)
2697             self.failUnlessIn(redir_url, res)
2698         d.addCallback(_check3)
2699         return d
2700
2701     def test_POST_DIRURL_check(self):
2702         foo_url = self.public_url + "/foo/"
2703         d = self.POST(foo_url, t="check")
2704         def _check(res):
2705             self.failUnlessIn("Healthy :", res)
2706         d.addCallback(_check)
2707         redir_url = "http://allmydata.org/TARGET"
2708         def _check2(statuscode, target):
2709             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2710             self.failUnlessReallyEqual(target, redir_url)
2711         d.addCallback(lambda res:
2712                       self.shouldRedirect2("test_POST_DIRURL_check",
2713                                            _check2,
2714                                            self.POST, foo_url,
2715                                            t="check",
2716                                            when_done=redir_url))
2717         d.addCallback(lambda res:
2718                       self.POST(foo_url, t="check", return_to=redir_url))
2719         def _check3(res):
2720             self.failUnlessIn("Healthy :", res)
2721             self.failUnlessIn("Return to file/directory", res)
2722             self.failUnlessIn(redir_url, res)
2723         d.addCallback(_check3)
2724
2725         d.addCallback(lambda res:
2726                       self.POST(foo_url, t="check", output="JSON"))
2727         def _check_json(res):
2728             data = simplejson.loads(res)
2729             self.failUnlessIn("storage-index", data)
2730             self.failUnless(data["results"]["healthy"])
2731         d.addCallback(_check_json)
2732
2733         return d
2734
2735     def test_POST_DIRURL_check_and_repair(self):
2736         foo_url = self.public_url + "/foo/"
2737         d = self.POST(foo_url, t="check", repair="true")
2738         def _check(res):
2739             self.failUnlessIn("Healthy :", res)
2740         d.addCallback(_check)
2741         redir_url = "http://allmydata.org/TARGET"
2742         def _check2(statuscode, target):
2743             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2744             self.failUnlessReallyEqual(target, redir_url)
2745         d.addCallback(lambda res:
2746                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2747                                            _check2,
2748                                            self.POST, foo_url,
2749                                            t="check", repair="true",
2750                                            when_done=redir_url))
2751         d.addCallback(lambda res:
2752                       self.POST(foo_url, t="check", return_to=redir_url))
2753         def _check3(res):
2754             self.failUnlessIn("Healthy :", res)
2755             self.failUnlessIn("Return to file/directory", res)
2756             self.failUnlessIn(redir_url, res)
2757         d.addCallback(_check3)
2758         return d
2759
2760     def test_POST_FILEURL_mdmf_check(self):
2761         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2762         d = self.POST(quux_url, t="check")
2763         def _check(res):
2764             self.failUnlessIn("Healthy", res)
2765         d.addCallback(_check)
2766         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2767         d.addCallback(lambda ignored:
2768                       self.POST(quux_extension_url, t="check"))
2769         d.addCallback(_check)
2770         return d
2771
2772     def test_POST_FILEURL_mdmf_check_and_repair(self):
2773         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2774         d = self.POST(quux_url, t="check", repair="true")
2775         def _check(res):
2776             self.failUnlessIn("Healthy", res)
2777         d.addCallback(_check)
2778         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2779         d.addCallback(lambda ignored:
2780                       self.POST(quux_extension_url, t="check", repair="true"))
2781         d.addCallback(_check)
2782         return d
2783
2784     def wait_for_operation(self, ignored, ophandle):
2785         url = "/operations/" + ophandle
2786         url += "?t=status&output=JSON"
2787         d = self.GET(url)
2788         def _got(res):
2789             data = simplejson.loads(res)
2790             if not data["finished"]:
2791                 d = self.stall(delay=1.0)
2792                 d.addCallback(self.wait_for_operation, ophandle)
2793                 return d
2794             return data
2795         d.addCallback(_got)
2796         return d
2797
2798     def get_operation_results(self, ignored, ophandle, output=None):
2799         url = "/operations/" + ophandle
2800         url += "?t=status"
2801         if output:
2802             url += "&output=" + output
2803         d = self.GET(url)
2804         def _got(res):
2805             if output and output.lower() == "json":
2806                 return simplejson.loads(res)
2807             return res
2808         d.addCallback(_got)
2809         return d
2810
2811     def test_POST_DIRURL_deepcheck_no_ophandle(self):
2812         d = self.shouldFail2(error.Error,
2813                              "test_POST_DIRURL_deepcheck_no_ophandle",
2814                              "400 Bad Request",
2815                              "slow operation requires ophandle=",
2816                              self.POST, self.public_url, t="start-deep-check")
2817         return d
2818
2819     def test_POST_DIRURL_deepcheck(self):
2820         def _check_redirect(statuscode, target):
2821             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2822             self.failUnless(target.endswith("/operations/123"))
2823         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2824                                  self.POST, self.public_url,
2825                                  t="start-deep-check", ophandle="123")
2826         d.addCallback(self.wait_for_operation, "123")
2827         def _check_json(data):
2828             self.failUnlessReallyEqual(data["finished"], True)
2829             self.failUnlessReallyEqual(data["count-objects-checked"], 11)
2830             self.failUnlessReallyEqual(data["count-objects-healthy"], 11)
2831         d.addCallback(_check_json)
2832         d.addCallback(self.get_operation_results, "123", "html")
2833         def _check_html(res):
2834             self.failUnlessIn("Objects Checked: <span>11</span>", res)
2835             self.failUnlessIn("Objects Healthy: <span>11</span>", res)
2836             self.failUnlessIn(FAVICON_MARKUP, res)
2837         d.addCallback(_check_html)
2838
2839         d.addCallback(lambda res:
2840                       self.GET("/operations/123/"))
2841         d.addCallback(_check_html) # should be the same as without the slash
2842
2843         d.addCallback(lambda res:
2844                       self.shouldFail2(error.Error, "one", "404 Not Found",
2845                                        "No detailed results for SI bogus",
2846                                        self.GET, "/operations/123/bogus"))
2847
2848         foo_si = self._foo_node.get_storage_index()
2849         foo_si_s = base32.b2a(foo_si)
2850         d.addCallback(lambda res:
2851                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2852         def _check_foo_json(res):
2853             data = simplejson.loads(res)
2854             self.failUnlessEqual(data["storage-index"], foo_si_s)
2855             self.failUnless(data["results"]["healthy"])
2856         d.addCallback(_check_foo_json)
2857         return d
2858
2859     def test_POST_DIRURL_deepcheck_and_repair(self):
2860         d = self.POST(self.public_url, t="start-deep-check", repair="true",
2861                       ophandle="124", output="json", followRedirect=True)
2862         d.addCallback(self.wait_for_operation, "124")
2863         def _check_json(data):
2864             self.failUnlessReallyEqual(data["finished"], True)
2865             self.failUnlessReallyEqual(data["count-objects-checked"], 11)
2866             self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 11)
2867             self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2868             self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2869             self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2870             self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2871             self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2872             self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 11)
2873             self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2874             self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2875         d.addCallback(_check_json)
2876         d.addCallback(self.get_operation_results, "124", "html")
2877         def _check_html(res):
2878             self.failUnlessIn("Objects Checked: <span>11</span>", res)
2879
2880             self.failUnlessIn("Objects Healthy (before repair): <span>11</span>", res)
2881             self.failUnlessIn("Objects Unhealthy (before repair): <span>0</span>", res)
2882             self.failUnlessIn("Corrupt Shares (before repair): <span>0</span>", res)
2883
2884             self.failUnlessIn("Repairs Attempted: <span>0</span>", res)
2885             self.failUnlessIn("Repairs Successful: <span>0</span>", res)
2886             self.failUnlessIn("Repairs Unsuccessful: <span>0</span>", res)
2887
2888             self.failUnlessIn("Objects Healthy (after repair): <span>11</span>", res)
2889             self.failUnlessIn("Objects Unhealthy (after repair): <span>0</span>", res)
2890             self.failUnlessIn("Corrupt Shares (after repair): <span>0</span>", res)
2891
2892             self.failUnlessIn(FAVICON_MARKUP, res)
2893         d.addCallback(_check_html)
2894         return d
2895
2896     def test_POST_FILEURL_bad_t(self):
2897         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2898                              "POST to file: bad t=bogus",
2899                              self.POST, self.public_url + "/foo/bar.txt",
2900                              t="bogus")
2901         return d
2902
2903     def test_POST_mkdir(self): # return value?
2904         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2905         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2906         d.addCallback(self.failUnlessNodeKeysAre, [])
2907         return d
2908
2909     def test_POST_mkdir_mdmf(self):
2910         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=mdmf")
2911         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2912         d.addCallback(lambda node:
2913             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2914         return d
2915
2916     def test_POST_mkdir_sdmf(self):
2917         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=sdmf")
2918         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2919         d.addCallback(lambda node:
2920             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2921         return d
2922
2923     def test_POST_mkdir_bad_format(self):
2924         return self.shouldHTTPError("POST_mkdir_bad_format",
2925                                     400, "Bad Request", "Unknown format: foo",
2926                                     self.POST, self.public_url +
2927                                     "/foo?t=mkdir&name=newdir&format=foo")
2928
2929     def test_POST_mkdir_initial_children(self):
2930         (newkids, caps) = self._create_initial_children()
2931         d = self.POST2(self.public_url +
2932                        "/foo?t=mkdir-with-children&name=newdir",
2933                        simplejson.dumps(newkids))
2934         d.addCallback(lambda res:
2935                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2936         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2937         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2938         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2939         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2940         return d
2941
2942     def test_POST_mkdir_initial_children_mdmf(self):
2943         (newkids, caps) = self._create_initial_children()
2944         d = self.POST2(self.public_url +
2945                        "/foo?t=mkdir-with-children&name=newdir&format=mdmf",
2946                        simplejson.dumps(newkids))
2947         d.addCallback(lambda res:
2948                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2949         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2950         d.addCallback(lambda node:
2951             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2952         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2953         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2954                        caps['filecap1'])
2955         return d
2956
2957     # XXX: Duplication.
2958     def test_POST_mkdir_initial_children_sdmf(self):
2959         (newkids, caps) = self._create_initial_children()
2960         d = self.POST2(self.public_url +
2961                        "/foo?t=mkdir-with-children&name=newdir&format=sdmf",
2962                        simplejson.dumps(newkids))
2963         d.addCallback(lambda res:
2964                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2965         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2966         d.addCallback(lambda node:
2967             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2968         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2969         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2970                        caps['filecap1'])
2971         return d
2972
2973     def test_POST_mkdir_initial_children_bad_format(self):
2974         (newkids, caps) = self._create_initial_children()
2975         return self.shouldHTTPError("POST_mkdir_initial_children_bad_format",
2976                                     400, "Bad Request", "Unknown format: foo",
2977                                     self.POST, self.public_url + \
2978                                     "/foo?t=mkdir-with-children&name=newdir&format=foo",
2979                                     simplejson.dumps(newkids))
2980
2981     def test_POST_mkdir_immutable(self):
2982         (newkids, caps) = self._create_immutable_children()
2983         d = self.POST2(self.public_url +
2984                        "/foo?t=mkdir-immutable&name=newdir",
2985                        simplejson.dumps(newkids))
2986         d.addCallback(lambda res:
2987                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2988         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2989         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2990         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2991         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2992         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2993         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2994         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2995         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2996         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2997         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2998         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2999         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
3000         return d
3001
3002     def test_POST_mkdir_immutable_bad(self):
3003         (newkids, caps) = self._create_initial_children()
3004         d = self.shouldFail2(error.Error, "POST_mkdir_immutable_bad",
3005                              "400 Bad Request",
3006                              "needed to be immutable but was not",
3007                              self.POST2,
3008                              self.public_url +
3009                              "/foo?t=mkdir-immutable&name=newdir",
3010                              simplejson.dumps(newkids))
3011         return d
3012
3013     def test_POST_mkdir_2(self):
3014         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
3015         d.addCallback(lambda res:
3016                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
3017         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3018         d.addCallback(self.failUnlessNodeKeysAre, [])
3019         return d
3020
3021     def test_POST_mkdirs_2(self):
3022         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
3023         d.addCallback(lambda res:
3024                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
3025         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
3026         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
3027         d.addCallback(self.failUnlessNodeKeysAre, [])
3028         return d
3029
3030     def test_POST_mkdir_no_parentdir_noredirect(self):
3031         d = self.POST("/uri?t=mkdir")
3032         def _after_mkdir(res):
3033             uri.DirectoryURI.init_from_string(res)
3034         d.addCallback(_after_mkdir)
3035         return d
3036
3037     def test_POST_mkdir_no_parentdir_noredirect_mdmf(self):
3038         d = self.POST("/uri?t=mkdir&format=mdmf")
3039         def _after_mkdir(res):
3040             u = uri.from_string(res)
3041             # Check that this is an MDMF writecap
3042             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
3043         d.addCallback(_after_mkdir)
3044         return d
3045
3046     def test_POST_mkdir_no_parentdir_noredirect_sdmf(self):
3047         d = self.POST("/uri?t=mkdir&format=sdmf")
3048         def _after_mkdir(res):
3049             u = uri.from_string(res)
3050             self.failUnlessIsInstance(u, uri.DirectoryURI)
3051         d.addCallback(_after_mkdir)
3052         return d
3053
3054     def test_POST_mkdir_no_parentdir_noredirect_bad_format(self):
3055         return self.shouldHTTPError("POST_mkdir_no_parentdir_noredirect_bad_format",
3056                                     400, "Bad Request", "Unknown format: foo",
3057                                     self.POST, self.public_url +
3058                                     "/uri?t=mkdir&format=foo")
3059
3060     def test_POST_mkdir_no_parentdir_noredirect2(self):
3061         # make sure form-based arguments (as on the welcome page) still work
3062         d = self.POST("/uri", t="mkdir")
3063         def _after_mkdir(res):
3064             uri.DirectoryURI.init_from_string(res)
3065         d.addCallback(_after_mkdir)
3066         d.addErrback(self.explain_web_error)
3067         return d
3068
3069     def test_POST_mkdir_no_parentdir_redirect(self):
3070         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
3071         d.addBoth(self.shouldRedirect, None, statuscode='303')
3072         def _check_target(target):
3073             target = urllib.unquote(target)
3074             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
3075         d.addCallback(_check_target)
3076         return d
3077
3078     def test_POST_mkdir_no_parentdir_redirect2(self):
3079         d = self.POST("/uri", t="mkdir", redirect_to_result="true")
3080         d.addBoth(self.shouldRedirect, None, statuscode='303')
3081         def _check_target(target):
3082             target = urllib.unquote(target)
3083             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
3084         d.addCallback(_check_target)
3085         d.addErrback(self.explain_web_error)
3086         return d
3087
3088     def _make_readonly(self, u):
3089         ro_uri = uri.from_string(u).get_readonly()
3090         if ro_uri is None:
3091             return None
3092         return ro_uri.to_string()
3093
3094     def _create_initial_children(self):
3095         contents, n, filecap1 = self.makefile(12)
3096         md1 = {"metakey1": "metavalue1"}
3097         filecap2 = make_mutable_file_uri()
3098         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
3099         filecap3 = node3.get_readonly_uri()
3100         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
3101         dircap = DirectoryNode(node4, None, None).get_uri()
3102         mdmfcap = make_mutable_file_uri(mdmf=True)
3103         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
3104         emptydircap = "URI:DIR2-LIT:"
3105         newkids = {u"child-imm":        ["filenode", {"rw_uri": filecap1,
3106                                                       "ro_uri": self._make_readonly(filecap1),
3107                                                       "metadata": md1, }],
3108                    u"child-mutable":    ["filenode", {"rw_uri": filecap2,
3109                                                       "ro_uri": self._make_readonly(filecap2)}],
3110                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
3111                    u"unknownchild-rw":  ["unknown",  {"rw_uri": unknown_rwcap,
3112                                                       "ro_uri": unknown_rocap}],
3113                    u"unknownchild-ro":  ["unknown",  {"ro_uri": unknown_rocap}],
3114                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
3115                    u"dirchild":         ["dirnode",  {"rw_uri": dircap,
3116                                                       "ro_uri": self._make_readonly(dircap)}],
3117                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
3118                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
3119                    u"child-mutable-mdmf": ["filenode", {"rw_uri": mdmfcap,
3120                                                         "ro_uri": self._make_readonly(mdmfcap)}],
3121                    }
3122         return newkids, {'filecap1': filecap1,
3123                          'filecap2': filecap2,
3124                          'filecap3': filecap3,
3125                          'unknown_rwcap': unknown_rwcap,
3126                          'unknown_rocap': unknown_rocap,
3127                          'unknown_immcap': unknown_immcap,
3128                          'dircap': dircap,
3129                          'litdircap': litdircap,
3130                          'emptydircap': emptydircap,
3131                          'mdmfcap': mdmfcap}
3132
3133     def _create_immutable_children(self):
3134         contents, n, filecap1 = self.makefile(12)
3135         md1 = {"metakey1": "metavalue1"}
3136         tnode = create_chk_filenode("immutable directory contents\n"*10,
3137                                     self.get_all_contents())
3138         dnode = DirectoryNode(tnode, None, None)
3139         assert not dnode.is_mutable()
3140         immdircap = dnode.get_uri()
3141         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
3142         emptydircap = "URI:DIR2-LIT:"
3143         newkids = {u"child-imm":        ["filenode", {"ro_uri": filecap1,
3144                                                       "metadata": md1, }],
3145                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
3146                    u"dirchild-imm":     ["dirnode",  {"ro_uri": immdircap}],
3147                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
3148                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
3149                    }
3150         return newkids, {'filecap1': filecap1,
3151                          'unknown_immcap': unknown_immcap,
3152                          'immdircap': immdircap,
3153                          'litdircap': litdircap,
3154                          'emptydircap': emptydircap}
3155
3156     def test_POST_mkdir_no_parentdir_initial_children(self):
3157         (newkids, caps) = self._create_initial_children()
3158         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
3159         def _after_mkdir(res):
3160             self.failUnless(res.startswith("URI:DIR"), res)
3161             n = self.s.create_node_from_uri(res)
3162             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3163             d2.addCallback(lambda ign:
3164                            self.failUnlessROChildURIIs(n, u"child-imm",
3165                                                        caps['filecap1']))
3166             d2.addCallback(lambda ign:
3167                            self.failUnlessRWChildURIIs(n, u"child-mutable",
3168                                                        caps['filecap2']))
3169             d2.addCallback(lambda ign:
3170                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
3171                                                        caps['filecap3']))
3172             d2.addCallback(lambda ign:
3173                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
3174                                                        caps['unknown_rwcap']))
3175             d2.addCallback(lambda ign:
3176                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
3177                                                        caps['unknown_rocap']))
3178             d2.addCallback(lambda ign:
3179                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3180                                                        caps['unknown_immcap']))
3181             d2.addCallback(lambda ign:
3182                            self.failUnlessRWChildURIIs(n, u"dirchild",
3183                                                        caps['dircap']))
3184             return d2
3185         d.addCallback(_after_mkdir)
3186         return d
3187
3188     def test_POST_mkdir_no_parentdir_unexpected_children(self):
3189         # the regular /uri?t=mkdir operation is specified to ignore its body.
3190         # Only t=mkdir-with-children pays attention to it.
3191         (newkids, caps) = self._create_initial_children()
3192         d = self.shouldHTTPError("POST_mkdir_no_parentdir_unexpected_children",
3193                                  400, "Bad Request",
3194                                  "t=mkdir does not accept children=, "
3195                                  "try t=mkdir-with-children instead",
3196                                  self.POST2, "/uri?t=mkdir", # without children
3197                                  simplejson.dumps(newkids))
3198         return d
3199
3200     def test_POST_noparent_bad(self):
3201         d = self.shouldHTTPError("POST_noparent_bad",
3202                                  400, "Bad Request",
3203                                  "/uri accepts only PUT, PUT?t=mkdir, "
3204                                  "POST?t=upload, and POST?t=mkdir",
3205                                  self.POST, "/uri?t=bogus")
3206         return d
3207
3208     def test_POST_mkdir_no_parentdir_immutable(self):
3209         (newkids, caps) = self._create_immutable_children()
3210         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
3211         def _after_mkdir(res):
3212             self.failUnless(res.startswith("URI:DIR"), res)
3213             n = self.s.create_node_from_uri(res)
3214             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3215             d2.addCallback(lambda ign:
3216                            self.failUnlessROChildURIIs(n, u"child-imm",
3217                                                           caps['filecap1']))
3218             d2.addCallback(lambda ign:
3219                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3220                                                           caps['unknown_immcap']))
3221             d2.addCallback(lambda ign:
3222                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
3223                                                           caps['immdircap']))
3224             d2.addCallback(lambda ign:
3225                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
3226                                                           caps['litdircap']))
3227             d2.addCallback(lambda ign:
3228                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
3229                                                           caps['emptydircap']))
3230             return d2
3231         d.addCallback(_after_mkdir)
3232         return d
3233
3234     def test_POST_mkdir_no_parentdir_immutable_bad(self):
3235         (newkids, caps) = self._create_initial_children()
3236         d = self.shouldFail2(error.Error,
3237                              "test_POST_mkdir_no_parentdir_immutable_bad",
3238                              "400 Bad Request",
3239                              "needed to be immutable but was not",
3240                              self.POST2,
3241                              "/uri?t=mkdir-immutable",
3242                              simplejson.dumps(newkids))
3243         return d
3244
3245     def test_welcome_page_mkdir_button(self):
3246         # Fetch the welcome page.
3247         d = self.GET("/")
3248         def _after_get_welcome_page(res):
3249             MKDIR_BUTTON_RE = re.compile(
3250                 '<form action="([^"]*)" method="post".*'
3251                 '<input type="hidden" name="t" value="([^"]*)" />[ ]*'
3252                 '<input type="hidden" name="([^"]*)" value="([^"]*)" />[ ]*'
3253                 '<input type="submit" class="btn" value="Create a directory[^"]*" />')
3254             html = res.replace('\n', ' ')
3255             mo = MKDIR_BUTTON_RE.search(html)
3256             self.failUnless(mo, html)
3257             formaction = mo.group(1)
3258             formt = mo.group(2)
3259             formaname = mo.group(3)
3260             formavalue = mo.group(4)
3261             return (formaction, formt, formaname, formavalue)
3262         d.addCallback(_after_get_welcome_page)
3263         def _after_parse_form(res):
3264             (formaction, formt, formaname, formavalue) = res
3265             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
3266         d.addCallback(_after_parse_form)
3267         d.addBoth(self.shouldRedirect, None, statuscode='303')
3268         return d
3269
3270     def test_POST_mkdir_replace(self): # return value?
3271         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
3272         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3273         d.addCallback(self.failUnlessNodeKeysAre, [])
3274         return d
3275
3276     def test_POST_mkdir_no_replace_queryarg(self): # return value?
3277         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
3278         d.addBoth(self.shouldFail, error.Error,
3279                   "POST_mkdir_no_replace_queryarg",
3280                   "409 Conflict",
3281                   "There was already a child by that name, and you asked me "
3282                   "to not replace it")
3283         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3284         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3285         return d
3286
3287     def test_POST_mkdir_no_replace_field(self): # return value?
3288         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
3289                       replace="false")
3290         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
3291                   "409 Conflict",
3292                   "There was already a child by that name, and you asked me "
3293                   "to not replace it")
3294         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3295         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3296         return d
3297
3298     def test_POST_mkdir_whendone_field(self):
3299         d = self.POST(self.public_url + "/foo",
3300                       t="mkdir", name="newdir", when_done="/THERE")
3301         d.addBoth(self.shouldRedirect, "/THERE")
3302         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3303         d.addCallback(self.failUnlessNodeKeysAre, [])
3304         return d
3305
3306     def test_POST_mkdir_whendone_queryarg(self):
3307         d = self.POST(self.public_url + "/foo?when_done=/THERE",
3308                       t="mkdir", name="newdir")
3309         d.addBoth(self.shouldRedirect, "/THERE")
3310         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3311         d.addCallback(self.failUnlessNodeKeysAre, [])
3312         return d
3313
3314     def test_POST_bad_t(self):
3315         d = self.shouldFail2(error.Error, "POST_bad_t",
3316                              "400 Bad Request",
3317                              "POST to a directory with bad t=BOGUS",
3318                              self.POST, self.public_url + "/foo", t="BOGUS")
3319         return d
3320
3321     def test_POST_set_children(self, command_name="set_children"):
3322         contents9, n9, newuri9 = self.makefile(9)
3323         contents10, n10, newuri10 = self.makefile(10)
3324         contents11, n11, newuri11 = self.makefile(11)
3325
3326         reqbody = """{
3327                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
3328                                                 "size": 0,
3329                                                 "metadata": {
3330                                                   "ctime": 1002777696.7564139,
3331                                                   "mtime": 1002777696.7564139
3332                                                  }
3333                                                } ],
3334                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
3335                                                 "size": 1,
3336                                                 "metadata": {
3337                                                   "ctime": 1002777696.7564139,
3338                                                   "mtime": 1002777696.7564139
3339                                                  }
3340                                                } ],
3341                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
3342                                                 "size": 2,
3343                                                 "metadata": {
3344                                                   "ctime": 1002777696.7564139,
3345                                                   "mtime": 1002777696.7564139
3346                                                  }
3347                                                } ]
3348                     }""" % (newuri9, newuri10, newuri11)
3349
3350         url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
3351
3352         d = client.getPage(url, method="POST", postdata=reqbody)
3353         def _then(res):
3354             self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
3355             self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
3356             self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
3357
3358         d.addCallback(_then)
3359         d.addErrback(self.dump_error)
3360         return d
3361
3362     def test_POST_set_children_with_hyphen(self):
3363         return self.test_POST_set_children(command_name="set-children")
3364
3365     def test_POST_link_uri(self):
3366         contents, n, newuri = self.makefile(8)
3367         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
3368         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
3369         d.addCallback(lambda res:
3370                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3371                                                       contents))
3372         return d
3373
3374     def test_POST_link_uri_replace(self):
3375         contents, n, newuri = self.makefile(8)
3376         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
3377         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
3378         d.addCallback(lambda res:
3379                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3380                                                       contents))
3381         return d
3382
3383     def test_POST_link_uri_unknown_bad(self):
3384         d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
3385         d.addBoth(self.shouldFail, error.Error,
3386                   "POST_link_uri_unknown_bad",
3387                   "400 Bad Request",
3388                   "unknown cap in a write slot")
3389         return d
3390
3391     def test_POST_link_uri_unknown_ro_good(self):
3392         d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
3393         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
3394         return d
3395
3396     def test_POST_link_uri_unknown_imm_good(self):
3397         d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
3398         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
3399         return d
3400
3401     def test_POST_link_uri_no_replace_queryarg(self):
3402         contents, n, newuri = self.makefile(8)
3403         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
3404                       name="bar.txt", uri=newuri)
3405         d.addBoth(self.shouldFail, error.Error,
3406                   "POST_link_uri_no_replace_queryarg",
3407                   "409 Conflict",
3408                   "There was already a child by that name, and you asked me "
3409                   "to not replace it")
3410         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3411         d.addCallback(self.failUnlessIsBarDotTxt)
3412         return d
3413
3414     def test_POST_link_uri_no_replace_field(self):
3415         contents, n, newuri = self.makefile(8)
3416         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
3417                       name="bar.txt", uri=newuri)
3418         d.addBoth(self.shouldFail, error.Error,
3419                   "POST_link_uri_no_replace_field",
3420                   "409 Conflict",
3421                   "There was already a child by that name, and you asked me "
3422                   "to not replace it")
3423         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3424         d.addCallback(self.failUnlessIsBarDotTxt)
3425         return d
3426
3427     def test_POST_delete(self, command_name='delete'):
3428         d = self._foo_node.list()
3429         def _check_before(children):
3430             self.failUnlessIn(u"bar.txt", children)
3431         d.addCallback(_check_before)
3432         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t=command_name, name="bar.txt"))
3433         d.addCallback(lambda res: self._foo_node.list())
3434         def _check_after(children):
3435             self.failIfIn(u"bar.txt", children)
3436         d.addCallback(_check_after)
3437         return d
3438
3439     def test_POST_unlink(self):
3440         return self.test_POST_delete(command_name='unlink')
3441
3442     def test_POST_rename_file(self):
3443         d = self.POST(self.public_url + "/foo", t="rename",
3444                       from_name="bar.txt", to_name='wibble.txt')
3445         d.addCallback(lambda res:
3446                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3447         d.addCallback(lambda res:
3448                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
3449         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
3450         d.addCallback(self.failUnlessIsBarDotTxt)
3451         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
3452         d.addCallback(self.failUnlessIsBarJSON)
3453         return d
3454
3455     def test_POST_rename_file_redundant(self):
3456         d = self.POST(self.public_url + "/foo", t="rename",
3457                       from_name="bar.txt", to_name='bar.txt')
3458         d.addCallback(lambda res:
3459                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3460         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3461         d.addCallback(self.failUnlessIsBarDotTxt)
3462         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3463         d.addCallback(self.failUnlessIsBarJSON)
3464         return d
3465
3466     def test_POST_rename_file_replace(self):
3467         # rename a file and replace a directory with it
3468         d = self.POST(self.public_url + "/foo", t="rename",
3469                       from_name="bar.txt", to_name='empty')
3470         d.addCallback(lambda res:
3471                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3472         d.addCallback(lambda res:
3473                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
3474         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
3475         d.addCallback(self.failUnlessIsBarDotTxt)
3476         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3477         d.addCallback(self.failUnlessIsBarJSON)
3478         return d
3479
3480     def test_POST_rename_file_no_replace_queryarg(self):
3481         # rename a file and replace a directory with it
3482         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
3483                       from_name="bar.txt", to_name='empty')
3484         d.addBoth(self.shouldFail, error.Error,
3485                   "POST_rename_file_no_replace_queryarg",
3486                   "409 Conflict",
3487                   "There was already a child by that name, and you asked me "
3488                   "to not replace it")
3489         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3490         d.addCallback(self.failUnlessIsEmptyJSON)
3491         return d
3492
3493     def test_POST_rename_file_no_replace_field(self):
3494         # rename a file and replace a directory with it
3495         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
3496                       from_name="bar.txt", to_name='empty')
3497         d.addBoth(self.shouldFail, error.Error,
3498                   "POST_rename_file_no_replace_field",
3499                   "409 Conflict",
3500                   "There was already a child by that name, and you asked me "
3501                   "to not replace it")
3502         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3503         d.addCallback(self.failUnlessIsEmptyJSON)
3504         return d
3505
3506     def test_POST_rename_file_no_replace_same_link(self):
3507         d = self.POST(self.public_url + "/foo", t="rename",
3508                       replace="false", from_name="bar.txt", to_name="bar.txt")
3509         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3510         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3511         d.addCallback(self.failUnlessIsBarDotTxt)
3512         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3513         d.addCallback(self.failUnlessIsBarJSON)
3514         return d
3515
3516     def test_POST_rename_file_replace_only_files(self):
3517         d = self.POST(self.public_url + "/foo", t="rename",
3518                       replace="only-files", from_name="bar.txt",
3519                       to_name="baz.txt")
3520         d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3521         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3522         d.addCallback(self.failUnlessIsBarDotTxt)
3523         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt?t=json"))
3524         d.addCallback(self.failUnlessIsBarJSON)
3525         return d
3526
3527     def test_POST_rename_file_replace_only_files_conflict(self):
3528         d = self.shouldFail2(error.Error, "POST_relink_file_replace_only_files_conflict",
3529                              "409 Conflict",
3530                              "There was already a child by that name, and you asked me to not replace it.",
3531                              self.POST, self.public_url + "/foo", t="relink",
3532                              replace="only-files", from_name="bar.txt",
3533                              to_name="empty")
3534         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3535         d.addCallback(self.failUnlessIsBarDotTxt)
3536         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3537         d.addCallback(self.failUnlessIsBarJSON)
3538         return d
3539
3540     def failUnlessIsEmptyJSON(self, res):
3541         data = simplejson.loads(res)
3542         self.failUnlessEqual(data[0], "dirnode", data)
3543         self.failUnlessReallyEqual(len(data[1]["children"]), 0)
3544
3545     def test_POST_rename_file_to_slash_fail(self):
3546         d = self.POST(self.public_url + "/foo", t="rename",
3547                       from_name="bar.txt", to_name='kirk/spock.txt')
3548         d.addBoth(self.shouldFail, error.Error,
3549                   "test_POST_rename_file_to_slash_fail",
3550                   "400 Bad Request",
3551                   "to_name= may not contain a slash",
3552                   )
3553         d.addCallback(lambda res:
3554                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3555         return d
3556
3557     def test_POST_rename_file_from_slash_fail(self):
3558         d = self.POST(self.public_url + "/foo", t="rename",
3559                       from_name="sub/bar.txt", to_name='spock.txt')
3560         d.addBoth(self.shouldFail, error.Error,
3561                   "test_POST_rename_from_file_slash_fail",
3562                   "400 Bad Request",
3563                   "from_name= may not contain a slash",
3564                   )
3565         d.addCallback(lambda res:
3566                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3567         return d
3568
3569     def test_POST_rename_dir(self):
3570         d = self.POST(self.public_url, t="rename",
3571                       from_name="foo", to_name='plunk')
3572         d.addCallback(lambda res:
3573                       self.failIfNodeHasChild(self.public_root, u"foo"))
3574         d.addCallback(lambda res:
3575                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
3576         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
3577         d.addCallback(self.failUnlessIsFooJSON)
3578         return d
3579
3580     def test_POST_relink_file(self):
3581         d = self.POST(self.public_url + "/foo", t="relink",
3582                       from_name="bar.txt",
3583                       to_dir=self.public_root.get_uri() + "/foo/sub")
3584         d.addCallback(lambda res:
3585                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3586         d.addCallback(lambda res:
3587                       self.failUnlessNodeHasChild(self._sub_node, u"bar.txt"))
3588         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3589         d.addCallback(self.failUnlessIsBarDotTxt)
3590         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3591         d.addCallback(self.failUnlessIsBarJSON)
3592         return d
3593
3594     def test_POST_relink_file_new_name(self):
3595         d = self.POST(self.public_url + "/foo", t="relink",
3596                       from_name="bar.txt",
3597                       to_name="wibble.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3598         d.addCallback(lambda res:
3599                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3600         d.addCallback(lambda res:
3601                       self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3602         d.addCallback(lambda res:
3603                       self.failUnlessNodeHasChild(self._sub_node, u"wibble.txt"))
3604         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt"))
3605         d.addCallback(self.failUnlessIsBarDotTxt)
3606         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt?t=json"))
3607         d.addCallback(self.failUnlessIsBarJSON)
3608         return d
3609
3610     def test_POST_relink_file_replace(self):
3611         d = self.POST(self.public_url + "/foo", t="relink",
3612                       from_name="bar.txt",
3613                       to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3614         d.addCallback(lambda res:
3615                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3616         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3617         d.addCallback(self.failUnlessIsBarDotTxt)
3618         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt?t=json"))
3619         d.addCallback(self.failUnlessIsBarJSON)
3620         return d
3621
3622     def test_POST_relink_file_no_replace(self):
3623         d = self.shouldFail2(error.Error, "POST_relink_file_no_replace",
3624                              "409 Conflict",
3625                              "There was already a child by that name, and you asked me to not replace it",
3626                              self.POST, self.public_url + "/foo", t="relink",
3627                              replace="false", from_name="bar.txt",
3628                              to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3629         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3630         d.addCallback(self.failUnlessIsBarDotTxt)
3631         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3632         d.addCallback(self.failUnlessIsBarJSON)
3633         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3634         d.addCallback(self.failUnlessIsSubBazDotTxt)
3635         return d
3636
3637     def test_POST_relink_file_no_replace_explicitly_same_link(self):
3638         d = self.POST(self.public_url + "/foo", t="relink",
3639                       replace="false", from_name="bar.txt",
3640                       to_name="bar.txt", to_dir=self.public_root.get_uri() + "/foo")
3641         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3642         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3643         d.addCallback(self.failUnlessIsBarDotTxt)
3644         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3645         d.addCallback(self.failUnlessIsBarJSON)
3646         return d
3647
3648     def test_POST_relink_file_replace_only_files(self):
3649         d = self.POST(self.public_url + "/foo", t="relink",
3650                       replace="only-files", from_name="bar.txt",
3651                       to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3652         d.addCallback(lambda res:
3653                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3654         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3655         d.addCallback(self.failUnlessIsBarDotTxt)
3656         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt?t=json"))
3657         d.addCallback(self.failUnlessIsBarJSON)
3658         return d
3659
3660     def test_POST_relink_file_replace_only_files_conflict(self):
3661         d = self.shouldFail2(error.Error, "POST_relink_file_replace_only_files_conflict",
3662                              "409 Conflict",
3663                              "There was already a child by that name, and you asked me to not replace it.",
3664                              self.POST, self.public_url + "/foo", t="relink",
3665                              replace="only-files", from_name="bar.txt",
3666                              to_name="sub", to_dir=self.public_root.get_uri() + "/foo")
3667         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3668         d.addCallback(self.failUnlessIsBarDotTxt)
3669         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3670         d.addCallback(self.failUnlessIsBarJSON)
3671         return d
3672
3673     def test_POST_relink_file_to_slash_fail(self):
3674         d = self.shouldFail2(error.Error, "test_POST_rename_file_slash_fail",
3675                              "400 Bad Request",
3676                              "to_name= may not contain a slash",
3677                              self.POST, self.public_url + "/foo", t="relink",
3678                              from_name="bar.txt",
3679                              to_name="slash/fail.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3680         d.addCallback(lambda res:
3681                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3682         d.addCallback(lambda res:
3683                       self.failIfNodeHasChild(self._sub_node, u"slash/fail.txt"))
3684         d.addCallback(lambda ign:
3685                       self.shouldFail2(error.Error,
3686                                        "test_POST_rename_file_slash_fail2",
3687                                        "400 Bad Request",
3688                                        "from_name= may not contain a slash",
3689                                        self.POST, self.public_url + "/foo",
3690                                        t="relink",
3691                                        from_name="nope/bar.txt",
3692                                        to_name="fail.txt",
3693                                        to_dir=self.public_root.get_uri() + "/foo/sub"))
3694         return d
3695
3696     def test_POST_relink_file_explicitly_same_link(self):
3697         d = self.POST(self.public_url + "/foo", t="relink",
3698                       from_name="bar.txt",
3699                       to_name="bar.txt", to_dir=self.public_root.get_uri() + "/foo")
3700         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3701         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3702         d.addCallback(self.failUnlessIsBarDotTxt)
3703         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3704         d.addCallback(self.failUnlessIsBarJSON)
3705         return d
3706
3707     def test_POST_relink_file_implicitly_same_link(self):
3708         d = self.POST(self.public_url + "/foo", t="relink",
3709                       from_name="bar.txt")
3710         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3711         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3712         d.addCallback(self.failUnlessIsBarDotTxt)
3713         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3714         d.addCallback(self.failUnlessIsBarJSON)
3715         return d
3716
3717     def test_POST_relink_file_same_dir(self):
3718         d = self.POST(self.public_url + "/foo", t="relink",
3719                       from_name="bar.txt",
3720                       to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo")
3721         d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3722         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._sub_node, u"baz.txt"))
3723         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3724         d.addCallback(self.failUnlessIsBarDotTxt)
3725         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt?t=json"))
3726         d.addCallback(self.failUnlessIsBarJSON)
3727         return d
3728
3729     def test_POST_relink_file_bad_replace(self):
3730         d = self.shouldFail2(error.Error, "test_POST_relink_file_bad_replace",
3731                              "400 Bad Request", "invalid replace= argument: 'boogabooga'",
3732                              self.POST,
3733                              self.public_url + "/foo", t="relink",
3734                              replace="boogabooga", from_name="bar.txt",
3735                              to_dir=self.public_root.get_uri() + "/foo/sub")
3736         return d
3737
3738     def test_POST_relink_file_multi_level(self):
3739         d = self.POST(self.public_url + "/foo/sub/level2?t=mkdir", "")
3740         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t="relink",
3741                       from_name="bar.txt", to_dir=self.public_root.get_uri() + "/foo/sub/level2"))
3742         d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3743         d.addCallback(lambda res: self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3744         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt"))
3745         d.addCallback(self.failUnlessIsBarDotTxt)
3746         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt?t=json"))
3747         d.addCallback(self.failUnlessIsBarJSON)
3748         return d
3749
3750     def test_POST_relink_file_to_uri(self):
3751         d = self.POST(self.public_url + "/foo", t="relink", target_type="uri",
3752                       from_name="bar.txt", to_dir=self._sub_uri)
3753         d.addCallback(lambda res:
3754                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3755         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3756         d.addCallback(self.failUnlessIsBarDotTxt)
3757         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3758         d.addCallback(self.failUnlessIsBarJSON)
3759         return d
3760
3761     def test_POST_relink_file_to_nonexistent_dir(self):
3762         d = self.shouldFail2(error.Error, "POST_relink_file_to_nonexistent_dir",
3763                             "404 Not Found", "No such child: nopechucktesta",
3764                             self.POST, self.public_url + "/foo", t="relink",
3765                             from_name="bar.txt",
3766                             to_dir=self.public_root.get_uri() + "/nopechucktesta")
3767         return d
3768
3769     def test_POST_relink_file_into_file(self):
3770         d = self.shouldFail2(error.Error, "POST_relink_file_into_file",
3771                              "400 Bad Request", "to_dir is not a directory",
3772                              self.POST, self.public_url + "/foo", t="relink",
3773                              from_name="bar.txt",
3774                              to_dir=self.public_root.get_uri() + "/foo/baz.txt")
3775         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3776         d.addCallback(self.failUnlessIsBazDotTxt)
3777         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3778         d.addCallback(self.failUnlessIsBarDotTxt)
3779         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3780         d.addCallback(self.failUnlessIsBarJSON)
3781         return d
3782
3783     def test_POST_relink_file_to_bad_uri(self):
3784         d =  self.shouldFail2(error.Error, "POST_relink_file_to_bad_uri",
3785                               "400 Bad Request", "to_dir is not a directory",
3786                               self.POST, self.public_url + "/foo", t="relink",
3787                               from_name="bar.txt",
3788                               to_dir="URI:DIR2:mn5jlyjnrjeuydyswlzyui72i:rmneifcj6k6sycjljjhj3f6majsq2zqffydnnul5hfa4j577arma")
3789         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3790         d.addCallback(self.failUnlessIsBarDotTxt)
3791         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3792         d.addCallback(self.failUnlessIsBarJSON)
3793         return d
3794
3795     def test_POST_relink_dir(self):
3796         d = self.POST(self.public_url + "/foo", t="relink",
3797                       from_name="bar.txt",
3798                       to_dir=self.public_root.get_uri() + "/foo/empty")
3799         d.addCallback(lambda res: self.POST(self.public_url + "/foo",
3800                       t="relink", from_name="empty",
3801                       to_dir=self.public_root.get_uri() + "/foo/sub"))
3802         d.addCallback(lambda res:
3803                       self.failIfNodeHasChild(self._foo_node, u"empty"))
3804         d.addCallback(lambda res:
3805                       self.failUnlessNodeHasChild(self._sub_node, u"empty"))
3806         d.addCallback(lambda res:
3807                       self._sub_node.get_child_at_path(u"empty"))
3808         d.addCallback(lambda node:
3809                       self.failUnlessNodeHasChild(node, u"bar.txt"))
3810         d.addCallback(lambda res:
3811                       self.GET(self.public_url + "/foo/sub/empty/bar.txt"))
3812         d.addCallback(self.failUnlessIsBarDotTxt)
3813         return d
3814
3815     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
3816         """ If target is not None then the redirection has to go to target.  If
3817         statuscode is not None then the redirection has to be accomplished with
3818         that HTTP status code."""
3819         if not isinstance(res, failure.Failure):
3820             to_where = (target is None) and "somewhere" or ("to " + target)
3821             self.fail("%s: we were expecting to get redirected %s, not get an"
3822                       " actual page: %s" % (which, to_where, res))
3823         res.trap(error.PageRedirect)
3824         if statuscode is not None:
3825             self.failUnlessReallyEqual(res.value.status, statuscode,
3826                                        "%s: not a redirect" % which)
3827         if target is not None:
3828             # the PageRedirect does not seem to capture the uri= query arg
3829             # properly, so we can't check for it.
3830             realtarget = self.webish_url + target
3831             self.failUnlessReallyEqual(res.value.location, realtarget,
3832                                        "%s: wrong target" % which)
3833         return res.value.location
3834
3835     def test_GET_URI_form(self):
3836         base = "/uri?uri=%s" % self._bar_txt_uri
3837         # this is supposed to give us a redirect to /uri/$URI, plus arguments
3838         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
3839         d = self.GET(base)
3840         d.addBoth(self.shouldRedirect, targetbase)
3841         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
3842         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
3843         d.addCallback(lambda res: self.GET(base+"&t=json"))
3844         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
3845         d.addCallback(self.log, "about to get file by uri")
3846         d.addCallback(lambda res: self.GET(base, followRedirect=True))
3847         d.addCallback(self.failUnlessIsBarDotTxt)
3848         d.addCallback(self.log, "got file by uri, about to get dir by uri")
3849         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
3850                                            followRedirect=True))
3851         d.addCallback(self.failUnlessIsFooJSON)
3852         d.addCallback(self.log, "got dir by uri")
3853
3854         return d
3855
3856     def test_GET_URI_form_bad(self):
3857         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
3858                              "400 Bad Request", "GET /uri requires uri=",
3859                              self.GET, "/uri")
3860         return d
3861
3862     def test_GET_rename_form(self):
3863         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
3864                      followRedirect=True)
3865         def _check(res):
3866             self.failUnlessIn('name="when_done" value="."', res)
3867             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
3868             self.failUnlessIn(FAVICON_MARKUP, res)
3869         d.addCallback(_check)
3870         return d
3871
3872     def log(self, res, msg):
3873         #print "MSG: %s  RES: %s" % (msg, res)
3874         log.msg(msg)
3875         return res
3876
3877     def test_GET_URI_URL(self):
3878         base = "/uri/%s" % self._bar_txt_uri
3879         d = self.GET(base)
3880         d.addCallback(self.failUnlessIsBarDotTxt)
3881         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
3882         d.addCallback(self.failUnlessIsBarDotTxt)
3883         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
3884         d.addCallback(self.failUnlessIsBarDotTxt)
3885         return d
3886
3887     def test_GET_URI_URL_dir(self):
3888         base = "/uri/%s?t=json" % self._foo_uri
3889         d = self.GET(base)
3890         d.addCallback(self.failUnlessIsFooJSON)
3891         return d
3892
3893     def test_GET_URI_URL_missing(self):
3894         base = "/uri/%s" % self._bad_file_uri
3895         d = self.shouldHTTPError("test_GET_URI_URL_missing",
3896                                  http.GONE, None, "NotEnoughSharesError",
3897                                  self.GET, base)
3898         # TODO: how can we exercise both sides of WebDownloadTarget.fail
3899         # here? we must arrange for a download to fail after target.open()
3900         # has been called, and then inspect the response to see that it is
3901         # shorter than we expected.
3902         return d
3903
3904     def test_PUT_DIRURL_uri(self):
3905         d = self.s.create_dirnode()
3906         def _made_dir(dn):
3907             new_uri = dn.get_uri()
3908             # replace /foo with a new (empty) directory
3909             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
3910             d.addCallback(lambda res:
3911                           self.failUnlessReallyEqual(res.strip(), new_uri))
3912             d.addCallback(lambda res:
3913                           self.failUnlessRWChildURIIs(self.public_root,
3914                                                       u"foo",
3915                                                       new_uri))
3916             return d
3917         d.addCallback(_made_dir)
3918         return d
3919
3920     def test_PUT_DIRURL_uri_noreplace(self):
3921         d = self.s.create_dirnode()
3922         def _made_dir(dn):
3923             new_uri = dn.get_uri()
3924             # replace /foo with a new (empty) directory, but ask that
3925             # replace=false, so it should fail
3926             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
3927                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
3928                                  self.PUT,
3929                                  self.public_url + "/foo?t=uri&replace=false",
3930                                  new_uri)
3931             d.addCallback(lambda res:
3932                           self.failUnlessRWChildURIIs(self.public_root,
3933                                                       u"foo",
3934                                                       self._foo_uri))
3935             return d
3936         d.addCallback(_made_dir)
3937         return d
3938
3939     def test_PUT_DIRURL_bad_t(self):
3940         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
3941                              "400 Bad Request", "PUT to a directory",
3942                              self.PUT, self.public_url + "/foo?t=BOGUS", "")
3943         d.addCallback(lambda res:
3944                       self.failUnlessRWChildURIIs(self.public_root,
3945                                                   u"foo",
3946                                                   self._foo_uri))
3947         return d
3948
3949     def test_PUT_NEWFILEURL_uri(self):
3950         contents, n, new_uri = self.makefile(8)
3951         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
3952         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3953         d.addCallback(lambda res:
3954                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3955                                                       contents))
3956         return d
3957
3958     def test_PUT_NEWFILEURL_mdmf(self):
3959         new_contents = self.NEWFILE_CONTENTS * 300000
3960         d = self.PUT(self.public_url + \
3961                      "/foo/mdmf.txt?format=mdmf",
3962                      new_contents)
3963         d.addCallback(lambda ignored:
3964             self.GET(self.public_url + "/foo/mdmf.txt?t=json"))
3965         def _got_json(json):
3966             data = simplejson.loads(json)
3967             data = data[1]
3968             self.failUnlessIn("format", data)
3969             self.failUnlessEqual(data["format"], "MDMF")
3970             self.failUnless(data['rw_uri'].startswith("URI:MDMF"))
3971             self.failUnless(data['ro_uri'].startswith("URI:MDMF"))
3972         d.addCallback(_got_json)
3973         return d
3974
3975     def test_PUT_NEWFILEURL_sdmf(self):
3976         new_contents = self.NEWFILE_CONTENTS * 300000
3977         d = self.PUT(self.public_url + \
3978                      "/foo/sdmf.txt?format=sdmf",
3979                      new_contents)
3980         d.addCallback(lambda ignored:
3981             self.GET(self.public_url + "/foo/sdmf.txt?t=json"))
3982         def _got_json(json):
3983             data = simplejson.loads(json)
3984             data = data[1]
3985             self.failUnlessIn("format", data)
3986             self.failUnlessEqual(data["format"], "SDMF")
3987         d.addCallback(_got_json)
3988         return d
3989
3990     def test_PUT_NEWFILEURL_bad_format(self):
3991         new_contents = self.NEWFILE_CONTENTS * 300000
3992         return self.shouldHTTPError("PUT_NEWFILEURL_bad_format",
3993                                     400, "Bad Request", "Unknown format: foo",
3994                                     self.PUT, self.public_url + \
3995                                     "/foo/foo.txt?format=foo",
3996                                     new_contents)
3997
3998     def test_PUT_NEWFILEURL_uri_replace(self):
3999         contents, n, new_uri = self.makefile(8)
4000         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
4001         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
4002         d.addCallback(lambda res:
4003                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
4004                                                       contents))
4005         return d
4006
4007     def test_PUT_NEWFILEURL_uri_no_replace(self):
4008         contents, n, new_uri = self.makefile(8)
4009         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
4010         d.addBoth(self.shouldFail, error.Error,
4011                   "PUT_NEWFILEURL_uri_no_replace",
4012                   "409 Conflict",
4013                   "There was already a child by that name, and you asked me "
4014                   "to not replace it")
4015         return d
4016
4017     def test_PUT_NEWFILEURL_uri_unknown_bad(self):
4018         d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
4019         d.addBoth(self.shouldFail, error.Error,
4020                   "POST_put_uri_unknown_bad",
4021                   "400 Bad Request",
4022                   "unknown cap in a write slot")
4023         return d
4024
4025     def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
4026         d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
4027         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
4028                       u"put-future-ro.txt")
4029         return d
4030
4031     def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
4032         d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
4033         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
4034                       u"put-future-imm.txt")
4035         return d
4036
4037     def test_PUT_NEWFILE_URI(self):
4038         file_contents = "New file contents here\n"
4039         d = self.PUT("/uri", file_contents)
4040         def _check(uri):
4041             assert isinstance(uri, str), uri
4042             self.failUnlessIn(uri, self.get_all_contents())
4043             self.failUnlessReallyEqual(self.get_all_contents()[uri],
4044                                        file_contents)
4045             return self.GET("/uri/%s" % uri)
4046         d.addCallback(_check)
4047         def _check2(res):
4048             self.failUnlessReallyEqual(res, file_contents)
4049         d.addCallback(_check2)
4050         return d
4051
4052     def test_PUT_NEWFILE_URI_not_mutable(self):
4053         file_contents = "New file contents here\n"
4054         d = self.PUT("/uri?mutable=false", file_contents)
4055         def _check(uri):
4056             assert isinstance(uri, str), uri
4057             self.failUnlessIn(uri, self.get_all_contents())
4058             self.failUnlessReallyEqual(self.get_all_contents()[uri],
4059                                        file_contents)
4060             return self.GET("/uri/%s" % uri)
4061         d.addCallback(_check)
4062         def _check2(res):
4063             self.failUnlessReallyEqual(res, file_contents)
4064         d.addCallback(_check2)
4065         return d
4066
4067     def test_PUT_NEWFILE_URI_only_PUT(self):
4068         d = self.PUT("/uri?t=bogus", "")
4069         d.addBoth(self.shouldFail, error.Error,
4070                   "PUT_NEWFILE_URI_only_PUT",
4071                   "400 Bad Request",
4072                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
4073         return d
4074
4075     def test_PUT_NEWFILE_URI_mutable(self):
4076         file_contents = "New file contents here\n"
4077         d = self.PUT("/uri?mutable=true", file_contents)
4078         def _check1(filecap):
4079             filecap = filecap.strip()
4080             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
4081             self.filecap = filecap
4082             u = uri.WriteableSSKFileURI.init_from_string(filecap)
4083             self.failUnlessIn(u.get_storage_index(), self.get_all_contents())
4084             n = self.s.create_node_from_uri(filecap)
4085             return n.download_best_version()
4086         d.addCallback(_check1)
4087         def _check2(data):
4088             self.failUnlessReallyEqual(data, file_contents)
4089             return self.GET("/uri/%s" % urllib.quote(self.filecap))
4090         d.addCallback(_check2)
4091         def _check3(res):
4092             self.failUnlessReallyEqual(res, file_contents)
4093         d.addCallback(_check3)
4094         return d
4095
4096     def test_PUT_mkdir(self):
4097         d = self.PUT("/uri?t=mkdir", "")
4098         def _check(uri):
4099             n = self.s.create_node_from_uri(uri.strip())
4100             d2 = self.failUnlessNodeKeysAre(n, [])
4101             d2.addCallback(lambda res:
4102                            self.GET("/uri/%s?t=json" % uri))
4103             return d2
4104         d.addCallback(_check)
4105         d.addCallback(self.failUnlessIsEmptyJSON)
4106         return d
4107
4108     def test_PUT_mkdir_mdmf(self):
4109         d = self.PUT("/uri?t=mkdir&format=mdmf", "")
4110         def _got(res):
4111             u = uri.from_string(res)
4112             # Check that this is an MDMF writecap
4113             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
4114         d.addCallback(_got)
4115         return d
4116
4117     def test_PUT_mkdir_sdmf(self):
4118         d = self.PUT("/uri?t=mkdir&format=sdmf", "")
4119         def _got(res):
4120             u = uri.from_string(res)
4121             self.failUnlessIsInstance(u, uri.DirectoryURI)
4122         d.addCallback(_got)
4123         return d
4124
4125     def test_PUT_mkdir_bad_format(self):
4126         return self.shouldHTTPError("PUT_mkdir_bad_format",
4127                                     400, "Bad Request", "Unknown format: foo",
4128                                     self.PUT, "/uri?t=mkdir&format=foo",
4129                                     "")
4130
4131     def test_POST_check(self):
4132         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
4133         def _done(res):
4134             # this returns a string form of the results, which are probably
4135             # None since we're using fake filenodes.
4136             # TODO: verify that the check actually happened, by changing
4137             # FakeCHKFileNode to count how many times .check() has been
4138             # called.
4139             pass
4140         d.addCallback(_done)
4141         return d
4142
4143
4144     def test_PUT_update_at_offset(self):
4145         file_contents = "test file" * 100000 # about 900 KiB
4146         d = self.PUT("/uri?mutable=true", file_contents)
4147         def _then(filecap):
4148             self.filecap = filecap
4149             new_data = file_contents[:100]
4150             new = "replaced and so on"
4151             new_data += new
4152             new_data += file_contents[len(new_data):]
4153             assert len(new_data) == len(file_contents)
4154             self.new_data = new_data
4155         d.addCallback(_then)
4156         d.addCallback(lambda ignored:
4157             self.PUT("/uri/%s?replace=True&offset=100" % self.filecap,
4158                      "replaced and so on"))
4159         def _get_data(filecap):
4160             n = self.s.create_node_from_uri(filecap)
4161             return n.download_best_version()
4162         d.addCallback(_get_data)
4163         d.addCallback(lambda results:
4164             self.failUnlessEqual(results, self.new_data))
4165         # Now try appending things to the file
4166         d.addCallback(lambda ignored:
4167             self.PUT("/uri/%s?offset=%d" % (self.filecap, len(self.new_data)),
4168                      "puppies" * 100))
4169         d.addCallback(_get_data)
4170         d.addCallback(lambda results:
4171             self.failUnlessEqual(results, self.new_data + ("puppies" * 100)))
4172         # and try replacing the beginning of the file
4173         d.addCallback(lambda ignored:
4174             self.PUT("/uri/%s?offset=0" % self.filecap, "begin"))
4175         d.addCallback(_get_data)
4176         d.addCallback(lambda results:
4177             self.failUnlessEqual(results, "begin"+self.new_data[len("begin"):]+("puppies"*100)))
4178         return d
4179
4180     def test_PUT_update_at_invalid_offset(self):
4181         file_contents = "test file" * 100000 # about 900 KiB
4182         d = self.PUT("/uri?mutable=true", file_contents)
4183         def _then(filecap):
4184             self.filecap = filecap
4185         d.addCallback(_then)
4186         # Negative offsets should cause an error.
4187         d.addCallback(lambda ignored:
4188             self.shouldHTTPError("PUT_update_at_invalid_offset",
4189                                  400, "Bad Request",
4190                                  "Invalid offset",
4191                                  self.PUT,
4192                                  "/uri/%s?offset=-1" % self.filecap,
4193                                  "foo"))
4194         return d
4195
4196     def test_PUT_update_at_offset_immutable(self):
4197         file_contents = "Test file" * 100000
4198         d = self.PUT("/uri", file_contents)
4199         def _then(filecap):
4200             self.filecap = filecap
4201         d.addCallback(_then)
4202         d.addCallback(lambda ignored:
4203             self.shouldHTTPError("PUT_update_at_offset_immutable",
4204                                  400, "Bad Request",
4205                                  "immutable",
4206                                  self.PUT,
4207                                  "/uri/%s?offset=50" % self.filecap,
4208                                  "foo"))
4209         return d
4210
4211
4212     def test_bad_method(self):
4213         url = self.webish_url + self.public_url + "/foo/bar.txt"
4214         d = self.shouldHTTPError("bad_method",
4215                                  501, "Not Implemented",
4216                                  "I don't know how to treat a BOGUS request.",
4217                                  client.getPage, url, method="BOGUS")
4218         return d
4219
4220     def test_short_url(self):
4221         url = self.webish_url + "/uri"
4222         d = self.shouldHTTPError("short_url", 501, "Not Implemented",
4223                                  "I don't know how to treat a DELETE request.",
4224                                  client.getPage, url, method="DELETE")
4225         return d
4226
4227     def test_ophandle_bad(self):
4228         url = self.webish_url + "/operations/bogus?t=status"
4229         d = self.shouldHTTPError("ophandle_bad", 404, "404 Not Found",
4230                                  "unknown/expired handle 'bogus'",
4231                                  client.getPage, url)
4232         return d
4233
4234     def test_ophandle_cancel(self):
4235         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
4236                       followRedirect=True)
4237         d.addCallback(lambda ignored:
4238                       self.GET("/operations/128?t=status&output=JSON"))
4239         def _check1(res):
4240             data = simplejson.loads(res)
4241             self.failUnless("finished" in data, res)
4242             monitor = self.ws.root.child_operations.handles["128"][0]
4243             d = self.POST("/operations/128?t=cancel&output=JSON")
4244             def _check2(res):
4245                 data = simplejson.loads(res)
4246                 self.failUnless("finished" in data, res)
4247                 # t=cancel causes the handle to be forgotten
4248                 self.failUnless(monitor.is_cancelled())
4249             d.addCallback(_check2)
4250             return d
4251         d.addCallback(_check1)
4252         d.addCallback(lambda ignored:
4253                       self.shouldHTTPError("ophandle_cancel",
4254                                            404, "404 Not Found",
4255                                            "unknown/expired handle '128'",
4256                                            self.GET,
4257                                            "/operations/128?t=status&output=JSON"))
4258         return d
4259
4260     def test_ophandle_retainfor(self):
4261         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
4262                       followRedirect=True)
4263         d.addCallback(lambda ignored:
4264                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
4265         def _check1(res):
4266             data = simplejson.loads(res)
4267             self.failUnless("finished" in data, res)
4268         d.addCallback(_check1)
4269         # the retain-for=0 will cause the handle to be expired very soon
4270         d.addCallback(lambda ign:
4271             self.clock.advance(2.0))
4272         d.addCallback(lambda ignored:
4273                       self.shouldHTTPError("ophandle_retainfor",
4274                                            404, "404 Not Found",
4275                                            "unknown/expired handle '129'",
4276                                            self.GET,
4277                                            "/operations/129?t=status&output=JSON"))
4278         return d
4279
4280     def test_ophandle_release_after_complete(self):
4281         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
4282                       followRedirect=True)
4283         d.addCallback(self.wait_for_operation, "130")
4284         d.addCallback(lambda ignored:
4285                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
4286         # the release-after-complete=true will cause the handle to be expired
4287         d.addCallback(lambda ignored:
4288                       self.shouldHTTPError("ophandle_release_after_complete",
4289                                            404, "404 Not Found",
4290                                            "unknown/expired handle '130'",
4291                                            self.GET,
4292                                            "/operations/130?t=status&output=JSON"))
4293         return d
4294
4295     def test_uncollected_ophandle_expiration(self):
4296         # uncollected ophandles should expire after 4 days
4297         def _make_uncollected_ophandle(ophandle):
4298             d = self.POST(self.public_url +
4299                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
4300                           followRedirect=False)
4301             # When we start the operation, the webapi server will want
4302             # to redirect us to the page for the ophandle, so we get
4303             # confirmation that the operation has started. If the
4304             # manifest operation has finished by the time we get there,
4305             # following that redirect (by setting followRedirect=True
4306             # above) has the side effect of collecting the ophandle that
4307             # we've just created, which means that we can't use the
4308             # ophandle to test the uncollected timeout anymore. So,
4309             # instead, catch the 302 here and don't follow it.
4310             d.addBoth(self.should302, "uncollected_ophandle_creation")
4311             return d
4312         # Create an ophandle, don't collect it, then advance the clock by
4313         # 4 days - 1 second and make sure that the ophandle is still there.
4314         d = _make_uncollected_ophandle(131)
4315         d.addCallback(lambda ign:
4316             self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
4317         d.addCallback(lambda ign:
4318             self.GET("/operations/131?t=status&output=JSON"))
4319         def _check1(res):
4320             data = simplejson.loads(res)
4321             self.failUnless("finished" in data, res)
4322         d.addCallback(_check1)
4323         # Create an ophandle, don't collect it, then try to collect it
4324         # after 4 days. It should be gone.
4325         d.addCallback(lambda ign:
4326             _make_uncollected_ophandle(132))
4327         d.addCallback(lambda ign:
4328             self.clock.advance(96*60*60))
4329         d.addCallback(lambda ign:
4330             self.shouldHTTPError("uncollected_ophandle_expired_after_100_hours",
4331                                  404, "404 Not Found",
4332                                  "unknown/expired handle '132'",
4333                                  self.GET,
4334                                  "/operations/132?t=status&output=JSON"))
4335         return d
4336
4337     def test_collected_ophandle_expiration(self):
4338         # collected ophandles should expire after 1 day
4339         def _make_collected_ophandle(ophandle):
4340             d = self.POST(self.public_url +
4341                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
4342                           followRedirect=True)
4343             # By following the initial redirect, we collect the ophandle
4344             # we've just created.
4345             return d
4346         # Create a collected ophandle, then collect it after 23 hours
4347         # and 59 seconds to make sure that it is still there.
4348         d = _make_collected_ophandle(133)
4349         d.addCallback(lambda ign:
4350             self.clock.advance((24*60*60) - 1))
4351         d.addCallback(lambda ign:
4352             self.GET("/operations/133?t=status&output=JSON"))
4353         def _check1(res):
4354             data = simplejson.loads(res)
4355             self.failUnless("finished" in data, res)
4356         d.addCallback(_check1)
4357         # Create another uncollected ophandle, then try to collect it
4358         # after 24 hours to make sure that it is gone.
4359         d.addCallback(lambda ign:
4360             _make_collected_ophandle(134))
4361         d.addCallback(lambda ign:
4362             self.clock.advance(24*60*60))
4363         d.addCallback(lambda ign:
4364             self.shouldHTTPError("collected_ophandle_expired_after_1_day",
4365                                  404, "404 Not Found",
4366                                  "unknown/expired handle '134'",
4367                                  self.GET,
4368                                  "/operations/134?t=status&output=JSON"))
4369         return d
4370
4371     def test_incident(self):
4372         d = self.POST("/report_incident", details="eek")
4373         def _done(res):
4374             self.failIfIn("<html>", res)
4375             self.failUnlessIn("Thank you for your report!", res)
4376         d.addCallback(_done)
4377         return d
4378
4379     def test_static(self):
4380         webdir = os.path.join(self.staticdir, "subdir")
4381         fileutil.make_dirs(webdir)
4382         f = open(os.path.join(webdir, "hello.txt"), "wb")
4383         f.write("hello")
4384         f.close()
4385
4386         d = self.GET("/static/subdir/hello.txt")
4387         def _check(res):
4388             self.failUnlessReallyEqual(res, "hello")
4389         d.addCallback(_check)
4390         return d
4391
4392
4393 class IntroducerWeb(unittest.TestCase):
4394     def setUp(self):
4395         self.node = None
4396
4397     def tearDown(self):
4398         d = defer.succeed(None)
4399         if self.node:
4400             d.addCallback(lambda ign: self.node.stopService())
4401         d.addCallback(flushEventualQueue)
4402         return d
4403
4404     def test_welcome(self):
4405         basedir = "web.IntroducerWeb.test_welcome"
4406         os.mkdir(basedir)
4407         fileutil.write(os.path.join(basedir, "tahoe.cfg"), "[node]\nweb.port = tcp:0\n")
4408         self.node = IntroducerNode(basedir)
4409         self.ws = self.node.getServiceNamed("webish")
4410
4411         d = fireEventually(None)
4412         d.addCallback(lambda ign: self.node.startService())
4413         d.addCallback(lambda ign: self.node.when_tub_ready())
4414
4415         d.addCallback(lambda ign: self.GET("/"))
4416         def _check(res):
4417             self.failUnlessIn('Welcome to the Tahoe-LAFS Introducer', res)
4418             self.failUnlessIn(FAVICON_MARKUP, res)
4419         d.addCallback(_check)
4420         return d
4421
4422     def GET(self, urlpath, followRedirect=False, return_response=False,
4423             **kwargs):
4424         # if return_response=True, this fires with (data, statuscode,
4425         # respheaders) instead of just data.
4426         assert not isinstance(urlpath, unicode)
4427         url = self.ws.getURL().rstrip('/') + urlpath
4428         factory = HTTPClientGETFactory(url, method="GET",
4429                                        followRedirect=followRedirect, **kwargs)
4430         reactor.connectTCP("localhost", self.ws.getPortnum(), factory)
4431         d = factory.deferred
4432         def _got_data(data):
4433             return (data, factory.status, factory.response_headers)
4434         if return_response:
4435             d.addCallback(_got_data)
4436         return factory.deferred
4437
4438
4439 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4440     def test_load_file(self):
4441         # This will raise an exception unless a well-formed XML file is found under that name.
4442         common.getxmlfile('directory.xhtml').load()
4443
4444     def test_parse_replace_arg(self):
4445         self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
4446         self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
4447         self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
4448                                    "only-files")
4449         self.failUnlessRaises(common.WebError, common.parse_replace_arg, "only_fles")
4450
4451     def test_abbreviate_time(self):
4452         self.failUnlessReallyEqual(common.abbreviate_time(None), "")
4453         self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
4454         self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
4455         self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
4456         self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
4457         self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
4458
4459     def test_compute_rate(self):
4460         self.failUnlessReallyEqual(common.compute_rate(None, None), None)
4461         self.failUnlessReallyEqual(common.compute_rate(None, 1), None)
4462         self.failUnlessReallyEqual(common.compute_rate(250000, None), None)
4463         self.failUnlessReallyEqual(common.compute_rate(250000, 0), None)
4464         self.failUnlessReallyEqual(common.compute_rate(250000, 10), 25000.0)
4465         self.failUnlessReallyEqual(common.compute_rate(0, 10), 0.0)
4466         self.shouldFail(AssertionError, "test_compute_rate", "",
4467                         common.compute_rate, -100, 10)
4468         self.shouldFail(AssertionError, "test_compute_rate", "",
4469                         common.compute_rate, 100, -10)
4470
4471         # Sanity check
4472         rate = common.compute_rate(10*1000*1000, 1)
4473         self.failUnlessReallyEqual(common.abbreviate_rate(rate), "10.00MBps")
4474
4475     def test_abbreviate_rate(self):
4476         self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
4477         self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
4478         self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
4479         self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
4480
4481     def test_abbreviate_size(self):
4482         self.failUnlessReallyEqual(common.abbreviate_size(None), "")
4483         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
4484         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
4485         self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
4486         self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
4487
4488     def test_plural(self):
4489         def convert(s):
4490             return "%d second%s" % (s, status.plural(s))
4491         self.failUnlessReallyEqual(convert(0), "0 seconds")
4492         self.failUnlessReallyEqual(convert(1), "1 second")
4493         self.failUnlessReallyEqual(convert(2), "2 seconds")
4494         def convert2(s):
4495             return "has share%s: %s" % (status.plural(s), ",".join(s))
4496         self.failUnlessReallyEqual(convert2([]), "has shares: ")
4497         self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
4498         self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
4499
4500
4501 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4502
4503     def CHECK(self, ign, which, args, clientnum=0):
4504         fileurl = self.fileurls[which]
4505         url = fileurl + "?" + args
4506         return self.GET(url, method="POST", clientnum=clientnum)
4507
4508     def test_filecheck(self):
4509         self.basedir = "web/Grid/filecheck"
4510         self.set_up_grid()
4511         c0 = self.g.clients[0]
4512         self.uris = {}
4513         DATA = "data" * 100
4514         d = c0.upload(upload.Data(DATA, convergence=""))
4515         def _stash_uri(ur, which):
4516             self.uris[which] = ur.get_uri()
4517         d.addCallback(_stash_uri, "good")
4518         d.addCallback(lambda ign:
4519                       c0.upload(upload.Data(DATA+"1", convergence="")))
4520         d.addCallback(_stash_uri, "sick")
4521         d.addCallback(lambda ign:
4522                       c0.upload(upload.Data(DATA+"2", convergence="")))
4523         d.addCallback(_stash_uri, "dead")
4524         def _stash_mutable_uri(n, which):
4525             self.uris[which] = n.get_uri()
4526             assert isinstance(self.uris[which], str)
4527         d.addCallback(lambda ign:
4528             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4529         d.addCallback(_stash_mutable_uri, "corrupt")
4530         d.addCallback(lambda ign:
4531                       c0.upload(upload.Data("literal", convergence="")))
4532         d.addCallback(_stash_uri, "small")
4533         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
4534         d.addCallback(_stash_mutable_uri, "smalldir")
4535
4536         def _compute_fileurls(ignored):
4537             self.fileurls = {}
4538             for which in self.uris:
4539                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4540         d.addCallback(_compute_fileurls)
4541
4542         def _clobber_shares(ignored):
4543             good_shares = self.find_uri_shares(self.uris["good"])
4544             self.failUnlessReallyEqual(len(good_shares), 10)
4545             sick_shares = self.find_uri_shares(self.uris["sick"])
4546             os.unlink(sick_shares[0][2])
4547             dead_shares = self.find_uri_shares(self.uris["dead"])
4548             for i in range(1, 10):
4549                 os.unlink(dead_shares[i][2])
4550             c_shares = self.find_uri_shares(self.uris["corrupt"])
4551             cso = CorruptShareOptions()
4552             cso.stdout = StringIO()
4553             cso.parseOptions([c_shares[0][2]])
4554             corrupt_share(cso)
4555         d.addCallback(_clobber_shares)
4556
4557         d.addCallback(self.CHECK, "good", "t=check")
4558         def _got_html_good(res):
4559             self.failUnlessIn("Healthy", res)
4560             self.failIfIn("Not Healthy", res)
4561             self.failUnlessIn(FAVICON_MARKUP, res)
4562         d.addCallback(_got_html_good)
4563         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
4564         def _got_html_good_return_to(res):
4565             self.failUnlessIn("Healthy", res)
4566             self.failIfIn("Not Healthy", res)
4567             self.failUnlessIn('<a href="somewhere">Return to file', res)
4568         d.addCallback(_got_html_good_return_to)
4569         d.addCallback(self.CHECK, "good", "t=check&output=json")
4570         def _got_json_good(res):
4571             r = simplejson.loads(res)
4572             self.failUnlessEqual(r["summary"], "Healthy")
4573             self.failUnless(r["results"]["healthy"])
4574             self.failIf(r["results"]["needs-rebalancing"])
4575             self.failUnless(r["results"]["recoverable"])
4576         d.addCallback(_got_json_good)
4577
4578         d.addCallback(self.CHECK, "small", "t=check")
4579         def _got_html_small(res):
4580             self.failUnlessIn("Literal files are always healthy", res)
4581             self.failIfIn("Not Healthy", res)
4582         d.addCallback(_got_html_small)
4583         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
4584         def _got_html_small_return_to(res):
4585             self.failUnlessIn("Literal files are always healthy", res)
4586             self.failIfIn("Not Healthy", res)
4587             self.failUnlessIn('<a href="somewhere">Return to file', res)
4588         d.addCallback(_got_html_small_return_to)
4589         d.addCallback(self.CHECK, "small", "t=check&output=json")
4590         def _got_json_small(res):
4591             r = simplejson.loads(res)
4592             self.failUnlessEqual(r["storage-index"], "")
4593             self.failUnless(r["results"]["healthy"])
4594         d.addCallback(_got_json_small)
4595
4596         d.addCallback(self.CHECK, "smalldir", "t=check")
4597         def _got_html_smalldir(res):
4598             self.failUnlessIn("Literal files are always healthy", res)
4599             self.failIfIn("Not Healthy", res)
4600         d.addCallback(_got_html_smalldir)
4601         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
4602         def _got_json_smalldir(res):
4603             r = simplejson.loads(res)
4604             self.failUnlessEqual(r["storage-index"], "")
4605             self.failUnless(r["results"]["healthy"])
4606         d.addCallback(_got_json_smalldir)
4607
4608         d.addCallback(self.CHECK, "sick", "t=check")
4609         def _got_html_sick(res):
4610             self.failUnlessIn("Not Healthy", res)
4611         d.addCallback(_got_html_sick)
4612         d.addCallback(self.CHECK, "sick", "t=check&output=json")
4613         def _got_json_sick(res):
4614             r = simplejson.loads(res)
4615             self.failUnlessEqual(r["summary"],
4616                                  "Not Healthy: 9 shares (enc 3-of-10)")
4617             self.failIf(r["results"]["healthy"])
4618             self.failIf(r["results"]["needs-rebalancing"])
4619             self.failUnless(r["results"]["recoverable"])
4620         d.addCallback(_got_json_sick)
4621
4622         d.addCallback(self.CHECK, "dead", "t=check")
4623         def _got_html_dead(res):
4624             self.failUnlessIn("Not Healthy", res)
4625         d.addCallback(_got_html_dead)
4626         d.addCallback(self.CHECK, "dead", "t=check&output=json")
4627         def _got_json_dead(res):
4628             r = simplejson.loads(res)
4629             self.failUnlessEqual(r["summary"],
4630                                  "Not Healthy: 1 shares (enc 3-of-10)")
4631             self.failIf(r["results"]["healthy"])
4632             self.failIf(r["results"]["needs-rebalancing"])
4633             self.failIf(r["results"]["recoverable"])
4634         d.addCallback(_got_json_dead)
4635
4636         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
4637         def _got_html_corrupt(res):
4638             self.failUnlessIn("Not Healthy! : Unhealthy", res)
4639         d.addCallback(_got_html_corrupt)
4640         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
4641         def _got_json_corrupt(res):
4642             r = simplejson.loads(res)
4643             self.failUnlessIn("Unhealthy: 9 shares (enc 3-of-10)", r["summary"])
4644             self.failIf(r["results"]["healthy"])
4645             self.failUnless(r["results"]["recoverable"])
4646             self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
4647             self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
4648         d.addCallback(_got_json_corrupt)
4649
4650         d.addErrback(self.explain_web_error)
4651         return d
4652
4653     def test_repair_html(self):
4654         self.basedir = "web/Grid/repair_html"
4655         self.set_up_grid()
4656         c0 = self.g.clients[0]
4657         self.uris = {}
4658         DATA = "data" * 100
4659         d = c0.upload(upload.Data(DATA, convergence=""))
4660         def _stash_uri(ur, which):
4661             self.uris[which] = ur.get_uri()
4662         d.addCallback(_stash_uri, "good")
4663         d.addCallback(lambda ign:
4664                       c0.upload(upload.Data(DATA+"1", convergence="")))
4665         d.addCallback(_stash_uri, "sick")
4666         d.addCallback(lambda ign:
4667                       c0.upload(upload.Data(DATA+"2", convergence="")))
4668         d.addCallback(_stash_uri, "dead")
4669         def _stash_mutable_uri(n, which):
4670             self.uris[which] = n.get_uri()
4671             assert isinstance(self.uris[which], str)
4672         d.addCallback(lambda ign:
4673             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4674         d.addCallback(_stash_mutable_uri, "corrupt")
4675
4676         def _compute_fileurls(ignored):
4677             self.fileurls = {}
4678             for which in self.uris:
4679                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4680         d.addCallback(_compute_fileurls)
4681
4682         def _clobber_shares(ignored):
4683             good_shares = self.find_uri_shares(self.uris["good"])
4684             self.failUnlessReallyEqual(len(good_shares), 10)
4685             sick_shares = self.find_uri_shares(self.uris["sick"])
4686             os.unlink(sick_shares[0][2])
4687             dead_shares = self.find_uri_shares(self.uris["dead"])
4688             for i in range(1, 10):
4689                 os.unlink(dead_shares[i][2])
4690             c_shares = self.find_uri_shares(self.uris["corrupt"])
4691             cso = CorruptShareOptions()
4692             cso.stdout = StringIO()
4693             cso.parseOptions([c_shares[0][2]])
4694             corrupt_share(cso)
4695         d.addCallback(_clobber_shares)
4696
4697         d.addCallback(self.CHECK, "good", "t=check&repair=true")
4698         def _got_html_good(res):
4699             self.failUnlessIn("Healthy", res)
4700             self.failIfIn("Not Healthy", res)
4701             self.failUnlessIn("No repair necessary", res)
4702             self.failUnlessIn(FAVICON_MARKUP, res)
4703         d.addCallback(_got_html_good)
4704
4705         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
4706         def _got_html_sick(res):
4707             self.failUnlessIn("Healthy : healthy", res)
4708             self.failIfIn("Not Healthy", res)
4709             self.failUnlessIn("Repair successful", res)
4710         d.addCallback(_got_html_sick)
4711
4712         # repair of a dead file will fail, of course, but it isn't yet
4713         # clear how this should be reported. Right now it shows up as
4714         # a "410 Gone".
4715         #
4716         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
4717         #def _got_html_dead(res):
4718         #    print res
4719         #    self.failUnlessIn("Healthy : healthy", res)
4720         #    self.failIfIn("Not Healthy", res)
4721         #    self.failUnlessIn("No repair necessary", res)
4722         #d.addCallback(_got_html_dead)
4723
4724         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
4725         def _got_html_corrupt(res):
4726             self.failUnlessIn("Healthy : Healthy", res)
4727             self.failIfIn("Not Healthy", res)
4728             self.failUnlessIn("Repair successful", res)
4729         d.addCallback(_got_html_corrupt)
4730
4731         d.addErrback(self.explain_web_error)
4732         return d
4733
4734     def test_repair_json(self):
4735         self.basedir = "web/Grid/repair_json"
4736         self.set_up_grid()
4737         c0 = self.g.clients[0]
4738         self.uris = {}
4739         DATA = "data" * 100
4740         d = c0.upload(upload.Data(DATA+"1", convergence=""))
4741         def _stash_uri(ur, which):
4742             self.uris[which] = ur.get_uri()
4743         d.addCallback(_stash_uri, "sick")
4744
4745         def _compute_fileurls(ignored):
4746             self.fileurls = {}
4747             for which in self.uris:
4748                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4749         d.addCallback(_compute_fileurls)
4750
4751         def _clobber_shares(ignored):
4752             sick_shares = self.find_uri_shares(self.uris["sick"])
4753             os.unlink(sick_shares[0][2])
4754         d.addCallback(_clobber_shares)
4755
4756         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
4757         def _got_json_sick(res):
4758             r = simplejson.loads(res)
4759             self.failUnlessReallyEqual(r["repair-attempted"], True)
4760             self.failUnlessReallyEqual(r["repair-successful"], True)
4761             self.failUnlessEqual(r["pre-repair-results"]["summary"],
4762                                  "Not Healthy: 9 shares (enc 3-of-10)")
4763             self.failIf(r["pre-repair-results"]["results"]["healthy"])
4764             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
4765             self.failUnless(r["post-repair-results"]["results"]["healthy"])
4766         d.addCallback(_got_json_sick)
4767
4768         d.addErrback(self.explain_web_error)
4769         return d
4770
4771     def test_unknown(self, immutable=False):
4772         self.basedir = "web/Grid/unknown"
4773         if immutable:
4774             self.basedir = "web/Grid/unknown-immutable"
4775
4776         self.set_up_grid()
4777         c0 = self.g.clients[0]
4778         self.uris = {}
4779         self.fileurls = {}
4780
4781         # the future cap format may contain slashes, which must be tolerated
4782         expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
4783                                                            safe="")
4784
4785         if immutable:
4786             name = u"future-imm"
4787             future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
4788             d = c0.create_immutable_dirnode({name: (future_node, {})})
4789         else:
4790             name = u"future"
4791             future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4792             d = c0.create_dirnode()
4793
4794         def _stash_root_and_create_file(n):
4795             self.rootnode = n
4796             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
4797             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
4798             if not immutable:
4799                 return self.rootnode.set_node(name, future_node)
4800         d.addCallback(_stash_root_and_create_file)
4801
4802         # make sure directory listing tolerates unknown nodes
4803         d.addCallback(lambda ign: self.GET(self.rooturl))
4804         def _check_directory_html(res, expected_type_suffix):
4805             pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
4806                                   '<td>%s</td>' % (expected_type_suffix, str(name)),
4807                                  re.DOTALL)
4808             self.failUnless(re.search(pattern, res), res)
4809             # find the More Info link for name, should be relative
4810             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4811             info_url = mo.group(1)
4812             self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
4813         if immutable:
4814             d.addCallback(_check_directory_html, "-IMM")
4815         else:
4816             d.addCallback(_check_directory_html, "")
4817
4818         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4819         def _check_directory_json(res, expect_rw_uri):
4820             data = simplejson.loads(res)
4821             self.failUnlessEqual(data[0], "dirnode")
4822             f = data[1]["children"][name]
4823             self.failUnlessEqual(f[0], "unknown")
4824             if expect_rw_uri:
4825                 self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data)
4826             else:
4827                 self.failIfIn("rw_uri", f[1])
4828             if immutable:
4829                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
4830             else:
4831                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
4832             self.failUnlessIn("metadata", f[1])
4833         d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
4834
4835         def _check_info(res, expect_rw_uri, expect_ro_uri):
4836             self.failUnlessIn("Object Type: <span>unknown</span>", res)
4837             if expect_rw_uri:
4838                 self.failUnlessIn(unknown_rwcap, res)
4839             if expect_ro_uri:
4840                 if immutable:
4841                     self.failUnlessIn(unknown_immcap, res)
4842                 else:
4843                     self.failUnlessIn(unknown_rocap, res)
4844             else:
4845                 self.failIfIn(unknown_rocap, res)
4846             self.failIfIn("Raw data as", res)
4847             self.failIfIn("Directory writecap", res)
4848             self.failIfIn("Checker Operations", res)
4849             self.failIfIn("Mutable File Operations", res)
4850             self.failIfIn("Directory Operations", res)
4851
4852         # FIXME: these should have expect_rw_uri=not immutable; I don't know
4853         # why they fail. Possibly related to ticket #922.
4854
4855         d.addCallback(lambda ign: self.GET(expected_info_url))
4856         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
4857         d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
4858         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
4859
4860         def _check_json(res, expect_rw_uri):
4861             data = simplejson.loads(res)
4862             self.failUnlessEqual(data[0], "unknown")
4863             if expect_rw_uri:
4864                 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data)
4865             else:
4866                 self.failIfIn("rw_uri", data[1])
4867
4868             if immutable:
4869                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data)
4870                 self.failUnlessReallyEqual(data[1]["mutable"], False)
4871             elif expect_rw_uri:
4872                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4873                 self.failUnlessReallyEqual(data[1]["mutable"], True)
4874             else:
4875                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4876                 self.failIfIn("mutable", data[1])
4877
4878             # TODO: check metadata contents
4879             self.failUnlessIn("metadata", data[1])
4880
4881         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
4882         d.addCallback(_check_json, expect_rw_uri=not immutable)
4883
4884         # and make sure that a read-only version of the directory can be
4885         # rendered too. This version will not have unknown_rwcap, whether
4886         # or not future_node was immutable.
4887         d.addCallback(lambda ign: self.GET(self.rourl))
4888         if immutable:
4889             d.addCallback(_check_directory_html, "-IMM")
4890         else:
4891             d.addCallback(_check_directory_html, "-RO")
4892
4893         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
4894         d.addCallback(_check_directory_json, expect_rw_uri=False)
4895
4896         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
4897         d.addCallback(_check_json, expect_rw_uri=False)
4898
4899         # TODO: check that getting t=info from the Info link in the ro directory
4900         # works, and does not include the writecap URI.
4901         return d
4902
4903     def test_immutable_unknown(self):
4904         return self.test_unknown(immutable=True)
4905
4906     def test_mutant_dirnodes_are_omitted(self):
4907         self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
4908
4909         self.set_up_grid()
4910         c = self.g.clients[0]
4911         nm = c.nodemaker
4912         self.uris = {}
4913         self.fileurls = {}
4914
4915         lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
4916         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
4917         mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
4918
4919         # This method tests mainly dirnode, but we'd have to duplicate code in order to
4920         # test the dirnode and web layers separately.
4921
4922         # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
4923         # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
4924         # When the directory is read, the mutants should be silently disposed of, leaving
4925         # their lonely sibling.
4926         # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
4927         # because immutable directories don't have a writecap and therefore that field
4928         # isn't (and can't be) decrypted.
4929         # TODO: The field still exists in the netstring. Technically we should check what
4930         # happens if something is put there (_unpack_contents should raise ValueError),
4931         # but that can wait.
4932
4933         lonely_child = nm.create_from_cap(lonely_uri)
4934         mutant_ro_child = nm.create_from_cap(mut_read_uri)
4935         mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
4936
4937         def _by_hook_or_by_crook():
4938             return True
4939         for n in [mutant_ro_child, mutant_write_in_ro_child]:
4940             n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
4941
4942         mutant_write_in_ro_child.get_write_uri    = lambda: None
4943         mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
4944
4945         kids = {u"lonely":      (lonely_child, {}),
4946                 u"ro":          (mutant_ro_child, {}),
4947                 u"write-in-ro": (mutant_write_in_ro_child, {}),
4948                 }
4949         d = c.create_immutable_dirnode(kids)
4950
4951         def _created(dn):
4952             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
4953             self.failIf(dn.is_mutable())
4954             self.failUnless(dn.is_readonly())
4955             # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
4956             self.failIf(hasattr(dn._node, 'get_writekey'))
4957             rep = str(dn)
4958             self.failUnlessIn("RO-IMM", rep)
4959             cap = dn.get_cap()
4960             self.failUnlessIn("CHK", cap.to_string())
4961             self.cap = cap
4962             self.rootnode = dn
4963             self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
4964             return download_to_data(dn._node)
4965         d.addCallback(_created)
4966
4967         def _check_data(data):
4968             # Decode the netstring representation of the directory to check that all children
4969             # are present. This is a bit of an abstraction violation, but there's not really
4970             # any other way to do it given that the real DirectoryNode._unpack_contents would
4971             # strip the mutant children out (which is what we're trying to test, later).
4972             position = 0
4973             numkids = 0
4974             while position < len(data):
4975                 entries, position = split_netstring(data, 1, position)
4976                 entry = entries[0]
4977                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
4978                 name = name_utf8.decode("utf-8")
4979                 self.failUnlessEqual(rwcapdata, "")
4980                 self.failUnlessIn(name, kids)
4981                 (expected_child, ign) = kids[name]
4982                 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
4983                 numkids += 1
4984
4985             self.failUnlessReallyEqual(numkids, 3)
4986             return self.rootnode.list()
4987         d.addCallback(_check_data)
4988
4989         # Now when we use the real directory listing code, the mutants should be absent.
4990         def _check_kids(children):
4991             self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
4992             lonely_node, lonely_metadata = children[u"lonely"]
4993
4994             self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
4995             self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
4996         d.addCallback(_check_kids)
4997
4998         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
4999         d.addCallback(lambda n: n.list())
5000         d.addCallback(_check_kids)  # again with dirnode recreated from cap
5001
5002         # Make sure the lonely child can be listed in HTML...
5003         d.addCallback(lambda ign: self.GET(self.rooturl))
5004         def _check_html(res):
5005             self.failIfIn("URI:SSK", res)
5006             get_lonely = "".join([r'<td>FILE</td>',
5007                                   r'\s+<td>',
5008                                   r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
5009                                   r'</td>',
5010                                   r'\s+<td align="right">%d</td>' % len("one"),
5011                                   ])
5012             self.failUnless(re.search(get_lonely, res), res)
5013
5014             # find the More Info link for name, should be relative
5015             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
5016             info_url = mo.group(1)
5017             self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
5018         d.addCallback(_check_html)
5019
5020         # ... and in JSON.
5021         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
5022         def _check_json(res):
5023             data = simplejson.loads(res)
5024             self.failUnlessEqual(data[0], "dirnode")
5025             listed_children = data[1]["children"]
5026             self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
5027             ll_type, ll_data = listed_children[u"lonely"]
5028             self.failUnlessEqual(ll_type, "filenode")
5029             self.failIfIn("rw_uri", ll_data)
5030             self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
5031         d.addCallback(_check_json)
5032         return d
5033
5034     def test_deep_check(self):
5035         self.basedir = "web/Grid/deep_check"
5036         self.set_up_grid()
5037         c0 = self.g.clients[0]
5038         self.uris = {}
5039         self.fileurls = {}
5040         DATA = "data" * 100
5041         d = c0.create_dirnode()
5042         def _stash_root_and_create_file(n):
5043             self.rootnode = n
5044             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5045             return n.add_file(u"good", upload.Data(DATA, convergence=""))
5046         d.addCallback(_stash_root_and_create_file)
5047         def _stash_uri(fn, which):
5048             self.uris[which] = fn.get_uri()
5049             return fn
5050         d.addCallback(_stash_uri, "good")
5051         d.addCallback(lambda ign:
5052                       self.rootnode.add_file(u"small",
5053                                              upload.Data("literal",
5054                                                         convergence="")))
5055         d.addCallback(_stash_uri, "small")
5056         d.addCallback(lambda ign:
5057                       self.rootnode.add_file(u"sick",
5058                                              upload.Data(DATA+"1",
5059                                                         convergence="")))
5060         d.addCallback(_stash_uri, "sick")
5061
5062         # this tests that deep-check and stream-manifest will ignore
5063         # UnknownNode instances. Hopefully this will also cover deep-stats.
5064         future_node = UnknownNode(unknown_rwcap, unknown_rocap)
5065         d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
5066
5067         def _clobber_shares(ignored):
5068             self.delete_shares_numbered(self.uris["sick"], [0,1])
5069         d.addCallback(_clobber_shares)
5070
5071         # root
5072         # root/good
5073         # root/small
5074         # root/sick
5075         # root/future
5076
5077         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
5078         def _done(res):
5079             try:
5080                 units = [simplejson.loads(line)
5081                          for line in res.splitlines()
5082                          if line]
5083             except ValueError:
5084                 print "response is:", res
5085                 print "undecodeable line was '%s'" % line
5086                 raise
5087             self.failUnlessReallyEqual(len(units), 5+1)
5088             # should be parent-first
5089             u0 = units[0]
5090             self.failUnlessEqual(u0["path"], [])
5091             self.failUnlessEqual(u0["type"], "directory")
5092             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
5093             u0cr = u0["check-results"]
5094             self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
5095
5096             ugood = [u for u in units
5097                      if u["type"] == "file" and u["path"] == [u"good"]][0]
5098             self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
5099             ugoodcr = ugood["check-results"]
5100             self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
5101
5102             stats = units[-1]
5103             self.failUnlessEqual(stats["type"], "stats")
5104             s = stats["stats"]
5105             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
5106             self.failUnlessReallyEqual(s["count-literal-files"], 1)
5107             self.failUnlessReallyEqual(s["count-directories"], 1)
5108             self.failUnlessReallyEqual(s["count-unknown"], 1)
5109         d.addCallback(_done)
5110
5111         d.addCallback(self.CHECK, "root", "t=stream-manifest")
5112         def _check_manifest(res):
5113             self.failUnless(res.endswith("\n"))
5114             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
5115             self.failUnlessReallyEqual(len(units), 5+1)
5116             self.failUnlessEqual(units[-1]["type"], "stats")
5117             first = units[0]
5118             self.failUnlessEqual(first["path"], [])
5119             self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri())
5120             self.failUnlessEqual(first["type"], "directory")
5121             stats = units[-1]["stats"]
5122             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
5123             self.failUnlessReallyEqual(stats["count-literal-files"], 1)
5124             self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
5125             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
5126             self.failUnlessReallyEqual(stats["count-unknown"], 1)
5127         d.addCallback(_check_manifest)
5128
5129         # now add root/subdir and root/subdir/grandchild, then make subdir
5130         # unrecoverable, then see what happens
5131
5132         d.addCallback(lambda ign:
5133                       self.rootnode.create_subdirectory(u"subdir"))
5134         d.addCallback(_stash_uri, "subdir")
5135         d.addCallback(lambda subdir_node:
5136                       subdir_node.add_file(u"grandchild",
5137                                            upload.Data(DATA+"2",
5138                                                        convergence="")))
5139         d.addCallback(_stash_uri, "grandchild")
5140
5141         d.addCallback(lambda ign:
5142                       self.delete_shares_numbered(self.uris["subdir"],
5143                                                   range(1, 10)))
5144
5145         # root
5146         # root/good
5147         # root/small
5148         # root/sick
5149         # root/future
5150         # root/subdir [unrecoverable]
5151         # root/subdir/grandchild
5152
5153         # how should a streaming-JSON API indicate fatal error?
5154         # answer: emit ERROR: instead of a JSON string
5155
5156         d.addCallback(self.CHECK, "root", "t=stream-manifest")
5157         def _check_broken_manifest(res):
5158             lines = res.splitlines()
5159             error_lines = [i
5160                            for (i,line) in enumerate(lines)
5161                            if line.startswith("ERROR:")]
5162             if not error_lines:
5163                 self.fail("no ERROR: in output: %s" % (res,))
5164             first_error = error_lines[0]
5165             error_line = lines[first_error]
5166             error_msg = lines[first_error+1:]
5167             error_msg_s = "\n".join(error_msg) + "\n"
5168             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
5169                               error_line)
5170             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
5171             units = [simplejson.loads(line) for line in lines[:first_error]]
5172             self.failUnlessReallyEqual(len(units), 6) # includes subdir
5173             last_unit = units[-1]
5174             self.failUnlessEqual(last_unit["path"], ["subdir"])
5175         d.addCallback(_check_broken_manifest)
5176
5177         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
5178         def _check_broken_deepcheck(res):
5179             lines = res.splitlines()
5180             error_lines = [i
5181                            for (i,line) in enumerate(lines)
5182                            if line.startswith("ERROR:")]
5183             if not error_lines:
5184                 self.fail("no ERROR: in output: %s" % (res,))
5185             first_error = error_lines[0]
5186             error_line = lines[first_error]
5187             error_msg = lines[first_error+1:]
5188             error_msg_s = "\n".join(error_msg) + "\n"
5189             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
5190                               error_line)
5191             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
5192             units = [simplejson.loads(line) for line in lines[:first_error]]
5193             self.failUnlessReallyEqual(len(units), 6) # includes subdir
5194             last_unit = units[-1]
5195             self.failUnlessEqual(last_unit["path"], ["subdir"])
5196             r = last_unit["check-results"]["results"]
5197             self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
5198             self.failUnlessReallyEqual(r["count-shares-good"], 1)
5199             self.failUnlessReallyEqual(r["recoverable"], False)
5200         d.addCallback(_check_broken_deepcheck)
5201
5202         d.addErrback(self.explain_web_error)
5203         return d
5204
5205     def test_deep_check_and_repair(self):
5206         self.basedir = "web/Grid/deep_check_and_repair"
5207         self.set_up_grid()
5208         c0 = self.g.clients[0]
5209         self.uris = {}
5210         self.fileurls = {}
5211         DATA = "data" * 100
5212         d = c0.create_dirnode()
5213         def _stash_root_and_create_file(n):
5214             self.rootnode = n
5215             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5216             return n.add_file(u"good", upload.Data(DATA, convergence=""))
5217         d.addCallback(_stash_root_and_create_file)
5218         def _stash_uri(fn, which):
5219             self.uris[which] = fn.get_uri()
5220         d.addCallback(_stash_uri, "good")
5221         d.addCallback(lambda ign:
5222                       self.rootnode.add_file(u"small",
5223                                              upload.Data("literal",
5224                                                         convergence="")))
5225         d.addCallback(_stash_uri, "small")
5226         d.addCallback(lambda ign:
5227                       self.rootnode.add_file(u"sick",
5228                                              upload.Data(DATA+"1",
5229                                                         convergence="")))
5230         d.addCallback(_stash_uri, "sick")
5231         #d.addCallback(lambda ign:
5232         #              self.rootnode.add_file(u"dead",
5233         #                                     upload.Data(DATA+"2",
5234         #                                                convergence="")))
5235         #d.addCallback(_stash_uri, "dead")
5236
5237         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
5238         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
5239         #d.addCallback(_stash_uri, "corrupt")
5240
5241         def _clobber_shares(ignored):
5242             good_shares = self.find_uri_shares(self.uris["good"])
5243             self.failUnlessReallyEqual(len(good_shares), 10)
5244             sick_shares = self.find_uri_shares(self.uris["sick"])
5245             os.unlink(sick_shares[0][2])
5246             #dead_shares = self.find_uri_shares(self.uris["dead"])
5247             #for i in range(1, 10):
5248             #    os.unlink(dead_shares[i][2])
5249
5250             #c_shares = self.find_uri_shares(self.uris["corrupt"])
5251             #cso = CorruptShareOptions()
5252             #cso.stdout = StringIO()
5253             #cso.parseOptions([c_shares[0][2]])
5254             #corrupt_share(cso)
5255         d.addCallback(_clobber_shares)
5256
5257         # root
5258         # root/good   CHK, 10 shares
5259         # root/small  LIT
5260         # root/sick   CHK, 9 shares
5261
5262         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
5263         def _done(res):
5264             units = [simplejson.loads(line)
5265                      for line in res.splitlines()
5266                      if line]
5267             self.failUnlessReallyEqual(len(units), 4+1)
5268             # should be parent-first
5269             u0 = units[0]
5270             self.failUnlessEqual(u0["path"], [])
5271             self.failUnlessEqual(u0["type"], "directory")
5272             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
5273             u0crr = u0["check-and-repair-results"]
5274             self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
5275             self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
5276
5277             ugood = [u for u in units
5278                      if u["type"] == "file" and u["path"] == [u"good"]][0]
5279             self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
5280             ugoodcrr = ugood["check-and-repair-results"]
5281             self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
5282             self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
5283
5284             usick = [u for u in units
5285                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
5286             self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"])
5287             usickcrr = usick["check-and-repair-results"]
5288             self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
5289             self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
5290             self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
5291             self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
5292
5293             stats = units[-1]
5294             self.failUnlessEqual(stats["type"], "stats")
5295             s = stats["stats"]
5296             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
5297             self.failUnlessReallyEqual(s["count-literal-files"], 1)
5298             self.failUnlessReallyEqual(s["count-directories"], 1)
5299         d.addCallback(_done)
5300
5301         d.addErrback(self.explain_web_error)
5302         return d
5303
5304     def _count_leases(self, ignored, which):
5305         u = self.uris[which]
5306         shares = self.find_uri_shares(u)
5307         lease_counts = []
5308         for shnum, serverid, fn in shares:
5309             sf = get_share_file(fn)
5310             num_leases = len(list(sf.get_leases()))
5311             lease_counts.append( (fn, num_leases) )
5312         return lease_counts
5313
5314     def _assert_leasecount(self, lease_counts, expected):
5315         for (fn, num_leases) in lease_counts:
5316             if num_leases != expected:
5317                 self.fail("expected %d leases, have %d, on %s" %
5318                           (expected, num_leases, fn))
5319
5320     def test_add_lease(self):
5321         self.basedir = "web/Grid/add_lease"
5322         self.set_up_grid(num_clients=2)
5323         c0 = self.g.clients[0]
5324         self.uris = {}
5325         DATA = "data" * 100
5326         d = c0.upload(upload.Data(DATA, convergence=""))
5327         def _stash_uri(ur, which):
5328             self.uris[which] = ur.get_uri()
5329         d.addCallback(_stash_uri, "one")
5330         d.addCallback(lambda ign:
5331                       c0.upload(upload.Data(DATA+"1", convergence="")))
5332         d.addCallback(_stash_uri, "two")
5333         def _stash_mutable_uri(n, which):
5334             self.uris[which] = n.get_uri()
5335             assert isinstance(self.uris[which], str)
5336         d.addCallback(lambda ign:
5337             c0.create_mutable_file(publish.MutableData(DATA+"2")))
5338         d.addCallback(_stash_mutable_uri, "mutable")
5339
5340         def _compute_fileurls(ignored):
5341             self.fileurls = {}
5342             for which in self.uris:
5343                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
5344         d.addCallback(_compute_fileurls)
5345
5346         d.addCallback(self._count_leases, "one")
5347         d.addCallback(self._assert_leasecount, 1)
5348         d.addCallback(self._count_leases, "two")
5349         d.addCallback(self._assert_leasecount, 1)
5350         d.addCallback(self._count_leases, "mutable")
5351         d.addCallback(self._assert_leasecount, 1)
5352
5353         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
5354         def _got_html_good(res):
5355             self.failUnlessIn("Healthy", res)
5356             self.failIfIn("Not Healthy", res)
5357         d.addCallback(_got_html_good)
5358
5359         d.addCallback(self._count_leases, "one")
5360         d.addCallback(self._assert_leasecount, 1)
5361         d.addCallback(self._count_leases, "two")
5362         d.addCallback(self._assert_leasecount, 1)
5363         d.addCallback(self._count_leases, "mutable")
5364         d.addCallback(self._assert_leasecount, 1)
5365
5366         # this CHECK uses the original client, which uses the same
5367         # lease-secrets, so it will just renew the original lease
5368         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
5369         d.addCallback(_got_html_good)
5370
5371         d.addCallback(self._count_leases, "one")
5372         d.addCallback(self._assert_leasecount, 1)
5373         d.addCallback(self._count_leases, "two")
5374         d.addCallback(self._assert_leasecount, 1)
5375         d.addCallback(self._count_leases, "mutable")
5376         d.addCallback(self._assert_leasecount, 1)
5377
5378         # this CHECK uses an alternate client, which adds a second lease
5379         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
5380         d.addCallback(_got_html_good)
5381
5382         d.addCallback(self._count_leases, "one")
5383         d.addCallback(self._assert_leasecount, 2)
5384         d.addCallback(self._count_leases, "two")
5385         d.addCallback(self._assert_leasecount, 1)
5386         d.addCallback(self._count_leases, "mutable")
5387         d.addCallback(self._assert_leasecount, 1)
5388
5389         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
5390         d.addCallback(_got_html_good)
5391
5392         d.addCallback(self._count_leases, "one")
5393         d.addCallback(self._assert_leasecount, 2)
5394         d.addCallback(self._count_leases, "two")
5395         d.addCallback(self._assert_leasecount, 1)
5396         d.addCallback(self._count_leases, "mutable")
5397         d.addCallback(self._assert_leasecount, 1)
5398
5399         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
5400                       clientnum=1)
5401         d.addCallback(_got_html_good)
5402
5403         d.addCallback(self._count_leases, "one")
5404         d.addCallback(self._assert_leasecount, 2)
5405         d.addCallback(self._count_leases, "two")
5406         d.addCallback(self._assert_leasecount, 1)
5407         d.addCallback(self._count_leases, "mutable")
5408         d.addCallback(self._assert_leasecount, 2)
5409
5410         d.addErrback(self.explain_web_error)
5411         return d
5412
5413     def test_deep_add_lease(self):
5414         self.basedir = "web/Grid/deep_add_lease"
5415         self.set_up_grid(num_clients=2)
5416         c0 = self.g.clients[0]
5417         self.uris = {}
5418         self.fileurls = {}
5419         DATA = "data" * 100
5420         d = c0.create_dirnode()
5421         def _stash_root_and_create_file(n):
5422             self.rootnode = n
5423             self.uris["root"] = n.get_uri()
5424             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5425             return n.add_file(u"one", upload.Data(DATA, convergence=""))
5426         d.addCallback(_stash_root_and_create_file)
5427         def _stash_uri(fn, which):
5428             self.uris[which] = fn.get_uri()
5429         d.addCallback(_stash_uri, "one")
5430         d.addCallback(lambda ign:
5431                       self.rootnode.add_file(u"small",
5432                                              upload.Data("literal",
5433                                                         convergence="")))
5434         d.addCallback(_stash_uri, "small")
5435
5436         d.addCallback(lambda ign:
5437             c0.create_mutable_file(publish.MutableData("mutable")))
5438         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
5439         d.addCallback(_stash_uri, "mutable")
5440
5441         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
5442         def _done(res):
5443             units = [simplejson.loads(line)
5444                      for line in res.splitlines()
5445                      if line]
5446             # root, one, small, mutable,   stats
5447             self.failUnlessReallyEqual(len(units), 4+1)
5448         d.addCallback(_done)
5449
5450         d.addCallback(self._count_leases, "root")
5451         d.addCallback(self._assert_leasecount, 1)
5452         d.addCallback(self._count_leases, "one")
5453         d.addCallback(self._assert_leasecount, 1)
5454         d.addCallback(self._count_leases, "mutable")
5455         d.addCallback(self._assert_leasecount, 1)
5456
5457         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
5458         d.addCallback(_done)
5459
5460         d.addCallback(self._count_leases, "root")
5461         d.addCallback(self._assert_leasecount, 1)
5462         d.addCallback(self._count_leases, "one")
5463         d.addCallback(self._assert_leasecount, 1)
5464         d.addCallback(self._count_leases, "mutable")
5465         d.addCallback(self._assert_leasecount, 1)
5466
5467         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
5468                       clientnum=1)
5469         d.addCallback(_done)
5470
5471         d.addCallback(self._count_leases, "root")
5472         d.addCallback(self._assert_leasecount, 2)
5473         d.addCallback(self._count_leases, "one")
5474         d.addCallback(self._assert_leasecount, 2)
5475         d.addCallback(self._count_leases, "mutable")
5476         d.addCallback(self._assert_leasecount, 2)
5477
5478         d.addErrback(self.explain_web_error)
5479         return d
5480
5481
5482     def test_exceptions(self):
5483         self.basedir = "web/Grid/exceptions"
5484         self.set_up_grid(num_clients=1, num_servers=2)
5485         c0 = self.g.clients[0]
5486         c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 2
5487         self.fileurls = {}
5488         DATA = "data" * 100
5489         d = c0.create_dirnode()
5490         def _stash_root(n):
5491             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5492             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
5493             return n
5494         d.addCallback(_stash_root)
5495         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
5496         def _stash_bad(ur):
5497             self.fileurls["1share"] = "uri/" + urllib.quote(ur.get_uri())
5498             self.delete_shares_numbered(ur.get_uri(), range(1,10))
5499
5500             u = uri.from_string(ur.get_uri())
5501             u.key = testutil.flip_bit(u.key, 0)
5502             baduri = u.to_string()
5503             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
5504         d.addCallback(_stash_bad)
5505         d.addCallback(lambda ign: c0.create_dirnode())
5506         def _mangle_dirnode_1share(n):
5507             u = n.get_uri()
5508             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
5509             self.fileurls["dir-1share-json"] = url + "?t=json"
5510             self.delete_shares_numbered(u, range(1,10))
5511         d.addCallback(_mangle_dirnode_1share)
5512         d.addCallback(lambda ign: c0.create_dirnode())
5513         def _mangle_dirnode_0share(n):
5514             u = n.get_uri()
5515             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
5516             self.fileurls["dir-0share-json"] = url + "?t=json"
5517             self.delete_shares_numbered(u, range(0,10))
5518         d.addCallback(_mangle_dirnode_0share)
5519
5520         # NotEnoughSharesError should be reported sensibly, with a
5521         # text/plain explanation of the problem, and perhaps some
5522         # information on which shares *could* be found.
5523
5524         d.addCallback(lambda ignored:
5525                       self.shouldHTTPError("GET unrecoverable",
5526                                            410, "Gone", "NoSharesError",
5527                                            self.GET, self.fileurls["0shares"]))
5528         def _check_zero_shares(body):
5529             self.failIfIn("<html>", body)
5530             body = " ".join(body.strip().split())
5531             exp = ("NoSharesError: no shares could be found. "
5532                    "Zero shares usually indicates a corrupt URI, or that "
5533                    "no servers were connected, but it might also indicate "
5534                    "severe corruption. You should perform a filecheck on "
5535                    "this object to learn more. The full error message is: "
5536                    "no shares (need 3). Last failure: None")
5537             self.failUnlessReallyEqual(exp, body)
5538         d.addCallback(_check_zero_shares)
5539
5540
5541         d.addCallback(lambda ignored:
5542                       self.shouldHTTPError("GET 1share",
5543                                            410, "Gone", "NotEnoughSharesError",
5544                                            self.GET, self.fileurls["1share"]))
5545         def _check_one_share(body):
5546             self.failIfIn("<html>", body)
5547             body = " ".join(body.strip().split())
5548             msgbase = ("NotEnoughSharesError: This indicates that some "
5549                        "servers were unavailable, or that shares have been "
5550                        "lost to server departure, hard drive failure, or disk "
5551                        "corruption. You should perform a filecheck on "
5552                        "this object to learn more. The full error message is:"
5553                        )
5554             msg1 = msgbase + (" ran out of shares:"
5555                               " complete=sh0"
5556                               " pending="
5557                               " overdue= unused= need 3. Last failure: None")
5558             msg2 = msgbase + (" ran out of shares:"
5559                               " complete="
5560                               " pending=Share(sh0-on-xgru5)"
5561                               " overdue= unused= need 3. Last failure: None")
5562             self.failUnless(body == msg1 or body == msg2, body)
5563         d.addCallback(_check_one_share)
5564
5565         d.addCallback(lambda ignored:
5566                       self.shouldHTTPError("GET imaginary",
5567                                            404, "Not Found", None,
5568                                            self.GET, self.fileurls["imaginary"]))
5569         def _missing_child(body):
5570             self.failUnlessIn("No such child: imaginary", body)
5571         d.addCallback(_missing_child)
5572
5573         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
5574         def _check_0shares_dir_html(body):
5575             self.failUnlessIn("<html>", body)
5576             # we should see the regular page, but without the child table or
5577             # the dirops forms
5578             body = " ".join(body.strip().split())
5579             self.failUnlessIn('href="?t=info">More info on this directory',
5580                               body)
5581             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5582                    "could not be retrieved, because there were insufficient "
5583                    "good shares. This might indicate that no servers were "
5584                    "connected, insufficient servers were connected, the URI "
5585                    "was corrupt, or that shares have been lost due to server "
5586                    "departure, hard drive failure, or disk corruption. You "
5587                    "should perform a filecheck on this object to learn more.")
5588             self.failUnlessIn(exp, body)
5589             self.failUnlessIn("No upload forms: directory is unreadable", body)
5590         d.addCallback(_check_0shares_dir_html)
5591
5592         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
5593         def _check_1shares_dir_html(body):
5594             # at some point, we'll split UnrecoverableFileError into 0-shares
5595             # and some-shares like we did for immutable files (since there
5596             # are different sorts of advice to offer in each case). For now,
5597             # they present the same way.
5598             self.failUnlessIn("<html>", body)
5599             body = " ".join(body.strip().split())
5600             self.failUnlessIn('href="?t=info">More info on this directory',
5601                               body)
5602             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5603                    "could not be retrieved, because there were insufficient "
5604                    "good shares. This might indicate that no servers were "
5605                    "connected, insufficient servers were connected, the URI "
5606                    "was corrupt, or that shares have been lost due to server "
5607                    "departure, hard drive failure, or disk corruption. You "
5608                    "should perform a filecheck on this object to learn more.")
5609             self.failUnlessIn(exp, body)
5610             self.failUnlessIn("No upload forms: directory is unreadable", body)
5611         d.addCallback(_check_1shares_dir_html)
5612
5613         d.addCallback(lambda ignored:
5614                       self.shouldHTTPError("GET dir-0share-json",
5615                                            410, "Gone", "UnrecoverableFileError",
5616                                            self.GET,
5617                                            self.fileurls["dir-0share-json"]))
5618         def _check_unrecoverable_file(body):
5619             self.failIfIn("<html>", body)
5620             body = " ".join(body.strip().split())
5621             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5622                    "could not be retrieved, because there were insufficient "
5623                    "good shares. This might indicate that no servers were "
5624                    "connected, insufficient servers were connected, the URI "
5625                    "was corrupt, or that shares have been lost due to server "
5626                    "departure, hard drive failure, or disk corruption. You "
5627                    "should perform a filecheck on this object to learn more.")
5628             self.failUnlessReallyEqual(exp, body)
5629         d.addCallback(_check_unrecoverable_file)
5630
5631         d.addCallback(lambda ignored:
5632                       self.shouldHTTPError("GET dir-1share-json",
5633                                            410, "Gone", "UnrecoverableFileError",
5634                                            self.GET,
5635                                            self.fileurls["dir-1share-json"]))
5636         d.addCallback(_check_unrecoverable_file)
5637
5638         d.addCallback(lambda ignored:
5639                       self.shouldHTTPError("GET imaginary",
5640                                            404, "Not Found", None,
5641                                            self.GET, self.fileurls["imaginary"]))
5642
5643         # attach a webapi child that throws a random error, to test how it
5644         # gets rendered.
5645         w = c0.getServiceNamed("webish")
5646         w.root.putChild("ERRORBOOM", ErrorBoom())
5647
5648         # "Accept: */*" :        should get a text/html stack trace
5649         # "Accept: text/plain" : should get a text/plain stack trace
5650         # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
5651         # no Accept header:      should get a text/html stack trace
5652
5653         d.addCallback(lambda ignored:
5654                       self.shouldHTTPError("GET errorboom_html",
5655                                            500, "Internal Server Error", None,
5656                                            self.GET, "ERRORBOOM",
5657                                            headers={"accept": "*/*"}))
5658         def _internal_error_html1(body):
5659             self.failUnlessIn("<html>", "expected HTML, not '%s'" % body)
5660         d.addCallback(_internal_error_html1)
5661
5662         d.addCallback(lambda ignored:
5663                       self.shouldHTTPError("GET errorboom_text",
5664                                            500, "Internal Server Error", None,
5665                                            self.GET, "ERRORBOOM",
5666                                            headers={"accept": "text/plain"}))
5667         def _internal_error_text2(body):
5668             self.failIfIn("<html>", body)
5669             self.failUnless(body.startswith("Traceback "), body)
5670         d.addCallback(_internal_error_text2)
5671
5672         CLI_accepts = "text/plain, application/octet-stream"
5673         d.addCallback(lambda ignored:
5674                       self.shouldHTTPError("GET errorboom_text",
5675                                            500, "Internal Server Error", None,
5676                                            self.GET, "ERRORBOOM",
5677                                            headers={"accept": CLI_accepts}))
5678         def _internal_error_text3(body):
5679             self.failIfIn("<html>", body)
5680             self.failUnless(body.startswith("Traceback "), body)
5681         d.addCallback(_internal_error_text3)
5682
5683         d.addCallback(lambda ignored:
5684                       self.shouldHTTPError("GET errorboom_text",
5685                                            500, "Internal Server Error", None,
5686                                            self.GET, "ERRORBOOM"))
5687         def _internal_error_html4(body):
5688             self.failUnlessIn("<html>", body)
5689         d.addCallback(_internal_error_html4)
5690
5691         def _flush_errors(res):
5692             # Trial: please ignore the CompletelyUnhandledError in the logs
5693             self.flushLoggedErrors(CompletelyUnhandledError)
5694             return res
5695         d.addBoth(_flush_errors)
5696
5697         return d
5698
5699     def test_blacklist(self):
5700         # download from a blacklisted URI, get an error
5701         self.basedir = "web/Grid/blacklist"
5702         self.set_up_grid()
5703         c0 = self.g.clients[0]
5704         c0_basedir = c0.basedir
5705         fn = os.path.join(c0_basedir, "access.blacklist")
5706         self.uris = {}
5707         DATA = "off-limits " * 50
5708
5709         d = c0.upload(upload.Data(DATA, convergence=""))
5710         def _stash_uri_and_create_dir(ur):
5711             self.uri = ur.get_uri()
5712             self.url = "uri/"+self.uri
5713             u = uri.from_string_filenode(self.uri)
5714             self.si = u.get_storage_index()
5715             childnode = c0.create_node_from_uri(self.uri, None)
5716             return c0.create_dirnode({u"blacklisted.txt": (childnode,{}) })
5717         d.addCallback(_stash_uri_and_create_dir)
5718         def _stash_dir(node):
5719             self.dir_node = node
5720             self.dir_uri = node.get_uri()
5721             self.dir_url = "uri/"+self.dir_uri
5722         d.addCallback(_stash_dir)
5723         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5724         def _check_dir_html(body):
5725             self.failUnlessIn("<html>", body)
5726             self.failUnlessIn("blacklisted.txt</a>", body)
5727         d.addCallback(_check_dir_html)
5728         d.addCallback(lambda ign: self.GET(self.url))
5729         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5730
5731         def _blacklist(ign):
5732             f = open(fn, "w")
5733             f.write(" # this is a comment\n")
5734             f.write(" \n")
5735             f.write("\n") # also exercise blank lines
5736             f.write("%s %s\n" % (base32.b2a(self.si), "off-limits to you"))
5737             f.close()
5738             # clients should be checking the blacklist each time, so we don't
5739             # need to restart the client
5740         d.addCallback(_blacklist)
5741         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_uri",
5742                                                        403, "Forbidden",
5743                                                        "Access Prohibited: off-limits",
5744                                                        self.GET, self.url))
5745
5746         # We should still be able to list the parent directory, in HTML...
5747         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5748         def _check_dir_html2(body):
5749             self.failUnlessIn("<html>", body)
5750             self.failUnlessIn("blacklisted.txt</strike>", body)
5751         d.addCallback(_check_dir_html2)
5752
5753         # ... and in JSON (used by CLI).
5754         d.addCallback(lambda ign: self.GET(self.dir_url+"?t=json", followRedirect=True))
5755         def _check_dir_json(res):
5756             data = simplejson.loads(res)
5757             self.failUnless(isinstance(data, list), data)
5758             self.failUnlessEqual(data[0], "dirnode")
5759             self.failUnless(isinstance(data[1], dict), data)
5760             self.failUnlessIn("children", data[1])
5761             self.failUnlessIn("blacklisted.txt", data[1]["children"])
5762             childdata = data[1]["children"]["blacklisted.txt"]
5763             self.failUnless(isinstance(childdata, list), data)
5764             self.failUnlessEqual(childdata[0], "filenode")
5765             self.failUnless(isinstance(childdata[1], dict), data)
5766         d.addCallback(_check_dir_json)
5767
5768         def _unblacklist(ign):
5769             open(fn, "w").close()
5770             # the Blacklist object watches mtime to tell when the file has
5771             # changed, but on windows this test will run faster than the
5772             # filesystem's mtime resolution. So we edit Blacklist.last_mtime
5773             # to force a reload.
5774             self.g.clients[0].blacklist.last_mtime -= 2.0
5775         d.addCallback(_unblacklist)
5776
5777         # now a read should work
5778         d.addCallback(lambda ign: self.GET(self.url))
5779         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5780
5781         # read again to exercise the blacklist-is-unchanged logic
5782         d.addCallback(lambda ign: self.GET(self.url))
5783         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5784
5785         # now add a blacklisted directory, and make sure files under it are
5786         # refused too
5787         def _add_dir(ign):
5788             childnode = c0.create_node_from_uri(self.uri, None)
5789             return c0.create_dirnode({u"child": (childnode,{}) })
5790         d.addCallback(_add_dir)
5791         def _get_dircap(dn):
5792             self.dir_si_b32 = base32.b2a(dn.get_storage_index())
5793             self.dir_url_base = "uri/"+dn.get_write_uri()
5794             self.dir_url_json1 = "uri/"+dn.get_write_uri()+"?t=json"
5795             self.dir_url_json2 = "uri/"+dn.get_write_uri()+"/?t=json"
5796             self.dir_url_json_ro = "uri/"+dn.get_readonly_uri()+"/?t=json"
5797             self.child_url = "uri/"+dn.get_readonly_uri()+"/child"
5798         d.addCallback(_get_dircap)
5799         d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True))
5800         d.addCallback(lambda body: self.failUnlessIn("<html>", body))
5801         d.addCallback(lambda ign: self.GET(self.dir_url_json1))
5802         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5803         d.addCallback(lambda ign: self.GET(self.dir_url_json2))
5804         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5805         d.addCallback(lambda ign: self.GET(self.dir_url_json_ro))
5806         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5807         d.addCallback(lambda ign: self.GET(self.child_url))
5808         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5809
5810         def _block_dir(ign):
5811             f = open(fn, "w")
5812             f.write("%s %s\n" % (self.dir_si_b32, "dir-off-limits to you"))
5813             f.close()
5814             self.g.clients[0].blacklist.last_mtime -= 2.0
5815         d.addCallback(_block_dir)
5816         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir base",
5817                                                        403, "Forbidden",
5818                                                        "Access Prohibited: dir-off-limits",
5819                                                        self.GET, self.dir_url_base))
5820         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json1",
5821                                                        403, "Forbidden",
5822                                                        "Access Prohibited: dir-off-limits",
5823                                                        self.GET, self.dir_url_json1))
5824         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json2",
5825                                                        403, "Forbidden",
5826                                                        "Access Prohibited: dir-off-limits",
5827                                                        self.GET, self.dir_url_json2))
5828         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json_ro",
5829                                                        403, "Forbidden",
5830                                                        "Access Prohibited: dir-off-limits",
5831                                                        self.GET, self.dir_url_json_ro))
5832         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir child",
5833                                                        403, "Forbidden",
5834                                                        "Access Prohibited: dir-off-limits",
5835                                                        self.GET, self.child_url))
5836         return d
5837
5838
5839 class CompletelyUnhandledError(Exception):
5840     pass
5841 class ErrorBoom(rend.Page):
5842     def beforeRender(self, ctx):
5843         raise CompletelyUnhandledError("whoops")