src/allmydata/test/test_web.py

   1 import os.path, re, urllib, time, cgi
   2 import simplejson
   3 from StringIO import StringIO
   4
   5 from twisted.application import service
   6 from twisted.trial import unittest
   7 from twisted.internet import defer, reactor
   8 from twisted.internet.task import Clock
   9 from twisted.web import client, error, http
  10 from twisted.python import failure, log
  11
  12 from foolscap.api import fireEventually, flushEventualQueue
  13
  14 from nevow.util import escapeToXML
  15 from nevow import rend
  16
  17 from allmydata import interfaces, uri, webish, dirnode
  18 from allmydata.storage.shares import get_share_file
  19 from allmydata.storage_client import StorageFarmBroker, StubServer
  20 from allmydata.immutable import upload
  21 from allmydata.immutable.downloader.status import DownloadStatus
  22 from allmydata.dirnode import DirectoryNode
  23 from allmydata.nodemaker import NodeMaker
  24 from allmydata.unknown import UnknownNode
  25 from allmydata.web import status, common
  26 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  27 from allmydata.util import fileutil, base32, hashutil
  28 from allmydata.util.consumer import download_to_data
  29 from allmydata.util.netstring import split_netstring
  30 from allmydata.util.encodingutil import to_str
  31 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  32      create_chk_filenode, WebErrorMixin, ShouldFailMixin, \
  33      make_mutable_file_uri, create_mutable_filenode
  34 from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
  35 from allmydata.mutable import servermap, publish, retrieve
  36 import allmydata.test.common_util as testutil
  37 from allmydata.test.no_network import GridTestMixin
  38 from allmydata.test.common_web import HTTPClientGETFactory, \
  39      HTTPClientHEADFactory
  40 from allmydata.client import Client, SecretHolder
  41 from allmydata.introducer import IntroducerNode
  42
  43 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  44 # create a webserver that works against them
  45
  46 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  47
  48 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
  49 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
  50 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
  51
  52 FAVICON_MARKUP = '<link href="/icon.png" rel="shortcut icon" />'
  53
  54
  55 class FakeStatsProvider:
  56     def get_stats(self):
  57         stats = {'stats': {}, 'counters': {}}
  58         return stats
  59
  60 class FakeNodeMaker(NodeMaker):
  61     encoding_params = {
  62         'k': 3,
  63         'n': 10,
  64         'happy': 7,
  65         'max_segment_size':128*1024 # 1024=KiB
  66     }
  67     def _create_lit(self, cap):
  68         return FakeCHKFileNode(cap, self.all_contents)
  69     def _create_immutable(self, cap):
  70         return FakeCHKFileNode(cap, self.all_contents)
  71     def _create_mutable(self, cap):
  72         return FakeMutableFileNode(None, None,
  73                                    self.encoding_params, None,
  74                                    self.all_contents).init_from_cap(cap)
  75     def create_mutable_file(self, contents="", keysize=None,
  76                             version=SDMF_VERSION):
  77         n = FakeMutableFileNode(None, None, self.encoding_params, None,
  78                                 self.all_contents)
  79         return n.create(contents, version=version)
  80
  81 class FakeUploader(service.Service):
  82     name = "uploader"
  83     helper_furl = None
  84     helper_connected = False
  85
  86     def upload(self, uploadable):
  87         d = uploadable.get_size()
  88         d.addCallback(lambda size: uploadable.read(size))
  89         def _got_data(datav):
  90             data = "".join(datav)
  91             n = create_chk_filenode(data, self.all_contents)
  92             ur = upload.UploadResults(file_size=len(data),
  93                                       ciphertext_fetched=0,
  94                                       preexisting_shares=0,
  95                                       pushed_shares=10,
  96                                       sharemap={},
  97                                       servermap={},
  98                                       timings={},
  99                                       uri_extension_data={},
 100                                       uri_extension_hash="fake",
 101                                       verifycapstr="fakevcap")
 102             ur.set_uri(n.get_uri())
 103             return ur
 104         d.addCallback(_got_data)
 105         return d
 106
 107     def get_helper_info(self):
 108         return (self.helper_furl, self.helper_connected)
 109
 110
 111 def build_one_ds():
 112     ds = DownloadStatus("storage_index", 1234)
 113     now = time.time()
 114
 115     serverA = StubServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
 116     serverB = StubServer(hashutil.tagged_hash("foo", "serverid_b")[:20])
 117     storage_index = hashutil.storage_index_hash("SI")
 118     e0 = ds.add_segment_request(0, now)
 119     e0.activate(now+0.5)
 120     e0.deliver(now+1, 0, 100, 0.5) # when, start,len, decodetime
 121     e1 = ds.add_segment_request(1, now+2)
 122     e1.error(now+3)
 123     # two outstanding requests
 124     e2 = ds.add_segment_request(2, now+4)
 125     e3 = ds.add_segment_request(3, now+5)
 126     del e2,e3 # hush pyflakes
 127
 128     # simulate a segment which gets delivered faster than a system clock tick (ticket #1166)
 129     e = ds.add_segment_request(4, now)
 130     e.activate(now)
 131     e.deliver(now, 0, 140, 0.5)
 132
 133     e = ds.add_dyhb_request(serverA, now)
 134     e.finished([1,2], now+1)
 135     e = ds.add_dyhb_request(serverB, now+2) # left unfinished
 136
 137     e = ds.add_read_event(0, 120, now)
 138     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 139     e.finished(now+1)
 140     e = ds.add_read_event(120, 30, now+2) # left unfinished
 141
 142     e = ds.add_block_request(serverA, 1, 100, 20, now)
 143     e.finished(20, now+1)
 144     e = ds.add_block_request(serverB, 1, 120, 30, now+1) # left unfinished
 145
 146     # make sure that add_read_event() can come first too
 147     ds1 = DownloadStatus(storage_index, 1234)
 148     e = ds1.add_read_event(0, 120, now)
 149     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 150     e.finished(now+1)
 151
 152     return ds
 153
 154 class FakeHistory:
 155     _all_upload_status = [upload.UploadStatus()]
 156     _all_download_status = [build_one_ds()]
 157     _all_mapupdate_statuses = [servermap.UpdateStatus()]
 158     _all_publish_statuses = [publish.PublishStatus()]
 159     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
 160
 161     def list_all_upload_statuses(self):
 162         return self._all_upload_status
 163     def list_all_download_statuses(self):
 164         return self._all_download_status
 165     def list_all_mapupdate_statuses(self):
 166         return self._all_mapupdate_statuses
 167     def list_all_publish_statuses(self):
 168         return self._all_publish_statuses
 169     def list_all_retrieve_statuses(self):
 170         return self._all_retrieve_statuses
 171     def list_all_helper_statuses(self):
 172         return []
 173
 174 class FakeDisplayableServer(StubServer):
 175     def __init__(self, serverid, nickname):
 176         StubServer.__init__(self, serverid)
 177         self.announcement = {"my-version": "allmydata-tahoe-fake",
 178                              "service-name": "storage",
 179                              "nickname": nickname}
 180     def is_connected(self):
 181         return True
 182     def get_permutation_seed(self):
 183         return ""
 184     def get_remote_host(self):
 185         return ""
 186     def get_last_loss_time(self):
 187         return None
 188     def get_announcement_time(self):
 189         return None
 190     def get_announcement(self):
 191         return self.announcement
 192     def get_nickname(self):
 193         return self.announcement["nickname"]
 194
 195 class FakeBucketCounter(object):
 196     def get_state(self):
 197         return {"last-complete-bucket-count": 0}
 198     def get_progress(self):
 199         return {"estimated-time-per-cycle": 0,
 200                 "cycle-in-progress": False,
 201                 "remaining-wait-time": 0}
 202
 203 class FakeLeaseChecker(object):
 204     def __init__(self):
 205         self.expiration_enabled = False
 206         self.mode = "age"
 207         self.override_lease_duration = None
 208         self.sharetypes_to_expire = {}
 209     def get_state(self):
 210         return {"history": None}
 211     def get_progress(self):
 212         return {"estimated-time-per-cycle": 0,
 213                 "cycle-in-progress": False,
 214                 "remaining-wait-time": 0}
 215
 216 class FakeStorageServer(service.MultiService):
 217     name = 'storage'
 218     def __init__(self, nodeid, nickname):
 219         service.MultiService.__init__(self)
 220         self.my_nodeid = nodeid
 221         self.nickname = nickname
 222         self.bucket_counter = FakeBucketCounter()
 223         self.lease_checker = FakeLeaseChecker()
 224     def get_stats(self):
 225         return {"storage_server.accepting_immutable_shares": False}
 226
 227 class FakeClient(Client):
 228     def __init__(self):
 229         # don't upcall to Client.__init__, since we only want to initialize a
 230         # minimal subset
 231         service.MultiService.__init__(self)
 232         self.all_contents = {}
 233         self.nodeid = "fake_nodeid"
 234         self.nickname = u"fake_nickname \u263A"
 235         self.introducer_furl = "None"
 236         self.stats_provider = FakeStatsProvider()
 237         self._secret_holder = SecretHolder("lease secret", "convergence secret")
 238         self.helper = None
 239         self.convergence = "some random string"
 240         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 241         # fake knowledge of another server
 242         self.storage_broker.test_add_server("other_nodeid",
 243                                             FakeDisplayableServer("other_nodeid", u"other_nickname \u263B"))
 244         self.introducer_client = None
 245         self.history = FakeHistory()
 246         self.uploader = FakeUploader()
 247         self.uploader.all_contents = self.all_contents
 248         self.uploader.setServiceParent(self)
 249         self.blacklist = None
 250         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 251                                        self.uploader, None,
 252                                        None, None, None)
 253         self.nodemaker.all_contents = self.all_contents
 254         self.mutable_file_default = SDMF_VERSION
 255         self.addService(FakeStorageServer(self.nodeid, self.nickname))
 256
 257     def get_long_nodeid(self):
 258         return "v0-nodeid"
 259     def get_long_tubid(self):
 260         return "tubid"
 261
 262     def startService(self):
 263         return service.MultiService.startService(self)
 264     def stopService(self):
 265         return service.MultiService.stopService(self)
 266
 267     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 268
 269 class WebMixin(object):
 270     def setUp(self):
 271         self.s = FakeClient()
 272         self.s.startService()
 273         self.staticdir = self.mktemp()
 274         self.clock = Clock()
 275         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
 276                                       clock=self.clock)
 277         self.ws.setServiceParent(self.s)
 278         self.webish_port = self.ws.getPortnum()
 279         self.webish_url = self.ws.getURL()
 280         assert self.webish_url.endswith("/")
 281         self.webish_url = self.webish_url[:-1] # these tests add their own /
 282
 283         l = [ self.s.create_dirnode() for x in range(6) ]
 284         d = defer.DeferredList(l)
 285         def _then(res):
 286             self.public_root = res[0][1]
 287             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 288             self.public_url = "/uri/" + self.public_root.get_uri()
 289             self.private_root = res[1][1]
 290
 291             foo = res[2][1]
 292             self._foo_node = foo
 293             self._foo_uri = foo.get_uri()
 294             self._foo_readonly_uri = foo.get_readonly_uri()
 295             self._foo_verifycap = foo.get_verify_cap().to_string()
 296             # NOTE: we ignore the deferred on all set_uri() calls, because we
 297             # know the fake nodes do these synchronously
 298             self.public_root.set_uri(u"foo", foo.get_uri(),
 299                                      foo.get_readonly_uri())
 300
 301             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 302             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 303             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 304
 305             # sdmf
 306             # XXX: Do we ever use this?
 307             self.BAZ_CONTENTS, n, self._baz_txt_uri, self._baz_txt_readonly_uri = self.makefile_mutable(0)
 308
 309             foo.set_uri(u"baz.txt", self._baz_txt_uri, self._baz_txt_readonly_uri)
 310
 311             # mdmf
 312             self.QUUX_CONTENTS, n, self._quux_txt_uri, self._quux_txt_readonly_uri = self.makefile_mutable(0, mdmf=True)
 313             assert self._quux_txt_uri.startswith("URI:MDMF")
 314             foo.set_uri(u"quux.txt", self._quux_txt_uri, self._quux_txt_readonly_uri)
 315
 316             foo.set_uri(u"empty", res[3][1].get_uri(),
 317                         res[3][1].get_readonly_uri())
 318             sub_uri = res[4][1].get_uri()
 319             self._sub_uri = sub_uri
 320             foo.set_uri(u"sub", sub_uri, sub_uri)
 321             sub = self.s.create_node_from_uri(sub_uri)
 322             self._sub_node = sub
 323
 324             _ign, n, blocking_uri = self.makefile(1)
 325             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 326
 327             # filenode to test for html encoding issues
 328             self._htmlname_unicode = u"<&weirdly'named\"file>>>_<iframe />.txt"
 329             self._htmlname_raw = self._htmlname_unicode.encode('utf-8')
 330             self._htmlname_urlencoded = urllib.quote(self._htmlname_raw, '')
 331             self._htmlname_escaped = escapeToXML(self._htmlname_raw)
 332             self._htmlname_escaped_attr = cgi.escape(self._htmlname_raw, quote=True)
 333             self._htmlname_escaped_double = escapeToXML(cgi.escape(self._htmlname_raw, quote=True))
 334             self.HTMLNAME_CONTENTS, n, self._htmlname_txt_uri = self.makefile(0)
 335             foo.set_uri(self._htmlname_unicode, self._htmlname_txt_uri, self._htmlname_txt_uri)
 336
 337             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 338             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 339             # still think of it as an umlaut
 340             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 341
 342             self.SUBBAZ_CONTENTS, n, baz_file = self.makefile(2)
 343             self._baz_file_uri = baz_file
 344             sub.set_uri(u"baz.txt", baz_file, baz_file)
 345
 346             _ign, n, self._bad_file_uri = self.makefile(3)
 347             # this uri should not be downloadable
 348             del self.s.all_contents[self._bad_file_uri]
 349
 350             rodir = res[5][1]
 351             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 352                                      rodir.get_readonly_uri())
 353             rodir.set_uri(u"nor", baz_file, baz_file)
 354
 355             # public/
 356             # public/foo/
 357             # public/foo/bar.txt
 358             # public/foo/baz.txt
 359             # public/foo/quux.txt
 360             # public/foo/blockingfile
 361             # public/foo/<&weirdly'named\"file>>>_<iframe />.txt
 362             # public/foo/empty/
 363             # public/foo/sub/
 364             # public/foo/sub/baz.txt
 365             # public/reedownlee/
 366             # public/reedownlee/nor
 367             self.NEWFILE_CONTENTS = "newfile contents\n"
 368
 369             return foo.get_metadata_for(u"bar.txt")
 370         d.addCallback(_then)
 371         def _got_metadata(metadata):
 372             self._bar_txt_metadata = metadata
 373         d.addCallback(_got_metadata)
 374         return d
 375
 376     def get_all_contents(self):
 377         return self.s.all_contents
 378
 379     def makefile(self, number):
 380         contents = "contents of file %s\n" % number
 381         n = create_chk_filenode(contents, self.get_all_contents())
 382         return contents, n, n.get_uri()
 383
 384     def makefile_mutable(self, number, mdmf=False):
 385         contents = "contents of mutable file %s\n" % number
 386         n = create_mutable_filenode(contents, mdmf, self.s.all_contents)
 387         return contents, n, n.get_uri(), n.get_readonly_uri()
 388
 389     def tearDown(self):
 390         return self.s.stopService()
 391
 392     def failUnlessIsBarDotTxt(self, res):
 393         self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
 394
 395     def failUnlessIsQuuxDotTxt(self, res):
 396         self.failUnlessReallyEqual(res, self.QUUX_CONTENTS, res)
 397
 398     def failUnlessIsBazDotTxt(self, res):
 399         self.failUnlessReallyEqual(res, self.BAZ_CONTENTS, res)
 400
 401     def failUnlessIsSubBazDotTxt(self, res):
 402         self.failUnlessReallyEqual(res, self.SUBBAZ_CONTENTS, res)
 403
 404     def failUnlessIsBarJSON(self, res):
 405         data = simplejson.loads(res)
 406         self.failUnless(isinstance(data, list))
 407         self.failUnlessEqual(data[0], "filenode")
 408         self.failUnless(isinstance(data[1], dict))
 409         self.failIf(data[1]["mutable"])
 410         self.failIfIn("rw_uri", data[1]) # immutable
 411         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
 412         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
 413         self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
 414
 415     def failUnlessIsQuuxJSON(self, res, readonly=False):
 416         data = simplejson.loads(res)
 417         self.failUnless(isinstance(data, list))
 418         self.failUnlessEqual(data[0], "filenode")
 419         self.failUnless(isinstance(data[1], dict))
 420         metadata = data[1]
 421         return self.failUnlessIsQuuxDotTxtMetadata(metadata, readonly)
 422
 423     def failUnlessIsQuuxDotTxtMetadata(self, metadata, readonly):
 424         self.failUnless(metadata['mutable'])
 425         if readonly:
 426             self.failIfIn("rw_uri", metadata)
 427         else:
 428             self.failUnlessIn("rw_uri", metadata)
 429             self.failUnlessEqual(metadata['rw_uri'], self._quux_txt_uri)
 430         self.failUnlessIn("ro_uri", metadata)
 431         self.failUnlessEqual(metadata['ro_uri'], self._quux_txt_readonly_uri)
 432         self.failUnlessReallyEqual(metadata['size'], len(self.QUUX_CONTENTS))
 433
 434     def failUnlessIsFooJSON(self, res):
 435         data = simplejson.loads(res)
 436         self.failUnless(isinstance(data, list))
 437         self.failUnlessEqual(data[0], "dirnode", res)
 438         self.failUnless(isinstance(data[1], dict))
 439         self.failUnless(data[1]["mutable"])
 440         self.failUnlessIn("rw_uri", data[1]) # mutable
 441         self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
 442         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
 443         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
 444
 445         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 446         self.failUnlessEqual(kidnames,
 447                              [self._htmlname_unicode, u"bar.txt", u"baz.txt",
 448                               u"blockingfile", u"empty", u"n\u00fc.txt", u"quux.txt", u"sub"])
 449         kids = dict( [(unicode(name),value)
 450                       for (name,value)
 451                       in data[1]["children"].iteritems()] )
 452         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 453         self.failUnlessIn("metadata", kids[u"sub"][1])
 454         self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
 455         tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
 456         self.failUnlessIn("linkcrtime", tahoe_md)
 457         self.failUnlessIn("linkmotime", tahoe_md)
 458         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 459         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 460         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri)
 461         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]),
 462                                    self._bar_txt_verifycap)
 463         self.failUnlessIn("metadata", kids[u"bar.txt"][1])
 464         self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
 465         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
 466                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
 467         self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
 468                                    self._bar_txt_uri)
 469         self.failUnlessIn("quux.txt", kids)
 470         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["rw_uri"]),
 471                                    self._quux_txt_uri)
 472         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["ro_uri"]),
 473                                    self._quux_txt_readonly_uri)
 474
 475     def GET(self, urlpath, followRedirect=False, return_response=False,
 476             **kwargs):
 477         # if return_response=True, this fires with (data, statuscode,
 478         # respheaders) instead of just data.
 479         assert not isinstance(urlpath, unicode)
 480         url = self.webish_url + urlpath
 481         factory = HTTPClientGETFactory(url, method="GET",
 482                                        followRedirect=followRedirect, **kwargs)
 483         reactor.connectTCP("localhost", self.webish_port, factory)
 484         d = factory.deferred
 485         def _got_data(data):
 486             return (data, factory.status, factory.response_headers)
 487         if return_response:
 488             d.addCallback(_got_data)
 489         return factory.deferred
 490
 491     def HEAD(self, urlpath, return_response=False, **kwargs):
 492         # this requires some surgery, because twisted.web.client doesn't want
 493         # to give us back the response headers.
 494         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 495         reactor.connectTCP("localhost", self.webish_port, factory)
 496         d = factory.deferred
 497         def _got_data(data):
 498             return (data, factory.status, factory.response_headers)
 499         if return_response:
 500             d.addCallback(_got_data)
 501         return factory.deferred
 502
 503     def PUT(self, urlpath, data, **kwargs):
 504         url = self.webish_url + urlpath
 505         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 506
 507     def DELETE(self, urlpath):
 508         url = self.webish_url + urlpath
 509         return client.getPage(url, method="DELETE")
 510
 511     def POST(self, urlpath, followRedirect=False, **fields):
 512         sepbase = "boogabooga"
 513         sep = "--" + sepbase
 514         form = []
 515         form.append(sep)
 516         form.append('Content-Disposition: form-data; name="_charset"')
 517         form.append('')
 518         form.append('UTF-8')
 519         form.append(sep)
 520         for name, value in fields.iteritems():
 521             if isinstance(value, tuple):
 522                 filename, value = value
 523                 form.append('Content-Disposition: form-data; name="%s"; '
 524                             'filename="%s"' % (name, filename.encode("utf-8")))
 525             else:
 526                 form.append('Content-Disposition: form-data; name="%s"' % name)
 527             form.append('')
 528             if isinstance(value, unicode):
 529                 value = value.encode("utf-8")
 530             else:
 531                 value = str(value)
 532             assert isinstance(value, str)
 533             form.append(value)
 534             form.append(sep)
 535         form[-1] += "--"
 536         body = ""
 537         headers = {}
 538         if fields:
 539             body = "\r\n".join(form) + "\r\n"
 540             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 541         return self.POST2(urlpath, body, headers, followRedirect)
 542
 543     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 544         url = self.webish_url + urlpath
 545         return client.getPage(url, method="POST", postdata=body,
 546                               headers=headers, followRedirect=followRedirect)
 547
 548     def shouldFail(self, res, expected_failure, which,
 549                    substring=None, response_substring=None):
 550         if isinstance(res, failure.Failure):
 551             res.trap(expected_failure)
 552             if substring:
 553                 self.failUnlessIn(substring, str(res), which)
 554             if response_substring:
 555                 self.failUnlessIn(response_substring, res.value.response, which)
 556         else:
 557             self.fail("%s was supposed to raise %s, not get '%s'" %
 558                       (which, expected_failure, res))
 559
 560     def shouldFail2(self, expected_failure, which, substring,
 561                     response_substring,
 562                     callable, *args, **kwargs):
 563         assert substring is None or isinstance(substring, str)
 564         assert response_substring is None or isinstance(response_substring, str)
 565         d = defer.maybeDeferred(callable, *args, **kwargs)
 566         def done(res):
 567             if isinstance(res, failure.Failure):
 568                 res.trap(expected_failure)
 569                 if substring:
 570                     self.failUnlessIn(substring, str(res),
 571                                       "'%s' not in '%s' (response is '%s') for test '%s'" % \
 572                                       (substring, str(res),
 573                                        getattr(res.value, "response", ""),
 574                                        which))
 575                 if response_substring:
 576                     self.failUnlessIn(response_substring, res.value.response,
 577                                       "'%s' not in '%s' for test '%s'" % \
 578                                       (response_substring, res.value.response,
 579                                        which))
 580             else:
 581                 self.fail("%s was supposed to raise %s, not get '%s'" %
 582                           (which, expected_failure, res))
 583         d.addBoth(done)
 584         return d
 585
 586     def should404(self, res, which):
 587         if isinstance(res, failure.Failure):
 588             res.trap(error.Error)
 589             self.failUnlessReallyEqual(res.value.status, "404")
 590         else:
 591             self.fail("%s was supposed to Error(404), not get '%s'" %
 592                       (which, res))
 593
 594     def should302(self, res, which):
 595         if isinstance(res, failure.Failure):
 596             res.trap(error.Error)
 597             self.failUnlessReallyEqual(res.value.status, "302")
 598         else:
 599             self.fail("%s was supposed to Error(302), not get '%s'" %
 600                       (which, res))
 601
 602
 603 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
 604     def test_create(self):
 605         pass
 606
 607     def test_welcome(self):
 608         d = self.GET("/")
 609         def _check(res):
 610             self.failUnlessIn('<title>Tahoe-LAFS - Welcome</title>', res)
 611             self.failUnlessIn(FAVICON_MARKUP, res)
 612             self.failUnlessIn('<a href="status">Recent and Active Operations</a>', res)
 613             self.failUnlessIn('<a href="statistics">Operational Statistics</a>', res)
 614             self.failUnlessIn('<input type="hidden" name="t" value="report-incident" />', res)
 615             self.failUnlessIn('Page rendered at', res)
 616             self.failUnlessIn('Tahoe-LAFS code imported from:', res)
 617             res_u = res.decode('utf-8')
 618             self.failUnlessIn(u'<td>fake_nickname \u263A</td>', res_u)
 619             self.failUnlessIn(u'<div class="nickname">other_nickname \u263B</div>', res_u)
 620             self.failUnlessIn(u'\u00A9 <a href="https://tahoe-lafs.org/">Tahoe-LAFS Software Foundation', res_u)
 621
 622             self.s.basedir = 'web/test_welcome'
 623             fileutil.make_dirs("web/test_welcome")
 624             fileutil.make_dirs("web/test_welcome/private")
 625             return self.GET("/")
 626         d.addCallback(_check)
 627         return d
 628
 629     def test_introducer_status(self):
 630         class MockIntroducerClient(object):
 631             def __init__(self, connected):
 632                 self.connected = connected
 633             def connected_to_introducer(self):
 634                 return self.connected
 635
 636         d = defer.succeed(None)
 637
 638         # introducer not connected, unguessable furl
 639         def _set_introducer_not_connected_unguessable(ign):
 640             self.s.introducer_furl = "pb://someIntroducer/secret"
 641             self.s.introducer_client = MockIntroducerClient(False)
 642             return self.GET("/")
 643         d.addCallback(_set_introducer_not_connected_unguessable)
 644         def _check_introducer_not_connected_unguessable(res):
 645             html = res.replace('\n', ' ')
 646             self.failUnlessIn('<div class="furl">pb://someIntroducer/[censored]</div>', html)
 647             self.failIfIn('pb://someIntroducer/secret', html)
 648             self.failUnless(re.search('<div class="status-indicator connected-no"></div>[ ]*<div>Introducer not connected</div>', html), res)
 649         d.addCallback(_check_introducer_not_connected_unguessable)
 650
 651         # introducer connected, unguessable furl
 652         def _set_introducer_connected_unguessable(ign):
 653             self.s.introducer_furl = "pb://someIntroducer/secret"
 654             self.s.introducer_client = MockIntroducerClient(True)
 655             return self.GET("/")
 656         d.addCallback(_set_introducer_connected_unguessable)
 657         def _check_introducer_connected_unguessable(res):
 658             html = res.replace('\n', ' ')
 659             self.failUnlessIn('<div class="furl">pb://someIntroducer/[censored]</div>', html)
 660             self.failIfIn('pb://someIntroducer/secret', html)
 661             self.failUnless(re.search('<div class="status-indicator connected-yes"></div>[ ]*<div>Introducer</div>', html), res)
 662         d.addCallback(_check_introducer_connected_unguessable)
 663
 664         # introducer connected, guessable furl
 665         def _set_introducer_connected_guessable(ign):
 666             self.s.introducer_furl = "pb://someIntroducer/introducer"
 667             self.s.introducer_client = MockIntroducerClient(True)
 668             return self.GET("/")
 669         d.addCallback(_set_introducer_connected_guessable)
 670         def _check_introducer_connected_guessable(res):
 671             html = res.replace('\n', ' ')
 672             self.failUnlessIn('<div class="furl">pb://someIntroducer/introducer</div>', html)
 673             self.failUnless(re.search('<div class="status-indicator connected-yes"></div>[ ]*<div>Introducer</div>', html), res)
 674         d.addCallback(_check_introducer_connected_guessable)
 675         return d
 676
 677     def test_helper_status(self):
 678         d = defer.succeed(None)
 679
 680         # set helper furl to None
 681         def _set_no_helper(ign):
 682             self.s.uploader.helper_furl = None
 683             return self.GET("/")
 684         d.addCallback(_set_no_helper)
 685         def _check_no_helper(res):
 686             html = res.replace('\n', ' ')
 687             self.failUnless(re.search('<div class="status-indicator connected-not-configured"></div>[ ]*<div>Helper</div>', html), res)
 688         d.addCallback(_check_no_helper)
 689
 690         # enable helper, not connected
 691         def _set_helper_not_connected(ign):
 692             self.s.uploader.helper_furl = "pb://someHelper/secret"
 693             self.s.uploader.helper_connected = False
 694             return self.GET("/")
 695         d.addCallback(_set_helper_not_connected)
 696         def _check_helper_not_connected(res):
 697             html = res.replace('\n', ' ')
 698             self.failUnlessIn('<div class="furl">pb://someHelper/[censored]</div>', html)
 699             self.failIfIn('pb://someHelper/secret', html)
 700             self.failUnless(re.search('<div class="status-indicator connected-no"></div>[ ]*<div>Helper not connected</div>', html), res)
 701         d.addCallback(_check_helper_not_connected)
 702
 703         # enable helper, connected
 704         def _set_helper_connected(ign):
 705             self.s.uploader.helper_furl = "pb://someHelper/secret"
 706             self.s.uploader.helper_connected = True
 707             return self.GET("/")
 708         d.addCallback(_set_helper_connected)
 709         def _check_helper_connected(res):
 710             html = res.replace('\n', ' ')
 711             self.failUnlessIn('<div class="furl">pb://someHelper/[censored]</div>', html)
 712             self.failIfIn('pb://someHelper/secret', html)
 713             self.failUnless(re.search('<div class="status-indicator connected-yes"></div>[ ]*<div>Helper</div>', html), res)
 714         d.addCallback(_check_helper_connected)
 715         return d
 716
 717     def test_storage(self):
 718         d = self.GET("/storage")
 719         def _check(res):
 720             self.failUnlessIn('Storage Server Status', res)
 721             self.failUnlessIn(FAVICON_MARKUP, res)
 722             res_u = res.decode('utf-8')
 723             self.failUnlessIn(u'<li>Server Nickname: <span class="nickname mine">fake_nickname \u263A</span></li>', res_u)
 724         d.addCallback(_check)
 725         return d
 726
 727     def test_status(self):
 728         h = self.s.get_history()
 729         dl_num = h.list_all_download_statuses()[0].get_counter()
 730         ul_num = h.list_all_upload_statuses()[0].get_counter()
 731         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 732         pub_num = h.list_all_publish_statuses()[0].get_counter()
 733         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 734         d = self.GET("/status", followRedirect=True)
 735         def _check(res):
 736             self.failUnlessIn('Recent and Active Operations', res)
 737             self.failUnlessIn('"down-%d"' % dl_num, res)
 738             self.failUnlessIn('"up-%d"' % ul_num, res)
 739             self.failUnlessIn('"mapupdate-%d"' % mu_num, res)
 740             self.failUnlessIn('"publish-%d"' % pub_num, res)
 741             self.failUnlessIn('"retrieve-%d"' % ret_num, res)
 742         d.addCallback(_check)
 743         d.addCallback(lambda res: self.GET("/status/?t=json"))
 744         def _check_json(res):
 745             data = simplejson.loads(res)
 746             self.failUnless(isinstance(data, dict))
 747             #active = data["active"]
 748             # TODO: test more. We need a way to fake an active operation
 749             # here.
 750         d.addCallback(_check_json)
 751
 752         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 753         def _check_dl(res):
 754             self.failUnlessIn("File Download Status", res)
 755         d.addCallback(_check_dl)
 756         d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
 757         def _check_dl_json(res):
 758             data = simplejson.loads(res)
 759             self.failUnless(isinstance(data, dict))
 760             self.failUnlessIn("read", data)
 761             self.failUnlessEqual(data["read"][0]["length"], 120)
 762             self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
 763             self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
 764             self.failUnlessEqual(data["segment"][2]["finish_time"], None)
 765             phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
 766             cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
 767             # serverids[] keys are strings, since that's what JSON does, but
 768             # we'd really like them to be ints
 769             self.failUnlessEqual(data["serverids"]["0"], "phwrsjte")
 770             self.failUnless(data["serverids"].has_key("1"),
 771                             str(data["serverids"]))
 772             self.failUnlessEqual(data["serverids"]["1"], "cmpuvkjm",
 773                                  str(data["serverids"]))
 774             self.failUnlessEqual(data["server_info"][phwr_id]["short"],
 775                                  "phwrsjte")
 776             self.failUnlessEqual(data["server_info"][cmpu_id]["short"],
 777                                  "cmpuvkjm")
 778             self.failUnlessIn("dyhb", data)
 779             self.failUnlessIn("misc", data)
 780         d.addCallback(_check_dl_json)
 781         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 782         def _check_ul(res):
 783             self.failUnlessIn("File Upload Status", res)
 784         d.addCallback(_check_ul)
 785         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 786         def _check_mapupdate(res):
 787             self.failUnlessIn("Mutable File Servermap Update Status", res)
 788         d.addCallback(_check_mapupdate)
 789         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 790         def _check_publish(res):
 791             self.failUnlessIn("Mutable File Publish Status", res)
 792         d.addCallback(_check_publish)
 793         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 794         def _check_retrieve(res):
 795             self.failUnlessIn("Mutable File Retrieve Status", res)
 796         d.addCallback(_check_retrieve)
 797
 798         return d
 799
 800     def test_status_numbers(self):
 801         drrm = status.DownloadResultsRendererMixin()
 802         self.failUnlessReallyEqual(drrm.render_time(None, None), "")
 803         self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
 804         self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
 805         self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
 806         self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
 807         self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
 808         self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 809         self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
 810         self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
 811
 812         urrm = status.UploadResultsRendererMixin()
 813         self.failUnlessReallyEqual(urrm.render_time(None, None), "")
 814         self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
 815         self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
 816         self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
 817         self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
 818         self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
 819         self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 820         self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
 821         self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
 822
 823     def test_GET_FILEURL(self):
 824         d = self.GET(self.public_url + "/foo/bar.txt")
 825         d.addCallback(self.failUnlessIsBarDotTxt)
 826         return d
 827
 828     def test_GET_FILEURL_range(self):
 829         headers = {"range": "bytes=1-10"}
 830         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 831                      return_response=True)
 832         def _got((res, status, headers)):
 833             self.failUnlessReallyEqual(int(status), 206)
 834             self.failUnless(headers.has_key("content-range"))
 835             self.failUnlessReallyEqual(headers["content-range"][0],
 836                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 837             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
 838         d.addCallback(_got)
 839         return d
 840
 841     def test_GET_FILEURL_partial_range(self):
 842         headers = {"range": "bytes=5-"}
 843         length  = len(self.BAR_CONTENTS)
 844         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 845                      return_response=True)
 846         def _got((res, status, headers)):
 847             self.failUnlessReallyEqual(int(status), 206)
 848             self.failUnless(headers.has_key("content-range"))
 849             self.failUnlessReallyEqual(headers["content-range"][0],
 850                                        "bytes 5-%d/%d" % (length-1, length))
 851             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
 852         d.addCallback(_got)
 853         return d
 854
 855     def test_GET_FILEURL_partial_end_range(self):
 856         headers = {"range": "bytes=-5"}
 857         length  = len(self.BAR_CONTENTS)
 858         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 859                      return_response=True)
 860         def _got((res, status, headers)):
 861             self.failUnlessReallyEqual(int(status), 206)
 862             self.failUnless(headers.has_key("content-range"))
 863             self.failUnlessReallyEqual(headers["content-range"][0],
 864                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 865             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
 866         d.addCallback(_got)
 867         return d
 868
 869     def test_GET_FILEURL_partial_range_overrun(self):
 870         headers = {"range": "bytes=100-200"}
 871         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
 872                              "416 Requested Range not satisfiable",
 873                              "First beyond end of file",
 874                              self.GET, self.public_url + "/foo/bar.txt",
 875                              headers=headers)
 876         return d
 877
 878     def test_HEAD_FILEURL_range(self):
 879         headers = {"range": "bytes=1-10"}
 880         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 881                      return_response=True)
 882         def _got((res, status, headers)):
 883             self.failUnlessReallyEqual(res, "")
 884             self.failUnlessReallyEqual(int(status), 206)
 885             self.failUnless(headers.has_key("content-range"))
 886             self.failUnlessReallyEqual(headers["content-range"][0],
 887                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 888         d.addCallback(_got)
 889         return d
 890
 891     def test_HEAD_FILEURL_partial_range(self):
 892         headers = {"range": "bytes=5-"}
 893         length  = len(self.BAR_CONTENTS)
 894         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 895                      return_response=True)
 896         def _got((res, status, headers)):
 897             self.failUnlessReallyEqual(int(status), 206)
 898             self.failUnless(headers.has_key("content-range"))
 899             self.failUnlessReallyEqual(headers["content-range"][0],
 900                                        "bytes 5-%d/%d" % (length-1, length))
 901         d.addCallback(_got)
 902         return d
 903
 904     def test_HEAD_FILEURL_partial_end_range(self):
 905         headers = {"range": "bytes=-5"}
 906         length  = len(self.BAR_CONTENTS)
 907         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 908                      return_response=True)
 909         def _got((res, status, headers)):
 910             self.failUnlessReallyEqual(int(status), 206)
 911             self.failUnless(headers.has_key("content-range"))
 912             self.failUnlessReallyEqual(headers["content-range"][0],
 913                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 914         d.addCallback(_got)
 915         return d
 916
 917     def test_HEAD_FILEURL_partial_range_overrun(self):
 918         headers = {"range": "bytes=100-200"}
 919         d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
 920                              "416 Requested Range not satisfiable",
 921                              "",
 922                              self.HEAD, self.public_url + "/foo/bar.txt",
 923                              headers=headers)
 924         return d
 925
 926     def test_GET_FILEURL_range_bad(self):
 927         headers = {"range": "BOGUS=fizbop-quarnak"}
 928         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 929                      return_response=True)
 930         def _got((res, status, headers)):
 931             self.failUnlessReallyEqual(int(status), 200)
 932             self.failUnless(not headers.has_key("content-range"))
 933             self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
 934         d.addCallback(_got)
 935         return d
 936
 937     def test_HEAD_FILEURL(self):
 938         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 939         def _got((res, status, headers)):
 940             self.failUnlessReallyEqual(res, "")
 941             self.failUnlessReallyEqual(headers["content-length"][0],
 942                                        str(len(self.BAR_CONTENTS)))
 943             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
 944         d.addCallback(_got)
 945         return d
 946
 947     def test_GET_FILEURL_named(self):
 948         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 949         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 950         d = self.GET(base + "/@@name=/blah.txt")
 951         d.addCallback(self.failUnlessIsBarDotTxt)
 952         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 953         d.addCallback(self.failUnlessIsBarDotTxt)
 954         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 955         d.addCallback(self.failUnlessIsBarDotTxt)
 956         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 957         d.addCallback(self.failUnlessIsBarDotTxt)
 958         save_url = base + "?save=true&filename=blah.txt"
 959         d.addCallback(lambda res: self.GET(save_url))
 960         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 961         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 962         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 963         u_url = base + "?save=true&filename=" + u_fn_e
 964         d.addCallback(lambda res: self.GET(u_url))
 965         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 966         return d
 967
 968     def test_PUT_FILEURL_named_bad(self):
 969         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 970         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 971                              "400 Bad Request",
 972                              "/file can only be used with GET or HEAD",
 973                              self.PUT, base + "/@@name=/blah.txt", "")
 974         return d
 975
 976
 977     def test_GET_DIRURL_named_bad(self):
 978         base = "/file/%s" % urllib.quote(self._foo_uri)
 979         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 980                              "400 Bad Request",
 981                              "is not a file-cap",
 982                              self.GET, base + "/@@name=/blah.txt")
 983         return d
 984
 985     def test_GET_slash_file_bad(self):
 986         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 987                              "404 Not Found",
 988                              "/file must be followed by a file-cap and a name",
 989                              self.GET, "/file")
 990         return d
 991
 992     def test_GET_unhandled_URI_named(self):
 993         contents, n, newuri = self.makefile(12)
 994         verifier_cap = n.get_verify_cap().to_string()
 995         base = "/file/%s" % urllib.quote(verifier_cap)
 996         # client.create_node_from_uri() can't handle verify-caps
 997         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 998                              "400 Bad Request", "is not a file-cap",
 999                              self.GET, base)
1000         return d
1001
1002     def test_GET_unhandled_URI(self):
1003         contents, n, newuri = self.makefile(12)
1004         verifier_cap = n.get_verify_cap().to_string()
1005         base = "/uri/%s" % urllib.quote(verifier_cap)
1006         # client.create_node_from_uri() can't handle verify-caps
1007         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
1008                              "400 Bad Request",
1009                              "GET unknown URI type: can only do t=info",
1010                              self.GET, base)
1011         return d
1012
1013     def test_GET_FILE_URI(self):
1014         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
1015         d = self.GET(base)
1016         d.addCallback(self.failUnlessIsBarDotTxt)
1017         return d
1018
1019     def test_GET_FILE_URI_mdmf(self):
1020         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
1021         d = self.GET(base)
1022         d.addCallback(self.failUnlessIsQuuxDotTxt)
1023         return d
1024
1025     def test_GET_FILE_URI_mdmf_extensions(self):
1026         base = "/uri/%s" % urllib.quote("%s:RANDOMSTUFF" % self._quux_txt_uri)
1027         d = self.GET(base)
1028         d.addCallback(self.failUnlessIsQuuxDotTxt)
1029         return d
1030
1031     def test_GET_FILE_URI_mdmf_readonly(self):
1032         base = "/uri/%s" % urllib.quote(self._quux_txt_readonly_uri)
1033         d = self.GET(base)
1034         d.addCallback(self.failUnlessIsQuuxDotTxt)
1035         return d
1036
1037     def test_GET_FILE_URI_badchild(self):
1038         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
1039         errmsg = "Files have no children, certainly not named 'boguschild'"
1040         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
1041                              "400 Bad Request", errmsg,
1042                              self.GET, base)
1043         return d
1044
1045     def test_PUT_FILE_URI_badchild(self):
1046         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
1047         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
1048         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
1049                              "400 Bad Request", errmsg,
1050                              self.PUT, base, "")
1051         return d
1052
1053     def test_PUT_FILE_URI_mdmf(self):
1054         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
1055         self._quux_new_contents = "new_contents"
1056         d = self.GET(base)
1057         d.addCallback(lambda res:
1058             self.failUnlessIsQuuxDotTxt(res))
1059         d.addCallback(lambda ignored:
1060             self.PUT(base, self._quux_new_contents))
1061         d.addCallback(lambda ignored:
1062             self.GET(base))
1063         d.addCallback(lambda res:
1064             self.failUnlessReallyEqual(res, self._quux_new_contents))
1065         return d
1066
1067     def test_PUT_FILE_URI_mdmf_extensions(self):
1068         base = "/uri/%s" % urllib.quote("%s:EXTENSIONSTUFF" % self._quux_txt_uri)
1069         self._quux_new_contents = "new_contents"
1070         d = self.GET(base)
1071         d.addCallback(lambda res: self.failUnlessIsQuuxDotTxt(res))
1072         d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
1073         d.addCallback(lambda ignored: self.GET(base))
1074         d.addCallback(lambda res: self.failUnlessEqual(self._quux_new_contents,
1075                                                        res))
1076         return d
1077
1078     def test_PUT_FILE_URI_mdmf_readonly(self):
1079         # We're not allowed to PUT things to a readonly cap.
1080         base = "/uri/%s" % self._quux_txt_readonly_uri
1081         d = self.GET(base)
1082         d.addCallback(lambda res:
1083             self.failUnlessIsQuuxDotTxt(res))
1084         # What should we get here? We get a 500 error now; that's not right.
1085         d.addCallback(lambda ignored:
1086             self.shouldFail2(error.Error, "test_PUT_FILE_URI_mdmf_readonly",
1087                              "400 Bad Request", "read-only cap",
1088                              self.PUT, base, "new data"))
1089         return d
1090
1091     def test_PUT_FILE_URI_sdmf_readonly(self):
1092         # We're not allowed to put things to a readonly cap.
1093         base = "/uri/%s" % self._baz_txt_readonly_uri
1094         d = self.GET(base)
1095         d.addCallback(lambda res:
1096             self.failUnlessIsBazDotTxt(res))
1097         d.addCallback(lambda ignored:
1098             self.shouldFail2(error.Error, "test_PUT_FILE_URI_sdmf_readonly",
1099                              "400 Bad Request", "read-only cap",
1100                              self.PUT, base, "new_data"))
1101         return d
1102
1103     def test_GET_etags(self):
1104
1105         def _check_etags(uri):
1106             d1 = _get_etag(uri)
1107             d2 = _get_etag(uri, 'json')
1108             d = defer.DeferredList([d1, d2], consumeErrors=True)
1109             def _check(results):
1110                 # All deferred must succeed
1111                 self.failUnless(all([r[0] for r in results]))
1112                 # the etag for the t=json form should be just like the etag
1113                 # fo the default t='' form, but with a 'json' suffix
1114                 self.failUnlessEqual(results[0][1] + 'json', results[1][1])
1115             d.addCallback(_check)
1116             return d
1117
1118         def _get_etag(uri, t=''):
1119             targetbase = "/uri/%s?t=%s" % (urllib.quote(uri.strip()), t)
1120             d = self.GET(targetbase, return_response=True, followRedirect=True)
1121             def _just_the_etag(result):
1122                 data, response, headers = result
1123                 etag = headers['etag'][0]
1124                 if uri.startswith('URI:DIR'):
1125                     self.failUnless(etag.startswith('DIR:'), etag)
1126                 return etag
1127             return d.addCallback(_just_the_etag)
1128
1129         # Check that etags work with immutable directories
1130         (newkids, caps) = self._create_immutable_children()
1131         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1132                       simplejson.dumps(newkids))
1133         def _stash_immdir_uri(uri):
1134             self._immdir_uri = uri
1135             return uri
1136         d.addCallback(_stash_immdir_uri)
1137         d.addCallback(_check_etags)
1138
1139         # Check that etags work with immutable files
1140         d.addCallback(lambda _: _check_etags(self._bar_txt_uri))
1141
1142         # use the ETag on GET
1143         def _check_match(ign):
1144             uri = "/uri/%s" % self._bar_txt_uri
1145             d = self.GET(uri, return_response=True)
1146             # extract the ETag
1147             d.addCallback(lambda (data, code, headers):
1148                           headers['etag'][0])
1149             # do a GET that's supposed to match the ETag
1150             d.addCallback(lambda etag:
1151                           self.GET(uri, return_response=True,
1152                                    headers={"If-None-Match": etag}))
1153             # make sure it short-circuited (304 instead of 200)
1154             d.addCallback(lambda (data, code, headers):
1155                           self.failUnlessEqual(int(code), http.NOT_MODIFIED))
1156             return d
1157         d.addCallback(_check_match)
1158
1159         def _no_etag(uri, t):
1160             target = "/uri/%s?t=%s" % (uri, t)
1161             d = self.GET(target, return_response=True, followRedirect=True)
1162             d.addCallback(lambda (data, code, headers):
1163                           self.failIf("etag" in headers, target))
1164             return d
1165         def _yes_etag(uri, t):
1166             target = "/uri/%s?t=%s" % (uri, t)
1167             d = self.GET(target, return_response=True, followRedirect=True)
1168             d.addCallback(lambda (data, code, headers):
1169                           self.failUnless("etag" in headers, target))
1170             return d
1171
1172         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, ""))
1173         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "json"))
1174         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "uri"))
1175         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "readonly-uri"))
1176         d.addCallback(lambda ign: _no_etag(self._bar_txt_uri, "info"))
1177
1178         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, ""))
1179         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "json"))
1180         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "uri"))
1181         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "readonly-uri"))
1182         d.addCallback(lambda ign: _no_etag(self._immdir_uri, "info"))
1183         d.addCallback(lambda ign: _no_etag(self._immdir_uri, "rename-form"))
1184
1185         return d
1186
1187     # TODO: version of this with a Unicode filename
1188     def test_GET_FILEURL_save(self):
1189         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
1190                      return_response=True)
1191         def _got((res, statuscode, headers)):
1192             content_disposition = headers["content-disposition"][0]
1193             self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
1194             self.failUnlessIsBarDotTxt(res)
1195         d.addCallback(_got)
1196         return d
1197
1198     def test_GET_FILEURL_missing(self):
1199         d = self.GET(self.public_url + "/foo/missing")
1200         d.addBoth(self.should404, "test_GET_FILEURL_missing")
1201         return d
1202
1203     def test_GET_FILEURL_info_mdmf(self):
1204         d = self.GET("/uri/%s?t=info" % self._quux_txt_uri)
1205         def _got(res):
1206             self.failUnlessIn("mutable file (mdmf)", res)
1207             self.failUnlessIn(self._quux_txt_uri, res)
1208             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1209         d.addCallback(_got)
1210         return d
1211
1212     def test_GET_FILEURL_info_mdmf_readonly(self):
1213         d = self.GET("/uri/%s?t=info" % self._quux_txt_readonly_uri)
1214         def _got(res):
1215             self.failUnlessIn("mutable file (mdmf)", res)
1216             self.failIfIn(self._quux_txt_uri, res)
1217             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1218         d.addCallback(_got)
1219         return d
1220
1221     def test_GET_FILEURL_info_sdmf(self):
1222         d = self.GET("/uri/%s?t=info" % self._baz_txt_uri)
1223         def _got(res):
1224             self.failUnlessIn("mutable file (sdmf)", res)
1225             self.failUnlessIn(self._baz_txt_uri, res)
1226         d.addCallback(_got)
1227         return d
1228
1229     def test_GET_FILEURL_info_mdmf_extensions(self):
1230         d = self.GET("/uri/%s:STUFF?t=info" % self._quux_txt_uri)
1231         def _got(res):
1232             self.failUnlessIn("mutable file (mdmf)", res)
1233             self.failUnlessIn(self._quux_txt_uri, res)
1234             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1235         d.addCallback(_got)
1236         return d
1237
1238     def test_PUT_overwrite_only_files(self):
1239         # create a directory, put a file in that directory.
1240         contents, n, filecap = self.makefile(8)
1241         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
1242         d.addCallback(lambda res:
1243             self.PUT(self.public_url + "/foo/dir/file1.txt",
1244                      self.NEWFILE_CONTENTS))
1245         # try to overwrite the file with replace=only-files
1246         # (this should work)
1247         d.addCallback(lambda res:
1248             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
1249                      filecap))
1250         d.addCallback(lambda res:
1251             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
1252                  "There was already a child by that name, and you asked me "
1253                  "to not replace it",
1254                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
1255                  filecap))
1256         return d
1257
1258     def test_PUT_NEWFILEURL(self):
1259         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
1260         # TODO: we lose the response code, so we can't check this
1261         #self.failUnlessReallyEqual(responsecode, 201)
1262         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1263         d.addCallback(lambda res:
1264                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1265                                                       self.NEWFILE_CONTENTS))
1266         return d
1267
1268     def test_PUT_NEWFILEURL_not_mutable(self):
1269         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
1270                      self.NEWFILE_CONTENTS)
1271         # TODO: we lose the response code, so we can't check this
1272         #self.failUnlessReallyEqual(responsecode, 201)
1273         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1274         d.addCallback(lambda res:
1275                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1276                                                       self.NEWFILE_CONTENTS))
1277         return d
1278
1279     def test_PUT_NEWFILEURL_unlinked_mdmf(self):
1280         # this should get us a few segments of an MDMF mutable file,
1281         # which we can then test for.
1282         contents = self.NEWFILE_CONTENTS * 300000
1283         d = self.PUT("/uri?format=mdmf",
1284                      contents)
1285         def _got_filecap(filecap):
1286             self.failUnless(filecap.startswith("URI:MDMF"))
1287             return filecap
1288         d.addCallback(_got_filecap)
1289         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1290         d.addCallback(lambda json: self.failUnlessIn("MDMF", json))
1291         return d
1292
1293     def test_PUT_NEWFILEURL_unlinked_sdmf(self):
1294         contents = self.NEWFILE_CONTENTS * 300000
1295         d = self.PUT("/uri?format=sdmf",
1296                      contents)
1297         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1298         d.addCallback(lambda json: self.failUnlessIn("SDMF", json))
1299         return d
1300
1301     def test_PUT_NEWFILEURL_unlinked_bad_format(self):
1302         contents = self.NEWFILE_CONTENTS * 300000
1303         return self.shouldHTTPError("PUT_NEWFILEURL_unlinked_bad_format",
1304                                     400, "Bad Request", "Unknown format: foo",
1305                                     self.PUT, "/uri?format=foo",
1306                                     contents)
1307
1308     def test_PUT_NEWFILEURL_range_bad(self):
1309         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
1310         target = self.public_url + "/foo/new.txt"
1311         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
1312                              "501 Not Implemented",
1313                              "Content-Range in PUT not yet supported",
1314                              # (and certainly not for immutable files)
1315                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
1316                              headers=headers)
1317         d.addCallback(lambda res:
1318                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
1319         return d
1320
1321     def test_PUT_NEWFILEURL_mutable(self):
1322         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1323                      self.NEWFILE_CONTENTS)
1324         # TODO: we lose the response code, so we can't check this
1325         #self.failUnlessReallyEqual(responsecode, 201)
1326         def _check_uri(res):
1327             u = uri.from_string_mutable_filenode(res)
1328             self.failUnless(u.is_mutable())
1329             self.failIf(u.is_readonly())
1330             return res
1331         d.addCallback(_check_uri)
1332         d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
1333         d.addCallback(lambda res:
1334                       self.failUnlessMutableChildContentsAre(self._foo_node,
1335                                                              u"new.txt",
1336                                                              self.NEWFILE_CONTENTS))
1337         return d
1338
1339     def test_PUT_NEWFILEURL_mutable_toobig(self):
1340         # It is okay to upload large mutable files, so we should be able
1341         # to do that.
1342         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1343                      "b" * (self.s.MUTABLE_SIZELIMIT + 1))
1344         return d
1345
1346     def test_PUT_NEWFILEURL_replace(self):
1347         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
1348         # TODO: we lose the response code, so we can't check this
1349         #self.failUnlessReallyEqual(responsecode, 200)
1350         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
1351         d.addCallback(lambda res:
1352                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1353                                                       self.NEWFILE_CONTENTS))
1354         return d
1355
1356     def test_PUT_NEWFILEURL_bad_t(self):
1357         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
1358                              "PUT to a file: bad t=bogus",
1359                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
1360                              "contents")
1361         return d
1362
1363     def test_PUT_NEWFILEURL_no_replace(self):
1364         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
1365                      self.NEWFILE_CONTENTS)
1366         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
1367                   "409 Conflict",
1368                   "There was already a child by that name, and you asked me "
1369                   "to not replace it")
1370         return d
1371
1372     def test_PUT_NEWFILEURL_mkdirs(self):
1373         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
1374         fn = self._foo_node
1375         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
1376         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
1377         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
1378         d.addCallback(lambda res:
1379                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
1380                                                       self.NEWFILE_CONTENTS))
1381         return d
1382
1383     def test_PUT_NEWFILEURL_blocked(self):
1384         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
1385                      self.NEWFILE_CONTENTS)
1386         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
1387                   "409 Conflict",
1388                   "Unable to create directory 'blockingfile': a file was in the way")
1389         return d
1390
1391     def test_PUT_NEWFILEURL_emptyname(self):
1392         # an empty pathname component (i.e. a double-slash) is disallowed
1393         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
1394                              "400 Bad Request",
1395                              "The webapi does not allow empty pathname components",
1396                              self.PUT, self.public_url + "/foo//new.txt", "")
1397         return d
1398
1399     def test_DELETE_FILEURL(self):
1400         d = self.DELETE(self.public_url + "/foo/bar.txt")
1401         d.addCallback(lambda res:
1402                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1403         return d
1404
1405     def test_DELETE_FILEURL_missing(self):
1406         d = self.DELETE(self.public_url + "/foo/missing")
1407         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
1408         return d
1409
1410     def test_DELETE_FILEURL_missing2(self):
1411         d = self.DELETE(self.public_url + "/missing/missing")
1412         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
1413         return d
1414
1415     def failUnlessHasBarDotTxtMetadata(self, res):
1416         data = simplejson.loads(res)
1417         self.failUnless(isinstance(data, list))
1418         self.failUnlessIn("metadata", data[1])
1419         self.failUnlessIn("tahoe", data[1]["metadata"])
1420         self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
1421         self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
1422         self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
1423                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
1424
1425     def test_GET_FILEURL_json(self):
1426         # twisted.web.http.parse_qs ignores any query args without an '=', so
1427         # I can't do "GET /path?json", I have to do "GET /path/t=json"
1428         # instead. This may make it tricky to emulate the S3 interface
1429         # completely.
1430         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
1431         def _check1(data):
1432             self.failUnlessIsBarJSON(data)
1433             self.failUnlessHasBarDotTxtMetadata(data)
1434             return
1435         d.addCallback(_check1)
1436         return d
1437
1438     def test_GET_FILEURL_json_mutable_type(self):
1439         # The JSON should include format, which says whether the
1440         # file is SDMF or MDMF
1441         d = self.PUT("/uri?format=mdmf",
1442                      self.NEWFILE_CONTENTS * 300000)
1443         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1444         def _got_json(json, version):
1445             data = simplejson.loads(json)
1446             assert "filenode" == data[0]
1447             data = data[1]
1448             assert isinstance(data, dict)
1449
1450             self.failUnlessIn("format", data)
1451             self.failUnlessEqual(data["format"], version)
1452
1453         d.addCallback(_got_json, "MDMF")
1454         # Now make an SDMF file and check that it is reported correctly.
1455         d.addCallback(lambda ignored:
1456             self.PUT("/uri?format=sdmf",
1457                       self.NEWFILE_CONTENTS * 300000))
1458         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1459         d.addCallback(_got_json, "SDMF")
1460         return d
1461
1462     def test_GET_FILEURL_json_mdmf(self):
1463         d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
1464         d.addCallback(self.failUnlessIsQuuxJSON)
1465         return d
1466
1467     def test_GET_FILEURL_json_missing(self):
1468         d = self.GET(self.public_url + "/foo/missing?json")
1469         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
1470         return d
1471
1472     def test_GET_FILEURL_uri(self):
1473         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
1474         def _check(res):
1475             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1476         d.addCallback(_check)
1477         d.addCallback(lambda res:
1478                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
1479         def _check2(res):
1480             # for now, for files, uris and readonly-uris are the same
1481             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1482         d.addCallback(_check2)
1483         return d
1484
1485     def test_GET_FILEURL_badtype(self):
1486         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
1487                                  "bad t=bogus",
1488                                  self.GET,
1489                                  self.public_url + "/foo/bar.txt?t=bogus")
1490         return d
1491
1492     def test_CSS_FILE(self):
1493         d = self.GET("/tahoe.css", followRedirect=True)
1494         def _check(res):
1495             CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
1496             self.failUnless(CSS_STYLE.search(res), res)
1497         d.addCallback(_check)
1498         return d
1499
1500     def test_GET_FILEURL_uri_missing(self):
1501         d = self.GET(self.public_url + "/foo/missing?t=uri")
1502         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
1503         return d
1504
1505     def _check_upload_and_mkdir_forms(self, html):
1506         # We should have a form to create a file, with radio buttons that allow
1507         # the user to toggle whether it is a CHK/LIT (default), SDMF, or MDMF file.
1508         self.failUnlessIn('name="t" value="upload"', html)
1509         self.failUnless(re.search('<input [^/]*id="upload-chk"', html), html)
1510         self.failUnless(re.search('<input [^/]*id="upload-sdmf"', html), html)
1511         self.failUnless(re.search('<input [^/]*id="upload-mdmf"', html), html)
1512
1513         # We should also have the ability to create a mutable directory, with
1514         # radio buttons that allow the user to toggle whether it is an SDMF (default)
1515         # or MDMF directory.
1516         self.failUnlessIn('name="t" value="mkdir"', html)
1517         self.failUnless(re.search('<input [^/]*id="mkdir-sdmf"', html), html)
1518         self.failUnless(re.search('<input [^/]*id="mkdir-mdmf"', html), html)
1519
1520         self.failUnlessIn(FAVICON_MARKUP, html)
1521
1522     def test_GET_DIRECTORY_html(self):
1523         d = self.GET(self.public_url + "/foo", followRedirect=True)
1524         def _check(html):
1525             self.failUnlessIn('<div class="toolbar-item"><a href="../../..">Return to Welcome page</a></div>', html)
1526             self._check_upload_and_mkdir_forms(html)
1527             self.failUnlessIn("quux", html)
1528         d.addCallback(_check)
1529         return d
1530
1531     def test_GET_DIRECTORY_html_filenode_encoding(self):
1532         d = self.GET(self.public_url + "/foo", followRedirect=True)
1533         def _check(html):
1534             # Check if encoded entries are there
1535             self.failUnlessIn('@@named=/' + self._htmlname_urlencoded + '">'
1536                               + self._htmlname_escaped + '</a>', html)
1537             self.failUnlessIn('value="' + self._htmlname_escaped_attr + '"', html)
1538             self.failIfIn(self._htmlname_escaped_double, html)
1539             # Make sure that Nevow escaping actually works by checking for unsafe characters
1540             # and that '&' is escaped.
1541             for entity in '<>':
1542                 self.failUnlessIn(entity, self._htmlname_raw)
1543                 self.failIfIn(entity, self._htmlname_escaped)
1544             self.failUnlessIn('&', re.sub(r'&(amp|lt|gt|quot|apos);', '', self._htmlname_raw))
1545             self.failIfIn('&', re.sub(r'&(amp|lt|gt|quot|apos);', '', self._htmlname_escaped))
1546         d.addCallback(_check)
1547         return d
1548
1549     def test_GET_root_html(self):
1550         d = self.GET("/")
1551         d.addCallback(self._check_upload_and_mkdir_forms)
1552         return d
1553
1554     def test_GET_DIRURL(self):
1555         # the addSlash means we get a redirect here
1556         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1557         ROOT = "../../.."
1558         d = self.GET(self.public_url + "/foo", followRedirect=True)
1559         def _check(res):
1560             self.failUnlessIn('<a href="%s">Return to Welcome page' % ROOT, res)
1561
1562             # the FILE reference points to a URI, but it should end in bar.txt
1563             bar_url = ("%s/file/%s/@@named=/bar.txt" %
1564                        (ROOT, urllib.quote(self._bar_txt_uri)))
1565             get_bar = "".join([r'<td>FILE</td>',
1566                                r'\s+<td>',
1567                                r'<a href="%s">bar.txt</a>' % bar_url,
1568                                r'</td>',
1569                                r'\s+<td align="right">%d</td>' % len(self.BAR_CONTENTS),
1570                                ])
1571             self.failUnless(re.search(get_bar, res), res)
1572             for label in ['unlink', 'rename/relink']:
1573                 for line in res.split("\n"):
1574                     # find the line that contains the relevant button for bar.txt
1575                     if ("form action" in line and
1576                         ('value="%s"' % (label,)) in line and
1577                         'value="bar.txt"' in line):
1578                         # the form target should use a relative URL
1579                         foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1580                         self.failUnlessIn('action="%s"' % foo_url, line)
1581                         # and the when_done= should too
1582                         #done_url = urllib.quote(???)
1583                         #self.failUnlessIn('name="when_done" value="%s"' % done_url, line)
1584
1585                         # 'unlink' needs to use POST because it directly has a side effect
1586                         if label == 'unlink':
1587                             self.failUnlessIn('method="post"', line)
1588                         break
1589                 else:
1590                     self.fail("unable to find '%s bar.txt' line" % (label,))
1591
1592             # the DIR reference just points to a URI
1593             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1594             get_sub = ((r'<td>DIR</td>')
1595                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1596             self.failUnless(re.search(get_sub, res), res)
1597         d.addCallback(_check)
1598
1599         # look at a readonly directory
1600         d.addCallback(lambda res:
1601                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
1602         def _check2(res):
1603             self.failUnlessIn("(read-only)", res)
1604             self.failIfIn("Upload a file", res)
1605         d.addCallback(_check2)
1606
1607         # and at a directory that contains a readonly directory
1608         d.addCallback(lambda res:
1609                       self.GET(self.public_url, followRedirect=True))
1610         def _check3(res):
1611             self.failUnless(re.search('<td>DIR-RO</td>'
1612                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1613         d.addCallback(_check3)
1614
1615         # and an empty directory
1616         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1617         def _check4(res):
1618             self.failUnlessIn("directory is empty", res)
1619             MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
1620             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1621         d.addCallback(_check4)
1622
1623         # and at a literal directory
1624         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1625         d.addCallback(lambda res:
1626                       self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1627         def _check5(res):
1628             self.failUnlessIn('(immutable)', res)
1629             self.failUnless(re.search('<td>FILE</td>'
1630                                       r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1631         d.addCallback(_check5)
1632         return d
1633
1634     def test_GET_DIRURL_badtype(self):
1635         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1636                                  400, "Bad Request",
1637                                  "bad t=bogus",
1638                                  self.GET,
1639                                  self.public_url + "/foo?t=bogus")
1640         return d
1641
1642     def test_GET_DIRURL_json(self):
1643         d = self.GET(self.public_url + "/foo?t=json")
1644         d.addCallback(self.failUnlessIsFooJSON)
1645         return d
1646
1647     def test_GET_DIRURL_json_format(self):
1648         d = self.PUT(self.public_url + \
1649                      "/foo/sdmf.txt?format=sdmf",
1650                      self.NEWFILE_CONTENTS * 300000)
1651         d.addCallback(lambda ignored:
1652             self.PUT(self.public_url + \
1653                      "/foo/mdmf.txt?format=mdmf",
1654                      self.NEWFILE_CONTENTS * 300000))
1655         # Now we have an MDMF and SDMF file in the directory. If we GET
1656         # its JSON, we should see their encodings.
1657         d.addCallback(lambda ignored:
1658             self.GET(self.public_url + "/foo?t=json"))
1659         def _got_json(json):
1660             data = simplejson.loads(json)
1661             assert data[0] == "dirnode"
1662
1663             data = data[1]
1664             kids = data['children']
1665
1666             mdmf_data = kids['mdmf.txt'][1]
1667             self.failUnlessIn("format", mdmf_data)
1668             self.failUnlessEqual(mdmf_data["format"], "MDMF")
1669
1670             sdmf_data = kids['sdmf.txt'][1]
1671             self.failUnlessIn("format", sdmf_data)
1672             self.failUnlessEqual(sdmf_data["format"], "SDMF")
1673         d.addCallback(_got_json)
1674         return d
1675
1676
1677     def test_POST_DIRURL_manifest_no_ophandle(self):
1678         d = self.shouldFail2(error.Error,
1679                              "test_POST_DIRURL_manifest_no_ophandle",
1680                              "400 Bad Request",
1681                              "slow operation requires ophandle=",
1682                              self.POST, self.public_url, t="start-manifest")
1683         return d
1684
1685     def test_POST_DIRURL_manifest(self):
1686         d = defer.succeed(None)
1687         def getman(ignored, output):
1688             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1689                           followRedirect=True)
1690             d.addCallback(self.wait_for_operation, "125")
1691             d.addCallback(self.get_operation_results, "125", output)
1692             return d
1693         d.addCallback(getman, None)
1694         def _got_html(manifest):
1695             self.failUnlessIn("Manifest of SI=", manifest)
1696             self.failUnlessIn("<td>sub</td>", manifest)
1697             self.failUnlessIn(self._sub_uri, manifest)
1698             self.failUnlessIn("<td>sub/baz.txt</td>", manifest)
1699             self.failUnlessIn(FAVICON_MARKUP, manifest)
1700         d.addCallback(_got_html)
1701
1702         # both t=status and unadorned GET should be identical
1703         d.addCallback(lambda res: self.GET("/operations/125"))
1704         d.addCallback(_got_html)
1705
1706         d.addCallback(getman, "html")
1707         d.addCallback(_got_html)
1708         d.addCallback(getman, "text")
1709         def _got_text(manifest):
1710             self.failUnlessIn("\nsub " + self._sub_uri + "\n", manifest)
1711             self.failUnlessIn("\nsub/baz.txt URI:CHK:", manifest)
1712         d.addCallback(_got_text)
1713         d.addCallback(getman, "JSON")
1714         def _got_json(res):
1715             data = res["manifest"]
1716             got = {}
1717             for (path_list, cap) in data:
1718                 got[tuple(path_list)] = cap
1719             self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
1720             self.failUnlessIn((u"sub", u"baz.txt"), got)
1721             self.failUnlessIn("finished", res)
1722             self.failUnlessIn("origin", res)
1723             self.failUnlessIn("storage-index", res)
1724             self.failUnlessIn("verifycaps", res)
1725             self.failUnlessIn("stats", res)
1726         d.addCallback(_got_json)
1727         return d
1728
1729     def test_POST_DIRURL_deepsize_no_ophandle(self):
1730         d = self.shouldFail2(error.Error,
1731                              "test_POST_DIRURL_deepsize_no_ophandle",
1732                              "400 Bad Request",
1733                              "slow operation requires ophandle=",
1734                              self.POST, self.public_url, t="start-deep-size")
1735         return d
1736
1737     def test_POST_DIRURL_deepsize(self):
1738         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1739                       followRedirect=True)
1740         d.addCallback(self.wait_for_operation, "126")
1741         d.addCallback(self.get_operation_results, "126", "json")
1742         def _got_json(data):
1743             self.failUnlessReallyEqual(data["finished"], True)
1744             size = data["size"]
1745             self.failUnless(size > 1000)
1746         d.addCallback(_got_json)
1747         d.addCallback(self.get_operation_results, "126", "text")
1748         def _got_text(res):
1749             mo = re.search(r'^size: (\d+)$', res, re.M)
1750             self.failUnless(mo, res)
1751             size = int(mo.group(1))
1752             # with directories, the size varies.
1753             self.failUnless(size > 1000)
1754         d.addCallback(_got_text)
1755         return d
1756
1757     def test_POST_DIRURL_deepstats_no_ophandle(self):
1758         d = self.shouldFail2(error.Error,
1759                              "test_POST_DIRURL_deepstats_no_ophandle",
1760                              "400 Bad Request",
1761                              "slow operation requires ophandle=",
1762                              self.POST, self.public_url, t="start-deep-stats")
1763         return d
1764
1765     def test_POST_DIRURL_deepstats(self):
1766         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1767                       followRedirect=True)
1768         d.addCallback(self.wait_for_operation, "127")
1769         d.addCallback(self.get_operation_results, "127", "json")
1770         def _got_json(stats):
1771             expected = {"count-immutable-files": 4,
1772                         "count-mutable-files": 2,
1773                         "count-literal-files": 0,
1774                         "count-files": 6,
1775                         "count-directories": 3,
1776                         "size-immutable-files": 76,
1777                         "size-literal-files": 0,
1778                         #"size-directories": 1912, # varies
1779                         #"largest-directory": 1590,
1780                         "largest-directory-children": 8,
1781                         "largest-immutable-file": 19,
1782                         }
1783             for k,v in expected.iteritems():
1784                 self.failUnlessReallyEqual(stats[k], v,
1785                                            "stats[%s] was %s, not %s" %
1786                                            (k, stats[k], v))
1787             self.failUnlessReallyEqual(stats["size-files-histogram"],
1788                                        [ [11, 31, 4] ])
1789         d.addCallback(_got_json)
1790         return d
1791
1792     def test_POST_DIRURL_stream_manifest(self):
1793         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1794         def _check(res):
1795             self.failUnless(res.endswith("\n"))
1796             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1797             self.failUnlessReallyEqual(len(units), 10)
1798             self.failUnlessEqual(units[-1]["type"], "stats")
1799             first = units[0]
1800             self.failUnlessEqual(first["path"], [])
1801             self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri)
1802             self.failUnlessEqual(first["type"], "directory")
1803             baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0]
1804             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1805             self.failIfEqual(baz["storage-index"], None)
1806             self.failIfEqual(baz["verifycap"], None)
1807             self.failIfEqual(baz["repaircap"], None)
1808             # XXX: Add quux and baz to this test.
1809             return
1810         d.addCallback(_check)
1811         return d
1812
1813     def test_GET_DIRURL_uri(self):
1814         d = self.GET(self.public_url + "/foo?t=uri")
1815         def _check(res):
1816             self.failUnlessReallyEqual(to_str(res), self._foo_uri)
1817         d.addCallback(_check)
1818         return d
1819
1820     def test_GET_DIRURL_readonly_uri(self):
1821         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1822         def _check(res):
1823             self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri)
1824         d.addCallback(_check)
1825         return d
1826
1827     def test_PUT_NEWDIRURL(self):
1828         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1829         d.addCallback(lambda res:
1830                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1831         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1832         d.addCallback(self.failUnlessNodeKeysAre, [])
1833         return d
1834
1835     def test_PUT_NEWDIRURL_mdmf(self):
1836         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1837         d.addCallback(lambda res:
1838                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1839         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1840         d.addCallback(lambda node:
1841             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1842         return d
1843
1844     def test_PUT_NEWDIRURL_sdmf(self):
1845         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=sdmf",
1846                      "")
1847         d.addCallback(lambda res:
1848                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1849         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1850         d.addCallback(lambda node:
1851             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1852         return d
1853
1854     def test_PUT_NEWDIRURL_bad_format(self):
1855         return self.shouldHTTPError("PUT_NEWDIRURL_bad_format",
1856                                     400, "Bad Request", "Unknown format: foo",
1857                                     self.PUT, self.public_url +
1858                                     "/foo/newdir=?t=mkdir&format=foo", "")
1859
1860     def test_POST_NEWDIRURL(self):
1861         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1862         d.addCallback(lambda res:
1863                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1864         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1865         d.addCallback(self.failUnlessNodeKeysAre, [])
1866         return d
1867
1868     def test_POST_NEWDIRURL_mdmf(self):
1869         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1870         d.addCallback(lambda res:
1871                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1872         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1873         d.addCallback(lambda node:
1874             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1875         return d
1876
1877     def test_POST_NEWDIRURL_sdmf(self):
1878         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=sdmf", "")
1879         d.addCallback(lambda res:
1880             self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1881         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1882         d.addCallback(lambda node:
1883             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1884         return d
1885
1886     def test_POST_NEWDIRURL_bad_format(self):
1887         return self.shouldHTTPError("POST_NEWDIRURL_bad_format",
1888                                     400, "Bad Request", "Unknown format: foo",
1889                                     self.POST2, self.public_url + \
1890                                     "/foo/newdir?t=mkdir&format=foo", "")
1891
1892     def test_POST_NEWDIRURL_emptyname(self):
1893         # an empty pathname component (i.e. a double-slash) is disallowed
1894         d = self.shouldFail2(error.Error, "POST_NEWDIRURL_emptyname",
1895                              "400 Bad Request",
1896                              "The webapi does not allow empty pathname components, i.e. a double slash",
1897                              self.POST, self.public_url + "//?t=mkdir")
1898         return d
1899
1900     def _do_POST_NEWDIRURL_initial_children_test(self, version=None):
1901         (newkids, caps) = self._create_initial_children()
1902         query = "/foo/newdir?t=mkdir-with-children"
1903         if version == MDMF_VERSION:
1904             query += "&format=mdmf"
1905         elif version == SDMF_VERSION:
1906             query += "&format=sdmf"
1907         else:
1908             version = SDMF_VERSION # for later
1909         d = self.POST2(self.public_url + query,
1910                        simplejson.dumps(newkids))
1911         def _check(uri):
1912             n = self.s.create_node_from_uri(uri.strip())
1913             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1914             self.failUnlessEqual(n._node.get_version(), version)
1915             d2.addCallback(lambda ign:
1916                            self.failUnlessROChildURIIs(n, u"child-imm",
1917                                                        caps['filecap1']))
1918             d2.addCallback(lambda ign:
1919                            self.failUnlessRWChildURIIs(n, u"child-mutable",
1920                                                        caps['filecap2']))
1921             d2.addCallback(lambda ign:
1922                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1923                                                        caps['filecap3']))
1924             d2.addCallback(lambda ign:
1925                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1926                                                        caps['unknown_rocap']))
1927             d2.addCallback(lambda ign:
1928                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1929                                                        caps['unknown_rwcap']))
1930             d2.addCallback(lambda ign:
1931                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1932                                                        caps['unknown_immcap']))
1933             d2.addCallback(lambda ign:
1934                            self.failUnlessRWChildURIIs(n, u"dirchild",
1935                                                        caps['dircap']))
1936             d2.addCallback(lambda ign:
1937                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1938                                                        caps['litdircap']))
1939             d2.addCallback(lambda ign:
1940                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1941                                                        caps['emptydircap']))
1942             return d2
1943         d.addCallback(_check)
1944         d.addCallback(lambda res:
1945                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1946         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1947         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1948         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1949         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1950         return d
1951
1952     def test_POST_NEWDIRURL_initial_children(self):
1953         return self._do_POST_NEWDIRURL_initial_children_test()
1954
1955     def test_POST_NEWDIRURL_initial_children_mdmf(self):
1956         return self._do_POST_NEWDIRURL_initial_children_test(MDMF_VERSION)
1957
1958     def test_POST_NEWDIRURL_initial_children_sdmf(self):
1959         return self._do_POST_NEWDIRURL_initial_children_test(SDMF_VERSION)
1960
1961     def test_POST_NEWDIRURL_initial_children_bad_format(self):
1962         (newkids, caps) = self._create_initial_children()
1963         return self.shouldHTTPError("POST_NEWDIRURL_initial_children_bad_format",
1964                                     400, "Bad Request", "Unknown format: foo",
1965                                     self.POST2, self.public_url + \
1966                                     "/foo/newdir?t=mkdir-with-children&format=foo",
1967                                     simplejson.dumps(newkids))
1968
1969     def test_POST_NEWDIRURL_immutable(self):
1970         (newkids, caps) = self._create_immutable_children()
1971         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1972                        simplejson.dumps(newkids))
1973         def _check(uri):
1974             n = self.s.create_node_from_uri(uri.strip())
1975             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1976             d2.addCallback(lambda ign:
1977                            self.failUnlessROChildURIIs(n, u"child-imm",
1978                                                        caps['filecap1']))
1979             d2.addCallback(lambda ign:
1980                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1981                                                        caps['unknown_immcap']))
1982             d2.addCallback(lambda ign:
1983                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
1984                                                        caps['immdircap']))
1985             d2.addCallback(lambda ign:
1986                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1987                                                        caps['litdircap']))
1988             d2.addCallback(lambda ign:
1989                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1990                                                        caps['emptydircap']))
1991             return d2
1992         d.addCallback(_check)
1993         d.addCallback(lambda res:
1994                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1995         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1996         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1997         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1998         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1999         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2000         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2001         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2002         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2003         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2004         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2005         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2006         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2007         d.addErrback(self.explain_web_error)
2008         return d
2009
2010     def test_POST_NEWDIRURL_immutable_bad(self):
2011         (newkids, caps) = self._create_initial_children()
2012         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
2013                              "400 Bad Request",
2014                              "needed to be immutable but was not",
2015                              self.POST2,
2016                              self.public_url + "/foo/newdir?t=mkdir-immutable",
2017                              simplejson.dumps(newkids))
2018         return d
2019
2020     def test_PUT_NEWDIRURL_exists(self):
2021         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
2022         d.addCallback(lambda res:
2023                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
2024         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2025         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2026         return d
2027
2028     def test_PUT_NEWDIRURL_blocked(self):
2029         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
2030                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
2031                              self.PUT,
2032                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
2033         d.addCallback(lambda res:
2034                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
2035         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2036         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2037         return d
2038
2039     def test_PUT_NEWDIRURL_mkdirs(self):
2040         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
2041         d.addCallback(lambda res:
2042                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
2043         d.addCallback(lambda res:
2044                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
2045         d.addCallback(lambda res:
2046                       self._foo_node.get_child_at_path(u"subdir/newdir"))
2047         d.addCallback(self.failUnlessNodeKeysAre, [])
2048         return d
2049
2050     def test_PUT_NEWDIRURL_mkdirs_mdmf(self):
2051         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=mdmf", "")
2052         d.addCallback(lambda ignored:
2053             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
2054         d.addCallback(lambda ignored:
2055             self.failIfNodeHasChild(self._foo_node, u"newdir"))
2056         d.addCallback(lambda ignored:
2057             self._foo_node.get_child_at_path(u"subdir"))
2058         def _got_subdir(subdir):
2059             # XXX: What we want?
2060             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
2061             self.failUnlessNodeHasChild(subdir, u"newdir")
2062             return subdir.get_child_at_path(u"newdir")
2063         d.addCallback(_got_subdir)
2064         d.addCallback(lambda newdir:
2065             self.failUnlessEqual(newdir._node.get_version(), MDMF_VERSION))
2066         return d
2067
2068     def test_PUT_NEWDIRURL_mkdirs_sdmf(self):
2069         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=sdmf", "")
2070         d.addCallback(lambda ignored:
2071             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
2072         d.addCallback(lambda ignored:
2073             self.failIfNodeHasChild(self._foo_node, u"newdir"))
2074         d.addCallback(lambda ignored:
2075             self._foo_node.get_child_at_path(u"subdir"))
2076         def _got_subdir(subdir):
2077             # XXX: What we want?
2078             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
2079             self.failUnlessNodeHasChild(subdir, u"newdir")
2080             return subdir.get_child_at_path(u"newdir")
2081         d.addCallback(_got_subdir)
2082         d.addCallback(lambda newdir:
2083             self.failUnlessEqual(newdir._node.get_version(), SDMF_VERSION))
2084         return d
2085
2086     def test_PUT_NEWDIRURL_mkdirs_bad_format(self):
2087         return self.shouldHTTPError("PUT_NEWDIRURL_mkdirs_bad_format",
2088                                     400, "Bad Request", "Unknown format: foo",
2089                                     self.PUT, self.public_url + \
2090                                     "/foo/subdir/newdir?t=mkdir&format=foo",
2091                                     "")
2092
2093     def test_DELETE_DIRURL(self):
2094         d = self.DELETE(self.public_url + "/foo")
2095         d.addCallback(lambda res:
2096                       self.failIfNodeHasChild(self.public_root, u"foo"))
2097         return d
2098
2099     def test_DELETE_DIRURL_missing(self):
2100         d = self.DELETE(self.public_url + "/foo/missing")
2101         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
2102         d.addCallback(lambda res:
2103                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
2104         return d
2105
2106     def test_DELETE_DIRURL_missing2(self):
2107         d = self.DELETE(self.public_url + "/missing")
2108         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
2109         return d
2110
2111     def dump_root(self):
2112         print "NODEWALK"
2113         w = webish.DirnodeWalkerMixin()
2114         def visitor(childpath, childnode, metadata):
2115             print childpath
2116         d = w.walk(self.public_root, visitor)
2117         return d
2118
2119     def failUnlessNodeKeysAre(self, node, expected_keys):
2120         for k in expected_keys:
2121             assert isinstance(k, unicode)
2122         d = node.list()
2123         def _check(children):
2124             self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
2125         d.addCallback(_check)
2126         return d
2127     def failUnlessNodeHasChild(self, node, name):
2128         assert isinstance(name, unicode)
2129         d = node.list()
2130         def _check(children):
2131             self.failUnlessIn(name, children)
2132         d.addCallback(_check)
2133         return d
2134     def failIfNodeHasChild(self, node, name):
2135         assert isinstance(name, unicode)
2136         d = node.list()
2137         def _check(children):
2138             self.failIfIn(name, children)
2139         d.addCallback(_check)
2140         return d
2141
2142     def failUnlessChildContentsAre(self, node, name, expected_contents):
2143         assert isinstance(name, unicode)
2144         d = node.get_child_at_path(name)
2145         d.addCallback(lambda node: download_to_data(node))
2146         def _check(contents):
2147             self.failUnlessReallyEqual(contents, expected_contents)
2148         d.addCallback(_check)
2149         return d
2150
2151     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
2152         assert isinstance(name, unicode)
2153         d = node.get_child_at_path(name)
2154         d.addCallback(lambda node: node.download_best_version())
2155         def _check(contents):
2156             self.failUnlessReallyEqual(contents, expected_contents)
2157         d.addCallback(_check)
2158         return d
2159
2160     def failUnlessRWChildURIIs(self, node, name, expected_uri):
2161         assert isinstance(name, unicode)
2162         d = node.get_child_at_path(name)
2163         def _check(child):
2164             self.failUnless(child.is_unknown() or not child.is_readonly())
2165             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
2166             self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
2167             expected_ro_uri = self._make_readonly(expected_uri)
2168             if expected_ro_uri:
2169                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
2170         d.addCallback(_check)
2171         return d
2172
2173     def failUnlessROChildURIIs(self, node, name, expected_uri):
2174         assert isinstance(name, unicode)
2175         d = node.get_child_at_path(name)
2176         def _check(child):
2177             self.failUnless(child.is_unknown() or child.is_readonly())
2178             self.failUnlessReallyEqual(child.get_write_uri(), None)
2179             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
2180             self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
2181         d.addCallback(_check)
2182         return d
2183
2184     def failUnlessURIMatchesRWChild(self, got_uri, node, name):
2185         assert isinstance(name, unicode)
2186         d = node.get_child_at_path(name)
2187         def _check(child):
2188             self.failUnless(child.is_unknown() or not child.is_readonly())
2189             self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
2190             self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
2191             expected_ro_uri = self._make_readonly(got_uri)
2192             if expected_ro_uri:
2193                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
2194         d.addCallback(_check)
2195         return d
2196
2197     def failUnlessURIMatchesROChild(self, got_uri, node, name):
2198         assert isinstance(name, unicode)
2199         d = node.get_child_at_path(name)
2200         def _check(child):
2201             self.failUnless(child.is_unknown() or child.is_readonly())
2202             self.failUnlessReallyEqual(child.get_write_uri(), None)
2203             self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
2204             self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
2205         d.addCallback(_check)
2206         return d
2207
2208     def failUnlessCHKURIHasContents(self, got_uri, contents):
2209         self.failUnless(self.get_all_contents()[got_uri] == contents)
2210
2211     def test_POST_upload(self):
2212         d = self.POST(self.public_url + "/foo", t="upload",
2213                       file=("new.txt", self.NEWFILE_CONTENTS))
2214         fn = self._foo_node
2215         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2216         d.addCallback(lambda res:
2217                       self.failUnlessChildContentsAre(fn, u"new.txt",
2218                                                       self.NEWFILE_CONTENTS))
2219         return d
2220
2221     def test_POST_upload_unicode(self):
2222         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2223         d = self.POST(self.public_url + "/foo", t="upload",
2224                       file=(filename, self.NEWFILE_CONTENTS))
2225         fn = self._foo_node
2226         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2227         d.addCallback(lambda res:
2228                       self.failUnlessChildContentsAre(fn, filename,
2229                                                       self.NEWFILE_CONTENTS))
2230         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2231         d.addCallback(lambda res: self.GET(target_url))
2232         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2233                                                                   self.NEWFILE_CONTENTS,
2234                                                                   contents))
2235         return d
2236
2237     def test_POST_upload_unicode_named(self):
2238         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2239         d = self.POST(self.public_url + "/foo", t="upload",
2240                       name=filename,
2241                       file=("overridden", self.NEWFILE_CONTENTS))
2242         fn = self._foo_node
2243         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2244         d.addCallback(lambda res:
2245                       self.failUnlessChildContentsAre(fn, filename,
2246                                                       self.NEWFILE_CONTENTS))
2247         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2248         d.addCallback(lambda res: self.GET(target_url))
2249         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2250                                                                   self.NEWFILE_CONTENTS,
2251                                                                   contents))
2252         return d
2253
2254     def test_POST_upload_no_link(self):
2255         d = self.POST("/uri", t="upload",
2256                       file=("new.txt", self.NEWFILE_CONTENTS))
2257         def _check_upload_results(page):
2258             # this should be a page which describes the results of the upload
2259             # that just finished.
2260             self.failUnlessIn("Upload Results:", page)
2261             self.failUnlessIn("URI:", page)
2262             uri_re = re.compile("URI: <tt><span>(.*)</span>")
2263             mo = uri_re.search(page)
2264             self.failUnless(mo, page)
2265             new_uri = mo.group(1)
2266             return new_uri
2267         d.addCallback(_check_upload_results)
2268         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
2269         return d
2270
2271     def test_POST_upload_no_link_whendone(self):
2272         d = self.POST("/uri", t="upload", when_done="/",
2273                       file=("new.txt", self.NEWFILE_CONTENTS))
2274         d.addBoth(self.shouldRedirect, "/")
2275         return d
2276
2277     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
2278         d = defer.maybeDeferred(callable, *args, **kwargs)
2279         def done(res):
2280             if isinstance(res, failure.Failure):
2281                 res.trap(error.PageRedirect)
2282                 statuscode = res.value.status
2283                 target = res.value.location
2284                 return checker(statuscode, target)
2285             self.fail("%s: callable was supposed to redirect, not return '%s'"
2286                       % (which, res))
2287         d.addBoth(done)
2288         return d
2289
2290     def test_POST_upload_no_link_whendone_results(self):
2291         def check(statuscode, target):
2292             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2293             self.failUnless(target.startswith(self.webish_url), target)
2294             return client.getPage(target, method="GET")
2295         # We encode "uri" as "%75ri" to exercise a case affected by ticket #1860.
2296         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
2297                                  check,
2298                                  self.POST, "/uri", t="upload",
2299                                  when_done="/%75ri/%(uri)s",
2300                                  file=("new.txt", self.NEWFILE_CONTENTS))
2301         d.addCallback(lambda res:
2302                       self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
2303         return d
2304
2305     def test_POST_upload_no_link_mutable(self):
2306         d = self.POST("/uri", t="upload", mutable="true",
2307                       file=("new.txt", self.NEWFILE_CONTENTS))
2308         def _check(filecap):
2309             filecap = filecap.strip()
2310             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2311             self.filecap = filecap
2312             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2313             self.failUnlessIn(u.get_storage_index(), self.get_all_contents())
2314             n = self.s.create_node_from_uri(filecap)
2315             return n.download_best_version()
2316         d.addCallback(_check)
2317         def _check2(data):
2318             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2319             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2320         d.addCallback(_check2)
2321         def _check3(data):
2322             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2323             return self.GET("/file/%s" % urllib.quote(self.filecap))
2324         d.addCallback(_check3)
2325         def _check4(data):
2326             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2327         d.addCallback(_check4)
2328         return d
2329
2330     def test_POST_upload_no_link_mutable_toobig(self):
2331         # The SDMF size limit is no longer in place, so we should be
2332         # able to upload mutable files that are as large as we want them
2333         # to be.
2334         d = self.POST("/uri", t="upload", mutable="true",
2335                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2336         return d
2337
2338
2339     def test_POST_upload_format_unlinked(self):
2340         def _check_upload_unlinked(ign, format, uri_prefix):
2341             filename = format + ".txt"
2342             d = self.POST("/uri?t=upload&format=" + format,
2343                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2344             def _got_results(results):
2345                 if format.upper() in ("SDMF", "MDMF"):
2346                     # webapi.rst says this returns a filecap
2347                     filecap = results
2348                 else:
2349                     # for immutable, it returns an "upload results page", and
2350                     # the filecap is buried inside
2351                     line = [l for l in results.split("\n") if "URI: " in l][0]
2352                     mo = re.search(r'<span>([^<]+)</span>', line)
2353                     filecap = mo.group(1)
2354                 self.failUnless(filecap.startswith(uri_prefix),
2355                                 (uri_prefix, filecap))
2356                 return self.GET("/uri/%s?t=json" % filecap)
2357             d.addCallback(_got_results)
2358             def _got_json(json):
2359                 data = simplejson.loads(json)
2360                 data = data[1]
2361                 self.failUnlessIn("format", data)
2362                 self.failUnlessEqual(data["format"], format.upper())
2363             d.addCallback(_got_json)
2364             return d
2365         d = defer.succeed(None)
2366         d.addCallback(_check_upload_unlinked, "chk", "URI:CHK")
2367         d.addCallback(_check_upload_unlinked, "CHK", "URI:CHK")
2368         d.addCallback(_check_upload_unlinked, "sdmf", "URI:SSK")
2369         d.addCallback(_check_upload_unlinked, "mdmf", "URI:MDMF")
2370         return d
2371
2372     def test_POST_upload_bad_format_unlinked(self):
2373         return self.shouldHTTPError("POST_upload_bad_format_unlinked",
2374                                     400, "Bad Request", "Unknown format: foo",
2375                                     self.POST,
2376                                     "/uri?t=upload&format=foo",
2377                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2378
2379     def test_POST_upload_format(self):
2380         def _check_upload(ign, format, uri_prefix, fn=None):
2381             filename = format + ".txt"
2382             d = self.POST(self.public_url +
2383                           "/foo?t=upload&format=" + format,
2384                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2385             def _got_filecap(filecap):
2386                 if fn is not None:
2387                     filenameu = unicode(filename)
2388                     self.failUnlessURIMatchesRWChild(filecap, fn, filenameu)
2389                 self.failUnless(filecap.startswith(uri_prefix))
2390                 return self.GET(self.public_url + "/foo/%s?t=json" % filename)
2391             d.addCallback(_got_filecap)
2392             def _got_json(json):
2393                 data = simplejson.loads(json)
2394                 data = data[1]
2395                 self.failUnlessIn("format", data)
2396                 self.failUnlessEqual(data["format"], format.upper())
2397             d.addCallback(_got_json)
2398             return d
2399
2400         d = defer.succeed(None)
2401         d.addCallback(_check_upload, "chk", "URI:CHK")
2402         d.addCallback(_check_upload, "sdmf", "URI:SSK", self._foo_node)
2403         d.addCallback(_check_upload, "mdmf", "URI:MDMF")
2404         d.addCallback(_check_upload, "MDMF", "URI:MDMF")
2405         return d
2406
2407     def test_POST_upload_bad_format(self):
2408         return self.shouldHTTPError("POST_upload_bad_format",
2409                                     400, "Bad Request", "Unknown format: foo",
2410                                     self.POST, self.public_url + \
2411                                     "/foo?t=upload&format=foo",
2412                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2413
2414     def test_POST_upload_mutable(self):
2415         # this creates a mutable file
2416         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
2417                       file=("new.txt", self.NEWFILE_CONTENTS))
2418         fn = self._foo_node
2419         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2420         d.addCallback(lambda res:
2421                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2422                                                              self.NEWFILE_CONTENTS))
2423         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2424         def _got(newnode):
2425             self.failUnless(IMutableFileNode.providedBy(newnode))
2426             self.failUnless(newnode.is_mutable())
2427             self.failIf(newnode.is_readonly())
2428             self._mutable_node = newnode
2429             self._mutable_uri = newnode.get_uri()
2430         d.addCallback(_got)
2431
2432         # now upload it again and make sure that the URI doesn't change
2433         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
2434         d.addCallback(lambda res:
2435                       self.POST(self.public_url + "/foo", t="upload",
2436                                 mutable="true",
2437                                 file=("new.txt", NEWER_CONTENTS)))
2438         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2439         d.addCallback(lambda res:
2440                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2441                                                              NEWER_CONTENTS))
2442         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2443         def _got2(newnode):
2444             self.failUnless(IMutableFileNode.providedBy(newnode))
2445             self.failUnless(newnode.is_mutable())
2446             self.failIf(newnode.is_readonly())
2447             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2448         d.addCallback(_got2)
2449
2450         # upload a second time, using PUT instead of POST
2451         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
2452         d.addCallback(lambda res:
2453                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
2454         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2455         d.addCallback(lambda res:
2456                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2457                                                              NEW2_CONTENTS))
2458
2459         # finally list the directory, since mutable files are displayed
2460         # slightly differently
2461
2462         d.addCallback(lambda res:
2463                       self.GET(self.public_url + "/foo/",
2464                                followRedirect=True))
2465         def _check_page(res):
2466             # TODO: assert more about the contents
2467             self.failUnlessIn("SSK", res)
2468             return res
2469         d.addCallback(_check_page)
2470
2471         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2472         def _got3(newnode):
2473             self.failUnless(IMutableFileNode.providedBy(newnode))
2474             self.failUnless(newnode.is_mutable())
2475             self.failIf(newnode.is_readonly())
2476             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2477         d.addCallback(_got3)
2478
2479         # look at the JSON form of the enclosing directory
2480         d.addCallback(lambda res:
2481                       self.GET(self.public_url + "/foo/?t=json",
2482                                followRedirect=True))
2483         def _check_page_json(res):
2484             parsed = simplejson.loads(res)
2485             self.failUnlessEqual(parsed[0], "dirnode")
2486             children = dict( [(unicode(name),value)
2487                               for (name,value)
2488                               in parsed[1]["children"].iteritems()] )
2489             self.failUnlessIn(u"new.txt", children)
2490             new_json = children[u"new.txt"]
2491             self.failUnlessEqual(new_json[0], "filenode")
2492             self.failUnless(new_json[1]["mutable"])
2493             self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri)
2494             ro_uri = self._mutable_node.get_readonly().to_string()
2495             self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri)
2496         d.addCallback(_check_page_json)
2497
2498         # and the JSON form of the file
2499         d.addCallback(lambda res:
2500                       self.GET(self.public_url + "/foo/new.txt?t=json"))
2501         def _check_file_json(res):
2502             parsed = simplejson.loads(res)
2503             self.failUnlessEqual(parsed[0], "filenode")
2504             self.failUnless(parsed[1]["mutable"])
2505             self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri)
2506             ro_uri = self._mutable_node.get_readonly().to_string()
2507             self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri)
2508         d.addCallback(_check_file_json)
2509
2510         # and look at t=uri and t=readonly-uri
2511         d.addCallback(lambda res:
2512                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
2513         d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
2514         d.addCallback(lambda res:
2515                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
2516         def _check_ro_uri(res):
2517             ro_uri = self._mutable_node.get_readonly().to_string()
2518             self.failUnlessReallyEqual(res, ro_uri)
2519         d.addCallback(_check_ro_uri)
2520
2521         # make sure we can get to it from /uri/URI
2522         d.addCallback(lambda res:
2523                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
2524         d.addCallback(lambda res:
2525                       self.failUnlessReallyEqual(res, NEW2_CONTENTS))
2526
2527         # and that HEAD computes the size correctly
2528         d.addCallback(lambda res:
2529                       self.HEAD(self.public_url + "/foo/new.txt",
2530                                 return_response=True))
2531         def _got_headers((res, status, headers)):
2532             self.failUnlessReallyEqual(res, "")
2533             self.failUnlessReallyEqual(headers["content-length"][0],
2534                                        str(len(NEW2_CONTENTS)))
2535             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
2536         d.addCallback(_got_headers)
2537
2538         # make sure that outdated size limits aren't enforced anymore.
2539         d.addCallback(lambda ignored:
2540             self.POST(self.public_url + "/foo", t="upload",
2541                       mutable="true",
2542                       file=("new.txt",
2543                             "b" * (self.s.MUTABLE_SIZELIMIT+1))))
2544         d.addErrback(self.dump_error)
2545         return d
2546
2547     def test_POST_upload_mutable_toobig(self):
2548         # SDMF had a size limti that was removed a while ago. MDMF has
2549         # never had a size limit. Test to make sure that we do not
2550         # encounter errors when trying to upload large mutable files,
2551         # since there should be no coded prohibitions regarding large
2552         # mutable files.
2553         d = self.POST(self.public_url + "/foo",
2554                       t="upload", mutable="true",
2555                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2556         return d
2557
2558     def dump_error(self, f):
2559         # if the web server returns an error code (like 400 Bad Request),
2560         # web.client.getPage puts the HTTP response body into the .response
2561         # attribute of the exception object that it gives back. It does not
2562         # appear in the Failure's repr(), so the ERROR that trial displays
2563         # will be rather terse and unhelpful. addErrback this method to the
2564         # end of your chain to get more information out of these errors.
2565         if f.check(error.Error):
2566             print "web.error.Error:"
2567             print f
2568             print f.value.response
2569         return f
2570
2571     def test_POST_upload_replace(self):
2572         d = self.POST(self.public_url + "/foo", t="upload",
2573                       file=("bar.txt", self.NEWFILE_CONTENTS))
2574         fn = self._foo_node
2575         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
2576         d.addCallback(lambda res:
2577                       self.failUnlessChildContentsAre(fn, u"bar.txt",
2578                                                       self.NEWFILE_CONTENTS))
2579         return d
2580
2581     def test_POST_upload_no_replace_ok(self):
2582         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2583                       file=("new.txt", self.NEWFILE_CONTENTS))
2584         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
2585         d.addCallback(lambda res: self.failUnlessReallyEqual(res,
2586                                                              self.NEWFILE_CONTENTS))
2587         return d
2588
2589     def test_POST_upload_no_replace_queryarg(self):
2590         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2591                       file=("bar.txt", self.NEWFILE_CONTENTS))
2592         d.addBoth(self.shouldFail, error.Error,
2593                   "POST_upload_no_replace_queryarg",
2594                   "409 Conflict",
2595                   "There was already a child by that name, and you asked me "
2596                   "to not replace it")
2597         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2598         d.addCallback(self.failUnlessIsBarDotTxt)
2599         return d
2600
2601     def test_POST_upload_no_replace_field(self):
2602         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
2603                       file=("bar.txt", self.NEWFILE_CONTENTS))
2604         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
2605                   "409 Conflict",
2606                   "There was already a child by that name, and you asked me "
2607                   "to not replace it")
2608         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2609         d.addCallback(self.failUnlessIsBarDotTxt)
2610         return d
2611
2612     def test_POST_upload_whendone(self):
2613         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
2614                       file=("new.txt", self.NEWFILE_CONTENTS))
2615         d.addBoth(self.shouldRedirect, "/THERE")
2616         fn = self._foo_node
2617         d.addCallback(lambda res:
2618                       self.failUnlessChildContentsAre(fn, u"new.txt",
2619                                                       self.NEWFILE_CONTENTS))
2620         return d
2621
2622     def test_POST_upload_named(self):
2623         fn = self._foo_node
2624         d = self.POST(self.public_url + "/foo", t="upload",
2625                       name="new.txt", file=self.NEWFILE_CONTENTS)
2626         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2627         d.addCallback(lambda res:
2628                       self.failUnlessChildContentsAre(fn, u"new.txt",
2629                                                       self.NEWFILE_CONTENTS))
2630         return d
2631
2632     def test_POST_upload_named_badfilename(self):
2633         d = self.POST(self.public_url + "/foo", t="upload",
2634                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
2635         d.addBoth(self.shouldFail, error.Error,
2636                   "test_POST_upload_named_badfilename",
2637                   "400 Bad Request",
2638                   "name= may not contain a slash",
2639                   )
2640         # make sure that nothing was added
2641         d.addCallback(lambda res:
2642                       self.failUnlessNodeKeysAre(self._foo_node,
2643                                                  [self._htmlname_unicode,
2644                                                   u"bar.txt", u"baz.txt", u"blockingfile",
2645                                                   u"empty", u"n\u00fc.txt", u"quux.txt",
2646                                                   u"sub"]))
2647         return d
2648
2649     def test_POST_FILEURL_check(self):
2650         bar_url = self.public_url + "/foo/bar.txt"
2651         d = self.POST(bar_url, t="check")
2652         def _check(res):
2653             self.failUnlessIn("Healthy :", res)
2654         d.addCallback(_check)
2655         redir_url = "http://allmydata.org/TARGET"
2656         def _check2(statuscode, target):
2657             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2658             self.failUnlessReallyEqual(target, redir_url)
2659         d.addCallback(lambda res:
2660                       self.shouldRedirect2("test_POST_FILEURL_check",
2661                                            _check2,
2662                                            self.POST, bar_url,
2663                                            t="check",
2664                                            when_done=redir_url))
2665         d.addCallback(lambda res:
2666                       self.POST(bar_url, t="check", return_to=redir_url))
2667         def _check3(res):
2668             self.failUnlessIn("Healthy :", res)
2669             self.failUnlessIn("Return to file", res)
2670             self.failUnlessIn(redir_url, res)
2671         d.addCallback(_check3)
2672
2673         d.addCallback(lambda res:
2674                       self.POST(bar_url, t="check", output="JSON"))
2675         def _check_json(res):
2676             data = simplejson.loads(res)
2677             self.failUnlessIn("storage-index", data)
2678             self.failUnless(data["results"]["healthy"])
2679         d.addCallback(_check_json)
2680
2681         return d
2682
2683     def test_POST_FILEURL_check_and_repair(self):
2684         bar_url = self.public_url + "/foo/bar.txt"
2685         d = self.POST(bar_url, t="check", repair="true")
2686         def _check(res):
2687             self.failUnlessIn("Healthy :", res)
2688         d.addCallback(_check)
2689         redir_url = "http://allmydata.org/TARGET"
2690         def _check2(statuscode, target):
2691             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2692             self.failUnlessReallyEqual(target, redir_url)
2693         d.addCallback(lambda res:
2694                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
2695                                            _check2,
2696                                            self.POST, bar_url,
2697                                            t="check", repair="true",
2698                                            when_done=redir_url))
2699         d.addCallback(lambda res:
2700                       self.POST(bar_url, t="check", return_to=redir_url))
2701         def _check3(res):
2702             self.failUnlessIn("Healthy :", res)
2703             self.failUnlessIn("Return to file", res)
2704             self.failUnlessIn(redir_url, res)
2705         d.addCallback(_check3)
2706         return d
2707
2708     def test_POST_DIRURL_check(self):
2709         foo_url = self.public_url + "/foo/"
2710         d = self.POST(foo_url, t="check")
2711         def _check(res):
2712             self.failUnlessIn("Healthy :", res)
2713         d.addCallback(_check)
2714         redir_url = "http://allmydata.org/TARGET"
2715         def _check2(statuscode, target):
2716             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2717             self.failUnlessReallyEqual(target, redir_url)
2718         d.addCallback(lambda res:
2719                       self.shouldRedirect2("test_POST_DIRURL_check",
2720                                            _check2,
2721                                            self.POST, foo_url,
2722                                            t="check",
2723                                            when_done=redir_url))
2724         d.addCallback(lambda res:
2725                       self.POST(foo_url, t="check", return_to=redir_url))
2726         def _check3(res):
2727             self.failUnlessIn("Healthy :", res)
2728             self.failUnlessIn("Return to file/directory", res)
2729             self.failUnlessIn(redir_url, res)
2730         d.addCallback(_check3)
2731
2732         d.addCallback(lambda res:
2733                       self.POST(foo_url, t="check", output="JSON"))
2734         def _check_json(res):
2735             data = simplejson.loads(res)
2736             self.failUnlessIn("storage-index", data)
2737             self.failUnless(data["results"]["healthy"])
2738         d.addCallback(_check_json)
2739
2740         return d
2741
2742     def test_POST_DIRURL_check_and_repair(self):
2743         foo_url = self.public_url + "/foo/"
2744         d = self.POST(foo_url, t="check", repair="true")
2745         def _check(res):
2746             self.failUnlessIn("Healthy :", res)
2747         d.addCallback(_check)
2748         redir_url = "http://allmydata.org/TARGET"
2749         def _check2(statuscode, target):
2750             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2751             self.failUnlessReallyEqual(target, redir_url)
2752         d.addCallback(lambda res:
2753                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2754                                            _check2,
2755                                            self.POST, foo_url,
2756                                            t="check", repair="true",
2757                                            when_done=redir_url))
2758         d.addCallback(lambda res:
2759                       self.POST(foo_url, t="check", return_to=redir_url))
2760         def _check3(res):
2761             self.failUnlessIn("Healthy :", res)
2762             self.failUnlessIn("Return to file/directory", res)
2763             self.failUnlessIn(redir_url, res)
2764         d.addCallback(_check3)
2765         return d
2766
2767     def test_POST_FILEURL_mdmf_check(self):
2768         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2769         d = self.POST(quux_url, t="check")
2770         def _check(res):
2771             self.failUnlessIn("Healthy", res)
2772         d.addCallback(_check)
2773         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2774         d.addCallback(lambda ignored:
2775                       self.POST(quux_extension_url, t="check"))
2776         d.addCallback(_check)
2777         return d
2778
2779     def test_POST_FILEURL_mdmf_check_and_repair(self):
2780         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2781         d = self.POST(quux_url, t="check", repair="true")
2782         def _check(res):
2783             self.failUnlessIn("Healthy", res)
2784         d.addCallback(_check)
2785         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2786         d.addCallback(lambda ignored:
2787                       self.POST(quux_extension_url, t="check", repair="true"))
2788         d.addCallback(_check)
2789         return d
2790
2791     def wait_for_operation(self, ignored, ophandle):
2792         url = "/operations/" + ophandle
2793         url += "?t=status&output=JSON"
2794         d = self.GET(url)
2795         def _got(res):
2796             data = simplejson.loads(res)
2797             if not data["finished"]:
2798                 d = self.stall(delay=1.0)
2799                 d.addCallback(self.wait_for_operation, ophandle)
2800                 return d
2801             return data
2802         d.addCallback(_got)
2803         return d
2804
2805     def get_operation_results(self, ignored, ophandle, output=None):
2806         url = "/operations/" + ophandle
2807         url += "?t=status"
2808         if output:
2809             url += "&output=" + output
2810         d = self.GET(url)
2811         def _got(res):
2812             if output and output.lower() == "json":
2813                 return simplejson.loads(res)
2814             return res
2815         d.addCallback(_got)
2816         return d
2817
2818     def test_POST_DIRURL_deepcheck_no_ophandle(self):
2819         d = self.shouldFail2(error.Error,
2820                              "test_POST_DIRURL_deepcheck_no_ophandle",
2821                              "400 Bad Request",
2822                              "slow operation requires ophandle=",
2823                              self.POST, self.public_url, t="start-deep-check")
2824         return d
2825
2826     def test_POST_DIRURL_deepcheck(self):
2827         def _check_redirect(statuscode, target):
2828             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2829             self.failUnless(target.endswith("/operations/123"))
2830         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2831                                  self.POST, self.public_url,
2832                                  t="start-deep-check", ophandle="123")
2833         d.addCallback(self.wait_for_operation, "123")
2834         def _check_json(data):
2835             self.failUnlessReallyEqual(data["finished"], True)
2836             self.failUnlessReallyEqual(data["count-objects-checked"], 11)
2837             self.failUnlessReallyEqual(data["count-objects-healthy"], 11)
2838         d.addCallback(_check_json)
2839         d.addCallback(self.get_operation_results, "123", "html")
2840         def _check_html(res):
2841             self.failUnlessIn("Objects Checked: <span>11</span>", res)
2842             self.failUnlessIn("Objects Healthy: <span>11</span>", res)
2843             self.failUnlessIn(FAVICON_MARKUP, res)
2844         d.addCallback(_check_html)
2845
2846         d.addCallback(lambda res:
2847                       self.GET("/operations/123/"))
2848         d.addCallback(_check_html) # should be the same as without the slash
2849
2850         d.addCallback(lambda res:
2851                       self.shouldFail2(error.Error, "one", "404 Not Found",
2852                                        "No detailed results for SI bogus",
2853                                        self.GET, "/operations/123/bogus"))
2854
2855         foo_si = self._foo_node.get_storage_index()
2856         foo_si_s = base32.b2a(foo_si)
2857         d.addCallback(lambda res:
2858                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2859         def _check_foo_json(res):
2860             data = simplejson.loads(res)
2861             self.failUnlessEqual(data["storage-index"], foo_si_s)
2862             self.failUnless(data["results"]["healthy"])
2863         d.addCallback(_check_foo_json)
2864         return d
2865
2866     def test_POST_DIRURL_deepcheck_and_repair(self):
2867         d = self.POST(self.public_url, t="start-deep-check", repair="true",
2868                       ophandle="124", output="json", followRedirect=True)
2869         d.addCallback(self.wait_for_operation, "124")
2870         def _check_json(data):
2871             self.failUnlessReallyEqual(data["finished"], True)
2872             self.failUnlessReallyEqual(data["count-objects-checked"], 11)
2873             self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 11)
2874             self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2875             self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2876             self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2877             self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2878             self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2879             self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 11)
2880             self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2881             self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2882         d.addCallback(_check_json)
2883         d.addCallback(self.get_operation_results, "124", "html")
2884         def _check_html(res):
2885             self.failUnlessIn("Objects Checked: <span>11</span>", res)
2886
2887             self.failUnlessIn("Objects Healthy (before repair): <span>11</span>", res)
2888             self.failUnlessIn("Objects Unhealthy (before repair): <span>0</span>", res)
2889             self.failUnlessIn("Corrupt Shares (before repair): <span>0</span>", res)
2890
2891             self.failUnlessIn("Repairs Attempted: <span>0</span>", res)
2892             self.failUnlessIn("Repairs Successful: <span>0</span>", res)
2893             self.failUnlessIn("Repairs Unsuccessful: <span>0</span>", res)
2894
2895             self.failUnlessIn("Objects Healthy (after repair): <span>11</span>", res)
2896             self.failUnlessIn("Objects Unhealthy (after repair): <span>0</span>", res)
2897             self.failUnlessIn("Corrupt Shares (after repair): <span>0</span>", res)
2898
2899             self.failUnlessIn(FAVICON_MARKUP, res)
2900         d.addCallback(_check_html)
2901         return d
2902
2903     def test_POST_FILEURL_bad_t(self):
2904         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2905                              "POST to file: bad t=bogus",
2906                              self.POST, self.public_url + "/foo/bar.txt",
2907                              t="bogus")
2908         return d
2909
2910     def test_POST_mkdir(self): # return value?
2911         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2912         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2913         d.addCallback(self.failUnlessNodeKeysAre, [])
2914         return d
2915
2916     def test_POST_mkdir_mdmf(self):
2917         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=mdmf")
2918         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2919         d.addCallback(lambda node:
2920             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2921         return d
2922
2923     def test_POST_mkdir_sdmf(self):
2924         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=sdmf")
2925         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2926         d.addCallback(lambda node:
2927             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2928         return d
2929
2930     def test_POST_mkdir_bad_format(self):
2931         return self.shouldHTTPError("POST_mkdir_bad_format",
2932                                     400, "Bad Request", "Unknown format: foo",
2933                                     self.POST, self.public_url +
2934                                     "/foo?t=mkdir&name=newdir&format=foo")
2935
2936     def test_POST_mkdir_initial_children(self):
2937         (newkids, caps) = self._create_initial_children()
2938         d = self.POST2(self.public_url +
2939                        "/foo?t=mkdir-with-children&name=newdir",
2940                        simplejson.dumps(newkids))
2941         d.addCallback(lambda res:
2942                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2943         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2944         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2945         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2946         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2947         return d
2948
2949     def test_POST_mkdir_initial_children_mdmf(self):
2950         (newkids, caps) = self._create_initial_children()
2951         d = self.POST2(self.public_url +
2952                        "/foo?t=mkdir-with-children&name=newdir&format=mdmf",
2953                        simplejson.dumps(newkids))
2954         d.addCallback(lambda res:
2955                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2956         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2957         d.addCallback(lambda node:
2958             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2959         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2960         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2961                        caps['filecap1'])
2962         return d
2963
2964     # XXX: Duplication.
2965     def test_POST_mkdir_initial_children_sdmf(self):
2966         (newkids, caps) = self._create_initial_children()
2967         d = self.POST2(self.public_url +
2968                        "/foo?t=mkdir-with-children&name=newdir&format=sdmf",
2969                        simplejson.dumps(newkids))
2970         d.addCallback(lambda res:
2971                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2972         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2973         d.addCallback(lambda node:
2974             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2975         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2976         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2977                        caps['filecap1'])
2978         return d
2979
2980     def test_POST_mkdir_initial_children_bad_format(self):
2981         (newkids, caps) = self._create_initial_children()
2982         return self.shouldHTTPError("POST_mkdir_initial_children_bad_format",
2983                                     400, "Bad Request", "Unknown format: foo",
2984                                     self.POST, self.public_url + \
2985                                     "/foo?t=mkdir-with-children&name=newdir&format=foo",
2986                                     simplejson.dumps(newkids))
2987
2988     def test_POST_mkdir_immutable(self):
2989         (newkids, caps) = self._create_immutable_children()
2990         d = self.POST2(self.public_url +
2991                        "/foo?t=mkdir-immutable&name=newdir",
2992                        simplejson.dumps(newkids))
2993         d.addCallback(lambda res:
2994                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2995         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2996         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2997         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2998         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2999         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3000         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
3001         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3002         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
3003         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3004         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
3005         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3006         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
3007         return d
3008
3009     def test_POST_mkdir_immutable_bad(self):
3010         (newkids, caps) = self._create_initial_children()
3011         d = self.shouldFail2(error.Error, "POST_mkdir_immutable_bad",
3012                              "400 Bad Request",
3013                              "needed to be immutable but was not",
3014                              self.POST2,
3015                              self.public_url +
3016                              "/foo?t=mkdir-immutable&name=newdir",
3017                              simplejson.dumps(newkids))
3018         return d
3019
3020     def test_POST_mkdir_2(self):
3021         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
3022         d.addCallback(lambda res:
3023                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
3024         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3025         d.addCallback(self.failUnlessNodeKeysAre, [])
3026         return d
3027
3028     def test_POST_mkdirs_2(self):
3029         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
3030         d.addCallback(lambda res:
3031                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
3032         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
3033         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
3034         d.addCallback(self.failUnlessNodeKeysAre, [])
3035         return d
3036
3037     def test_POST_mkdir_no_parentdir_noredirect(self):
3038         d = self.POST("/uri?t=mkdir")
3039         def _after_mkdir(res):
3040             uri.DirectoryURI.init_from_string(res)
3041         d.addCallback(_after_mkdir)
3042         return d
3043
3044     def test_POST_mkdir_no_parentdir_noredirect_mdmf(self):
3045         d = self.POST("/uri?t=mkdir&format=mdmf")
3046         def _after_mkdir(res):
3047             u = uri.from_string(res)
3048             # Check that this is an MDMF writecap
3049             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
3050         d.addCallback(_after_mkdir)
3051         return d
3052
3053     def test_POST_mkdir_no_parentdir_noredirect_sdmf(self):
3054         d = self.POST("/uri?t=mkdir&format=sdmf")
3055         def _after_mkdir(res):
3056             u = uri.from_string(res)
3057             self.failUnlessIsInstance(u, uri.DirectoryURI)
3058         d.addCallback(_after_mkdir)
3059         return d
3060
3061     def test_POST_mkdir_no_parentdir_noredirect_bad_format(self):
3062         return self.shouldHTTPError("POST_mkdir_no_parentdir_noredirect_bad_format",
3063                                     400, "Bad Request", "Unknown format: foo",
3064                                     self.POST, self.public_url +
3065                                     "/uri?t=mkdir&format=foo")
3066
3067     def test_POST_mkdir_no_parentdir_noredirect2(self):
3068         # make sure form-based arguments (as on the welcome page) still work
3069         d = self.POST("/uri", t="mkdir")
3070         def _after_mkdir(res):
3071             uri.DirectoryURI.init_from_string(res)
3072         d.addCallback(_after_mkdir)
3073         d.addErrback(self.explain_web_error)
3074         return d
3075
3076     def test_POST_mkdir_no_parentdir_redirect(self):
3077         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
3078         d.addBoth(self.shouldRedirect, None, statuscode='303')
3079         def _check_target(target):
3080             target = urllib.unquote(target)
3081             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
3082         d.addCallback(_check_target)
3083         return d
3084
3085     def test_POST_mkdir_no_parentdir_redirect2(self):
3086         d = self.POST("/uri", t="mkdir", redirect_to_result="true")
3087         d.addBoth(self.shouldRedirect, None, statuscode='303')
3088         def _check_target(target):
3089             target = urllib.unquote(target)
3090             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
3091         d.addCallback(_check_target)
3092         d.addErrback(self.explain_web_error)
3093         return d
3094
3095     def _make_readonly(self, u):
3096         ro_uri = uri.from_string(u).get_readonly()
3097         if ro_uri is None:
3098             return None
3099         return ro_uri.to_string()
3100
3101     def _create_initial_children(self):
3102         contents, n, filecap1 = self.makefile(12)
3103         md1 = {"metakey1": "metavalue1"}
3104         filecap2 = make_mutable_file_uri()
3105         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
3106         filecap3 = node3.get_readonly_uri()
3107         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
3108         dircap = DirectoryNode(node4, None, None).get_uri()
3109         mdmfcap = make_mutable_file_uri(mdmf=True)
3110         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
3111         emptydircap = "URI:DIR2-LIT:"
3112         newkids = {u"child-imm":        ["filenode", {"rw_uri": filecap1,
3113                                                       "ro_uri": self._make_readonly(filecap1),
3114                                                       "metadata": md1, }],
3115                    u"child-mutable":    ["filenode", {"rw_uri": filecap2,
3116                                                       "ro_uri": self._make_readonly(filecap2)}],
3117                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
3118                    u"unknownchild-rw":  ["unknown",  {"rw_uri": unknown_rwcap,
3119                                                       "ro_uri": unknown_rocap}],
3120                    u"unknownchild-ro":  ["unknown",  {"ro_uri": unknown_rocap}],
3121                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
3122                    u"dirchild":         ["dirnode",  {"rw_uri": dircap,
3123                                                       "ro_uri": self._make_readonly(dircap)}],
3124                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
3125                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
3126                    u"child-mutable-mdmf": ["filenode", {"rw_uri": mdmfcap,
3127                                                         "ro_uri": self._make_readonly(mdmfcap)}],
3128                    }
3129         return newkids, {'filecap1': filecap1,
3130                          'filecap2': filecap2,
3131                          'filecap3': filecap3,
3132                          'unknown_rwcap': unknown_rwcap,
3133                          'unknown_rocap': unknown_rocap,
3134                          'unknown_immcap': unknown_immcap,
3135                          'dircap': dircap,
3136                          'litdircap': litdircap,
3137                          'emptydircap': emptydircap,
3138                          'mdmfcap': mdmfcap}
3139
3140     def _create_immutable_children(self):
3141         contents, n, filecap1 = self.makefile(12)
3142         md1 = {"metakey1": "metavalue1"}
3143         tnode = create_chk_filenode("immutable directory contents\n"*10,
3144                                     self.get_all_contents())
3145         dnode = DirectoryNode(tnode, None, None)
3146         assert not dnode.is_mutable()
3147         immdircap = dnode.get_uri()
3148         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
3149         emptydircap = "URI:DIR2-LIT:"
3150         newkids = {u"child-imm":        ["filenode", {"ro_uri": filecap1,
3151                                                       "metadata": md1, }],
3152                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
3153                    u"dirchild-imm":     ["dirnode",  {"ro_uri": immdircap}],
3154                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
3155                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
3156                    }
3157         return newkids, {'filecap1': filecap1,
3158                          'unknown_immcap': unknown_immcap,
3159                          'immdircap': immdircap,
3160                          'litdircap': litdircap,
3161                          'emptydircap': emptydircap}
3162
3163     def test_POST_mkdir_no_parentdir_initial_children(self):
3164         (newkids, caps) = self._create_initial_children()
3165         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
3166         def _after_mkdir(res):
3167             self.failUnless(res.startswith("URI:DIR"), res)
3168             n = self.s.create_node_from_uri(res)
3169             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3170             d2.addCallback(lambda ign:
3171                            self.failUnlessROChildURIIs(n, u"child-imm",
3172                                                        caps['filecap1']))
3173             d2.addCallback(lambda ign:
3174                            self.failUnlessRWChildURIIs(n, u"child-mutable",
3175                                                        caps['filecap2']))
3176             d2.addCallback(lambda ign:
3177                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
3178                                                        caps['filecap3']))
3179             d2.addCallback(lambda ign:
3180                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
3181                                                        caps['unknown_rwcap']))
3182             d2.addCallback(lambda ign:
3183                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
3184                                                        caps['unknown_rocap']))
3185             d2.addCallback(lambda ign:
3186                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3187                                                        caps['unknown_immcap']))
3188             d2.addCallback(lambda ign:
3189                            self.failUnlessRWChildURIIs(n, u"dirchild",
3190                                                        caps['dircap']))
3191             return d2
3192         d.addCallback(_after_mkdir)
3193         return d
3194
3195     def test_POST_mkdir_no_parentdir_unexpected_children(self):
3196         # the regular /uri?t=mkdir operation is specified to ignore its body.
3197         # Only t=mkdir-with-children pays attention to it.
3198         (newkids, caps) = self._create_initial_children()
3199         d = self.shouldHTTPError("POST_mkdir_no_parentdir_unexpected_children",
3200                                  400, "Bad Request",
3201                                  "t=mkdir does not accept children=, "
3202                                  "try t=mkdir-with-children instead",
3203                                  self.POST2, "/uri?t=mkdir", # without children
3204                                  simplejson.dumps(newkids))
3205         return d
3206
3207     def test_POST_noparent_bad(self):
3208         d = self.shouldHTTPError("POST_noparent_bad",
3209                                  400, "Bad Request",
3210                                  "/uri accepts only PUT, PUT?t=mkdir, "
3211                                  "POST?t=upload, and POST?t=mkdir",
3212                                  self.POST, "/uri?t=bogus")
3213         return d
3214
3215     def test_POST_mkdir_no_parentdir_immutable(self):
3216         (newkids, caps) = self._create_immutable_children()
3217         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
3218         def _after_mkdir(res):
3219             self.failUnless(res.startswith("URI:DIR"), res)
3220             n = self.s.create_node_from_uri(res)
3221             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3222             d2.addCallback(lambda ign:
3223                            self.failUnlessROChildURIIs(n, u"child-imm",
3224                                                           caps['filecap1']))
3225             d2.addCallback(lambda ign:
3226                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3227                                                           caps['unknown_immcap']))
3228             d2.addCallback(lambda ign:
3229                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
3230                                                           caps['immdircap']))
3231             d2.addCallback(lambda ign:
3232                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
3233                                                           caps['litdircap']))
3234             d2.addCallback(lambda ign:
3235                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
3236                                                           caps['emptydircap']))
3237             return d2
3238         d.addCallback(_after_mkdir)
3239         return d
3240
3241     def test_POST_mkdir_no_parentdir_immutable_bad(self):
3242         (newkids, caps) = self._create_initial_children()
3243         d = self.shouldFail2(error.Error,
3244                              "test_POST_mkdir_no_parentdir_immutable_bad",
3245                              "400 Bad Request",
3246                              "needed to be immutable but was not",
3247                              self.POST2,
3248                              "/uri?t=mkdir-immutable",
3249                              simplejson.dumps(newkids))
3250         return d
3251
3252     def test_welcome_page_mkdir_button(self):
3253         # Fetch the welcome page.
3254         d = self.GET("/")
3255         def _after_get_welcome_page(res):
3256             MKDIR_BUTTON_RE = re.compile(
3257                 '<form action="([^"]*)" method="post".*'
3258                 '<input type="hidden" name="t" value="([^"]*)" />[ ]*'
3259                 '<input type="hidden" name="([^"]*)" value="([^"]*)" />[ ]*'
3260                 '<input type="submit" class="btn" value="Create a directory[^"]*" />')
3261             html = res.replace('\n', ' ')
3262             mo = MKDIR_BUTTON_RE.search(html)
3263             self.failUnless(mo, html)
3264             formaction = mo.group(1)
3265             formt = mo.group(2)
3266             formaname = mo.group(3)
3267             formavalue = mo.group(4)
3268             return (formaction, formt, formaname, formavalue)
3269         d.addCallback(_after_get_welcome_page)
3270         def _after_parse_form(res):
3271             (formaction, formt, formaname, formavalue) = res
3272             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
3273         d.addCallback(_after_parse_form)
3274         d.addBoth(self.shouldRedirect, None, statuscode='303')
3275         return d
3276
3277     def test_POST_mkdir_replace(self): # return value?
3278         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
3279         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3280         d.addCallback(self.failUnlessNodeKeysAre, [])
3281         return d
3282
3283     def test_POST_mkdir_no_replace_queryarg(self): # return value?
3284         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
3285         d.addBoth(self.shouldFail, error.Error,
3286                   "POST_mkdir_no_replace_queryarg",
3287                   "409 Conflict",
3288                   "There was already a child by that name, and you asked me "
3289                   "to not replace it")
3290         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3291         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3292         return d
3293
3294     def test_POST_mkdir_no_replace_field(self): # return value?
3295         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
3296                       replace="false")
3297         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
3298                   "409 Conflict",
3299                   "There was already a child by that name, and you asked me "
3300                   "to not replace it")
3301         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3302         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3303         return d
3304
3305     def test_POST_mkdir_whendone_field(self):
3306         d = self.POST(self.public_url + "/foo",
3307                       t="mkdir", name="newdir", when_done="/THERE")
3308         d.addBoth(self.shouldRedirect, "/THERE")
3309         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3310         d.addCallback(self.failUnlessNodeKeysAre, [])
3311         return d
3312
3313     def test_POST_mkdir_whendone_queryarg(self):
3314         d = self.POST(self.public_url + "/foo?when_done=/THERE",
3315                       t="mkdir", name="newdir")
3316         d.addBoth(self.shouldRedirect, "/THERE")
3317         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3318         d.addCallback(self.failUnlessNodeKeysAre, [])
3319         return d
3320
3321     def test_POST_bad_t(self):
3322         d = self.shouldFail2(error.Error, "POST_bad_t",
3323                              "400 Bad Request",
3324                              "POST to a directory with bad t=BOGUS",
3325                              self.POST, self.public_url + "/foo", t="BOGUS")
3326         return d
3327
3328     def test_POST_set_children(self, command_name="set_children"):
3329         contents9, n9, newuri9 = self.makefile(9)
3330         contents10, n10, newuri10 = self.makefile(10)
3331         contents11, n11, newuri11 = self.makefile(11)
3332
3333         reqbody = """{
3334                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
3335                                                 "size": 0,
3336                                                 "metadata": {
3337                                                   "ctime": 1002777696.7564139,
3338                                                   "mtime": 1002777696.7564139
3339                                                  }
3340                                                } ],
3341                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
3342                                                 "size": 1,
3343                                                 "metadata": {
3344                                                   "ctime": 1002777696.7564139,
3345                                                   "mtime": 1002777696.7564139
3346                                                  }
3347                                                } ],
3348                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
3349                                                 "size": 2,
3350                                                 "metadata": {
3351                                                   "ctime": 1002777696.7564139,
3352                                                   "mtime": 1002777696.7564139
3353                                                  }
3354                                                } ]
3355                     }""" % (newuri9, newuri10, newuri11)
3356
3357         url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
3358
3359         d = client.getPage(url, method="POST", postdata=reqbody)
3360         def _then(res):
3361             self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
3362             self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
3363             self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
3364
3365         d.addCallback(_then)
3366         d.addErrback(self.dump_error)
3367         return d
3368
3369     def test_POST_set_children_with_hyphen(self):
3370         return self.test_POST_set_children(command_name="set-children")
3371
3372     def test_POST_link_uri(self):
3373         contents, n, newuri = self.makefile(8)
3374         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
3375         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
3376         d.addCallback(lambda res:
3377                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3378                                                       contents))
3379         return d
3380
3381     def test_POST_link_uri_replace(self):
3382         contents, n, newuri = self.makefile(8)
3383         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
3384         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
3385         d.addCallback(lambda res:
3386                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3387                                                       contents))
3388         return d
3389
3390     def test_POST_link_uri_unknown_bad(self):
3391         d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
3392         d.addBoth(self.shouldFail, error.Error,
3393                   "POST_link_uri_unknown_bad",
3394                   "400 Bad Request",
3395                   "unknown cap in a write slot")
3396         return d
3397
3398     def test_POST_link_uri_unknown_ro_good(self):
3399         d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
3400         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
3401         return d
3402
3403     def test_POST_link_uri_unknown_imm_good(self):
3404         d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
3405         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
3406         return d
3407
3408     def test_POST_link_uri_no_replace_queryarg(self):
3409         contents, n, newuri = self.makefile(8)
3410         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
3411                       name="bar.txt", uri=newuri)
3412         d.addBoth(self.shouldFail, error.Error,
3413                   "POST_link_uri_no_replace_queryarg",
3414                   "409 Conflict",
3415                   "There was already a child by that name, and you asked me "
3416                   "to not replace it")
3417         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3418         d.addCallback(self.failUnlessIsBarDotTxt)
3419         return d
3420
3421     def test_POST_link_uri_no_replace_field(self):
3422         contents, n, newuri = self.makefile(8)
3423         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
3424                       name="bar.txt", uri=newuri)
3425         d.addBoth(self.shouldFail, error.Error,
3426                   "POST_link_uri_no_replace_field",
3427                   "409 Conflict",
3428                   "There was already a child by that name, and you asked me "
3429                   "to not replace it")
3430         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3431         d.addCallback(self.failUnlessIsBarDotTxt)
3432         return d
3433
3434     def test_POST_delete(self, command_name='delete'):
3435         d = self._foo_node.list()
3436         def _check_before(children):
3437             self.failUnlessIn(u"bar.txt", children)
3438         d.addCallback(_check_before)
3439         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t=command_name, name="bar.txt"))
3440         d.addCallback(lambda res: self._foo_node.list())
3441         def _check_after(children):
3442             self.failIfIn(u"bar.txt", children)
3443         d.addCallback(_check_after)
3444         return d
3445
3446     def test_POST_unlink(self):
3447         return self.test_POST_delete(command_name='unlink')
3448
3449     def test_POST_rename_file(self):
3450         d = self.POST(self.public_url + "/foo", t="rename",
3451                       from_name="bar.txt", to_name='wibble.txt')
3452         d.addCallback(lambda res:
3453                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3454         d.addCallback(lambda res:
3455                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
3456         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
3457         d.addCallback(self.failUnlessIsBarDotTxt)
3458         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
3459         d.addCallback(self.failUnlessIsBarJSON)
3460         return d
3461
3462     def test_POST_rename_file_redundant(self):
3463         d = self.POST(self.public_url + "/foo", t="rename",
3464                       from_name="bar.txt", to_name='bar.txt')
3465         d.addCallback(lambda res:
3466                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3467         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3468         d.addCallback(self.failUnlessIsBarDotTxt)
3469         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3470         d.addCallback(self.failUnlessIsBarJSON)
3471         return d
3472
3473     def test_POST_rename_file_replace(self):
3474         # rename a file and replace a directory with it
3475         d = self.POST(self.public_url + "/foo", t="rename",
3476                       from_name="bar.txt", to_name='empty')
3477         d.addCallback(lambda res:
3478                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3479         d.addCallback(lambda res:
3480                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
3481         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
3482         d.addCallback(self.failUnlessIsBarDotTxt)
3483         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3484         d.addCallback(self.failUnlessIsBarJSON)
3485         return d
3486
3487     def test_POST_rename_file_no_replace_queryarg(self):
3488         # rename a file and replace a directory with it
3489         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
3490                       from_name="bar.txt", to_name='empty')
3491         d.addBoth(self.shouldFail, error.Error,
3492                   "POST_rename_file_no_replace_queryarg",
3493                   "409 Conflict",
3494                   "There was already a child by that name, and you asked me "
3495                   "to not replace it")
3496         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3497         d.addCallback(self.failUnlessIsEmptyJSON)
3498         return d
3499
3500     def test_POST_rename_file_no_replace_field(self):
3501         # rename a file and replace a directory with it
3502         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
3503                       from_name="bar.txt", to_name='empty')
3504         d.addBoth(self.shouldFail, error.Error,
3505                   "POST_rename_file_no_replace_field",
3506                   "409 Conflict",
3507                   "There was already a child by that name, and you asked me "
3508                   "to not replace it")
3509         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3510         d.addCallback(self.failUnlessIsEmptyJSON)
3511         return d
3512
3513     def test_POST_rename_file_no_replace_same_link(self):
3514         d = self.POST(self.public_url + "/foo", t="rename",
3515                       replace="false", from_name="bar.txt", to_name="bar.txt")
3516         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3517         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3518         d.addCallback(self.failUnlessIsBarDotTxt)
3519         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3520         d.addCallback(self.failUnlessIsBarJSON)
3521         return d
3522
3523     def test_POST_rename_file_replace_only_files(self):
3524         d = self.POST(self.public_url + "/foo", t="rename",
3525                       replace="only-files", from_name="bar.txt",
3526                       to_name="baz.txt")
3527         d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3528         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3529         d.addCallback(self.failUnlessIsBarDotTxt)
3530         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt?t=json"))
3531         d.addCallback(self.failUnlessIsBarJSON)
3532         return d
3533
3534     def test_POST_rename_file_replace_only_files_conflict(self):
3535         d = self.shouldFail2(error.Error, "POST_relink_file_replace_only_files_conflict",
3536                              "409 Conflict",
3537                              "There was already a child by that name, and you asked me to not replace it.",
3538                              self.POST, self.public_url + "/foo", t="relink",
3539                              replace="only-files", from_name="bar.txt",
3540                              to_name="empty")
3541         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3542         d.addCallback(self.failUnlessIsBarDotTxt)
3543         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3544         d.addCallback(self.failUnlessIsBarJSON)
3545         return d
3546
3547     def failUnlessIsEmptyJSON(self, res):
3548         data = simplejson.loads(res)
3549         self.failUnlessEqual(data[0], "dirnode", data)
3550         self.failUnlessReallyEqual(len(data[1]["children"]), 0)
3551
3552     def test_POST_rename_file_to_slash_fail(self):
3553         d = self.POST(self.public_url + "/foo", t="rename",
3554                       from_name="bar.txt", to_name='kirk/spock.txt')
3555         d.addBoth(self.shouldFail, error.Error,
3556                   "test_POST_rename_file_to_slash_fail",
3557                   "400 Bad Request",
3558                   "to_name= may not contain a slash",
3559                   )
3560         d.addCallback(lambda res:
3561                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3562         return d
3563
3564     def test_POST_rename_file_from_slash_fail(self):
3565         d = self.POST(self.public_url + "/foo", t="rename",
3566                       from_name="sub/bar.txt", to_name='spock.txt')
3567         d.addBoth(self.shouldFail, error.Error,
3568                   "test_POST_rename_from_file_slash_fail",
3569                   "400 Bad Request",
3570                   "from_name= may not contain a slash",
3571                   )
3572         d.addCallback(lambda res:
3573                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3574         return d
3575
3576     def test_POST_rename_dir(self):
3577         d = self.POST(self.public_url, t="rename",
3578                       from_name="foo", to_name='plunk')
3579         d.addCallback(lambda res:
3580                       self.failIfNodeHasChild(self.public_root, u"foo"))
3581         d.addCallback(lambda res:
3582                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
3583         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
3584         d.addCallback(self.failUnlessIsFooJSON)
3585         return d
3586
3587     def test_POST_relink_file(self):
3588         d = self.POST(self.public_url + "/foo", t="relink",
3589                       from_name="bar.txt",
3590                       to_dir=self.public_root.get_uri() + "/foo/sub")
3591         d.addCallback(lambda res:
3592                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3593         d.addCallback(lambda res:
3594                       self.failUnlessNodeHasChild(self._sub_node, u"bar.txt"))
3595         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3596         d.addCallback(self.failUnlessIsBarDotTxt)
3597         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3598         d.addCallback(self.failUnlessIsBarJSON)
3599         return d
3600
3601     def test_POST_relink_file_new_name(self):
3602         d = self.POST(self.public_url + "/foo", t="relink",
3603                       from_name="bar.txt",
3604                       to_name="wibble.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3605         d.addCallback(lambda res:
3606                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3607         d.addCallback(lambda res:
3608                       self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3609         d.addCallback(lambda res:
3610                       self.failUnlessNodeHasChild(self._sub_node, u"wibble.txt"))
3611         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt"))
3612         d.addCallback(self.failUnlessIsBarDotTxt)
3613         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt?t=json"))
3614         d.addCallback(self.failUnlessIsBarJSON)
3615         return d
3616
3617     def test_POST_relink_file_replace(self):
3618         d = self.POST(self.public_url + "/foo", t="relink",
3619                       from_name="bar.txt",
3620                       to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3621         d.addCallback(lambda res:
3622                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3623         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3624         d.addCallback(self.failUnlessIsBarDotTxt)
3625         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt?t=json"))
3626         d.addCallback(self.failUnlessIsBarJSON)
3627         return d
3628
3629     def test_POST_relink_file_no_replace(self):
3630         d = self.shouldFail2(error.Error, "POST_relink_file_no_replace",
3631                              "409 Conflict",
3632                              "There was already a child by that name, and you asked me to not replace it",
3633                              self.POST, self.public_url + "/foo", t="relink",
3634                              replace="false", from_name="bar.txt",
3635                              to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3636         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3637         d.addCallback(self.failUnlessIsBarDotTxt)
3638         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3639         d.addCallback(self.failUnlessIsBarJSON)
3640         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3641         d.addCallback(self.failUnlessIsSubBazDotTxt)
3642         return d
3643
3644     def test_POST_relink_file_no_replace_explicitly_same_link(self):
3645         d = self.POST(self.public_url + "/foo", t="relink",
3646                       replace="false", from_name="bar.txt",
3647                       to_name="bar.txt", to_dir=self.public_root.get_uri() + "/foo")
3648         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3649         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3650         d.addCallback(self.failUnlessIsBarDotTxt)
3651         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3652         d.addCallback(self.failUnlessIsBarJSON)
3653         return d
3654
3655     def test_POST_relink_file_replace_only_files(self):
3656         d = self.POST(self.public_url + "/foo", t="relink",
3657                       replace="only-files", from_name="bar.txt",
3658                       to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3659         d.addCallback(lambda res:
3660                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3661         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3662         d.addCallback(self.failUnlessIsBarDotTxt)
3663         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt?t=json"))
3664         d.addCallback(self.failUnlessIsBarJSON)
3665         return d
3666
3667     def test_POST_relink_file_replace_only_files_conflict(self):
3668         d = self.shouldFail2(error.Error, "POST_relink_file_replace_only_files_conflict",
3669                              "409 Conflict",
3670                              "There was already a child by that name, and you asked me to not replace it.",
3671                              self.POST, self.public_url + "/foo", t="relink",
3672                              replace="only-files", from_name="bar.txt",
3673                              to_name="sub", to_dir=self.public_root.get_uri() + "/foo")
3674         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3675         d.addCallback(self.failUnlessIsBarDotTxt)
3676         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3677         d.addCallback(self.failUnlessIsBarJSON)
3678         return d
3679
3680     def test_POST_relink_file_to_slash_fail(self):
3681         d = self.shouldFail2(error.Error, "test_POST_rename_file_slash_fail",
3682                              "400 Bad Request",
3683                              "to_name= may not contain a slash",
3684                              self.POST, self.public_url + "/foo", t="relink",
3685                              from_name="bar.txt",
3686                              to_name="slash/fail.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3687         d.addCallback(lambda res:
3688                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3689         d.addCallback(lambda res:
3690                       self.failIfNodeHasChild(self._sub_node, u"slash/fail.txt"))
3691         d.addCallback(lambda ign:
3692                       self.shouldFail2(error.Error,
3693                                        "test_POST_rename_file_slash_fail2",
3694                                        "400 Bad Request",
3695                                        "from_name= may not contain a slash",
3696                                        self.POST, self.public_url + "/foo",
3697                                        t="relink",
3698                                        from_name="nope/bar.txt",
3699                                        to_name="fail.txt",
3700                                        to_dir=self.public_root.get_uri() + "/foo/sub"))
3701         return d
3702
3703     def test_POST_relink_file_explicitly_same_link(self):
3704         d = self.POST(self.public_url + "/foo", t="relink",
3705                       from_name="bar.txt",
3706                       to_name="bar.txt", to_dir=self.public_root.get_uri() + "/foo")
3707         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3708         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3709         d.addCallback(self.failUnlessIsBarDotTxt)
3710         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3711         d.addCallback(self.failUnlessIsBarJSON)
3712         return d
3713
3714     def test_POST_relink_file_implicitly_same_link(self):
3715         d = self.POST(self.public_url + "/foo", t="relink",
3716                       from_name="bar.txt")
3717         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3718         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3719         d.addCallback(self.failUnlessIsBarDotTxt)
3720         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3721         d.addCallback(self.failUnlessIsBarJSON)
3722         return d
3723
3724     def test_POST_relink_file_same_dir(self):
3725         d = self.POST(self.public_url + "/foo", t="relink",
3726                       from_name="bar.txt",
3727                       to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo")
3728         d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3729         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._sub_node, u"baz.txt"))
3730         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3731         d.addCallback(self.failUnlessIsBarDotTxt)
3732         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt?t=json"))
3733         d.addCallback(self.failUnlessIsBarJSON)
3734         return d
3735
3736     def test_POST_relink_file_bad_replace(self):
3737         d = self.shouldFail2(error.Error, "test_POST_relink_file_bad_replace",
3738                              "400 Bad Request", "invalid replace= argument: 'boogabooga'",
3739                              self.POST,
3740                              self.public_url + "/foo", t="relink",
3741                              replace="boogabooga", from_name="bar.txt",
3742                              to_dir=self.public_root.get_uri() + "/foo/sub")
3743         return d
3744
3745     def test_POST_relink_file_multi_level(self):
3746         d = self.POST(self.public_url + "/foo/sub/level2?t=mkdir", "")
3747         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t="relink",
3748                       from_name="bar.txt", to_dir=self.public_root.get_uri() + "/foo/sub/level2"))
3749         d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3750         d.addCallback(lambda res: self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3751         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt"))
3752         d.addCallback(self.failUnlessIsBarDotTxt)
3753         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt?t=json"))
3754         d.addCallback(self.failUnlessIsBarJSON)
3755         return d
3756
3757     def test_POST_relink_file_to_uri(self):
3758         d = self.POST(self.public_url + "/foo", t="relink", target_type="uri",
3759                       from_name="bar.txt", to_dir=self._sub_uri)
3760         d.addCallback(lambda res:
3761                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3762         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3763         d.addCallback(self.failUnlessIsBarDotTxt)
3764         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3765         d.addCallback(self.failUnlessIsBarJSON)
3766         return d
3767
3768     def test_POST_relink_file_to_nonexistent_dir(self):
3769         d = self.shouldFail2(error.Error, "POST_relink_file_to_nonexistent_dir",
3770                             "404 Not Found", "No such child: nopechucktesta",
3771                             self.POST, self.public_url + "/foo", t="relink",
3772                             from_name="bar.txt",
3773                             to_dir=self.public_root.get_uri() + "/nopechucktesta")
3774         return d
3775
3776     def test_POST_relink_file_into_file(self):
3777         d = self.shouldFail2(error.Error, "POST_relink_file_into_file",
3778                              "400 Bad Request", "to_dir is not a directory",
3779                              self.POST, self.public_url + "/foo", t="relink",
3780                              from_name="bar.txt",
3781                              to_dir=self.public_root.get_uri() + "/foo/baz.txt")
3782         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3783         d.addCallback(self.failUnlessIsBazDotTxt)
3784         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3785         d.addCallback(self.failUnlessIsBarDotTxt)
3786         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3787         d.addCallback(self.failUnlessIsBarJSON)
3788         return d
3789
3790     def test_POST_relink_file_to_bad_uri(self):
3791         d =  self.shouldFail2(error.Error, "POST_relink_file_to_bad_uri",
3792                               "400 Bad Request", "to_dir is not a directory",
3793                               self.POST, self.public_url + "/foo", t="relink",
3794                               from_name="bar.txt",
3795                               to_dir="URI:DIR2:mn5jlyjnrjeuydyswlzyui72i:rmneifcj6k6sycjljjhj3f6majsq2zqffydnnul5hfa4j577arma")
3796         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3797         d.addCallback(self.failUnlessIsBarDotTxt)
3798         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3799         d.addCallback(self.failUnlessIsBarJSON)
3800         return d
3801
3802     def test_POST_relink_dir(self):
3803         d = self.POST(self.public_url + "/foo", t="relink",
3804                       from_name="bar.txt",
3805                       to_dir=self.public_root.get_uri() + "/foo/empty")
3806         d.addCallback(lambda res: self.POST(self.public_url + "/foo",
3807                       t="relink", from_name="empty",
3808                       to_dir=self.public_root.get_uri() + "/foo/sub"))
3809         d.addCallback(lambda res:
3810                       self.failIfNodeHasChild(self._foo_node, u"empty"))
3811         d.addCallback(lambda res:
3812                       self.failUnlessNodeHasChild(self._sub_node, u"empty"))
3813         d.addCallback(lambda res:
3814                       self._sub_node.get_child_at_path(u"empty"))
3815         d.addCallback(lambda node:
3816                       self.failUnlessNodeHasChild(node, u"bar.txt"))
3817         d.addCallback(lambda res:
3818                       self.GET(self.public_url + "/foo/sub/empty/bar.txt"))
3819         d.addCallback(self.failUnlessIsBarDotTxt)
3820         return d
3821
3822     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
3823         """ If target is not None then the redirection has to go to target.  If
3824         statuscode is not None then the redirection has to be accomplished with
3825         that HTTP status code."""
3826         if not isinstance(res, failure.Failure):
3827             to_where = (target is None) and "somewhere" or ("to " + target)
3828             self.fail("%s: we were expecting to get redirected %s, not get an"
3829                       " actual page: %s" % (which, to_where, res))
3830         res.trap(error.PageRedirect)
3831         if statuscode is not None:
3832             self.failUnlessReallyEqual(res.value.status, statuscode,
3833                                        "%s: not a redirect" % which)
3834         if target is not None:
3835             # the PageRedirect does not seem to capture the uri= query arg
3836             # properly, so we can't check for it.
3837             realtarget = self.webish_url + target
3838             self.failUnlessReallyEqual(res.value.location, realtarget,
3839                                        "%s: wrong target" % which)
3840         return res.value.location
3841
3842     def test_GET_URI_form(self):
3843         base = "/uri?uri=%s" % self._bar_txt_uri
3844         # this is supposed to give us a redirect to /uri/$URI, plus arguments
3845         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
3846         d = self.GET(base)
3847         d.addBoth(self.shouldRedirect, targetbase)
3848         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
3849         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
3850         d.addCallback(lambda res: self.GET(base+"&t=json"))
3851         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
3852         d.addCallback(self.log, "about to get file by uri")
3853         d.addCallback(lambda res: self.GET(base, followRedirect=True))
3854         d.addCallback(self.failUnlessIsBarDotTxt)
3855         d.addCallback(self.log, "got file by uri, about to get dir by uri")
3856         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
3857                                            followRedirect=True))
3858         d.addCallback(self.failUnlessIsFooJSON)
3859         d.addCallback(self.log, "got dir by uri")
3860
3861         return d
3862
3863     def test_GET_URI_form_bad(self):
3864         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
3865                              "400 Bad Request", "GET /uri requires uri=",
3866                              self.GET, "/uri")
3867         return d
3868
3869     def test_GET_rename_form(self):
3870         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
3871                      followRedirect=True)
3872         def _check(res):
3873             self.failUnlessIn('name="when_done" value="."', res)
3874             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
3875             self.failUnlessIn(FAVICON_MARKUP, res)
3876         d.addCallback(_check)
3877         return d
3878
3879     def log(self, res, msg):
3880         #print "MSG: %s  RES: %s" % (msg, res)
3881         log.msg(msg)
3882         return res
3883
3884     def test_GET_URI_URL(self):
3885         base = "/uri/%s" % self._bar_txt_uri
3886         d = self.GET(base)
3887         d.addCallback(self.failUnlessIsBarDotTxt)
3888         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
3889         d.addCallback(self.failUnlessIsBarDotTxt)
3890         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
3891         d.addCallback(self.failUnlessIsBarDotTxt)
3892         return d
3893
3894     def test_GET_URI_URL_dir(self):
3895         base = "/uri/%s?t=json" % self._foo_uri
3896         d = self.GET(base)
3897         d.addCallback(self.failUnlessIsFooJSON)
3898         return d
3899
3900     def test_GET_URI_URL_missing(self):
3901         base = "/uri/%s" % self._bad_file_uri
3902         d = self.shouldHTTPError("test_GET_URI_URL_missing",
3903                                  http.GONE, None, "NotEnoughSharesError",
3904                                  self.GET, base)
3905         # TODO: how can we exercise both sides of WebDownloadTarget.fail
3906         # here? we must arrange for a download to fail after target.open()
3907         # has been called, and then inspect the response to see that it is
3908         # shorter than we expected.
3909         return d
3910
3911     def test_PUT_DIRURL_uri(self):
3912         d = self.s.create_dirnode()
3913         def _made_dir(dn):
3914             new_uri = dn.get_uri()
3915             # replace /foo with a new (empty) directory
3916             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
3917             d.addCallback(lambda res:
3918                           self.failUnlessReallyEqual(res.strip(), new_uri))
3919             d.addCallback(lambda res:
3920                           self.failUnlessRWChildURIIs(self.public_root,
3921                                                       u"foo",
3922                                                       new_uri))
3923             return d
3924         d.addCallback(_made_dir)
3925         return d
3926
3927     def test_PUT_DIRURL_uri_noreplace(self):
3928         d = self.s.create_dirnode()
3929         def _made_dir(dn):
3930             new_uri = dn.get_uri()
3931             # replace /foo with a new (empty) directory, but ask that
3932             # replace=false, so it should fail
3933             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
3934                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
3935                                  self.PUT,
3936                                  self.public_url + "/foo?t=uri&replace=false",
3937                                  new_uri)
3938             d.addCallback(lambda res:
3939                           self.failUnlessRWChildURIIs(self.public_root,
3940                                                       u"foo",
3941                                                       self._foo_uri))
3942             return d
3943         d.addCallback(_made_dir)
3944         return d
3945
3946     def test_PUT_DIRURL_bad_t(self):
3947         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
3948                              "400 Bad Request", "PUT to a directory",
3949                              self.PUT, self.public_url + "/foo?t=BOGUS", "")
3950         d.addCallback(lambda res:
3951                       self.failUnlessRWChildURIIs(self.public_root,
3952                                                   u"foo",
3953                                                   self._foo_uri))
3954         return d
3955
3956     def test_PUT_NEWFILEURL_uri(self):
3957         contents, n, new_uri = self.makefile(8)
3958         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
3959         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3960         d.addCallback(lambda res:
3961                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3962                                                       contents))
3963         return d
3964
3965     def test_PUT_NEWFILEURL_mdmf(self):
3966         new_contents = self.NEWFILE_CONTENTS * 300000
3967         d = self.PUT(self.public_url + \
3968                      "/foo/mdmf.txt?format=mdmf",
3969                      new_contents)
3970         d.addCallback(lambda ignored:
3971             self.GET(self.public_url + "/foo/mdmf.txt?t=json"))
3972         def _got_json(json):
3973             data = simplejson.loads(json)
3974             data = data[1]
3975             self.failUnlessIn("format", data)
3976             self.failUnlessEqual(data["format"], "MDMF")
3977             self.failUnless(data['rw_uri'].startswith("URI:MDMF"))
3978             self.failUnless(data['ro_uri'].startswith("URI:MDMF"))
3979         d.addCallback(_got_json)
3980         return d
3981
3982     def test_PUT_NEWFILEURL_sdmf(self):
3983         new_contents = self.NEWFILE_CONTENTS * 300000
3984         d = self.PUT(self.public_url + \
3985                      "/foo/sdmf.txt?format=sdmf",
3986                      new_contents)
3987         d.addCallback(lambda ignored:
3988             self.GET(self.public_url + "/foo/sdmf.txt?t=json"))
3989         def _got_json(json):
3990             data = simplejson.loads(json)
3991             data = data[1]
3992             self.failUnlessIn("format", data)
3993             self.failUnlessEqual(data["format"], "SDMF")
3994         d.addCallback(_got_json)
3995         return d
3996
3997     def test_PUT_NEWFILEURL_bad_format(self):
3998         new_contents = self.NEWFILE_CONTENTS * 300000
3999         return self.shouldHTTPError("PUT_NEWFILEURL_bad_format",
4000                                     400, "Bad Request", "Unknown format: foo",
4001                                     self.PUT, self.public_url + \
4002                                     "/foo/foo.txt?format=foo",
4003                                     new_contents)
4004
4005     def test_PUT_NEWFILEURL_uri_replace(self):
4006         contents, n, new_uri = self.makefile(8)
4007         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
4008         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
4009         d.addCallback(lambda res:
4010                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
4011                                                       contents))
4012         return d
4013
4014     def test_PUT_NEWFILEURL_uri_no_replace(self):
4015         contents, n, new_uri = self.makefile(8)
4016         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
4017         d.addBoth(self.shouldFail, error.Error,
4018                   "PUT_NEWFILEURL_uri_no_replace",
4019                   "409 Conflict",
4020                   "There was already a child by that name, and you asked me "
4021                   "to not replace it")
4022         return d
4023
4024     def test_PUT_NEWFILEURL_uri_unknown_bad(self):
4025         d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
4026         d.addBoth(self.shouldFail, error.Error,
4027                   "POST_put_uri_unknown_bad",
4028                   "400 Bad Request",
4029                   "unknown cap in a write slot")
4030         return d
4031
4032     def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
4033         d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
4034         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
4035                       u"put-future-ro.txt")
4036         return d
4037
4038     def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
4039         d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
4040         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
4041                       u"put-future-imm.txt")
4042         return d
4043
4044     def test_PUT_NEWFILE_URI(self):
4045         file_contents = "New file contents here\n"
4046         d = self.PUT("/uri", file_contents)
4047         def _check(uri):
4048             assert isinstance(uri, str), uri
4049             self.failUnlessIn(uri, self.get_all_contents())
4050             self.failUnlessReallyEqual(self.get_all_contents()[uri],
4051                                        file_contents)
4052             return self.GET("/uri/%s" % uri)
4053         d.addCallback(_check)
4054         def _check2(res):
4055             self.failUnlessReallyEqual(res, file_contents)
4056         d.addCallback(_check2)
4057         return d
4058
4059     def test_PUT_NEWFILE_URI_not_mutable(self):
4060         file_contents = "New file contents here\n"
4061         d = self.PUT("/uri?mutable=false", file_contents)
4062         def _check(uri):
4063             assert isinstance(uri, str), uri
4064             self.failUnlessIn(uri, self.get_all_contents())
4065             self.failUnlessReallyEqual(self.get_all_contents()[uri],
4066                                        file_contents)
4067             return self.GET("/uri/%s" % uri)
4068         d.addCallback(_check)
4069         def _check2(res):
4070             self.failUnlessReallyEqual(res, file_contents)
4071         d.addCallback(_check2)
4072         return d
4073
4074     def test_PUT_NEWFILE_URI_only_PUT(self):
4075         d = self.PUT("/uri?t=bogus", "")
4076         d.addBoth(self.shouldFail, error.Error,
4077                   "PUT_NEWFILE_URI_only_PUT",
4078                   "400 Bad Request",
4079                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
4080         return d
4081
4082     def test_PUT_NEWFILE_URI_mutable(self):
4083         file_contents = "New file contents here\n"
4084         d = self.PUT("/uri?mutable=true", file_contents)
4085         def _check1(filecap):
4086             filecap = filecap.strip()
4087             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
4088             self.filecap = filecap
4089             u = uri.WriteableSSKFileURI.init_from_string(filecap)
4090             self.failUnlessIn(u.get_storage_index(), self.get_all_contents())
4091             n = self.s.create_node_from_uri(filecap)
4092             return n.download_best_version()
4093         d.addCallback(_check1)
4094         def _check2(data):
4095             self.failUnlessReallyEqual(data, file_contents)
4096             return self.GET("/uri/%s" % urllib.quote(self.filecap))
4097         d.addCallback(_check2)
4098         def _check3(res):
4099             self.failUnlessReallyEqual(res, file_contents)
4100         d.addCallback(_check3)
4101         return d
4102
4103     def test_PUT_mkdir(self):
4104         d = self.PUT("/uri?t=mkdir", "")
4105         def _check(uri):
4106             n = self.s.create_node_from_uri(uri.strip())
4107             d2 = self.failUnlessNodeKeysAre(n, [])
4108             d2.addCallback(lambda res:
4109                            self.GET("/uri/%s?t=json" % uri))
4110             return d2
4111         d.addCallback(_check)
4112         d.addCallback(self.failUnlessIsEmptyJSON)
4113         return d
4114
4115     def test_PUT_mkdir_mdmf(self):
4116         d = self.PUT("/uri?t=mkdir&format=mdmf", "")
4117         def _got(res):
4118             u = uri.from_string(res)
4119             # Check that this is an MDMF writecap
4120             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
4121         d.addCallback(_got)
4122         return d
4123
4124     def test_PUT_mkdir_sdmf(self):
4125         d = self.PUT("/uri?t=mkdir&format=sdmf", "")
4126         def _got(res):
4127             u = uri.from_string(res)
4128             self.failUnlessIsInstance(u, uri.DirectoryURI)
4129         d.addCallback(_got)
4130         return d
4131
4132     def test_PUT_mkdir_bad_format(self):
4133         return self.shouldHTTPError("PUT_mkdir_bad_format",
4134                                     400, "Bad Request", "Unknown format: foo",
4135                                     self.PUT, "/uri?t=mkdir&format=foo",
4136                                     "")
4137
4138     def test_POST_check(self):
4139         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
4140         def _done(res):
4141             # this returns a string form of the results, which are probably
4142             # None since we're using fake filenodes.
4143             # TODO: verify that the check actually happened, by changing
4144             # FakeCHKFileNode to count how many times .check() has been
4145             # called.
4146             pass
4147         d.addCallback(_done)
4148         return d
4149
4150
4151     def test_PUT_update_at_offset(self):
4152         file_contents = "test file" * 100000 # about 900 KiB
4153         d = self.PUT("/uri?mutable=true", file_contents)
4154         def _then(filecap):
4155             self.filecap = filecap
4156             new_data = file_contents[:100]
4157             new = "replaced and so on"
4158             new_data += new
4159             new_data += file_contents[len(new_data):]
4160             assert len(new_data) == len(file_contents)
4161             self.new_data = new_data
4162         d.addCallback(_then)
4163         d.addCallback(lambda ignored:
4164             self.PUT("/uri/%s?replace=True&offset=100" % self.filecap,
4165                      "replaced and so on"))
4166         def _get_data(filecap):
4167             n = self.s.create_node_from_uri(filecap)
4168             return n.download_best_version()
4169         d.addCallback(_get_data)
4170         d.addCallback(lambda results:
4171             self.failUnlessEqual(results, self.new_data))
4172         # Now try appending things to the file
4173         d.addCallback(lambda ignored:
4174             self.PUT("/uri/%s?offset=%d" % (self.filecap, len(self.new_data)),
4175                      "puppies" * 100))
4176         d.addCallback(_get_data)
4177         d.addCallback(lambda results:
4178             self.failUnlessEqual(results, self.new_data + ("puppies" * 100)))
4179         # and try replacing the beginning of the file
4180         d.addCallback(lambda ignored:
4181             self.PUT("/uri/%s?offset=0" % self.filecap, "begin"))
4182         d.addCallback(_get_data)
4183         d.addCallback(lambda results:
4184             self.failUnlessEqual(results, "begin"+self.new_data[len("begin"):]+("puppies"*100)))
4185         return d
4186
4187     def test_PUT_update_at_invalid_offset(self):
4188         file_contents = "test file" * 100000 # about 900 KiB
4189         d = self.PUT("/uri?mutable=true", file_contents)
4190         def _then(filecap):
4191             self.filecap = filecap
4192         d.addCallback(_then)
4193         # Negative offsets should cause an error.
4194         d.addCallback(lambda ignored:
4195             self.shouldHTTPError("PUT_update_at_invalid_offset",
4196                                  400, "Bad Request",
4197                                  "Invalid offset",
4198                                  self.PUT,
4199                                  "/uri/%s?offset=-1" % self.filecap,
4200                                  "foo"))
4201         return d
4202
4203     def test_PUT_update_at_offset_immutable(self):
4204         file_contents = "Test file" * 100000
4205         d = self.PUT("/uri", file_contents)
4206         def _then(filecap):
4207             self.filecap = filecap
4208         d.addCallback(_then)
4209         d.addCallback(lambda ignored:
4210             self.shouldHTTPError("PUT_update_at_offset_immutable",
4211                                  400, "Bad Request",
4212                                  "immutable",
4213                                  self.PUT,
4214                                  "/uri/%s?offset=50" % self.filecap,
4215                                  "foo"))
4216         return d
4217
4218
4219     def test_bad_method(self):
4220         url = self.webish_url + self.public_url + "/foo/bar.txt"
4221         d = self.shouldHTTPError("bad_method",
4222                                  501, "Not Implemented",
4223                                  "I don't know how to treat a BOGUS request.",
4224                                  client.getPage, url, method="BOGUS")
4225         return d
4226
4227     def test_short_url(self):
4228         url = self.webish_url + "/uri"
4229         d = self.shouldHTTPError("short_url", 501, "Not Implemented",
4230                                  "I don't know how to treat a DELETE request.",
4231                                  client.getPage, url, method="DELETE")
4232         return d
4233
4234     def test_ophandle_bad(self):
4235         url = self.webish_url + "/operations/bogus?t=status"
4236         d = self.shouldHTTPError("ophandle_bad", 404, "404 Not Found",
4237                                  "unknown/expired handle 'bogus'",
4238                                  client.getPage, url)
4239         return d
4240
4241     def test_ophandle_cancel(self):
4242         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
4243                       followRedirect=True)
4244         d.addCallback(lambda ignored:
4245                       self.GET("/operations/128?t=status&output=JSON"))
4246         def _check1(res):
4247             data = simplejson.loads(res)
4248             self.failUnless("finished" in data, res)
4249             monitor = self.ws.root.child_operations.handles["128"][0]
4250             d = self.POST("/operations/128?t=cancel&output=JSON")
4251             def _check2(res):
4252                 data = simplejson.loads(res)
4253                 self.failUnless("finished" in data, res)
4254                 # t=cancel causes the handle to be forgotten
4255                 self.failUnless(monitor.is_cancelled())
4256             d.addCallback(_check2)
4257             return d
4258         d.addCallback(_check1)
4259         d.addCallback(lambda ignored:
4260                       self.shouldHTTPError("ophandle_cancel",
4261                                            404, "404 Not Found",
4262                                            "unknown/expired handle '128'",
4263                                            self.GET,
4264                                            "/operations/128?t=status&output=JSON"))
4265         return d
4266
4267     def test_ophandle_retainfor(self):
4268         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
4269                       followRedirect=True)
4270         d.addCallback(lambda ignored:
4271                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
4272         def _check1(res):
4273             data = simplejson.loads(res)
4274             self.failUnless("finished" in data, res)
4275         d.addCallback(_check1)
4276         # the retain-for=0 will cause the handle to be expired very soon
4277         d.addCallback(lambda ign:
4278             self.clock.advance(2.0))
4279         d.addCallback(lambda ignored:
4280                       self.shouldHTTPError("ophandle_retainfor",
4281                                            404, "404 Not Found",
4282                                            "unknown/expired handle '129'",
4283                                            self.GET,
4284                                            "/operations/129?t=status&output=JSON"))
4285         return d
4286
4287     def test_ophandle_release_after_complete(self):
4288         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
4289                       followRedirect=True)
4290         d.addCallback(self.wait_for_operation, "130")
4291         d.addCallback(lambda ignored:
4292                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
4293         # the release-after-complete=true will cause the handle to be expired
4294         d.addCallback(lambda ignored:
4295                       self.shouldHTTPError("ophandle_release_after_complete",
4296                                            404, "404 Not Found",
4297                                            "unknown/expired handle '130'",
4298                                            self.GET,
4299                                            "/operations/130?t=status&output=JSON"))
4300         return d
4301
4302     def test_uncollected_ophandle_expiration(self):
4303         # uncollected ophandles should expire after 4 days
4304         def _make_uncollected_ophandle(ophandle):
4305             d = self.POST(self.public_url +
4306                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
4307                           followRedirect=False)
4308             # When we start the operation, the webapi server will want
4309             # to redirect us to the page for the ophandle, so we get
4310             # confirmation that the operation has started. If the
4311             # manifest operation has finished by the time we get there,
4312             # following that redirect (by setting followRedirect=True
4313             # above) has the side effect of collecting the ophandle that
4314             # we've just created, which means that we can't use the
4315             # ophandle to test the uncollected timeout anymore. So,
4316             # instead, catch the 302 here and don't follow it.
4317             d.addBoth(self.should302, "uncollected_ophandle_creation")
4318             return d
4319         # Create an ophandle, don't collect it, then advance the clock by
4320         # 4 days - 1 second and make sure that the ophandle is still there.
4321         d = _make_uncollected_ophandle(131)
4322         d.addCallback(lambda ign:
4323             self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
4324         d.addCallback(lambda ign:
4325             self.GET("/operations/131?t=status&output=JSON"))
4326         def _check1(res):
4327             data = simplejson.loads(res)
4328             self.failUnless("finished" in data, res)
4329         d.addCallback(_check1)
4330         # Create an ophandle, don't collect it, then try to collect it
4331         # after 4 days. It should be gone.
4332         d.addCallback(lambda ign:
4333             _make_uncollected_ophandle(132))
4334         d.addCallback(lambda ign:
4335             self.clock.advance(96*60*60))
4336         d.addCallback(lambda ign:
4337             self.shouldHTTPError("uncollected_ophandle_expired_after_100_hours",
4338                                  404, "404 Not Found",
4339                                  "unknown/expired handle '132'",
4340                                  self.GET,
4341                                  "/operations/132?t=status&output=JSON"))
4342         return d
4343
4344     def test_collected_ophandle_expiration(self):
4345         # collected ophandles should expire after 1 day
4346         def _make_collected_ophandle(ophandle):
4347             d = self.POST(self.public_url +
4348                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
4349                           followRedirect=True)
4350             # By following the initial redirect, we collect the ophandle
4351             # we've just created.
4352             return d
4353         # Create a collected ophandle, then collect it after 23 hours
4354         # and 59 seconds to make sure that it is still there.
4355         d = _make_collected_ophandle(133)
4356         d.addCallback(lambda ign:
4357             self.clock.advance((24*60*60) - 1))
4358         d.addCallback(lambda ign:
4359             self.GET("/operations/133?t=status&output=JSON"))
4360         def _check1(res):
4361             data = simplejson.loads(res)
4362             self.failUnless("finished" in data, res)
4363         d.addCallback(_check1)
4364         # Create another uncollected ophandle, then try to collect it
4365         # after 24 hours to make sure that it is gone.
4366         d.addCallback(lambda ign:
4367             _make_collected_ophandle(134))
4368         d.addCallback(lambda ign:
4369             self.clock.advance(24*60*60))
4370         d.addCallback(lambda ign:
4371             self.shouldHTTPError("collected_ophandle_expired_after_1_day",
4372                                  404, "404 Not Found",
4373                                  "unknown/expired handle '134'",
4374                                  self.GET,
4375                                  "/operations/134?t=status&output=JSON"))
4376         return d
4377
4378     def test_incident(self):
4379         d = self.POST("/report_incident", details="eek")
4380         def _done(res):
4381             self.failIfIn("<html>", res)
4382             self.failUnlessIn("An incident report has been saved", res)
4383         d.addCallback(_done)
4384         return d
4385
4386     def test_static(self):
4387         webdir = os.path.join(self.staticdir, "subdir")
4388         fileutil.make_dirs(webdir)
4389         f = open(os.path.join(webdir, "hello.txt"), "wb")
4390         f.write("hello")
4391         f.close()
4392
4393         d = self.GET("/static/subdir/hello.txt")
4394         def _check(res):
4395             self.failUnlessReallyEqual(res, "hello")
4396         d.addCallback(_check)
4397         return d
4398
4399
4400 class IntroducerWeb(unittest.TestCase):
4401     def setUp(self):
4402         self.node = None
4403
4404     def tearDown(self):
4405         d = defer.succeed(None)
4406         if self.node:
4407             d.addCallback(lambda ign: self.node.stopService())
4408         d.addCallback(flushEventualQueue)
4409         return d
4410
4411     def test_welcome(self):
4412         basedir = "web.IntroducerWeb.test_welcome"
4413         os.mkdir(basedir)
4414         fileutil.write(os.path.join(basedir, "tahoe.cfg"), "[node]\nweb.port = tcp:0\n")
4415         self.node = IntroducerNode(basedir)
4416         self.ws = self.node.getServiceNamed("webish")
4417
4418         d = fireEventually(None)
4419         d.addCallback(lambda ign: self.node.startService())
4420         d.addCallback(lambda ign: self.node.when_tub_ready())
4421
4422         d.addCallback(lambda ign: self.GET("/"))
4423         def _check(res):
4424             self.failUnlessIn('Welcome to the Tahoe-LAFS Introducer', res)
4425             self.failUnlessIn(FAVICON_MARKUP, res)
4426             self.failUnlessIn('Page rendered at', res)
4427             self.failUnlessIn('Tahoe-LAFS code imported from:', res)
4428         d.addCallback(_check)
4429         return d
4430
4431     def GET(self, urlpath, followRedirect=False, return_response=False,
4432             **kwargs):
4433         # if return_response=True, this fires with (data, statuscode,
4434         # respheaders) instead of just data.
4435         assert not isinstance(urlpath, unicode)
4436         url = self.ws.getURL().rstrip('/') + urlpath
4437         factory = HTTPClientGETFactory(url, method="GET",
4438                                        followRedirect=followRedirect, **kwargs)
4439         reactor.connectTCP("localhost", self.ws.getPortnum(), factory)
4440         d = factory.deferred
4441         def _got_data(data):
4442             return (data, factory.status, factory.response_headers)
4443         if return_response:
4444             d.addCallback(_got_data)
4445         return factory.deferred
4446
4447
4448 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4449     def test_load_file(self):
4450         # This will raise an exception unless a well-formed XML file is found under that name.
4451         common.getxmlfile('directory.xhtml').load()
4452
4453     def test_parse_replace_arg(self):
4454         self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
4455         self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
4456         self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
4457                                    "only-files")
4458         self.failUnlessRaises(common.WebError, common.parse_replace_arg, "only_fles")
4459
4460     def test_abbreviate_time(self):
4461         self.failUnlessReallyEqual(common.abbreviate_time(None), "")
4462         self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
4463         self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
4464         self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
4465         self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
4466         self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
4467
4468     def test_compute_rate(self):
4469         self.failUnlessReallyEqual(common.compute_rate(None, None), None)
4470         self.failUnlessReallyEqual(common.compute_rate(None, 1), None)
4471         self.failUnlessReallyEqual(common.compute_rate(250000, None), None)
4472         self.failUnlessReallyEqual(common.compute_rate(250000, 0), None)
4473         self.failUnlessReallyEqual(common.compute_rate(250000, 10), 25000.0)
4474         self.failUnlessReallyEqual(common.compute_rate(0, 10), 0.0)
4475         self.shouldFail(AssertionError, "test_compute_rate", "",
4476                         common.compute_rate, -100, 10)
4477         self.shouldFail(AssertionError, "test_compute_rate", "",
4478                         common.compute_rate, 100, -10)
4479
4480         # Sanity check
4481         rate = common.compute_rate(10*1000*1000, 1)
4482         self.failUnlessReallyEqual(common.abbreviate_rate(rate), "10.00MBps")
4483
4484     def test_abbreviate_rate(self):
4485         self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
4486         self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
4487         self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
4488         self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
4489
4490     def test_abbreviate_size(self):
4491         self.failUnlessReallyEqual(common.abbreviate_size(None), "")
4492         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
4493         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
4494         self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
4495         self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
4496
4497     def test_plural(self):
4498         def convert(s):
4499             return "%d second%s" % (s, status.plural(s))
4500         self.failUnlessReallyEqual(convert(0), "0 seconds")
4501         self.failUnlessReallyEqual(convert(1), "1 second")
4502         self.failUnlessReallyEqual(convert(2), "2 seconds")
4503         def convert2(s):
4504             return "has share%s: %s" % (status.plural(s), ",".join(s))
4505         self.failUnlessReallyEqual(convert2([]), "has shares: ")
4506         self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
4507         self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
4508
4509
4510 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4511
4512     def CHECK(self, ign, which, args, clientnum=0):
4513         fileurl = self.fileurls[which]
4514         url = fileurl + "?" + args
4515         return self.GET(url, method="POST", clientnum=clientnum)
4516
4517     def test_filecheck(self):
4518         self.basedir = "web/Grid/filecheck"
4519         self.set_up_grid()
4520         c0 = self.g.clients[0]
4521         self.uris = {}
4522         DATA = "data" * 100
4523         d = c0.upload(upload.Data(DATA, convergence=""))
4524         def _stash_uri(ur, which):
4525             self.uris[which] = ur.get_uri()
4526         d.addCallback(_stash_uri, "good")
4527         d.addCallback(lambda ign:
4528                       c0.upload(upload.Data(DATA+"1", convergence="")))
4529         d.addCallback(_stash_uri, "sick")
4530         d.addCallback(lambda ign:
4531                       c0.upload(upload.Data(DATA+"2", convergence="")))
4532         d.addCallback(_stash_uri, "dead")
4533         def _stash_mutable_uri(n, which):
4534             self.uris[which] = n.get_uri()
4535             assert isinstance(self.uris[which], str)
4536         d.addCallback(lambda ign:
4537             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4538         d.addCallback(_stash_mutable_uri, "corrupt")
4539         d.addCallback(lambda ign:
4540                       c0.upload(upload.Data("literal", convergence="")))
4541         d.addCallback(_stash_uri, "small")
4542         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
4543         d.addCallback(_stash_mutable_uri, "smalldir")
4544
4545         def _compute_fileurls(ignored):
4546             self.fileurls = {}
4547             for which in self.uris:
4548                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4549         d.addCallback(_compute_fileurls)
4550
4551         def _clobber_shares(ignored):
4552             good_shares = self.find_uri_shares(self.uris["good"])
4553             self.failUnlessReallyEqual(len(good_shares), 10)
4554             sick_shares = self.find_uri_shares(self.uris["sick"])
4555             os.unlink(sick_shares[0][2])
4556             dead_shares = self.find_uri_shares(self.uris["dead"])
4557             for i in range(1, 10):
4558                 os.unlink(dead_shares[i][2])
4559             c_shares = self.find_uri_shares(self.uris["corrupt"])
4560             cso = CorruptShareOptions()
4561             cso.stdout = StringIO()
4562             cso.parseOptions([c_shares[0][2]])
4563             corrupt_share(cso)
4564         d.addCallback(_clobber_shares)
4565
4566         d.addCallback(self.CHECK, "good", "t=check")
4567         def _got_html_good(res):
4568             self.failUnlessIn("Healthy", res)
4569             self.failIfIn("Not Healthy", res)
4570             self.failUnlessIn(FAVICON_MARKUP, res)
4571         d.addCallback(_got_html_good)
4572         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
4573         def _got_html_good_return_to(res):
4574             self.failUnlessIn("Healthy", res)
4575             self.failIfIn("Not Healthy", res)
4576             self.failUnlessIn('<a href="somewhere">Return to file', res)
4577         d.addCallback(_got_html_good_return_to)
4578         d.addCallback(self.CHECK, "good", "t=check&output=json")
4579         def _got_json_good(res):
4580             r = simplejson.loads(res)
4581             self.failUnlessEqual(r["summary"], "Healthy")
4582             self.failUnless(r["results"]["healthy"])
4583             self.failIfIn("needs-rebalancing", r["results"])
4584             self.failUnless(r["results"]["recoverable"])
4585         d.addCallback(_got_json_good)
4586
4587         d.addCallback(self.CHECK, "small", "t=check")
4588         def _got_html_small(res):
4589             self.failUnlessIn("Literal files are always healthy", res)
4590             self.failIfIn("Not Healthy", res)
4591         d.addCallback(_got_html_small)
4592         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
4593         def _got_html_small_return_to(res):
4594             self.failUnlessIn("Literal files are always healthy", res)
4595             self.failIfIn("Not Healthy", res)
4596             self.failUnlessIn('<a href="somewhere">Return to file', res)
4597         d.addCallback(_got_html_small_return_to)
4598         d.addCallback(self.CHECK, "small", "t=check&output=json")
4599         def _got_json_small(res):
4600             r = simplejson.loads(res)
4601             self.failUnlessEqual(r["storage-index"], "")
4602             self.failUnless(r["results"]["healthy"])
4603         d.addCallback(_got_json_small)
4604
4605         d.addCallback(self.CHECK, "smalldir", "t=check")
4606         def _got_html_smalldir(res):
4607             self.failUnlessIn("Literal files are always healthy", res)
4608             self.failIfIn("Not Healthy", res)
4609         d.addCallback(_got_html_smalldir)
4610         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
4611         def _got_json_smalldir(res):
4612             r = simplejson.loads(res)
4613             self.failUnlessEqual(r["storage-index"], "")
4614             self.failUnless(r["results"]["healthy"])
4615         d.addCallback(_got_json_smalldir)
4616
4617         d.addCallback(self.CHECK, "sick", "t=check")
4618         def _got_html_sick(res):
4619             self.failUnlessIn("Not Healthy", res)
4620         d.addCallback(_got_html_sick)
4621         d.addCallback(self.CHECK, "sick", "t=check&output=json")
4622         def _got_json_sick(res):
4623             r = simplejson.loads(res)
4624             self.failUnlessEqual(r["summary"],
4625                                  "Not Healthy: 9 shares (enc 3-of-10)")
4626             self.failIf(r["results"]["healthy"])
4627             self.failUnless(r["results"]["recoverable"])
4628             self.failIfIn("needs-rebalancing", r["results"])
4629         d.addCallback(_got_json_sick)
4630
4631         d.addCallback(self.CHECK, "dead", "t=check")
4632         def _got_html_dead(res):
4633             self.failUnlessIn("Not Healthy", res)
4634         d.addCallback(_got_html_dead)
4635         d.addCallback(self.CHECK, "dead", "t=check&output=json")
4636         def _got_json_dead(res):
4637             r = simplejson.loads(res)
4638             self.failUnlessEqual(r["summary"],
4639                                  "Not Healthy: 1 shares (enc 3-of-10)")
4640             self.failIf(r["results"]["healthy"])
4641             self.failIf(r["results"]["recoverable"])
4642             self.failIfIn("needs-rebalancing", r["results"])
4643         d.addCallback(_got_json_dead)
4644
4645         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
4646         def _got_html_corrupt(res):
4647             self.failUnlessIn("Not Healthy! : Unhealthy", res)
4648         d.addCallback(_got_html_corrupt)
4649         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
4650         def _got_json_corrupt(res):
4651             r = simplejson.loads(res)
4652             self.failUnlessIn("Unhealthy: 9 shares (enc 3-of-10)", r["summary"])
4653             self.failIf(r["results"]["healthy"])
4654             self.failUnless(r["results"]["recoverable"])
4655             self.failIfIn("needs-rebalancing", r["results"])
4656             self.failUnlessReallyEqual(r["results"]["count-happiness"], 9)
4657             self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
4658             self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
4659         d.addCallback(_got_json_corrupt)
4660
4661         d.addErrback(self.explain_web_error)
4662         return d
4663
4664     def test_repair_html(self):
4665         self.basedir = "web/Grid/repair_html"
4666         self.set_up_grid()
4667         c0 = self.g.clients[0]
4668         self.uris = {}
4669         DATA = "data" * 100
4670         d = c0.upload(upload.Data(DATA, convergence=""))
4671         def _stash_uri(ur, which):
4672             self.uris[which] = ur.get_uri()
4673         d.addCallback(_stash_uri, "good")
4674         d.addCallback(lambda ign:
4675                       c0.upload(upload.Data(DATA+"1", convergence="")))
4676         d.addCallback(_stash_uri, "sick")
4677         d.addCallback(lambda ign:
4678                       c0.upload(upload.Data(DATA+"2", convergence="")))
4679         d.addCallback(_stash_uri, "dead")
4680         def _stash_mutable_uri(n, which):
4681             self.uris[which] = n.get_uri()
4682             assert isinstance(self.uris[which], str)
4683         d.addCallback(lambda ign:
4684             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4685         d.addCallback(_stash_mutable_uri, "corrupt")
4686
4687         def _compute_fileurls(ignored):
4688             self.fileurls = {}
4689             for which in self.uris:
4690                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4691         d.addCallback(_compute_fileurls)
4692
4693         def _clobber_shares(ignored):
4694             good_shares = self.find_uri_shares(self.uris["good"])
4695             self.failUnlessReallyEqual(len(good_shares), 10)
4696             sick_shares = self.find_uri_shares(self.uris["sick"])
4697             os.unlink(sick_shares[0][2])
4698             dead_shares = self.find_uri_shares(self.uris["dead"])
4699             for i in range(1, 10):
4700                 os.unlink(dead_shares[i][2])
4701             c_shares = self.find_uri_shares(self.uris["corrupt"])
4702             cso = CorruptShareOptions()
4703             cso.stdout = StringIO()
4704             cso.parseOptions([c_shares[0][2]])
4705             corrupt_share(cso)
4706         d.addCallback(_clobber_shares)
4707
4708         d.addCallback(self.CHECK, "good", "t=check&repair=true")
4709         def _got_html_good(res):
4710             self.failUnlessIn("Healthy", res)
4711             self.failIfIn("Not Healthy", res)
4712             self.failUnlessIn("No repair necessary", res)
4713             self.failUnlessIn(FAVICON_MARKUP, res)
4714         d.addCallback(_got_html_good)
4715
4716         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
4717         def _got_html_sick(res):
4718             self.failUnlessIn("Healthy : healthy", res)
4719             self.failIfIn("Not Healthy", res)
4720             self.failUnlessIn("Repair successful", res)
4721         d.addCallback(_got_html_sick)
4722
4723         # repair of a dead file will fail, of course, but it isn't yet
4724         # clear how this should be reported. Right now it shows up as
4725         # a "410 Gone".
4726         #
4727         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
4728         #def _got_html_dead(res):
4729         #    print res
4730         #    self.failUnlessIn("Healthy : healthy", res)
4731         #    self.failIfIn("Not Healthy", res)
4732         #    self.failUnlessIn("No repair necessary", res)
4733         #d.addCallback(_got_html_dead)
4734
4735         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
4736         def _got_html_corrupt(res):
4737             self.failUnlessIn("Healthy : Healthy", res)
4738             self.failIfIn("Not Healthy", res)
4739             self.failUnlessIn("Repair successful", res)
4740         d.addCallback(_got_html_corrupt)
4741
4742         d.addErrback(self.explain_web_error)
4743         return d
4744
4745     def test_repair_json(self):
4746         self.basedir = "web/Grid/repair_json"
4747         self.set_up_grid()
4748         c0 = self.g.clients[0]
4749         self.uris = {}
4750         DATA = "data" * 100
4751         d = c0.upload(upload.Data(DATA+"1", convergence=""))
4752         def _stash_uri(ur, which):
4753             self.uris[which] = ur.get_uri()
4754         d.addCallback(_stash_uri, "sick")
4755
4756         def _compute_fileurls(ignored):
4757             self.fileurls = {}
4758             for which in self.uris:
4759                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4760         d.addCallback(_compute_fileurls)
4761
4762         def _clobber_shares(ignored):
4763             sick_shares = self.find_uri_shares(self.uris["sick"])
4764             os.unlink(sick_shares[0][2])
4765         d.addCallback(_clobber_shares)
4766
4767         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
4768         def _got_json_sick(res):
4769             r = simplejson.loads(res)
4770             self.failUnlessReallyEqual(r["repair-attempted"], True)
4771             self.failUnlessReallyEqual(r["repair-successful"], True)
4772             self.failUnlessEqual(r["pre-repair-results"]["summary"],
4773                                  "Not Healthy: 9 shares (enc 3-of-10)")
4774             self.failIf(r["pre-repair-results"]["results"]["healthy"])
4775             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
4776             self.failUnless(r["post-repair-results"]["results"]["healthy"])
4777         d.addCallback(_got_json_sick)
4778
4779         d.addErrback(self.explain_web_error)
4780         return d
4781
4782     def test_unknown(self, immutable=False):
4783         self.basedir = "web/Grid/unknown"
4784         if immutable:
4785             self.basedir = "web/Grid/unknown-immutable"
4786
4787         self.set_up_grid()
4788         c0 = self.g.clients[0]
4789         self.uris = {}
4790         self.fileurls = {}
4791
4792         # the future cap format may contain slashes, which must be tolerated
4793         expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
4794                                                            safe="")
4795
4796         if immutable:
4797             name = u"future-imm"
4798             future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
4799             d = c0.create_immutable_dirnode({name: (future_node, {})})
4800         else:
4801             name = u"future"
4802             future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4803             d = c0.create_dirnode()
4804
4805         def _stash_root_and_create_file(n):
4806             self.rootnode = n
4807             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
4808             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
4809             if not immutable:
4810                 return self.rootnode.set_node(name, future_node)
4811         d.addCallback(_stash_root_and_create_file)
4812
4813         # make sure directory listing tolerates unknown nodes
4814         d.addCallback(lambda ign: self.GET(self.rooturl))
4815         def _check_directory_html(res, expected_type_suffix):
4816             pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
4817                                   '<td>%s</td>' % (expected_type_suffix, str(name)),
4818                                  re.DOTALL)
4819             self.failUnless(re.search(pattern, res), res)
4820             # find the More Info link for name, should be relative
4821             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4822             info_url = mo.group(1)
4823             self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
4824         if immutable:
4825             d.addCallback(_check_directory_html, "-IMM")
4826         else:
4827             d.addCallback(_check_directory_html, "")
4828
4829         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4830         def _check_directory_json(res, expect_rw_uri):
4831             data = simplejson.loads(res)
4832             self.failUnlessEqual(data[0], "dirnode")
4833             f = data[1]["children"][name]
4834             self.failUnlessEqual(f[0], "unknown")
4835             if expect_rw_uri:
4836                 self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data)
4837             else:
4838                 self.failIfIn("rw_uri", f[1])
4839             if immutable:
4840                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
4841             else:
4842                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
4843             self.failUnlessIn("metadata", f[1])
4844         d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
4845
4846         def _check_info(res, expect_rw_uri, expect_ro_uri):
4847             self.failUnlessIn("Object Type: <span>unknown</span>", res)
4848             if expect_rw_uri:
4849                 self.failUnlessIn(unknown_rwcap, res)
4850             if expect_ro_uri:
4851                 if immutable:
4852                     self.failUnlessIn(unknown_immcap, res)
4853                 else:
4854                     self.failUnlessIn(unknown_rocap, res)
4855             else:
4856                 self.failIfIn(unknown_rocap, res)
4857             self.failIfIn("Raw data as", res)
4858             self.failIfIn("Directory writecap", res)
4859             self.failIfIn("Checker Operations", res)
4860             self.failIfIn("Mutable File Operations", res)
4861             self.failIfIn("Directory Operations", res)
4862
4863         # FIXME: these should have expect_rw_uri=not immutable; I don't know
4864         # why they fail. Possibly related to ticket #922.
4865
4866         d.addCallback(lambda ign: self.GET(expected_info_url))
4867         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
4868         d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
4869         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
4870
4871         def _check_json(res, expect_rw_uri):
4872             data = simplejson.loads(res)
4873             self.failUnlessEqual(data[0], "unknown")
4874             if expect_rw_uri:
4875                 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data)
4876             else:
4877                 self.failIfIn("rw_uri", data[1])
4878
4879             if immutable:
4880                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data)
4881                 self.failUnlessReallyEqual(data[1]["mutable"], False)
4882             elif expect_rw_uri:
4883                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4884                 self.failUnlessReallyEqual(data[1]["mutable"], True)
4885             else:
4886                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4887                 self.failIfIn("mutable", data[1])
4888
4889             # TODO: check metadata contents
4890             self.failUnlessIn("metadata", data[1])
4891
4892         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
4893         d.addCallback(_check_json, expect_rw_uri=not immutable)
4894
4895         # and make sure that a read-only version of the directory can be
4896         # rendered too. This version will not have unknown_rwcap, whether
4897         # or not future_node was immutable.
4898         d.addCallback(lambda ign: self.GET(self.rourl))
4899         if immutable:
4900             d.addCallback(_check_directory_html, "-IMM")
4901         else:
4902             d.addCallback(_check_directory_html, "-RO")
4903
4904         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
4905         d.addCallback(_check_directory_json, expect_rw_uri=False)
4906
4907         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
4908         d.addCallback(_check_json, expect_rw_uri=False)
4909
4910         # TODO: check that getting t=info from the Info link in the ro directory
4911         # works, and does not include the writecap URI.
4912         return d
4913
4914     def test_immutable_unknown(self):
4915         return self.test_unknown(immutable=True)
4916
4917     def test_mutant_dirnodes_are_omitted(self):
4918         self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
4919
4920         self.set_up_grid()
4921         c = self.g.clients[0]
4922         nm = c.nodemaker
4923         self.uris = {}
4924         self.fileurls = {}
4925
4926         lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
4927         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
4928         mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
4929
4930         # This method tests mainly dirnode, but we'd have to duplicate code in order to
4931         # test the dirnode and web layers separately.
4932
4933         # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
4934         # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
4935         # When the directory is read, the mutants should be silently disposed of, leaving
4936         # their lonely sibling.
4937         # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
4938         # because immutable directories don't have a writecap and therefore that field
4939         # isn't (and can't be) decrypted.
4940         # TODO: The field still exists in the netstring. Technically we should check what
4941         # happens if something is put there (_unpack_contents should raise ValueError),
4942         # but that can wait.
4943
4944         lonely_child = nm.create_from_cap(lonely_uri)
4945         mutant_ro_child = nm.create_from_cap(mut_read_uri)
4946         mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
4947
4948         def _by_hook_or_by_crook():
4949             return True
4950         for n in [mutant_ro_child, mutant_write_in_ro_child]:
4951             n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
4952
4953         mutant_write_in_ro_child.get_write_uri    = lambda: None
4954         mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
4955
4956         kids = {u"lonely":      (lonely_child, {}),
4957                 u"ro":          (mutant_ro_child, {}),
4958                 u"write-in-ro": (mutant_write_in_ro_child, {}),
4959                 }
4960         d = c.create_immutable_dirnode(kids)
4961
4962         def _created(dn):
4963             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
4964             self.failIf(dn.is_mutable())
4965             self.failUnless(dn.is_readonly())
4966             # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
4967             self.failIf(hasattr(dn._node, 'get_writekey'))
4968             rep = str(dn)
4969             self.failUnlessIn("RO-IMM", rep)
4970             cap = dn.get_cap()
4971             self.failUnlessIn("CHK", cap.to_string())
4972             self.cap = cap
4973             self.rootnode = dn
4974             self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
4975             return download_to_data(dn._node)
4976         d.addCallback(_created)
4977
4978         def _check_data(data):
4979             # Decode the netstring representation of the directory to check that all children
4980             # are present. This is a bit of an abstraction violation, but there's not really
4981             # any other way to do it given that the real DirectoryNode._unpack_contents would
4982             # strip the mutant children out (which is what we're trying to test, later).
4983             position = 0
4984             numkids = 0
4985             while position < len(data):
4986                 entries, position = split_netstring(data, 1, position)
4987                 entry = entries[0]
4988                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
4989                 name = name_utf8.decode("utf-8")
4990                 self.failUnlessEqual(rwcapdata, "")
4991                 self.failUnlessIn(name, kids)
4992                 (expected_child, ign) = kids[name]
4993                 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
4994                 numkids += 1
4995
4996             self.failUnlessReallyEqual(numkids, 3)
4997             return self.rootnode.list()
4998         d.addCallback(_check_data)
4999
5000         # Now when we use the real directory listing code, the mutants should be absent.
5001         def _check_kids(children):
5002             self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
5003             lonely_node, lonely_metadata = children[u"lonely"]
5004
5005             self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
5006             self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
5007         d.addCallback(_check_kids)
5008
5009         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
5010         d.addCallback(lambda n: n.list())
5011         d.addCallback(_check_kids)  # again with dirnode recreated from cap
5012
5013         # Make sure the lonely child can be listed in HTML...
5014         d.addCallback(lambda ign: self.GET(self.rooturl))
5015         def _check_html(res):
5016             self.failIfIn("URI:SSK", res)
5017             get_lonely = "".join([r'<td>FILE</td>',
5018                                   r'\s+<td>',
5019                                   r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
5020                                   r'</td>',
5021                                   r'\s+<td align="right">%d</td>' % len("one"),
5022                                   ])
5023             self.failUnless(re.search(get_lonely, res), res)
5024
5025             # find the More Info link for name, should be relative
5026             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
5027             info_url = mo.group(1)
5028             self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
5029         d.addCallback(_check_html)
5030
5031         # ... and in JSON.
5032         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
5033         def _check_json(res):
5034             data = simplejson.loads(res)
5035             self.failUnlessEqual(data[0], "dirnode")
5036             listed_children = data[1]["children"]
5037             self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
5038             ll_type, ll_data = listed_children[u"lonely"]
5039             self.failUnlessEqual(ll_type, "filenode")
5040             self.failIfIn("rw_uri", ll_data)
5041             self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
5042         d.addCallback(_check_json)
5043         return d
5044
5045     def test_deep_check(self):
5046         self.basedir = "web/Grid/deep_check"
5047         self.set_up_grid()
5048         c0 = self.g.clients[0]
5049         self.uris = {}
5050         self.fileurls = {}
5051         DATA = "data" * 100
5052         d = c0.create_dirnode()
5053         def _stash_root_and_create_file(n):
5054             self.rootnode = n
5055             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5056             return n.add_file(u"good", upload.Data(DATA, convergence=""))
5057         d.addCallback(_stash_root_and_create_file)
5058         def _stash_uri(fn, which):
5059             self.uris[which] = fn.get_uri()
5060             return fn
5061         d.addCallback(_stash_uri, "good")
5062         d.addCallback(lambda ign:
5063                       self.rootnode.add_file(u"small",
5064                                              upload.Data("literal",
5065                                                         convergence="")))
5066         d.addCallback(_stash_uri, "small")
5067         d.addCallback(lambda ign:
5068                       self.rootnode.add_file(u"sick",
5069                                              upload.Data(DATA+"1",
5070                                                         convergence="")))
5071         d.addCallback(_stash_uri, "sick")
5072
5073         # this tests that deep-check and stream-manifest will ignore
5074         # UnknownNode instances. Hopefully this will also cover deep-stats.
5075         future_node = UnknownNode(unknown_rwcap, unknown_rocap)
5076         d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
5077
5078         def _clobber_shares(ignored):
5079             self.delete_shares_numbered(self.uris["sick"], [0,1])
5080         d.addCallback(_clobber_shares)
5081
5082         # root
5083         # root/good
5084         # root/small
5085         # root/sick
5086         # root/future
5087
5088         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
5089         def _done(res):
5090             try:
5091                 units = [simplejson.loads(line)
5092                          for line in res.splitlines()
5093                          if line]
5094             except ValueError:
5095                 print "response is:", res
5096                 print "undecodeable line was '%s'" % line
5097                 raise
5098             self.failUnlessReallyEqual(len(units), 5+1)
5099             # should be parent-first
5100             u0 = units[0]
5101             self.failUnlessEqual(u0["path"], [])
5102             self.failUnlessEqual(u0["type"], "directory")
5103             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
5104             u0cr = u0["check-results"]
5105             self.failUnlessReallyEqual(u0cr["results"]["count-happiness"], 10)
5106             self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
5107
5108             ugood = [u for u in units
5109                      if u["type"] == "file" and u["path"] == [u"good"]][0]
5110             self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
5111             ugoodcr = ugood["check-results"]
5112             self.failUnlessReallyEqual(ugoodcr["results"]["count-happiness"], 10)
5113             self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
5114
5115             stats = units[-1]
5116             self.failUnlessEqual(stats["type"], "stats")
5117             s = stats["stats"]
5118             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
5119             self.failUnlessReallyEqual(s["count-literal-files"], 1)
5120             self.failUnlessReallyEqual(s["count-directories"], 1)
5121             self.failUnlessReallyEqual(s["count-unknown"], 1)
5122         d.addCallback(_done)
5123
5124         d.addCallback(self.CHECK, "root", "t=stream-manifest")
5125         def _check_manifest(res):
5126             self.failUnless(res.endswith("\n"))
5127             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
5128             self.failUnlessReallyEqual(len(units), 5+1)
5129             self.failUnlessEqual(units[-1]["type"], "stats")
5130             first = units[0]
5131             self.failUnlessEqual(first["path"], [])
5132             self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri())
5133             self.failUnlessEqual(first["type"], "directory")
5134             stats = units[-1]["stats"]
5135             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
5136             self.failUnlessReallyEqual(stats["count-literal-files"], 1)
5137             self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
5138             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
5139             self.failUnlessReallyEqual(stats["count-unknown"], 1)
5140         d.addCallback(_check_manifest)
5141
5142         # now add root/subdir and root/subdir/grandchild, then make subdir
5143         # unrecoverable, then see what happens
5144
5145         d.addCallback(lambda ign:
5146                       self.rootnode.create_subdirectory(u"subdir"))
5147         d.addCallback(_stash_uri, "subdir")
5148         d.addCallback(lambda subdir_node:
5149                       subdir_node.add_file(u"grandchild",
5150                                            upload.Data(DATA+"2",
5151                                                        convergence="")))
5152         d.addCallback(_stash_uri, "grandchild")
5153
5154         d.addCallback(lambda ign:
5155                       self.delete_shares_numbered(self.uris["subdir"],
5156                                                   range(1, 10)))
5157
5158         # root
5159         # root/good
5160         # root/small
5161         # root/sick
5162         # root/future
5163         # root/subdir [unrecoverable]
5164         # root/subdir/grandchild
5165
5166         # how should a streaming-JSON API indicate fatal error?
5167         # answer: emit ERROR: instead of a JSON string
5168
5169         d.addCallback(self.CHECK, "root", "t=stream-manifest")
5170         def _check_broken_manifest(res):
5171             lines = res.splitlines()
5172             error_lines = [i
5173                            for (i,line) in enumerate(lines)
5174                            if line.startswith("ERROR:")]
5175             if not error_lines:
5176                 self.fail("no ERROR: in output: %s" % (res,))
5177             first_error = error_lines[0]
5178             error_line = lines[first_error]
5179             error_msg = lines[first_error+1:]
5180             error_msg_s = "\n".join(error_msg) + "\n"
5181             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
5182                               error_line)
5183             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
5184             units = [simplejson.loads(line) for line in lines[:first_error]]
5185             self.failUnlessReallyEqual(len(units), 6) # includes subdir
5186             last_unit = units[-1]
5187             self.failUnlessEqual(last_unit["path"], ["subdir"])
5188         d.addCallback(_check_broken_manifest)
5189
5190         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
5191         def _check_broken_deepcheck(res):
5192             lines = res.splitlines()
5193             error_lines = [i
5194                            for (i,line) in enumerate(lines)
5195                            if line.startswith("ERROR:")]
5196             if not error_lines:
5197                 self.fail("no ERROR: in output: %s" % (res,))
5198             first_error = error_lines[0]
5199             error_line = lines[first_error]
5200             error_msg = lines[first_error+1:]
5201             error_msg_s = "\n".join(error_msg) + "\n"
5202             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
5203                               error_line)
5204             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
5205             units = [simplejson.loads(line) for line in lines[:first_error]]
5206             self.failUnlessReallyEqual(len(units), 6) # includes subdir
5207             last_unit = units[-1]
5208             self.failUnlessEqual(last_unit["path"], ["subdir"])
5209             r = last_unit["check-results"]["results"]
5210             self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
5211             self.failUnlessReallyEqual(r["count-happiness"], 1)
5212             self.failUnlessReallyEqual(r["count-shares-good"], 1)
5213             self.failUnlessReallyEqual(r["recoverable"], False)
5214         d.addCallback(_check_broken_deepcheck)
5215
5216         d.addErrback(self.explain_web_error)
5217         return d
5218
5219     def test_deep_check_and_repair(self):
5220         self.basedir = "web/Grid/deep_check_and_repair"
5221         self.set_up_grid()
5222         c0 = self.g.clients[0]
5223         self.uris = {}
5224         self.fileurls = {}
5225         DATA = "data" * 100
5226         d = c0.create_dirnode()
5227         def _stash_root_and_create_file(n):
5228             self.rootnode = n
5229             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5230             return n.add_file(u"good", upload.Data(DATA, convergence=""))
5231         d.addCallback(_stash_root_and_create_file)
5232         def _stash_uri(fn, which):
5233             self.uris[which] = fn.get_uri()
5234         d.addCallback(_stash_uri, "good")
5235         d.addCallback(lambda ign:
5236                       self.rootnode.add_file(u"small",
5237                                              upload.Data("literal",
5238                                                         convergence="")))
5239         d.addCallback(_stash_uri, "small")
5240         d.addCallback(lambda ign:
5241                       self.rootnode.add_file(u"sick",
5242                                              upload.Data(DATA+"1",
5243                                                         convergence="")))
5244         d.addCallback(_stash_uri, "sick")
5245         #d.addCallback(lambda ign:
5246         #              self.rootnode.add_file(u"dead",
5247         #                                     upload.Data(DATA+"2",
5248         #                                                convergence="")))
5249         #d.addCallback(_stash_uri, "dead")
5250
5251         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
5252         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
5253         #d.addCallback(_stash_uri, "corrupt")
5254
5255         def _clobber_shares(ignored):
5256             good_shares = self.find_uri_shares(self.uris["good"])
5257             self.failUnlessReallyEqual(len(good_shares), 10)
5258             sick_shares = self.find_uri_shares(self.uris["sick"])
5259             os.unlink(sick_shares[0][2])
5260             #dead_shares = self.find_uri_shares(self.uris["dead"])
5261             #for i in range(1, 10):
5262             #    os.unlink(dead_shares[i][2])
5263
5264             #c_shares = self.find_uri_shares(self.uris["corrupt"])
5265             #cso = CorruptShareOptions()
5266             #cso.stdout = StringIO()
5267             #cso.parseOptions([c_shares[0][2]])
5268             #corrupt_share(cso)
5269         d.addCallback(_clobber_shares)
5270
5271         # root
5272         # root/good   CHK, 10 shares
5273         # root/small  LIT
5274         # root/sick   CHK, 9 shares
5275
5276         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
5277         def _done(res):
5278             units = [simplejson.loads(line)
5279                      for line in res.splitlines()
5280                      if line]
5281             self.failUnlessReallyEqual(len(units), 4+1)
5282             # should be parent-first
5283             u0 = units[0]
5284             self.failUnlessEqual(u0["path"], [])
5285             self.failUnlessEqual(u0["type"], "directory")
5286             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
5287             u0crr = u0["check-and-repair-results"]
5288             self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
5289             self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-happiness"], 10)
5290             self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
5291
5292             ugood = [u for u in units
5293                      if u["type"] == "file" and u["path"] == [u"good"]][0]
5294             self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
5295             ugoodcrr = ugood["check-and-repair-results"]
5296             self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
5297             self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-happiness"], 10)
5298             self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
5299
5300             usick = [u for u in units
5301                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
5302             self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"])
5303             usickcrr = usick["check-and-repair-results"]
5304             self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
5305             self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
5306             self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-happiness"], 9)
5307             self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
5308             self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-happiness"], 10)
5309             self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
5310
5311             stats = units[-1]
5312             self.failUnlessEqual(stats["type"], "stats")
5313             s = stats["stats"]
5314             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
5315             self.failUnlessReallyEqual(s["count-literal-files"], 1)
5316             self.failUnlessReallyEqual(s["count-directories"], 1)
5317         d.addCallback(_done)
5318
5319         d.addErrback(self.explain_web_error)
5320         return d
5321
5322     def _count_leases(self, ignored, which):
5323         u = self.uris[which]
5324         shares = self.find_uri_shares(u)
5325         lease_counts = []
5326         for shnum, serverid, fn in shares:
5327             sf = get_share_file(fn)
5328             num_leases = len(list(sf.get_leases()))
5329             lease_counts.append( (fn, num_leases) )
5330         return lease_counts
5331
5332     def _assert_leasecount(self, lease_counts, expected):
5333         for (fn, num_leases) in lease_counts:
5334             if num_leases != expected:
5335                 self.fail("expected %d leases, have %d, on %s" %
5336                           (expected, num_leases, fn))
5337
5338     def test_add_lease(self):
5339         self.basedir = "web/Grid/add_lease"
5340         self.set_up_grid(num_clients=2)
5341         c0 = self.g.clients[0]
5342         self.uris = {}
5343         DATA = "data" * 100
5344         d = c0.upload(upload.Data(DATA, convergence=""))
5345         def _stash_uri(ur, which):
5346             self.uris[which] = ur.get_uri()
5347         d.addCallback(_stash_uri, "one")
5348         d.addCallback(lambda ign:
5349                       c0.upload(upload.Data(DATA+"1", convergence="")))
5350         d.addCallback(_stash_uri, "two")
5351         def _stash_mutable_uri(n, which):
5352             self.uris[which] = n.get_uri()
5353             assert isinstance(self.uris[which], str)
5354         d.addCallback(lambda ign:
5355             c0.create_mutable_file(publish.MutableData(DATA+"2")))
5356         d.addCallback(_stash_mutable_uri, "mutable")
5357
5358         def _compute_fileurls(ignored):
5359             self.fileurls = {}
5360             for which in self.uris:
5361                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
5362         d.addCallback(_compute_fileurls)
5363
5364         d.addCallback(self._count_leases, "one")
5365         d.addCallback(self._assert_leasecount, 1)
5366         d.addCallback(self._count_leases, "two")
5367         d.addCallback(self._assert_leasecount, 1)
5368         d.addCallback(self._count_leases, "mutable")
5369         d.addCallback(self._assert_leasecount, 1)
5370
5371         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
5372         def _got_html_good(res):
5373             self.failUnlessIn("Healthy", res)
5374             self.failIfIn("Not Healthy", res)
5375         d.addCallback(_got_html_good)
5376
5377         d.addCallback(self._count_leases, "one")
5378         d.addCallback(self._assert_leasecount, 1)
5379         d.addCallback(self._count_leases, "two")
5380         d.addCallback(self._assert_leasecount, 1)
5381         d.addCallback(self._count_leases, "mutable")
5382         d.addCallback(self._assert_leasecount, 1)
5383
5384         # this CHECK uses the original client, which uses the same
5385         # lease-secrets, so it will just renew the original lease
5386         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
5387         d.addCallback(_got_html_good)
5388
5389         d.addCallback(self._count_leases, "one")
5390         d.addCallback(self._assert_leasecount, 1)
5391         d.addCallback(self._count_leases, "two")
5392         d.addCallback(self._assert_leasecount, 1)
5393         d.addCallback(self._count_leases, "mutable")
5394         d.addCallback(self._assert_leasecount, 1)
5395
5396         # this CHECK uses an alternate client, which adds a second lease
5397         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
5398         d.addCallback(_got_html_good)
5399
5400         d.addCallback(self._count_leases, "one")
5401         d.addCallback(self._assert_leasecount, 2)
5402         d.addCallback(self._count_leases, "two")
5403         d.addCallback(self._assert_leasecount, 1)
5404         d.addCallback(self._count_leases, "mutable")
5405         d.addCallback(self._assert_leasecount, 1)
5406
5407         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
5408         d.addCallback(_got_html_good)
5409
5410         d.addCallback(self._count_leases, "one")
5411         d.addCallback(self._assert_leasecount, 2)
5412         d.addCallback(self._count_leases, "two")
5413         d.addCallback(self._assert_leasecount, 1)
5414         d.addCallback(self._count_leases, "mutable")
5415         d.addCallback(self._assert_leasecount, 1)
5416
5417         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
5418                       clientnum=1)
5419         d.addCallback(_got_html_good)
5420
5421         d.addCallback(self._count_leases, "one")
5422         d.addCallback(self._assert_leasecount, 2)
5423         d.addCallback(self._count_leases, "two")
5424         d.addCallback(self._assert_leasecount, 1)
5425         d.addCallback(self._count_leases, "mutable")
5426         d.addCallback(self._assert_leasecount, 2)
5427
5428         d.addErrback(self.explain_web_error)
5429         return d
5430
5431     def test_deep_add_lease(self):
5432         self.basedir = "web/Grid/deep_add_lease"
5433         self.set_up_grid(num_clients=2)
5434         c0 = self.g.clients[0]
5435         self.uris = {}
5436         self.fileurls = {}
5437         DATA = "data" * 100
5438         d = c0.create_dirnode()
5439         def _stash_root_and_create_file(n):
5440             self.rootnode = n
5441             self.uris["root"] = n.get_uri()
5442             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5443             return n.add_file(u"one", upload.Data(DATA, convergence=""))
5444         d.addCallback(_stash_root_and_create_file)
5445         def _stash_uri(fn, which):
5446             self.uris[which] = fn.get_uri()
5447         d.addCallback(_stash_uri, "one")
5448         d.addCallback(lambda ign:
5449                       self.rootnode.add_file(u"small",
5450                                              upload.Data("literal",
5451                                                         convergence="")))
5452         d.addCallback(_stash_uri, "small")
5453
5454         d.addCallback(lambda ign:
5455             c0.create_mutable_file(publish.MutableData("mutable")))
5456         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
5457         d.addCallback(_stash_uri, "mutable")
5458
5459         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
5460         def _done(res):
5461             units = [simplejson.loads(line)
5462                      for line in res.splitlines()
5463                      if line]
5464             # root, one, small, mutable,   stats
5465             self.failUnlessReallyEqual(len(units), 4+1)
5466         d.addCallback(_done)
5467
5468         d.addCallback(self._count_leases, "root")
5469         d.addCallback(self._assert_leasecount, 1)
5470         d.addCallback(self._count_leases, "one")
5471         d.addCallback(self._assert_leasecount, 1)
5472         d.addCallback(self._count_leases, "mutable")
5473         d.addCallback(self._assert_leasecount, 1)
5474
5475         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
5476         d.addCallback(_done)
5477
5478         d.addCallback(self._count_leases, "root")
5479         d.addCallback(self._assert_leasecount, 1)
5480         d.addCallback(self._count_leases, "one")
5481         d.addCallback(self._assert_leasecount, 1)
5482         d.addCallback(self._count_leases, "mutable")
5483         d.addCallback(self._assert_leasecount, 1)
5484
5485         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
5486                       clientnum=1)
5487         d.addCallback(_done)
5488
5489         d.addCallback(self._count_leases, "root")
5490         d.addCallback(self._assert_leasecount, 2)
5491         d.addCallback(self._count_leases, "one")
5492         d.addCallback(self._assert_leasecount, 2)
5493         d.addCallback(self._count_leases, "mutable")
5494         d.addCallback(self._assert_leasecount, 2)
5495
5496         d.addErrback(self.explain_web_error)
5497         return d
5498
5499
5500     def test_exceptions(self):
5501         self.basedir = "web/Grid/exceptions"
5502         self.set_up_grid(num_clients=1, num_servers=2)
5503         c0 = self.g.clients[0]
5504         c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 2
5505         self.fileurls = {}
5506         DATA = "data" * 100
5507         d = c0.create_dirnode()
5508         def _stash_root(n):
5509             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5510             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
5511             return n
5512         d.addCallback(_stash_root)
5513         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
5514         def _stash_bad(ur):
5515             self.fileurls["1share"] = "uri/" + urllib.quote(ur.get_uri())
5516             self.delete_shares_numbered(ur.get_uri(), range(1,10))
5517
5518             u = uri.from_string(ur.get_uri())
5519             u.key = testutil.flip_bit(u.key, 0)
5520             baduri = u.to_string()
5521             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
5522         d.addCallback(_stash_bad)
5523         d.addCallback(lambda ign: c0.create_dirnode())
5524         def _mangle_dirnode_1share(n):
5525             u = n.get_uri()
5526             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
5527             self.fileurls["dir-1share-json"] = url + "?t=json"
5528             self.delete_shares_numbered(u, range(1,10))
5529         d.addCallback(_mangle_dirnode_1share)
5530         d.addCallback(lambda ign: c0.create_dirnode())
5531         def _mangle_dirnode_0share(n):
5532             u = n.get_uri()
5533             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
5534             self.fileurls["dir-0share-json"] = url + "?t=json"
5535             self.delete_shares_numbered(u, range(0,10))
5536         d.addCallback(_mangle_dirnode_0share)
5537
5538         # NotEnoughSharesError should be reported sensibly, with a
5539         # text/plain explanation of the problem, and perhaps some
5540         # information on which shares *could* be found.
5541
5542         d.addCallback(lambda ignored:
5543                       self.shouldHTTPError("GET unrecoverable",
5544                                            410, "Gone", "NoSharesError",
5545                                            self.GET, self.fileurls["0shares"]))
5546         def _check_zero_shares(body):
5547             self.failIfIn("<html>", body)
5548             body = " ".join(body.strip().split())
5549             exp = ("NoSharesError: no shares could be found. "
5550                    "Zero shares usually indicates a corrupt URI, or that "
5551                    "no servers were connected, but it might also indicate "
5552                    "severe corruption. You should perform a filecheck on "
5553                    "this object to learn more. The full error message is: "
5554                    "no shares (need 3). Last failure: None")
5555             self.failUnlessReallyEqual(exp, body)
5556         d.addCallback(_check_zero_shares)
5557
5558
5559         d.addCallback(lambda ignored:
5560                       self.shouldHTTPError("GET 1share",
5561                                            410, "Gone", "NotEnoughSharesError",
5562                                            self.GET, self.fileurls["1share"]))
5563         def _check_one_share(body):
5564             self.failIfIn("<html>", body)
5565             body = " ".join(body.strip().split())
5566             msgbase = ("NotEnoughSharesError: This indicates that some "
5567                        "servers were unavailable, or that shares have been "
5568                        "lost to server departure, hard drive failure, or disk "
5569                        "corruption. You should perform a filecheck on "
5570                        "this object to learn more. The full error message is:"
5571                        )
5572             msg1 = msgbase + (" ran out of shares:"
5573                               " complete=sh0"
5574                               " pending="
5575                               " overdue= unused= need 3. Last failure: None")
5576             msg2 = msgbase + (" ran out of shares:"
5577                               " complete="
5578                               " pending=Share(sh0-on-xgru5)"
5579                               " overdue= unused= need 3. Last failure: None")
5580             self.failUnless(body == msg1 or body == msg2, body)
5581         d.addCallback(_check_one_share)
5582
5583         d.addCallback(lambda ignored:
5584                       self.shouldHTTPError("GET imaginary",
5585                                            404, "Not Found", None,
5586                                            self.GET, self.fileurls["imaginary"]))
5587         def _missing_child(body):
5588             self.failUnlessIn("No such child: imaginary", body)
5589         d.addCallback(_missing_child)
5590
5591         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
5592         def _check_0shares_dir_html(body):
5593             self.failUnlessIn("<html>", body)
5594             # we should see the regular page, but without the child table or
5595             # the dirops forms
5596             body = " ".join(body.strip().split())
5597             self.failUnlessIn('href="?t=info">More info on this directory',
5598                               body)
5599             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5600                    "could not be retrieved, because there were insufficient "
5601                    "good shares. This might indicate that no servers were "
5602                    "connected, insufficient servers were connected, the URI "
5603                    "was corrupt, or that shares have been lost due to server "
5604                    "departure, hard drive failure, or disk corruption. You "
5605                    "should perform a filecheck on this object to learn more.")
5606             self.failUnlessIn(exp, body)
5607             self.failUnlessIn("No upload forms: directory is unreadable", body)
5608         d.addCallback(_check_0shares_dir_html)
5609
5610         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
5611         def _check_1shares_dir_html(body):
5612             # at some point, we'll split UnrecoverableFileError into 0-shares
5613             # and some-shares like we did for immutable files (since there
5614             # are different sorts of advice to offer in each case). For now,
5615             # they present the same way.
5616             self.failUnlessIn("<html>", body)
5617             body = " ".join(body.strip().split())
5618             self.failUnlessIn('href="?t=info">More info on this directory',
5619                               body)
5620             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5621                    "could not be retrieved, because there were insufficient "
5622                    "good shares. This might indicate that no servers were "
5623                    "connected, insufficient servers were connected, the URI "
5624                    "was corrupt, or that shares have been lost due to server "
5625                    "departure, hard drive failure, or disk corruption. You "
5626                    "should perform a filecheck on this object to learn more.")
5627             self.failUnlessIn(exp, body)
5628             self.failUnlessIn("No upload forms: directory is unreadable", body)
5629         d.addCallback(_check_1shares_dir_html)
5630
5631         d.addCallback(lambda ignored:
5632                       self.shouldHTTPError("GET dir-0share-json",
5633                                            410, "Gone", "UnrecoverableFileError",
5634                                            self.GET,
5635                                            self.fileurls["dir-0share-json"]))
5636         def _check_unrecoverable_file(body):
5637             self.failIfIn("<html>", body)
5638             body = " ".join(body.strip().split())
5639             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5640                    "could not be retrieved, because there were insufficient "
5641                    "good shares. This might indicate that no servers were "
5642                    "connected, insufficient servers were connected, the URI "
5643                    "was corrupt, or that shares have been lost due to server "
5644                    "departure, hard drive failure, or disk corruption. You "
5645                    "should perform a filecheck on this object to learn more.")
5646             self.failUnlessReallyEqual(exp, body)
5647         d.addCallback(_check_unrecoverable_file)
5648
5649         d.addCallback(lambda ignored:
5650                       self.shouldHTTPError("GET dir-1share-json",
5651                                            410, "Gone", "UnrecoverableFileError",
5652                                            self.GET,
5653                                            self.fileurls["dir-1share-json"]))
5654         d.addCallback(_check_unrecoverable_file)
5655
5656         d.addCallback(lambda ignored:
5657                       self.shouldHTTPError("GET imaginary",
5658                                            404, "Not Found", None,
5659                                            self.GET, self.fileurls["imaginary"]))
5660
5661         # attach a webapi child that throws a random error, to test how it
5662         # gets rendered.
5663         w = c0.getServiceNamed("webish")
5664         w.root.putChild("ERRORBOOM", ErrorBoom())
5665
5666         # "Accept: */*" :        should get a text/html stack trace
5667         # "Accept: text/plain" : should get a text/plain stack trace
5668         # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
5669         # no Accept header:      should get a text/html stack trace
5670
5671         d.addCallback(lambda ignored:
5672                       self.shouldHTTPError("GET errorboom_html",
5673                                            500, "Internal Server Error", None,
5674                                            self.GET, "ERRORBOOM",
5675                                            headers={"accept": "*/*"}))
5676         def _internal_error_html1(body):
5677             self.failUnlessIn("<html>", "expected HTML, not '%s'" % body)
5678         d.addCallback(_internal_error_html1)
5679
5680         d.addCallback(lambda ignored:
5681                       self.shouldHTTPError("GET errorboom_text",
5682                                            500, "Internal Server Error", None,
5683                                            self.GET, "ERRORBOOM",
5684                                            headers={"accept": "text/plain"}))
5685         def _internal_error_text2(body):
5686             self.failIfIn("<html>", body)
5687             self.failUnless(body.startswith("Traceback "), body)
5688         d.addCallback(_internal_error_text2)
5689
5690         CLI_accepts = "text/plain, application/octet-stream"
5691         d.addCallback(lambda ignored:
5692                       self.shouldHTTPError("GET errorboom_text",
5693                                            500, "Internal Server Error", None,
5694                                            self.GET, "ERRORBOOM",
5695                                            headers={"accept": CLI_accepts}))
5696         def _internal_error_text3(body):
5697             self.failIfIn("<html>", body)
5698             self.failUnless(body.startswith("Traceback "), body)
5699         d.addCallback(_internal_error_text3)
5700
5701         d.addCallback(lambda ignored:
5702                       self.shouldHTTPError("GET errorboom_text",
5703                                            500, "Internal Server Error", None,
5704                                            self.GET, "ERRORBOOM"))
5705         def _internal_error_html4(body):
5706             self.failUnlessIn("<html>", body)
5707         d.addCallback(_internal_error_html4)
5708
5709         def _flush_errors(res):
5710             # Trial: please ignore the CompletelyUnhandledError in the logs
5711             self.flushLoggedErrors(CompletelyUnhandledError)
5712             return res
5713         d.addBoth(_flush_errors)
5714
5715         return d
5716
5717     def test_blacklist(self):
5718         # download from a blacklisted URI, get an error
5719         self.basedir = "web/Grid/blacklist"
5720         self.set_up_grid()
5721         c0 = self.g.clients[0]
5722         c0_basedir = c0.basedir
5723         fn = os.path.join(c0_basedir, "access.blacklist")
5724         self.uris = {}
5725         DATA = "off-limits " * 50
5726
5727         d = c0.upload(upload.Data(DATA, convergence=""))
5728         def _stash_uri_and_create_dir(ur):
5729             self.uri = ur.get_uri()
5730             self.url = "uri/"+self.uri
5731             u = uri.from_string_filenode(self.uri)
5732             self.si = u.get_storage_index()
5733             childnode = c0.create_node_from_uri(self.uri, None)
5734             return c0.create_dirnode({u"blacklisted.txt": (childnode,{}) })
5735         d.addCallback(_stash_uri_and_create_dir)
5736         def _stash_dir(node):
5737             self.dir_node = node
5738             self.dir_uri = node.get_uri()
5739             self.dir_url = "uri/"+self.dir_uri
5740         d.addCallback(_stash_dir)
5741         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5742         def _check_dir_html(body):
5743             self.failUnlessIn("<html>", body)
5744             self.failUnlessIn("blacklisted.txt</a>", body)
5745         d.addCallback(_check_dir_html)
5746         d.addCallback(lambda ign: self.GET(self.url))
5747         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5748
5749         def _blacklist(ign):
5750             f = open(fn, "w")
5751             f.write(" # this is a comment\n")
5752             f.write(" \n")
5753             f.write("\n") # also exercise blank lines
5754             f.write("%s %s\n" % (base32.b2a(self.si), "off-limits to you"))
5755             f.close()
5756             # clients should be checking the blacklist each time, so we don't
5757             # need to restart the client
5758         d.addCallback(_blacklist)
5759         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_uri",
5760                                                        403, "Forbidden",
5761                                                        "Access Prohibited: off-limits",
5762                                                        self.GET, self.url))
5763
5764         # We should still be able to list the parent directory, in HTML...
5765         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5766         def _check_dir_html2(body):
5767             self.failUnlessIn("<html>", body)
5768             self.failUnlessIn("blacklisted.txt</strike>", body)
5769         d.addCallback(_check_dir_html2)
5770
5771         # ... and in JSON (used by CLI).
5772         d.addCallback(lambda ign: self.GET(self.dir_url+"?t=json", followRedirect=True))
5773         def _check_dir_json(res):
5774             data = simplejson.loads(res)
5775             self.failUnless(isinstance(data, list), data)
5776             self.failUnlessEqual(data[0], "dirnode")
5777             self.failUnless(isinstance(data[1], dict), data)
5778             self.failUnlessIn("children", data[1])
5779             self.failUnlessIn("blacklisted.txt", data[1]["children"])
5780             childdata = data[1]["children"]["blacklisted.txt"]
5781             self.failUnless(isinstance(childdata, list), data)
5782             self.failUnlessEqual(childdata[0], "filenode")
5783             self.failUnless(isinstance(childdata[1], dict), data)
5784         d.addCallback(_check_dir_json)
5785
5786         def _unblacklist(ign):
5787             open(fn, "w").close()
5788             # the Blacklist object watches mtime to tell when the file has
5789             # changed, but on windows this test will run faster than the
5790             # filesystem's mtime resolution. So we edit Blacklist.last_mtime
5791             # to force a reload.
5792             self.g.clients[0].blacklist.last_mtime -= 2.0
5793         d.addCallback(_unblacklist)
5794
5795         # now a read should work
5796         d.addCallback(lambda ign: self.GET(self.url))
5797         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5798
5799         # read again to exercise the blacklist-is-unchanged logic
5800         d.addCallback(lambda ign: self.GET(self.url))
5801         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5802
5803         # now add a blacklisted directory, and make sure files under it are
5804         # refused too
5805         def _add_dir(ign):
5806             childnode = c0.create_node_from_uri(self.uri, None)
5807             return c0.create_dirnode({u"child": (childnode,{}) })
5808         d.addCallback(_add_dir)
5809         def _get_dircap(dn):
5810             self.dir_si_b32 = base32.b2a(dn.get_storage_index())
5811             self.dir_url_base = "uri/"+dn.get_write_uri()
5812             self.dir_url_json1 = "uri/"+dn.get_write_uri()+"?t=json"
5813             self.dir_url_json2 = "uri/"+dn.get_write_uri()+"/?t=json"
5814             self.dir_url_json_ro = "uri/"+dn.get_readonly_uri()+"/?t=json"
5815             self.child_url = "uri/"+dn.get_readonly_uri()+"/child"
5816         d.addCallback(_get_dircap)
5817         d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True))
5818         d.addCallback(lambda body: self.failUnlessIn("<html>", body))
5819         d.addCallback(lambda ign: self.GET(self.dir_url_json1))
5820         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5821         d.addCallback(lambda ign: self.GET(self.dir_url_json2))
5822         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5823         d.addCallback(lambda ign: self.GET(self.dir_url_json_ro))
5824         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5825         d.addCallback(lambda ign: self.GET(self.child_url))
5826         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5827
5828         def _block_dir(ign):
5829             f = open(fn, "w")
5830             f.write("%s %s\n" % (self.dir_si_b32, "dir-off-limits to you"))
5831             f.close()
5832             self.g.clients[0].blacklist.last_mtime -= 2.0
5833         d.addCallback(_block_dir)
5834         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir base",
5835                                                        403, "Forbidden",
5836                                                        "Access Prohibited: dir-off-limits",
5837                                                        self.GET, self.dir_url_base))
5838         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json1",
5839                                                        403, "Forbidden",
5840                                                        "Access Prohibited: dir-off-limits",
5841                                                        self.GET, self.dir_url_json1))
5842         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json2",
5843                                                        403, "Forbidden",
5844                                                        "Access Prohibited: dir-off-limits",
5845                                                        self.GET, self.dir_url_json2))
5846         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json_ro",
5847                                                        403, "Forbidden",
5848                                                        "Access Prohibited: dir-off-limits",
5849                                                        self.GET, self.dir_url_json_ro))
5850         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir child",
5851                                                        403, "Forbidden",
5852                                                        "Access Prohibited: dir-off-limits",
5853                                                        self.GET, self.child_url))
5854         return d
5855
5856
5857 class CompletelyUnhandledError(Exception):
5858     pass
5859 class ErrorBoom(rend.Page):
5860     def beforeRender(self, ctx):
5861         raise CompletelyUnhandledError("whoops")