src/allmydata/test/test_web.py

   1 import os.path, re, urllib, time, cgi
   2 import simplejson
   3 from StringIO import StringIO
   4
   5 from twisted.application import service
   6 from twisted.trial import unittest
   7 from twisted.internet import defer, reactor
   8 from twisted.internet.task import Clock
   9 from twisted.web import client, error, http
  10 from twisted.python import failure, log
  11
  12 from foolscap.api import fireEventually, flushEventualQueue
  13
  14 from nevow.util import escapeToXML
  15 from nevow import rend
  16
  17 from allmydata import interfaces, uri, webish, dirnode
  18 from allmydata.storage.shares import get_share_file
  19 from allmydata.storage_client import StorageFarmBroker, StubServer
  20 from allmydata.immutable import upload
  21 from allmydata.immutable.downloader.status import DownloadStatus
  22 from allmydata.dirnode import DirectoryNode
  23 from allmydata.nodemaker import NodeMaker
  24 from allmydata.unknown import UnknownNode
  25 from allmydata.web import status, common
  26 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  27 from allmydata.util import fileutil, base32, hashutil
  28 from allmydata.util.consumer import download_to_data
  29 from allmydata.util.netstring import split_netstring
  30 from allmydata.util.encodingutil import to_str
  31 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  32      create_chk_filenode, WebErrorMixin, ShouldFailMixin, \
  33      make_mutable_file_uri, create_mutable_filenode
  34 from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
  35 from allmydata.mutable import servermap, publish, retrieve
  36 import allmydata.test.common_util as testutil
  37 from allmydata.test.no_network import GridTestMixin
  38 from allmydata.test.common_web import HTTPClientGETFactory, \
  39      HTTPClientHEADFactory
  40 from allmydata.client import Client, SecretHolder
  41 from allmydata.introducer import IntroducerNode
  42
  43 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  44 # create a webserver that works against them
  45
  46 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  47
  48 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
  49 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
  50 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
  51
  52 FAVICON_MARKUP = '<link href="/icon.png" rel="shortcut icon" />'
  53
  54
  55 class FakeStatsProvider:
  56     def get_stats(self):
  57         stats = {'stats': {}, 'counters': {}}
  58         return stats
  59
  60 class FakeNodeMaker(NodeMaker):
  61     encoding_params = {
  62         'k': 3,
  63         'n': 10,
  64         'happy': 7,
  65         'max_segment_size':128*1024 # 1024=KiB
  66     }
  67     def _create_lit(self, cap):
  68         return FakeCHKFileNode(cap, self.all_contents)
  69     def _create_immutable(self, cap):
  70         return FakeCHKFileNode(cap, self.all_contents)
  71     def _create_mutable(self, cap):
  72         return FakeMutableFileNode(None, None,
  73                                    self.encoding_params, None,
  74                                    self.all_contents).init_from_cap(cap)
  75     def create_mutable_file(self, contents="", keysize=None,
  76                             version=SDMF_VERSION):
  77         n = FakeMutableFileNode(None, None, self.encoding_params, None,
  78                                 self.all_contents)
  79         return n.create(contents, version=version)
  80
  81 class FakeUploader(service.Service):
  82     name = "uploader"
  83     helper_furl = None
  84     helper_connected = False
  85
  86     def upload(self, uploadable):
  87         d = uploadable.get_size()
  88         d.addCallback(lambda size: uploadable.read(size))
  89         def _got_data(datav):
  90             data = "".join(datav)
  91             n = create_chk_filenode(data, self.all_contents)
  92             ur = upload.UploadResults(file_size=len(data),
  93                                       ciphertext_fetched=0,
  94                                       preexisting_shares=0,
  95                                       pushed_shares=10,
  96                                       sharemap={},
  97                                       servermap={},
  98                                       timings={},
  99                                       uri_extension_data={},
 100                                       uri_extension_hash="fake",
 101                                       verifycapstr="fakevcap")
 102             ur.set_uri(n.get_uri())
 103             return ur
 104         d.addCallback(_got_data)
 105         return d
 106
 107     def get_helper_info(self):
 108         return (self.helper_furl, self.helper_connected)
 109
 110
 111 def build_one_ds():
 112     ds = DownloadStatus("storage_index", 1234)
 113     now = time.time()
 114
 115     serverA = StubServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
 116     serverB = StubServer(hashutil.tagged_hash("foo", "serverid_b")[:20])
 117     storage_index = hashutil.storage_index_hash("SI")
 118     e0 = ds.add_segment_request(0, now)
 119     e0.activate(now+0.5)
 120     e0.deliver(now+1, 0, 100, 0.5) # when, start,len, decodetime
 121     e1 = ds.add_segment_request(1, now+2)
 122     e1.error(now+3)
 123     # two outstanding requests
 124     e2 = ds.add_segment_request(2, now+4)
 125     e3 = ds.add_segment_request(3, now+5)
 126     del e2,e3 # hush pyflakes
 127
 128     # simulate a segment which gets delivered faster than a system clock tick (ticket #1166)
 129     e = ds.add_segment_request(4, now)
 130     e.activate(now)
 131     e.deliver(now, 0, 140, 0.5)
 132
 133     e = ds.add_dyhb_request(serverA, now)
 134     e.finished([1,2], now+1)
 135     e = ds.add_dyhb_request(serverB, now+2) # left unfinished
 136
 137     e = ds.add_read_event(0, 120, now)
 138     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 139     e.finished(now+1)
 140     e = ds.add_read_event(120, 30, now+2) # left unfinished
 141
 142     e = ds.add_block_request(serverA, 1, 100, 20, now)
 143     e.finished(20, now+1)
 144     e = ds.add_block_request(serverB, 1, 120, 30, now+1) # left unfinished
 145
 146     # make sure that add_read_event() can come first too
 147     ds1 = DownloadStatus(storage_index, 1234)
 148     e = ds1.add_read_event(0, 120, now)
 149     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 150     e.finished(now+1)
 151
 152     return ds
 153
 154 class FakeHistory:
 155     _all_upload_status = [upload.UploadStatus()]
 156     _all_download_status = [build_one_ds()]
 157     _all_mapupdate_statuses = [servermap.UpdateStatus()]
 158     _all_publish_statuses = [publish.PublishStatus()]
 159     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
 160
 161     def list_all_upload_statuses(self):
 162         return self._all_upload_status
 163     def list_all_download_statuses(self):
 164         return self._all_download_status
 165     def list_all_mapupdate_statuses(self):
 166         return self._all_mapupdate_statuses
 167     def list_all_publish_statuses(self):
 168         return self._all_publish_statuses
 169     def list_all_retrieve_statuses(self):
 170         return self._all_retrieve_statuses
 171     def list_all_helper_statuses(self):
 172         return []
 173
 174 class FakeDisplayableServer(StubServer):
 175     def __init__(self, serverid, nickname):
 176         StubServer.__init__(self, serverid)
 177         self.announcement = {"my-version": "allmydata-tahoe-fake",
 178                              "service-name": "storage",
 179                              "nickname": nickname}
 180     def is_connected(self):
 181         return True
 182     def get_permutation_seed(self):
 183         return ""
 184     def get_remote_host(self):
 185         return ""
 186     def get_last_loss_time(self):
 187         return None
 188     def get_announcement_time(self):
 189         return None
 190     def get_announcement(self):
 191         return self.announcement
 192     def get_nickname(self):
 193         return self.announcement["nickname"]
 194
 195 class FakeBucketCounter(object):
 196     def get_state(self):
 197         return {"last-complete-bucket-count": 0}
 198     def get_progress(self):
 199         return {"estimated-time-per-cycle": 0,
 200                 "cycle-in-progress": False,
 201                 "remaining-wait-time": 0}
 202
 203 class FakeLeaseChecker(object):
 204     def __init__(self):
 205         self.expiration_enabled = False
 206         self.mode = "age"
 207         self.override_lease_duration = None
 208         self.sharetypes_to_expire = {}
 209     def get_state(self):
 210         return {"history": None}
 211     def get_progress(self):
 212         return {"estimated-time-per-cycle": 0,
 213                 "cycle-in-progress": False,
 214                 "remaining-wait-time": 0}
 215
 216 class FakeStorageServer(service.MultiService):
 217     name = 'storage'
 218     def __init__(self, nodeid, nickname):
 219         service.MultiService.__init__(self)
 220         self.my_nodeid = nodeid
 221         self.nickname = nickname
 222         self.bucket_counter = FakeBucketCounter()
 223         self.lease_checker = FakeLeaseChecker()
 224     def get_stats(self):
 225         return {"storage_server.accepting_immutable_shares": False}
 226
 227 class FakeClient(Client):
 228     def __init__(self):
 229         # don't upcall to Client.__init__, since we only want to initialize a
 230         # minimal subset
 231         service.MultiService.__init__(self)
 232         self.all_contents = {}
 233         self.nodeid = "fake_nodeid"
 234         self.nickname = u"fake_nickname \u263A"
 235         self.introducer_furl = "None"
 236         self.stats_provider = FakeStatsProvider()
 237         self._secret_holder = SecretHolder("lease secret", "convergence secret")
 238         self.helper = None
 239         self.convergence = "some random string"
 240         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 241         # fake knowledge of another server
 242         self.storage_broker.test_add_server("other_nodeid",
 243                                             FakeDisplayableServer("other_nodeid", u"other_nickname \u263B"))
 244         self.introducer_client = None
 245         self.history = FakeHistory()
 246         self.uploader = FakeUploader()
 247         self.uploader.all_contents = self.all_contents
 248         self.uploader.setServiceParent(self)
 249         self.blacklist = None
 250         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 251                                        self.uploader, None,
 252                                        None, None, None)
 253         self.nodemaker.all_contents = self.all_contents
 254         self.mutable_file_default = SDMF_VERSION
 255         self.addService(FakeStorageServer(self.nodeid, self.nickname))
 256
 257     def get_long_nodeid(self):
 258         return "v0-nodeid"
 259     def get_long_tubid(self):
 260         return "tubid"
 261
 262     def startService(self):
 263         return service.MultiService.startService(self)
 264     def stopService(self):
 265         return service.MultiService.stopService(self)
 266
 267     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 268
 269 class WebMixin(object):
 270     def setUp(self):
 271         self.s = FakeClient()
 272         self.s.startService()
 273         self.staticdir = self.mktemp()
 274         self.clock = Clock()
 275         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
 276                                       clock=self.clock)
 277         self.ws.setServiceParent(self.s)
 278         self.webish_port = self.ws.getPortnum()
 279         self.webish_url = self.ws.getURL()
 280         assert self.webish_url.endswith("/")
 281         self.webish_url = self.webish_url[:-1] # these tests add their own /
 282
 283         l = [ self.s.create_dirnode() for x in range(6) ]
 284         d = defer.DeferredList(l)
 285         def _then(res):
 286             self.public_root = res[0][1]
 287             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 288             self.public_url = "/uri/" + self.public_root.get_uri()
 289             self.private_root = res[1][1]
 290
 291             foo = res[2][1]
 292             self._foo_node = foo
 293             self._foo_uri = foo.get_uri()
 294             self._foo_readonly_uri = foo.get_readonly_uri()
 295             self._foo_verifycap = foo.get_verify_cap().to_string()
 296             # NOTE: we ignore the deferred on all set_uri() calls, because we
 297             # know the fake nodes do these synchronously
 298             self.public_root.set_uri(u"foo", foo.get_uri(),
 299                                      foo.get_readonly_uri())
 300
 301             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 302             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 303             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 304
 305             # sdmf
 306             # XXX: Do we ever use this?
 307             self.BAZ_CONTENTS, n, self._baz_txt_uri, self._baz_txt_readonly_uri = self.makefile_mutable(0)
 308
 309             foo.set_uri(u"baz.txt", self._baz_txt_uri, self._baz_txt_readonly_uri)
 310
 311             # mdmf
 312             self.QUUX_CONTENTS, n, self._quux_txt_uri, self._quux_txt_readonly_uri = self.makefile_mutable(0, mdmf=True)
 313             assert self._quux_txt_uri.startswith("URI:MDMF")
 314             foo.set_uri(u"quux.txt", self._quux_txt_uri, self._quux_txt_readonly_uri)
 315
 316             foo.set_uri(u"empty", res[3][1].get_uri(),
 317                         res[3][1].get_readonly_uri())
 318             sub_uri = res[4][1].get_uri()
 319             self._sub_uri = sub_uri
 320             foo.set_uri(u"sub", sub_uri, sub_uri)
 321             sub = self.s.create_node_from_uri(sub_uri)
 322             self._sub_node = sub
 323
 324             _ign, n, blocking_uri = self.makefile(1)
 325             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 326
 327             # filenode to test for html encoding issues
 328             self._htmlname_unicode = u"<&weirdly'named\"file>>>_<iframe />.txt"
 329             self._htmlname_raw = self._htmlname_unicode.encode('utf-8')
 330             self._htmlname_urlencoded = urllib.quote(self._htmlname_raw, '')
 331             self._htmlname_escaped = escapeToXML(self._htmlname_raw)
 332             self._htmlname_escaped_attr = cgi.escape(self._htmlname_raw, quote=True)
 333             self._htmlname_escaped_double = escapeToXML(cgi.escape(self._htmlname_raw, quote=True))
 334             self.HTMLNAME_CONTENTS, n, self._htmlname_txt_uri = self.makefile(0)
 335             foo.set_uri(self._htmlname_unicode, self._htmlname_txt_uri, self._htmlname_txt_uri)
 336
 337             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 338             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 339             # still think of it as an umlaut
 340             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 341
 342             self.SUBBAZ_CONTENTS, n, baz_file = self.makefile(2)
 343             self._baz_file_uri = baz_file
 344             sub.set_uri(u"baz.txt", baz_file, baz_file)
 345
 346             _ign, n, self._bad_file_uri = self.makefile(3)
 347             # this uri should not be downloadable
 348             del self.s.all_contents[self._bad_file_uri]
 349
 350             rodir = res[5][1]
 351             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 352                                      rodir.get_readonly_uri())
 353             rodir.set_uri(u"nor", baz_file, baz_file)
 354
 355             # public/
 356             # public/foo/
 357             # public/foo/bar.txt
 358             # public/foo/baz.txt
 359             # public/foo/quux.txt
 360             # public/foo/blockingfile
 361             # public/foo/<&weirdly'named\"file>>>_<iframe />.txt
 362             # public/foo/empty/
 363             # public/foo/sub/
 364             # public/foo/sub/baz.txt
 365             # public/reedownlee/
 366             # public/reedownlee/nor
 367             self.NEWFILE_CONTENTS = "newfile contents\n"
 368
 369             return foo.get_metadata_for(u"bar.txt")
 370         d.addCallback(_then)
 371         def _got_metadata(metadata):
 372             self._bar_txt_metadata = metadata
 373         d.addCallback(_got_metadata)
 374         return d
 375
 376     def get_all_contents(self):
 377         return self.s.all_contents
 378
 379     def makefile(self, number):
 380         contents = "contents of file %s\n" % number
 381         n = create_chk_filenode(contents, self.get_all_contents())
 382         return contents, n, n.get_uri()
 383
 384     def makefile_mutable(self, number, mdmf=False):
 385         contents = "contents of mutable file %s\n" % number
 386         n = create_mutable_filenode(contents, mdmf, self.s.all_contents)
 387         return contents, n, n.get_uri(), n.get_readonly_uri()
 388
 389     def tearDown(self):
 390         return self.s.stopService()
 391
 392     def failUnlessIsBarDotTxt(self, res):
 393         self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
 394
 395     def failUnlessIsQuuxDotTxt(self, res):
 396         self.failUnlessReallyEqual(res, self.QUUX_CONTENTS, res)
 397
 398     def failUnlessIsBazDotTxt(self, res):
 399         self.failUnlessReallyEqual(res, self.BAZ_CONTENTS, res)
 400
 401     def failUnlessIsSubBazDotTxt(self, res):
 402         self.failUnlessReallyEqual(res, self.SUBBAZ_CONTENTS, res)
 403
 404     def failUnlessIsBarJSON(self, res):
 405         data = simplejson.loads(res)
 406         self.failUnless(isinstance(data, list))
 407         self.failUnlessEqual(data[0], "filenode")
 408         self.failUnless(isinstance(data[1], dict))
 409         self.failIf(data[1]["mutable"])
 410         self.failIfIn("rw_uri", data[1]) # immutable
 411         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
 412         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
 413         self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
 414
 415     def failUnlessIsQuuxJSON(self, res, readonly=False):
 416         data = simplejson.loads(res)
 417         self.failUnless(isinstance(data, list))
 418         self.failUnlessEqual(data[0], "filenode")
 419         self.failUnless(isinstance(data[1], dict))
 420         metadata = data[1]
 421         return self.failUnlessIsQuuxDotTxtMetadata(metadata, readonly)
 422
 423     def failUnlessIsQuuxDotTxtMetadata(self, metadata, readonly):
 424         self.failUnless(metadata['mutable'])
 425         if readonly:
 426             self.failIfIn("rw_uri", metadata)
 427         else:
 428             self.failUnlessIn("rw_uri", metadata)
 429             self.failUnlessEqual(metadata['rw_uri'], self._quux_txt_uri)
 430         self.failUnlessIn("ro_uri", metadata)
 431         self.failUnlessEqual(metadata['ro_uri'], self._quux_txt_readonly_uri)
 432         self.failUnlessReallyEqual(metadata['size'], len(self.QUUX_CONTENTS))
 433
 434     def failUnlessIsFooJSON(self, res):
 435         data = simplejson.loads(res)
 436         self.failUnless(isinstance(data, list))
 437         self.failUnlessEqual(data[0], "dirnode", res)
 438         self.failUnless(isinstance(data[1], dict))
 439         self.failUnless(data[1]["mutable"])
 440         self.failUnlessIn("rw_uri", data[1]) # mutable
 441         self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
 442         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
 443         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
 444
 445         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 446         self.failUnlessEqual(kidnames,
 447                              [self._htmlname_unicode, u"bar.txt", u"baz.txt",
 448                               u"blockingfile", u"empty", u"n\u00fc.txt", u"quux.txt", u"sub"])
 449         kids = dict( [(unicode(name),value)
 450                       for (name,value)
 451                       in data[1]["children"].iteritems()] )
 452         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 453         self.failUnlessIn("metadata", kids[u"sub"][1])
 454         self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
 455         tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
 456         self.failUnlessIn("linkcrtime", tahoe_md)
 457         self.failUnlessIn("linkmotime", tahoe_md)
 458         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 459         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 460         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri)
 461         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]),
 462                                    self._bar_txt_verifycap)
 463         self.failUnlessIn("metadata", kids[u"bar.txt"][1])
 464         self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
 465         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
 466                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
 467         self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
 468                                    self._bar_txt_uri)
 469         self.failUnlessIn("quux.txt", kids)
 470         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["rw_uri"]),
 471                                    self._quux_txt_uri)
 472         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["ro_uri"]),
 473                                    self._quux_txt_readonly_uri)
 474
 475     def GET(self, urlpath, followRedirect=False, return_response=False,
 476             **kwargs):
 477         # if return_response=True, this fires with (data, statuscode,
 478         # respheaders) instead of just data.
 479         assert not isinstance(urlpath, unicode)
 480         url = self.webish_url + urlpath
 481         factory = HTTPClientGETFactory(url, method="GET",
 482                                        followRedirect=followRedirect, **kwargs)
 483         reactor.connectTCP("localhost", self.webish_port, factory)
 484         d = factory.deferred
 485         def _got_data(data):
 486             return (data, factory.status, factory.response_headers)
 487         if return_response:
 488             d.addCallback(_got_data)
 489         return factory.deferred
 490
 491     def HEAD(self, urlpath, return_response=False, **kwargs):
 492         # this requires some surgery, because twisted.web.client doesn't want
 493         # to give us back the response headers.
 494         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 495         reactor.connectTCP("localhost", self.webish_port, factory)
 496         d = factory.deferred
 497         def _got_data(data):
 498             return (data, factory.status, factory.response_headers)
 499         if return_response:
 500             d.addCallback(_got_data)
 501         return factory.deferred
 502
 503     def PUT(self, urlpath, data, **kwargs):
 504         url = self.webish_url + urlpath
 505         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 506
 507     def DELETE(self, urlpath):
 508         url = self.webish_url + urlpath
 509         return client.getPage(url, method="DELETE")
 510
 511     def POST(self, urlpath, followRedirect=False, **fields):
 512         sepbase = "boogabooga"
 513         sep = "--" + sepbase
 514         form = []
 515         form.append(sep)
 516         form.append('Content-Disposition: form-data; name="_charset"')
 517         form.append('')
 518         form.append('UTF-8')
 519         form.append(sep)
 520         for name, value in fields.iteritems():
 521             if isinstance(value, tuple):
 522                 filename, value = value
 523                 form.append('Content-Disposition: form-data; name="%s"; '
 524                             'filename="%s"' % (name, filename.encode("utf-8")))
 525             else:
 526                 form.append('Content-Disposition: form-data; name="%s"' % name)
 527             form.append('')
 528             if isinstance(value, unicode):
 529                 value = value.encode("utf-8")
 530             else:
 531                 value = str(value)
 532             assert isinstance(value, str)
 533             form.append(value)
 534             form.append(sep)
 535         form[-1] += "--"
 536         body = ""
 537         headers = {}
 538         if fields:
 539             body = "\r\n".join(form) + "\r\n"
 540             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 541         return self.POST2(urlpath, body, headers, followRedirect)
 542
 543     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 544         url = self.webish_url + urlpath
 545         return client.getPage(url, method="POST", postdata=body,
 546                               headers=headers, followRedirect=followRedirect)
 547
 548     def shouldFail(self, res, expected_failure, which,
 549                    substring=None, response_substring=None):
 550         if isinstance(res, failure.Failure):
 551             res.trap(expected_failure)
 552             if substring:
 553                 self.failUnlessIn(substring, str(res), which)
 554             if response_substring:
 555                 self.failUnlessIn(response_substring, res.value.response, which)
 556         else:
 557             self.fail("%s was supposed to raise %s, not get '%s'" %
 558                       (which, expected_failure, res))
 559
 560     def shouldFail2(self, expected_failure, which, substring,
 561                     response_substring,
 562                     callable, *args, **kwargs):
 563         assert substring is None or isinstance(substring, str)
 564         assert response_substring is None or isinstance(response_substring, str)
 565         d = defer.maybeDeferred(callable, *args, **kwargs)
 566         def done(res):
 567             if isinstance(res, failure.Failure):
 568                 res.trap(expected_failure)
 569                 if substring:
 570                     self.failUnlessIn(substring, str(res),
 571                                       "'%s' not in '%s' (response is '%s') for test '%s'" % \
 572                                       (substring, str(res),
 573                                        getattr(res.value, "response", ""),
 574                                        which))
 575                 if response_substring:
 576                     self.failUnlessIn(response_substring, res.value.response,
 577                                       "'%s' not in '%s' for test '%s'" % \
 578                                       (response_substring, res.value.response,
 579                                        which))
 580             else:
 581                 self.fail("%s was supposed to raise %s, not get '%s'" %
 582                           (which, expected_failure, res))
 583         d.addBoth(done)
 584         return d
 585
 586     def should404(self, res, which):
 587         if isinstance(res, failure.Failure):
 588             res.trap(error.Error)
 589             self.failUnlessReallyEqual(res.value.status, "404")
 590         else:
 591             self.fail("%s was supposed to Error(404), not get '%s'" %
 592                       (which, res))
 593
 594     def should302(self, res, which):
 595         if isinstance(res, failure.Failure):
 596             res.trap(error.Error)
 597             self.failUnlessReallyEqual(res.value.status, "302")
 598         else:
 599             self.fail("%s was supposed to Error(302), not get '%s'" %
 600                       (which, res))
 601
 602
 603 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
 604     def test_create(self):
 605         pass
 606
 607     def test_welcome(self):
 608         d = self.GET("/")
 609         def _check(res):
 610             self.failUnlessIn('<title>Tahoe-LAFS - Welcome</title>', res)
 611             self.failUnlessIn(FAVICON_MARKUP, res)
 612             self.failUnlessIn('<a href="status">Recent and Active Operations</a>', res)
 613             self.failUnlessIn('<a href="statistics">Operational Statistics</a>', res)
 614             self.failUnlessIn('<input type="hidden" name="t" value="report-incident" />', res)
 615             res_u = res.decode('utf-8')
 616             self.failUnlessIn(u'<td>fake_nickname \u263A</td>', res_u)
 617             self.failUnlessIn(u'<div class="nickname">other_nickname \u263B</div>', res_u)
 618             self.failUnlessIn(u'\u00A9 <a href="https://tahoe-lafs.org/">Tahoe-LAFS Software Foundation', res_u)
 619
 620             self.s.basedir = 'web/test_welcome'
 621             fileutil.make_dirs("web/test_welcome")
 622             fileutil.make_dirs("web/test_welcome/private")
 623             return self.GET("/")
 624         d.addCallback(_check)
 625         return d
 626
 627     def test_introducer_status(self):
 628         class MockIntroducerClient(object):
 629             def __init__(self, connected):
 630                 self.connected = connected
 631             def connected_to_introducer(self):
 632                 return self.connected
 633
 634         d = defer.succeed(None)
 635
 636         # introducer not connected, unguessable furl
 637         def _set_introducer_not_connected_unguessable(ign):
 638             self.s.introducer_furl = "pb://someIntroducer/secret"
 639             self.s.introducer_client = MockIntroducerClient(False)
 640             return self.GET("/")
 641         d.addCallback(_set_introducer_not_connected_unguessable)
 642         def _check_introducer_not_connected_unguessable(res):
 643             html = res.replace('\n', ' ')
 644             self.failUnlessIn('<div class="furl">pb://someIntroducer/[censored]</div>', html)
 645             self.failIfIn('pb://someIntroducer/secret', html)
 646             self.failUnless(re.search('<div class="status-indicator connected-no"></div>[ ]*<div>Introducer not connected</div>', html), res)
 647         d.addCallback(_check_introducer_not_connected_unguessable)
 648
 649         # introducer connected, unguessable furl
 650         def _set_introducer_connected_unguessable(ign):
 651             self.s.introducer_furl = "pb://someIntroducer/secret"
 652             self.s.introducer_client = MockIntroducerClient(True)
 653             return self.GET("/")
 654         d.addCallback(_set_introducer_connected_unguessable)
 655         def _check_introducer_connected_unguessable(res):
 656             html = res.replace('\n', ' ')
 657             self.failUnlessIn('<div class="furl">pb://someIntroducer/[censored]</div>', html)
 658             self.failIfIn('pb://someIntroducer/secret', html)
 659             self.failUnless(re.search('<div class="status-indicator connected-yes"></div>[ ]*<div>Introducer</div>', html), res)
 660         d.addCallback(_check_introducer_connected_unguessable)
 661
 662         # introducer connected, guessable furl
 663         def _set_introducer_connected_guessable(ign):
 664             self.s.introducer_furl = "pb://someIntroducer/introducer"
 665             self.s.introducer_client = MockIntroducerClient(True)
 666             return self.GET("/")
 667         d.addCallback(_set_introducer_connected_guessable)
 668         def _check_introducer_connected_guessable(res):
 669             html = res.replace('\n', ' ')
 670             self.failUnlessIn('<div class="furl">pb://someIntroducer/introducer</div>', html)
 671             self.failUnless(re.search('<div class="status-indicator connected-yes"></div>[ ]*<div>Introducer</div>', html), res)
 672         d.addCallback(_check_introducer_connected_guessable)
 673         return d
 674
 675     def test_helper_status(self):
 676         d = defer.succeed(None)
 677
 678         # set helper furl to None
 679         def _set_no_helper(ign):
 680             self.s.uploader.helper_furl = None
 681             return self.GET("/")
 682         d.addCallback(_set_no_helper)
 683         def _check_no_helper(res):
 684             html = res.replace('\n', ' ')
 685             self.failUnless(re.search('<div class="status-indicator connected-not-configured"></div>[ ]*<div>Helper</div>', html), res)
 686         d.addCallback(_check_no_helper)
 687
 688         # enable helper, not connected
 689         def _set_helper_not_connected(ign):
 690             self.s.uploader.helper_furl = "pb://someHelper/secret"
 691             self.s.uploader.helper_connected = False
 692             return self.GET("/")
 693         d.addCallback(_set_helper_not_connected)
 694         def _check_helper_not_connected(res):
 695             html = res.replace('\n', ' ')
 696             self.failUnlessIn('<div class="furl">pb://someHelper/[censored]</div>', html)
 697             self.failIfIn('pb://someHelper/secret', html)
 698             self.failUnless(re.search('<div class="status-indicator connected-no"></div>[ ]*<div>Helper not connected</div>', html), res)
 699         d.addCallback(_check_helper_not_connected)
 700
 701         # enable helper, connected
 702         def _set_helper_connected(ign):
 703             self.s.uploader.helper_furl = "pb://someHelper/secret"
 704             self.s.uploader.helper_connected = True
 705             return self.GET("/")
 706         d.addCallback(_set_helper_connected)
 707         def _check_helper_connected(res):
 708             html = res.replace('\n', ' ')
 709             self.failUnlessIn('<div class="furl">pb://someHelper/[censored]</div>', html)
 710             self.failIfIn('pb://someHelper/secret', html)
 711             self.failUnless(re.search('<div class="status-indicator connected-yes"></div>[ ]*<div>Helper</div>', html), res)
 712         d.addCallback(_check_helper_connected)
 713         return d
 714
 715     def test_storage(self):
 716         d = self.GET("/storage")
 717         def _check(res):
 718             self.failUnlessIn('Storage Server Status', res)
 719             self.failUnlessIn(FAVICON_MARKUP, res)
 720             res_u = res.decode('utf-8')
 721             self.failUnlessIn(u'<li>Server Nickname: <span class="nickname mine">fake_nickname \u263A</span></li>', res_u)
 722         d.addCallback(_check)
 723         return d
 724
 725     def test_status(self):
 726         h = self.s.get_history()
 727         dl_num = h.list_all_download_statuses()[0].get_counter()
 728         ul_num = h.list_all_upload_statuses()[0].get_counter()
 729         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 730         pub_num = h.list_all_publish_statuses()[0].get_counter()
 731         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 732         d = self.GET("/status", followRedirect=True)
 733         def _check(res):
 734             self.failUnlessIn('Recent and Active Operations', res)
 735             self.failUnlessIn('"down-%d"' % dl_num, res)
 736             self.failUnlessIn('"up-%d"' % ul_num, res)
 737             self.failUnlessIn('"mapupdate-%d"' % mu_num, res)
 738             self.failUnlessIn('"publish-%d"' % pub_num, res)
 739             self.failUnlessIn('"retrieve-%d"' % ret_num, res)
 740         d.addCallback(_check)
 741         d.addCallback(lambda res: self.GET("/status/?t=json"))
 742         def _check_json(res):
 743             data = simplejson.loads(res)
 744             self.failUnless(isinstance(data, dict))
 745             #active = data["active"]
 746             # TODO: test more. We need a way to fake an active operation
 747             # here.
 748         d.addCallback(_check_json)
 749
 750         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 751         def _check_dl(res):
 752             self.failUnlessIn("File Download Status", res)
 753         d.addCallback(_check_dl)
 754         d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
 755         def _check_dl_json(res):
 756             data = simplejson.loads(res)
 757             self.failUnless(isinstance(data, dict))
 758             self.failUnlessIn("read", data)
 759             self.failUnlessEqual(data["read"][0]["length"], 120)
 760             self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
 761             self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
 762             self.failUnlessEqual(data["segment"][2]["finish_time"], None)
 763             phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
 764             cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
 765             # serverids[] keys are strings, since that's what JSON does, but
 766             # we'd really like them to be ints
 767             self.failUnlessEqual(data["serverids"]["0"], "phwrsjte")
 768             self.failUnless(data["serverids"].has_key("1"),
 769                             str(data["serverids"]))
 770             self.failUnlessEqual(data["serverids"]["1"], "cmpuvkjm",
 771                                  str(data["serverids"]))
 772             self.failUnlessEqual(data["server_info"][phwr_id]["short"],
 773                                  "phwrsjte")
 774             self.failUnlessEqual(data["server_info"][cmpu_id]["short"],
 775                                  "cmpuvkjm")
 776             self.failUnlessIn("dyhb", data)
 777             self.failUnlessIn("misc", data)
 778         d.addCallback(_check_dl_json)
 779         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 780         def _check_ul(res):
 781             self.failUnlessIn("File Upload Status", res)
 782         d.addCallback(_check_ul)
 783         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 784         def _check_mapupdate(res):
 785             self.failUnlessIn("Mutable File Servermap Update Status", res)
 786         d.addCallback(_check_mapupdate)
 787         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 788         def _check_publish(res):
 789             self.failUnlessIn("Mutable File Publish Status", res)
 790         d.addCallback(_check_publish)
 791         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 792         def _check_retrieve(res):
 793             self.failUnlessIn("Mutable File Retrieve Status", res)
 794         d.addCallback(_check_retrieve)
 795
 796         return d
 797
 798     def test_status_numbers(self):
 799         drrm = status.DownloadResultsRendererMixin()
 800         self.failUnlessReallyEqual(drrm.render_time(None, None), "")
 801         self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
 802         self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
 803         self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
 804         self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
 805         self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
 806         self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 807         self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
 808         self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
 809
 810         urrm = status.UploadResultsRendererMixin()
 811         self.failUnlessReallyEqual(urrm.render_time(None, None), "")
 812         self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
 813         self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
 814         self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
 815         self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
 816         self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
 817         self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 818         self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
 819         self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
 820
 821     def test_GET_FILEURL(self):
 822         d = self.GET(self.public_url + "/foo/bar.txt")
 823         d.addCallback(self.failUnlessIsBarDotTxt)
 824         return d
 825
 826     def test_GET_FILEURL_range(self):
 827         headers = {"range": "bytes=1-10"}
 828         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 829                      return_response=True)
 830         def _got((res, status, headers)):
 831             self.failUnlessReallyEqual(int(status), 206)
 832             self.failUnless(headers.has_key("content-range"))
 833             self.failUnlessReallyEqual(headers["content-range"][0],
 834                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 835             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
 836         d.addCallback(_got)
 837         return d
 838
 839     def test_GET_FILEURL_partial_range(self):
 840         headers = {"range": "bytes=5-"}
 841         length  = len(self.BAR_CONTENTS)
 842         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 843                      return_response=True)
 844         def _got((res, status, headers)):
 845             self.failUnlessReallyEqual(int(status), 206)
 846             self.failUnless(headers.has_key("content-range"))
 847             self.failUnlessReallyEqual(headers["content-range"][0],
 848                                        "bytes 5-%d/%d" % (length-1, length))
 849             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
 850         d.addCallback(_got)
 851         return d
 852
 853     def test_GET_FILEURL_partial_end_range(self):
 854         headers = {"range": "bytes=-5"}
 855         length  = len(self.BAR_CONTENTS)
 856         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 857                      return_response=True)
 858         def _got((res, status, headers)):
 859             self.failUnlessReallyEqual(int(status), 206)
 860             self.failUnless(headers.has_key("content-range"))
 861             self.failUnlessReallyEqual(headers["content-range"][0],
 862                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 863             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
 864         d.addCallback(_got)
 865         return d
 866
 867     def test_GET_FILEURL_partial_range_overrun(self):
 868         headers = {"range": "bytes=100-200"}
 869         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
 870                              "416 Requested Range not satisfiable",
 871                              "First beyond end of file",
 872                              self.GET, self.public_url + "/foo/bar.txt",
 873                              headers=headers)
 874         return d
 875
 876     def test_HEAD_FILEURL_range(self):
 877         headers = {"range": "bytes=1-10"}
 878         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 879                      return_response=True)
 880         def _got((res, status, headers)):
 881             self.failUnlessReallyEqual(res, "")
 882             self.failUnlessReallyEqual(int(status), 206)
 883             self.failUnless(headers.has_key("content-range"))
 884             self.failUnlessReallyEqual(headers["content-range"][0],
 885                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 886         d.addCallback(_got)
 887         return d
 888
 889     def test_HEAD_FILEURL_partial_range(self):
 890         headers = {"range": "bytes=5-"}
 891         length  = len(self.BAR_CONTENTS)
 892         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 893                      return_response=True)
 894         def _got((res, status, headers)):
 895             self.failUnlessReallyEqual(int(status), 206)
 896             self.failUnless(headers.has_key("content-range"))
 897             self.failUnlessReallyEqual(headers["content-range"][0],
 898                                        "bytes 5-%d/%d" % (length-1, length))
 899         d.addCallback(_got)
 900         return d
 901
 902     def test_HEAD_FILEURL_partial_end_range(self):
 903         headers = {"range": "bytes=-5"}
 904         length  = len(self.BAR_CONTENTS)
 905         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 906                      return_response=True)
 907         def _got((res, status, headers)):
 908             self.failUnlessReallyEqual(int(status), 206)
 909             self.failUnless(headers.has_key("content-range"))
 910             self.failUnlessReallyEqual(headers["content-range"][0],
 911                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 912         d.addCallback(_got)
 913         return d
 914
 915     def test_HEAD_FILEURL_partial_range_overrun(self):
 916         headers = {"range": "bytes=100-200"}
 917         d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
 918                              "416 Requested Range not satisfiable",
 919                              "",
 920                              self.HEAD, self.public_url + "/foo/bar.txt",
 921                              headers=headers)
 922         return d
 923
 924     def test_GET_FILEURL_range_bad(self):
 925         headers = {"range": "BOGUS=fizbop-quarnak"}
 926         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 927                      return_response=True)
 928         def _got((res, status, headers)):
 929             self.failUnlessReallyEqual(int(status), 200)
 930             self.failUnless(not headers.has_key("content-range"))
 931             self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
 932         d.addCallback(_got)
 933         return d
 934
 935     def test_HEAD_FILEURL(self):
 936         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 937         def _got((res, status, headers)):
 938             self.failUnlessReallyEqual(res, "")
 939             self.failUnlessReallyEqual(headers["content-length"][0],
 940                                        str(len(self.BAR_CONTENTS)))
 941             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
 942         d.addCallback(_got)
 943         return d
 944
 945     def test_GET_FILEURL_named(self):
 946         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 947         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 948         d = self.GET(base + "/@@name=/blah.txt")
 949         d.addCallback(self.failUnlessIsBarDotTxt)
 950         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 951         d.addCallback(self.failUnlessIsBarDotTxt)
 952         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 953         d.addCallback(self.failUnlessIsBarDotTxt)
 954         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 955         d.addCallback(self.failUnlessIsBarDotTxt)
 956         save_url = base + "?save=true&filename=blah.txt"
 957         d.addCallback(lambda res: self.GET(save_url))
 958         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 959         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 960         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 961         u_url = base + "?save=true&filename=" + u_fn_e
 962         d.addCallback(lambda res: self.GET(u_url))
 963         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 964         return d
 965
 966     def test_PUT_FILEURL_named_bad(self):
 967         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 968         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 969                              "400 Bad Request",
 970                              "/file can only be used with GET or HEAD",
 971                              self.PUT, base + "/@@name=/blah.txt", "")
 972         return d
 973
 974
 975     def test_GET_DIRURL_named_bad(self):
 976         base = "/file/%s" % urllib.quote(self._foo_uri)
 977         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 978                              "400 Bad Request",
 979                              "is not a file-cap",
 980                              self.GET, base + "/@@name=/blah.txt")
 981         return d
 982
 983     def test_GET_slash_file_bad(self):
 984         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 985                              "404 Not Found",
 986                              "/file must be followed by a file-cap and a name",
 987                              self.GET, "/file")
 988         return d
 989
 990     def test_GET_unhandled_URI_named(self):
 991         contents, n, newuri = self.makefile(12)
 992         verifier_cap = n.get_verify_cap().to_string()
 993         base = "/file/%s" % urllib.quote(verifier_cap)
 994         # client.create_node_from_uri() can't handle verify-caps
 995         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 996                              "400 Bad Request", "is not a file-cap",
 997                              self.GET, base)
 998         return d
 999
1000     def test_GET_unhandled_URI(self):
1001         contents, n, newuri = self.makefile(12)
1002         verifier_cap = n.get_verify_cap().to_string()
1003         base = "/uri/%s" % urllib.quote(verifier_cap)
1004         # client.create_node_from_uri() can't handle verify-caps
1005         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
1006                              "400 Bad Request",
1007                              "GET unknown URI type: can only do t=info",
1008                              self.GET, base)
1009         return d
1010
1011     def test_GET_FILE_URI(self):
1012         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
1013         d = self.GET(base)
1014         d.addCallback(self.failUnlessIsBarDotTxt)
1015         return d
1016
1017     def test_GET_FILE_URI_mdmf(self):
1018         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
1019         d = self.GET(base)
1020         d.addCallback(self.failUnlessIsQuuxDotTxt)
1021         return d
1022
1023     def test_GET_FILE_URI_mdmf_extensions(self):
1024         base = "/uri/%s" % urllib.quote("%s:RANDOMSTUFF" % self._quux_txt_uri)
1025         d = self.GET(base)
1026         d.addCallback(self.failUnlessIsQuuxDotTxt)
1027         return d
1028
1029     def test_GET_FILE_URI_mdmf_readonly(self):
1030         base = "/uri/%s" % urllib.quote(self._quux_txt_readonly_uri)
1031         d = self.GET(base)
1032         d.addCallback(self.failUnlessIsQuuxDotTxt)
1033         return d
1034
1035     def test_GET_FILE_URI_badchild(self):
1036         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
1037         errmsg = "Files have no children, certainly not named 'boguschild'"
1038         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
1039                              "400 Bad Request", errmsg,
1040                              self.GET, base)
1041         return d
1042
1043     def test_PUT_FILE_URI_badchild(self):
1044         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
1045         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
1046         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
1047                              "400 Bad Request", errmsg,
1048                              self.PUT, base, "")
1049         return d
1050
1051     def test_PUT_FILE_URI_mdmf(self):
1052         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
1053         self._quux_new_contents = "new_contents"
1054         d = self.GET(base)
1055         d.addCallback(lambda res:
1056             self.failUnlessIsQuuxDotTxt(res))
1057         d.addCallback(lambda ignored:
1058             self.PUT(base, self._quux_new_contents))
1059         d.addCallback(lambda ignored:
1060             self.GET(base))
1061         d.addCallback(lambda res:
1062             self.failUnlessReallyEqual(res, self._quux_new_contents))
1063         return d
1064
1065     def test_PUT_FILE_URI_mdmf_extensions(self):
1066         base = "/uri/%s" % urllib.quote("%s:EXTENSIONSTUFF" % self._quux_txt_uri)
1067         self._quux_new_contents = "new_contents"
1068         d = self.GET(base)
1069         d.addCallback(lambda res: self.failUnlessIsQuuxDotTxt(res))
1070         d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
1071         d.addCallback(lambda ignored: self.GET(base))
1072         d.addCallback(lambda res: self.failUnlessEqual(self._quux_new_contents,
1073                                                        res))
1074         return d
1075
1076     def test_PUT_FILE_URI_mdmf_readonly(self):
1077         # We're not allowed to PUT things to a readonly cap.
1078         base = "/uri/%s" % self._quux_txt_readonly_uri
1079         d = self.GET(base)
1080         d.addCallback(lambda res:
1081             self.failUnlessIsQuuxDotTxt(res))
1082         # What should we get here? We get a 500 error now; that's not right.
1083         d.addCallback(lambda ignored:
1084             self.shouldFail2(error.Error, "test_PUT_FILE_URI_mdmf_readonly",
1085                              "400 Bad Request", "read-only cap",
1086                              self.PUT, base, "new data"))
1087         return d
1088
1089     def test_PUT_FILE_URI_sdmf_readonly(self):
1090         # We're not allowed to put things to a readonly cap.
1091         base = "/uri/%s" % self._baz_txt_readonly_uri
1092         d = self.GET(base)
1093         d.addCallback(lambda res:
1094             self.failUnlessIsBazDotTxt(res))
1095         d.addCallback(lambda ignored:
1096             self.shouldFail2(error.Error, "test_PUT_FILE_URI_sdmf_readonly",
1097                              "400 Bad Request", "read-only cap",
1098                              self.PUT, base, "new_data"))
1099         return d
1100
1101     def test_GET_etags(self):
1102
1103         def _check_etags(uri):
1104             d1 = _get_etag(uri)
1105             d2 = _get_etag(uri, 'json')
1106             d = defer.DeferredList([d1, d2], consumeErrors=True)
1107             def _check(results):
1108                 # All deferred must succeed
1109                 self.failUnless(all([r[0] for r in results]))
1110                 # the etag for the t=json form should be just like the etag
1111                 # fo the default t='' form, but with a 'json' suffix
1112                 self.failUnlessEqual(results[0][1] + 'json', results[1][1])
1113             d.addCallback(_check)
1114             return d
1115
1116         def _get_etag(uri, t=''):
1117             targetbase = "/uri/%s?t=%s" % (urllib.quote(uri.strip()), t)
1118             d = self.GET(targetbase, return_response=True, followRedirect=True)
1119             def _just_the_etag(result):
1120                 data, response, headers = result
1121                 etag = headers['etag'][0]
1122                 if uri.startswith('URI:DIR'):
1123                     self.failUnless(etag.startswith('DIR:'), etag)
1124                 return etag
1125             return d.addCallback(_just_the_etag)
1126
1127         # Check that etags work with immutable directories
1128         (newkids, caps) = self._create_immutable_children()
1129         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1130                       simplejson.dumps(newkids))
1131         def _stash_immdir_uri(uri):
1132             self._immdir_uri = uri
1133             return uri
1134         d.addCallback(_stash_immdir_uri)
1135         d.addCallback(_check_etags)
1136
1137         # Check that etags work with immutable files
1138         d.addCallback(lambda _: _check_etags(self._bar_txt_uri))
1139
1140         # use the ETag on GET
1141         def _check_match(ign):
1142             uri = "/uri/%s" % self._bar_txt_uri
1143             d = self.GET(uri, return_response=True)
1144             # extract the ETag
1145             d.addCallback(lambda (data, code, headers):
1146                           headers['etag'][0])
1147             # do a GET that's supposed to match the ETag
1148             d.addCallback(lambda etag:
1149                           self.GET(uri, return_response=True,
1150                                    headers={"If-None-Match": etag}))
1151             # make sure it short-circuited (304 instead of 200)
1152             d.addCallback(lambda (data, code, headers):
1153                           self.failUnlessEqual(int(code), http.NOT_MODIFIED))
1154             return d
1155         d.addCallback(_check_match)
1156
1157         def _no_etag(uri, t):
1158             target = "/uri/%s?t=%s" % (uri, t)
1159             d = self.GET(target, return_response=True, followRedirect=True)
1160             d.addCallback(lambda (data, code, headers):
1161                           self.failIf("etag" in headers, target))
1162             return d
1163         def _yes_etag(uri, t):
1164             target = "/uri/%s?t=%s" % (uri, t)
1165             d = self.GET(target, return_response=True, followRedirect=True)
1166             d.addCallback(lambda (data, code, headers):
1167                           self.failUnless("etag" in headers, target))
1168             return d
1169
1170         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, ""))
1171         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "json"))
1172         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "uri"))
1173         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "readonly-uri"))
1174         d.addCallback(lambda ign: _no_etag(self._bar_txt_uri, "info"))
1175
1176         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, ""))
1177         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "json"))
1178         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "uri"))
1179         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "readonly-uri"))
1180         d.addCallback(lambda ign: _no_etag(self._immdir_uri, "info"))
1181         d.addCallback(lambda ign: _no_etag(self._immdir_uri, "rename-form"))
1182
1183         return d
1184
1185     # TODO: version of this with a Unicode filename
1186     def test_GET_FILEURL_save(self):
1187         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
1188                      return_response=True)
1189         def _got((res, statuscode, headers)):
1190             content_disposition = headers["content-disposition"][0]
1191             self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
1192             self.failUnlessIsBarDotTxt(res)
1193         d.addCallback(_got)
1194         return d
1195
1196     def test_GET_FILEURL_missing(self):
1197         d = self.GET(self.public_url + "/foo/missing")
1198         d.addBoth(self.should404, "test_GET_FILEURL_missing")
1199         return d
1200
1201     def test_GET_FILEURL_info_mdmf(self):
1202         d = self.GET("/uri/%s?t=info" % self._quux_txt_uri)
1203         def _got(res):
1204             self.failUnlessIn("mutable file (mdmf)", res)
1205             self.failUnlessIn(self._quux_txt_uri, res)
1206             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1207         d.addCallback(_got)
1208         return d
1209
1210     def test_GET_FILEURL_info_mdmf_readonly(self):
1211         d = self.GET("/uri/%s?t=info" % self._quux_txt_readonly_uri)
1212         def _got(res):
1213             self.failUnlessIn("mutable file (mdmf)", res)
1214             self.failIfIn(self._quux_txt_uri, res)
1215             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1216         d.addCallback(_got)
1217         return d
1218
1219     def test_GET_FILEURL_info_sdmf(self):
1220         d = self.GET("/uri/%s?t=info" % self._baz_txt_uri)
1221         def _got(res):
1222             self.failUnlessIn("mutable file (sdmf)", res)
1223             self.failUnlessIn(self._baz_txt_uri, res)
1224         d.addCallback(_got)
1225         return d
1226
1227     def test_GET_FILEURL_info_mdmf_extensions(self):
1228         d = self.GET("/uri/%s:STUFF?t=info" % self._quux_txt_uri)
1229         def _got(res):
1230             self.failUnlessIn("mutable file (mdmf)", res)
1231             self.failUnlessIn(self._quux_txt_uri, res)
1232             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1233         d.addCallback(_got)
1234         return d
1235
1236     def test_PUT_overwrite_only_files(self):
1237         # create a directory, put a file in that directory.
1238         contents, n, filecap = self.makefile(8)
1239         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
1240         d.addCallback(lambda res:
1241             self.PUT(self.public_url + "/foo/dir/file1.txt",
1242                      self.NEWFILE_CONTENTS))
1243         # try to overwrite the file with replace=only-files
1244         # (this should work)
1245         d.addCallback(lambda res:
1246             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
1247                      filecap))
1248         d.addCallback(lambda res:
1249             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
1250                  "There was already a child by that name, and you asked me "
1251                  "to not replace it",
1252                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
1253                  filecap))
1254         return d
1255
1256     def test_PUT_NEWFILEURL(self):
1257         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
1258         # TODO: we lose the response code, so we can't check this
1259         #self.failUnlessReallyEqual(responsecode, 201)
1260         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1261         d.addCallback(lambda res:
1262                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1263                                                       self.NEWFILE_CONTENTS))
1264         return d
1265
1266     def test_PUT_NEWFILEURL_not_mutable(self):
1267         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
1268                      self.NEWFILE_CONTENTS)
1269         # TODO: we lose the response code, so we can't check this
1270         #self.failUnlessReallyEqual(responsecode, 201)
1271         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1272         d.addCallback(lambda res:
1273                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1274                                                       self.NEWFILE_CONTENTS))
1275         return d
1276
1277     def test_PUT_NEWFILEURL_unlinked_mdmf(self):
1278         # this should get us a few segments of an MDMF mutable file,
1279         # which we can then test for.
1280         contents = self.NEWFILE_CONTENTS * 300000
1281         d = self.PUT("/uri?format=mdmf",
1282                      contents)
1283         def _got_filecap(filecap):
1284             self.failUnless(filecap.startswith("URI:MDMF"))
1285             return filecap
1286         d.addCallback(_got_filecap)
1287         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1288         d.addCallback(lambda json: self.failUnlessIn("MDMF", json))
1289         return d
1290
1291     def test_PUT_NEWFILEURL_unlinked_sdmf(self):
1292         contents = self.NEWFILE_CONTENTS * 300000
1293         d = self.PUT("/uri?format=sdmf",
1294                      contents)
1295         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1296         d.addCallback(lambda json: self.failUnlessIn("SDMF", json))
1297         return d
1298
1299     def test_PUT_NEWFILEURL_unlinked_bad_format(self):
1300         contents = self.NEWFILE_CONTENTS * 300000
1301         return self.shouldHTTPError("PUT_NEWFILEURL_unlinked_bad_format",
1302                                     400, "Bad Request", "Unknown format: foo",
1303                                     self.PUT, "/uri?format=foo",
1304                                     contents)
1305
1306     def test_PUT_NEWFILEURL_range_bad(self):
1307         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
1308         target = self.public_url + "/foo/new.txt"
1309         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
1310                              "501 Not Implemented",
1311                              "Content-Range in PUT not yet supported",
1312                              # (and certainly not for immutable files)
1313                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
1314                              headers=headers)
1315         d.addCallback(lambda res:
1316                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
1317         return d
1318
1319     def test_PUT_NEWFILEURL_mutable(self):
1320         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1321                      self.NEWFILE_CONTENTS)
1322         # TODO: we lose the response code, so we can't check this
1323         #self.failUnlessReallyEqual(responsecode, 201)
1324         def _check_uri(res):
1325             u = uri.from_string_mutable_filenode(res)
1326             self.failUnless(u.is_mutable())
1327             self.failIf(u.is_readonly())
1328             return res
1329         d.addCallback(_check_uri)
1330         d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
1331         d.addCallback(lambda res:
1332                       self.failUnlessMutableChildContentsAre(self._foo_node,
1333                                                              u"new.txt",
1334                                                              self.NEWFILE_CONTENTS))
1335         return d
1336
1337     def test_PUT_NEWFILEURL_mutable_toobig(self):
1338         # It is okay to upload large mutable files, so we should be able
1339         # to do that.
1340         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1341                      "b" * (self.s.MUTABLE_SIZELIMIT + 1))
1342         return d
1343
1344     def test_PUT_NEWFILEURL_replace(self):
1345         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
1346         # TODO: we lose the response code, so we can't check this
1347         #self.failUnlessReallyEqual(responsecode, 200)
1348         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
1349         d.addCallback(lambda res:
1350                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1351                                                       self.NEWFILE_CONTENTS))
1352         return d
1353
1354     def test_PUT_NEWFILEURL_bad_t(self):
1355         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
1356                              "PUT to a file: bad t=bogus",
1357                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
1358                              "contents")
1359         return d
1360
1361     def test_PUT_NEWFILEURL_no_replace(self):
1362         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
1363                      self.NEWFILE_CONTENTS)
1364         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
1365                   "409 Conflict",
1366                   "There was already a child by that name, and you asked me "
1367                   "to not replace it")
1368         return d
1369
1370     def test_PUT_NEWFILEURL_mkdirs(self):
1371         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
1372         fn = self._foo_node
1373         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
1374         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
1375         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
1376         d.addCallback(lambda res:
1377                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
1378                                                       self.NEWFILE_CONTENTS))
1379         return d
1380
1381     def test_PUT_NEWFILEURL_blocked(self):
1382         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
1383                      self.NEWFILE_CONTENTS)
1384         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
1385                   "409 Conflict",
1386                   "Unable to create directory 'blockingfile': a file was in the way")
1387         return d
1388
1389     def test_PUT_NEWFILEURL_emptyname(self):
1390         # an empty pathname component (i.e. a double-slash) is disallowed
1391         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
1392                              "400 Bad Request",
1393                              "The webapi does not allow empty pathname components",
1394                              self.PUT, self.public_url + "/foo//new.txt", "")
1395         return d
1396
1397     def test_DELETE_FILEURL(self):
1398         d = self.DELETE(self.public_url + "/foo/bar.txt")
1399         d.addCallback(lambda res:
1400                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1401         return d
1402
1403     def test_DELETE_FILEURL_missing(self):
1404         d = self.DELETE(self.public_url + "/foo/missing")
1405         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
1406         return d
1407
1408     def test_DELETE_FILEURL_missing2(self):
1409         d = self.DELETE(self.public_url + "/missing/missing")
1410         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
1411         return d
1412
1413     def failUnlessHasBarDotTxtMetadata(self, res):
1414         data = simplejson.loads(res)
1415         self.failUnless(isinstance(data, list))
1416         self.failUnlessIn("metadata", data[1])
1417         self.failUnlessIn("tahoe", data[1]["metadata"])
1418         self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
1419         self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
1420         self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
1421                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
1422
1423     def test_GET_FILEURL_json(self):
1424         # twisted.web.http.parse_qs ignores any query args without an '=', so
1425         # I can't do "GET /path?json", I have to do "GET /path/t=json"
1426         # instead. This may make it tricky to emulate the S3 interface
1427         # completely.
1428         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
1429         def _check1(data):
1430             self.failUnlessIsBarJSON(data)
1431             self.failUnlessHasBarDotTxtMetadata(data)
1432             return
1433         d.addCallback(_check1)
1434         return d
1435
1436     def test_GET_FILEURL_json_mutable_type(self):
1437         # The JSON should include format, which says whether the
1438         # file is SDMF or MDMF
1439         d = self.PUT("/uri?format=mdmf",
1440                      self.NEWFILE_CONTENTS * 300000)
1441         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1442         def _got_json(json, version):
1443             data = simplejson.loads(json)
1444             assert "filenode" == data[0]
1445             data = data[1]
1446             assert isinstance(data, dict)
1447
1448             self.failUnlessIn("format", data)
1449             self.failUnlessEqual(data["format"], version)
1450
1451         d.addCallback(_got_json, "MDMF")
1452         # Now make an SDMF file and check that it is reported correctly.
1453         d.addCallback(lambda ignored:
1454             self.PUT("/uri?format=sdmf",
1455                       self.NEWFILE_CONTENTS * 300000))
1456         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1457         d.addCallback(_got_json, "SDMF")
1458         return d
1459
1460     def test_GET_FILEURL_json_mdmf(self):
1461         d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
1462         d.addCallback(self.failUnlessIsQuuxJSON)
1463         return d
1464
1465     def test_GET_FILEURL_json_missing(self):
1466         d = self.GET(self.public_url + "/foo/missing?json")
1467         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
1468         return d
1469
1470     def test_GET_FILEURL_uri(self):
1471         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
1472         def _check(res):
1473             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1474         d.addCallback(_check)
1475         d.addCallback(lambda res:
1476                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
1477         def _check2(res):
1478             # for now, for files, uris and readonly-uris are the same
1479             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1480         d.addCallback(_check2)
1481         return d
1482
1483     def test_GET_FILEURL_badtype(self):
1484         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
1485                                  "bad t=bogus",
1486                                  self.GET,
1487                                  self.public_url + "/foo/bar.txt?t=bogus")
1488         return d
1489
1490     def test_CSS_FILE(self):
1491         d = self.GET("/tahoe.css", followRedirect=True)
1492         def _check(res):
1493             CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
1494             self.failUnless(CSS_STYLE.search(res), res)
1495         d.addCallback(_check)
1496         return d
1497
1498     def test_GET_FILEURL_uri_missing(self):
1499         d = self.GET(self.public_url + "/foo/missing?t=uri")
1500         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
1501         return d
1502
1503     def _check_upload_and_mkdir_forms(self, html):
1504         # We should have a form to create a file, with radio buttons that allow
1505         # the user to toggle whether it is a CHK/LIT (default), SDMF, or MDMF file.
1506         self.failUnlessIn('name="t" value="upload"', html)
1507         self.failUnless(re.search('<input [^/]*id="upload-chk"', html), html)
1508         self.failUnless(re.search('<input [^/]*id="upload-sdmf"', html), html)
1509         self.failUnless(re.search('<input [^/]*id="upload-mdmf"', html), html)
1510
1511         # We should also have the ability to create a mutable directory, with
1512         # radio buttons that allow the user to toggle whether it is an SDMF (default)
1513         # or MDMF directory.
1514         self.failUnlessIn('name="t" value="mkdir"', html)
1515         self.failUnless(re.search('<input [^/]*id="mkdir-sdmf"', html), html)
1516         self.failUnless(re.search('<input [^/]*id="mkdir-mdmf"', html), html)
1517
1518         self.failUnlessIn(FAVICON_MARKUP, html)
1519
1520     def test_GET_DIRECTORY_html(self):
1521         d = self.GET(self.public_url + "/foo", followRedirect=True)
1522         def _check(html):
1523             self.failUnlessIn('<div class="toolbar-item"><a href="../../..">Return to Welcome page</a></div>', html)
1524             self._check_upload_and_mkdir_forms(html)
1525             self.failUnlessIn("quux", html)
1526         d.addCallback(_check)
1527         return d
1528
1529     def test_GET_DIRECTORY_html_filenode_encoding(self):
1530         d = self.GET(self.public_url + "/foo", followRedirect=True)
1531         def _check(html):
1532             # Check if encoded entries are there
1533             self.failUnlessIn('@@named=/' + self._htmlname_urlencoded + '">'
1534                               + self._htmlname_escaped + '</a>', html)
1535             self.failUnlessIn('value="' + self._htmlname_escaped_attr + '"', html)
1536             self.failIfIn(self._htmlname_escaped_double, html)
1537             # Make sure that Nevow escaping actually works by checking for unsafe characters
1538             # and that '&' is escaped.
1539             for entity in '<>':
1540                 self.failUnlessIn(entity, self._htmlname_raw)
1541                 self.failIfIn(entity, self._htmlname_escaped)
1542             self.failUnlessIn('&', re.sub(r'&(amp|lt|gt|quot|apos);', '', self._htmlname_raw))
1543             self.failIfIn('&', re.sub(r'&(amp|lt|gt|quot|apos);', '', self._htmlname_escaped))
1544         d.addCallback(_check)
1545         return d
1546
1547     def test_GET_root_html(self):
1548         d = self.GET("/")
1549         d.addCallback(self._check_upload_and_mkdir_forms)
1550         return d
1551
1552     def test_GET_DIRURL(self):
1553         # the addSlash means we get a redirect here
1554         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1555         ROOT = "../../.."
1556         d = self.GET(self.public_url + "/foo", followRedirect=True)
1557         def _check(res):
1558             self.failUnlessIn('<a href="%s">Return to Welcome page' % ROOT, res)
1559
1560             # the FILE reference points to a URI, but it should end in bar.txt
1561             bar_url = ("%s/file/%s/@@named=/bar.txt" %
1562                        (ROOT, urllib.quote(self._bar_txt_uri)))
1563             get_bar = "".join([r'<td>FILE</td>',
1564                                r'\s+<td>',
1565                                r'<a href="%s">bar.txt</a>' % bar_url,
1566                                r'</td>',
1567                                r'\s+<td align="right">%d</td>' % len(self.BAR_CONTENTS),
1568                                ])
1569             self.failUnless(re.search(get_bar, res), res)
1570             for label in ['unlink', 'rename/relink']:
1571                 for line in res.split("\n"):
1572                     # find the line that contains the relevant button for bar.txt
1573                     if ("form action" in line and
1574                         ('value="%s"' % (label,)) in line and
1575                         'value="bar.txt"' in line):
1576                         # the form target should use a relative URL
1577                         foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1578                         self.failUnlessIn('action="%s"' % foo_url, line)
1579                         # and the when_done= should too
1580                         #done_url = urllib.quote(???)
1581                         #self.failUnlessIn('name="when_done" value="%s"' % done_url, line)
1582
1583                         # 'unlink' needs to use POST because it directly has a side effect
1584                         if label == 'unlink':
1585                             self.failUnlessIn('method="post"', line)
1586                         break
1587                 else:
1588                     self.fail("unable to find '%s bar.txt' line" % (label,))
1589
1590             # the DIR reference just points to a URI
1591             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1592             get_sub = ((r'<td>DIR</td>')
1593                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1594             self.failUnless(re.search(get_sub, res), res)
1595         d.addCallback(_check)
1596
1597         # look at a readonly directory
1598         d.addCallback(lambda res:
1599                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
1600         def _check2(res):
1601             self.failUnlessIn("(read-only)", res)
1602             self.failIfIn("Upload a file", res)
1603         d.addCallback(_check2)
1604
1605         # and at a directory that contains a readonly directory
1606         d.addCallback(lambda res:
1607                       self.GET(self.public_url, followRedirect=True))
1608         def _check3(res):
1609             self.failUnless(re.search('<td>DIR-RO</td>'
1610                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1611         d.addCallback(_check3)
1612
1613         # and an empty directory
1614         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1615         def _check4(res):
1616             self.failUnlessIn("directory is empty", res)
1617             MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
1618             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1619         d.addCallback(_check4)
1620
1621         # and at a literal directory
1622         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1623         d.addCallback(lambda res:
1624                       self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1625         def _check5(res):
1626             self.failUnlessIn('(immutable)', res)
1627             self.failUnless(re.search('<td>FILE</td>'
1628                                       r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1629         d.addCallback(_check5)
1630         return d
1631
1632     def test_GET_DIRURL_badtype(self):
1633         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1634                                  400, "Bad Request",
1635                                  "bad t=bogus",
1636                                  self.GET,
1637                                  self.public_url + "/foo?t=bogus")
1638         return d
1639
1640     def test_GET_DIRURL_json(self):
1641         d = self.GET(self.public_url + "/foo?t=json")
1642         d.addCallback(self.failUnlessIsFooJSON)
1643         return d
1644
1645     def test_GET_DIRURL_json_format(self):
1646         d = self.PUT(self.public_url + \
1647                      "/foo/sdmf.txt?format=sdmf",
1648                      self.NEWFILE_CONTENTS * 300000)
1649         d.addCallback(lambda ignored:
1650             self.PUT(self.public_url + \
1651                      "/foo/mdmf.txt?format=mdmf",
1652                      self.NEWFILE_CONTENTS * 300000))
1653         # Now we have an MDMF and SDMF file in the directory. If we GET
1654         # its JSON, we should see their encodings.
1655         d.addCallback(lambda ignored:
1656             self.GET(self.public_url + "/foo?t=json"))
1657         def _got_json(json):
1658             data = simplejson.loads(json)
1659             assert data[0] == "dirnode"
1660
1661             data = data[1]
1662             kids = data['children']
1663
1664             mdmf_data = kids['mdmf.txt'][1]
1665             self.failUnlessIn("format", mdmf_data)
1666             self.failUnlessEqual(mdmf_data["format"], "MDMF")
1667
1668             sdmf_data = kids['sdmf.txt'][1]
1669             self.failUnlessIn("format", sdmf_data)
1670             self.failUnlessEqual(sdmf_data["format"], "SDMF")
1671         d.addCallback(_got_json)
1672         return d
1673
1674
1675     def test_POST_DIRURL_manifest_no_ophandle(self):
1676         d = self.shouldFail2(error.Error,
1677                              "test_POST_DIRURL_manifest_no_ophandle",
1678                              "400 Bad Request",
1679                              "slow operation requires ophandle=",
1680                              self.POST, self.public_url, t="start-manifest")
1681         return d
1682
1683     def test_POST_DIRURL_manifest(self):
1684         d = defer.succeed(None)
1685         def getman(ignored, output):
1686             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1687                           followRedirect=True)
1688             d.addCallback(self.wait_for_operation, "125")
1689             d.addCallback(self.get_operation_results, "125", output)
1690             return d
1691         d.addCallback(getman, None)
1692         def _got_html(manifest):
1693             self.failUnlessIn("Manifest of SI=", manifest)
1694             self.failUnlessIn("<td>sub</td>", manifest)
1695             self.failUnlessIn(self._sub_uri, manifest)
1696             self.failUnlessIn("<td>sub/baz.txt</td>", manifest)
1697             self.failUnlessIn(FAVICON_MARKUP, manifest)
1698         d.addCallback(_got_html)
1699
1700         # both t=status and unadorned GET should be identical
1701         d.addCallback(lambda res: self.GET("/operations/125"))
1702         d.addCallback(_got_html)
1703
1704         d.addCallback(getman, "html")
1705         d.addCallback(_got_html)
1706         d.addCallback(getman, "text")
1707         def _got_text(manifest):
1708             self.failUnlessIn("\nsub " + self._sub_uri + "\n", manifest)
1709             self.failUnlessIn("\nsub/baz.txt URI:CHK:", manifest)
1710         d.addCallback(_got_text)
1711         d.addCallback(getman, "JSON")
1712         def _got_json(res):
1713             data = res["manifest"]
1714             got = {}
1715             for (path_list, cap) in data:
1716                 got[tuple(path_list)] = cap
1717             self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
1718             self.failUnlessIn((u"sub", u"baz.txt"), got)
1719             self.failUnlessIn("finished", res)
1720             self.failUnlessIn("origin", res)
1721             self.failUnlessIn("storage-index", res)
1722             self.failUnlessIn("verifycaps", res)
1723             self.failUnlessIn("stats", res)
1724         d.addCallback(_got_json)
1725         return d
1726
1727     def test_POST_DIRURL_deepsize_no_ophandle(self):
1728         d = self.shouldFail2(error.Error,
1729                              "test_POST_DIRURL_deepsize_no_ophandle",
1730                              "400 Bad Request",
1731                              "slow operation requires ophandle=",
1732                              self.POST, self.public_url, t="start-deep-size")
1733         return d
1734
1735     def test_POST_DIRURL_deepsize(self):
1736         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1737                       followRedirect=True)
1738         d.addCallback(self.wait_for_operation, "126")
1739         d.addCallback(self.get_operation_results, "126", "json")
1740         def _got_json(data):
1741             self.failUnlessReallyEqual(data["finished"], True)
1742             size = data["size"]
1743             self.failUnless(size > 1000)
1744         d.addCallback(_got_json)
1745         d.addCallback(self.get_operation_results, "126", "text")
1746         def _got_text(res):
1747             mo = re.search(r'^size: (\d+)$', res, re.M)
1748             self.failUnless(mo, res)
1749             size = int(mo.group(1))
1750             # with directories, the size varies.
1751             self.failUnless(size > 1000)
1752         d.addCallback(_got_text)
1753         return d
1754
1755     def test_POST_DIRURL_deepstats_no_ophandle(self):
1756         d = self.shouldFail2(error.Error,
1757                              "test_POST_DIRURL_deepstats_no_ophandle",
1758                              "400 Bad Request",
1759                              "slow operation requires ophandle=",
1760                              self.POST, self.public_url, t="start-deep-stats")
1761         return d
1762
1763     def test_POST_DIRURL_deepstats(self):
1764         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1765                       followRedirect=True)
1766         d.addCallback(self.wait_for_operation, "127")
1767         d.addCallback(self.get_operation_results, "127", "json")
1768         def _got_json(stats):
1769             expected = {"count-immutable-files": 4,
1770                         "count-mutable-files": 2,
1771                         "count-literal-files": 0,
1772                         "count-files": 6,
1773                         "count-directories": 3,
1774                         "size-immutable-files": 76,
1775                         "size-literal-files": 0,
1776                         #"size-directories": 1912, # varies
1777                         #"largest-directory": 1590,
1778                         "largest-directory-children": 8,
1779                         "largest-immutable-file": 19,
1780                         }
1781             for k,v in expected.iteritems():
1782                 self.failUnlessReallyEqual(stats[k], v,
1783                                            "stats[%s] was %s, not %s" %
1784                                            (k, stats[k], v))
1785             self.failUnlessReallyEqual(stats["size-files-histogram"],
1786                                        [ [11, 31, 4] ])
1787         d.addCallback(_got_json)
1788         return d
1789
1790     def test_POST_DIRURL_stream_manifest(self):
1791         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1792         def _check(res):
1793             self.failUnless(res.endswith("\n"))
1794             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1795             self.failUnlessReallyEqual(len(units), 10)
1796             self.failUnlessEqual(units[-1]["type"], "stats")
1797             first = units[0]
1798             self.failUnlessEqual(first["path"], [])
1799             self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri)
1800             self.failUnlessEqual(first["type"], "directory")
1801             baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0]
1802             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1803             self.failIfEqual(baz["storage-index"], None)
1804             self.failIfEqual(baz["verifycap"], None)
1805             self.failIfEqual(baz["repaircap"], None)
1806             # XXX: Add quux and baz to this test.
1807             return
1808         d.addCallback(_check)
1809         return d
1810
1811     def test_GET_DIRURL_uri(self):
1812         d = self.GET(self.public_url + "/foo?t=uri")
1813         def _check(res):
1814             self.failUnlessReallyEqual(to_str(res), self._foo_uri)
1815         d.addCallback(_check)
1816         return d
1817
1818     def test_GET_DIRURL_readonly_uri(self):
1819         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1820         def _check(res):
1821             self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri)
1822         d.addCallback(_check)
1823         return d
1824
1825     def test_PUT_NEWDIRURL(self):
1826         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1827         d.addCallback(lambda res:
1828                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1829         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1830         d.addCallback(self.failUnlessNodeKeysAre, [])
1831         return d
1832
1833     def test_PUT_NEWDIRURL_mdmf(self):
1834         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1835         d.addCallback(lambda res:
1836                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1837         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1838         d.addCallback(lambda node:
1839             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1840         return d
1841
1842     def test_PUT_NEWDIRURL_sdmf(self):
1843         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=sdmf",
1844                      "")
1845         d.addCallback(lambda res:
1846                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1847         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1848         d.addCallback(lambda node:
1849             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1850         return d
1851
1852     def test_PUT_NEWDIRURL_bad_format(self):
1853         return self.shouldHTTPError("PUT_NEWDIRURL_bad_format",
1854                                     400, "Bad Request", "Unknown format: foo",
1855                                     self.PUT, self.public_url +
1856                                     "/foo/newdir=?t=mkdir&format=foo", "")
1857
1858     def test_POST_NEWDIRURL(self):
1859         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1860         d.addCallback(lambda res:
1861                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1862         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1863         d.addCallback(self.failUnlessNodeKeysAre, [])
1864         return d
1865
1866     def test_POST_NEWDIRURL_mdmf(self):
1867         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1868         d.addCallback(lambda res:
1869                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1870         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1871         d.addCallback(lambda node:
1872             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1873         return d
1874
1875     def test_POST_NEWDIRURL_sdmf(self):
1876         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=sdmf", "")
1877         d.addCallback(lambda res:
1878             self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1879         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1880         d.addCallback(lambda node:
1881             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1882         return d
1883
1884     def test_POST_NEWDIRURL_bad_format(self):
1885         return self.shouldHTTPError("POST_NEWDIRURL_bad_format",
1886                                     400, "Bad Request", "Unknown format: foo",
1887                                     self.POST2, self.public_url + \
1888                                     "/foo/newdir?t=mkdir&format=foo", "")
1889
1890     def test_POST_NEWDIRURL_emptyname(self):
1891         # an empty pathname component (i.e. a double-slash) is disallowed
1892         d = self.shouldFail2(error.Error, "POST_NEWDIRURL_emptyname",
1893                              "400 Bad Request",
1894                              "The webapi does not allow empty pathname components, i.e. a double slash",
1895                              self.POST, self.public_url + "//?t=mkdir")
1896         return d
1897
1898     def _do_POST_NEWDIRURL_initial_children_test(self, version=None):
1899         (newkids, caps) = self._create_initial_children()
1900         query = "/foo/newdir?t=mkdir-with-children"
1901         if version == MDMF_VERSION:
1902             query += "&format=mdmf"
1903         elif version == SDMF_VERSION:
1904             query += "&format=sdmf"
1905         else:
1906             version = SDMF_VERSION # for later
1907         d = self.POST2(self.public_url + query,
1908                        simplejson.dumps(newkids))
1909         def _check(uri):
1910             n = self.s.create_node_from_uri(uri.strip())
1911             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1912             self.failUnlessEqual(n._node.get_version(), version)
1913             d2.addCallback(lambda ign:
1914                            self.failUnlessROChildURIIs(n, u"child-imm",
1915                                                        caps['filecap1']))
1916             d2.addCallback(lambda ign:
1917                            self.failUnlessRWChildURIIs(n, u"child-mutable",
1918                                                        caps['filecap2']))
1919             d2.addCallback(lambda ign:
1920                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1921                                                        caps['filecap3']))
1922             d2.addCallback(lambda ign:
1923                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1924                                                        caps['unknown_rocap']))
1925             d2.addCallback(lambda ign:
1926                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1927                                                        caps['unknown_rwcap']))
1928             d2.addCallback(lambda ign:
1929                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1930                                                        caps['unknown_immcap']))
1931             d2.addCallback(lambda ign:
1932                            self.failUnlessRWChildURIIs(n, u"dirchild",
1933                                                        caps['dircap']))
1934             d2.addCallback(lambda ign:
1935                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1936                                                        caps['litdircap']))
1937             d2.addCallback(lambda ign:
1938                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1939                                                        caps['emptydircap']))
1940             return d2
1941         d.addCallback(_check)
1942         d.addCallback(lambda res:
1943                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1944         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1945         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1946         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1947         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1948         return d
1949
1950     def test_POST_NEWDIRURL_initial_children(self):
1951         return self._do_POST_NEWDIRURL_initial_children_test()
1952
1953     def test_POST_NEWDIRURL_initial_children_mdmf(self):
1954         return self._do_POST_NEWDIRURL_initial_children_test(MDMF_VERSION)
1955
1956     def test_POST_NEWDIRURL_initial_children_sdmf(self):
1957         return self._do_POST_NEWDIRURL_initial_children_test(SDMF_VERSION)
1958
1959     def test_POST_NEWDIRURL_initial_children_bad_format(self):
1960         (newkids, caps) = self._create_initial_children()
1961         return self.shouldHTTPError("POST_NEWDIRURL_initial_children_bad_format",
1962                                     400, "Bad Request", "Unknown format: foo",
1963                                     self.POST2, self.public_url + \
1964                                     "/foo/newdir?t=mkdir-with-children&format=foo",
1965                                     simplejson.dumps(newkids))
1966
1967     def test_POST_NEWDIRURL_immutable(self):
1968         (newkids, caps) = self._create_immutable_children()
1969         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1970                        simplejson.dumps(newkids))
1971         def _check(uri):
1972             n = self.s.create_node_from_uri(uri.strip())
1973             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1974             d2.addCallback(lambda ign:
1975                            self.failUnlessROChildURIIs(n, u"child-imm",
1976                                                        caps['filecap1']))
1977             d2.addCallback(lambda ign:
1978                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1979                                                        caps['unknown_immcap']))
1980             d2.addCallback(lambda ign:
1981                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
1982                                                        caps['immdircap']))
1983             d2.addCallback(lambda ign:
1984                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1985                                                        caps['litdircap']))
1986             d2.addCallback(lambda ign:
1987                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1988                                                        caps['emptydircap']))
1989             return d2
1990         d.addCallback(_check)
1991         d.addCallback(lambda res:
1992                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1993         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1994         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1995         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1996         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1997         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1998         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1999         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2000         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2001         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2002         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2003         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2004         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2005         d.addErrback(self.explain_web_error)
2006         return d
2007
2008     def test_POST_NEWDIRURL_immutable_bad(self):
2009         (newkids, caps) = self._create_initial_children()
2010         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
2011                              "400 Bad Request",
2012                              "needed to be immutable but was not",
2013                              self.POST2,
2014                              self.public_url + "/foo/newdir?t=mkdir-immutable",
2015                              simplejson.dumps(newkids))
2016         return d
2017
2018     def test_PUT_NEWDIRURL_exists(self):
2019         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
2020         d.addCallback(lambda res:
2021                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
2022         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2023         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2024         return d
2025
2026     def test_PUT_NEWDIRURL_blocked(self):
2027         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
2028                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
2029                              self.PUT,
2030                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
2031         d.addCallback(lambda res:
2032                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
2033         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2034         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2035         return d
2036
2037     def test_PUT_NEWDIRURL_mkdirs(self):
2038         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
2039         d.addCallback(lambda res:
2040                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
2041         d.addCallback(lambda res:
2042                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
2043         d.addCallback(lambda res:
2044                       self._foo_node.get_child_at_path(u"subdir/newdir"))
2045         d.addCallback(self.failUnlessNodeKeysAre, [])
2046         return d
2047
2048     def test_PUT_NEWDIRURL_mkdirs_mdmf(self):
2049         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=mdmf", "")
2050         d.addCallback(lambda ignored:
2051             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
2052         d.addCallback(lambda ignored:
2053             self.failIfNodeHasChild(self._foo_node, u"newdir"))
2054         d.addCallback(lambda ignored:
2055             self._foo_node.get_child_at_path(u"subdir"))
2056         def _got_subdir(subdir):
2057             # XXX: What we want?
2058             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
2059             self.failUnlessNodeHasChild(subdir, u"newdir")
2060             return subdir.get_child_at_path(u"newdir")
2061         d.addCallback(_got_subdir)
2062         d.addCallback(lambda newdir:
2063             self.failUnlessEqual(newdir._node.get_version(), MDMF_VERSION))
2064         return d
2065
2066     def test_PUT_NEWDIRURL_mkdirs_sdmf(self):
2067         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=sdmf", "")
2068         d.addCallback(lambda ignored:
2069             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
2070         d.addCallback(lambda ignored:
2071             self.failIfNodeHasChild(self._foo_node, u"newdir"))
2072         d.addCallback(lambda ignored:
2073             self._foo_node.get_child_at_path(u"subdir"))
2074         def _got_subdir(subdir):
2075             # XXX: What we want?
2076             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
2077             self.failUnlessNodeHasChild(subdir, u"newdir")
2078             return subdir.get_child_at_path(u"newdir")
2079         d.addCallback(_got_subdir)
2080         d.addCallback(lambda newdir:
2081             self.failUnlessEqual(newdir._node.get_version(), SDMF_VERSION))
2082         return d
2083
2084     def test_PUT_NEWDIRURL_mkdirs_bad_format(self):
2085         return self.shouldHTTPError("PUT_NEWDIRURL_mkdirs_bad_format",
2086                                     400, "Bad Request", "Unknown format: foo",
2087                                     self.PUT, self.public_url + \
2088                                     "/foo/subdir/newdir?t=mkdir&format=foo",
2089                                     "")
2090
2091     def test_DELETE_DIRURL(self):
2092         d = self.DELETE(self.public_url + "/foo")
2093         d.addCallback(lambda res:
2094                       self.failIfNodeHasChild(self.public_root, u"foo"))
2095         return d
2096
2097     def test_DELETE_DIRURL_missing(self):
2098         d = self.DELETE(self.public_url + "/foo/missing")
2099         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
2100         d.addCallback(lambda res:
2101                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
2102         return d
2103
2104     def test_DELETE_DIRURL_missing2(self):
2105         d = self.DELETE(self.public_url + "/missing")
2106         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
2107         return d
2108
2109     def dump_root(self):
2110         print "NODEWALK"
2111         w = webish.DirnodeWalkerMixin()
2112         def visitor(childpath, childnode, metadata):
2113             print childpath
2114         d = w.walk(self.public_root, visitor)
2115         return d
2116
2117     def failUnlessNodeKeysAre(self, node, expected_keys):
2118         for k in expected_keys:
2119             assert isinstance(k, unicode)
2120         d = node.list()
2121         def _check(children):
2122             self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
2123         d.addCallback(_check)
2124         return d
2125     def failUnlessNodeHasChild(self, node, name):
2126         assert isinstance(name, unicode)
2127         d = node.list()
2128         def _check(children):
2129             self.failUnlessIn(name, children)
2130         d.addCallback(_check)
2131         return d
2132     def failIfNodeHasChild(self, node, name):
2133         assert isinstance(name, unicode)
2134         d = node.list()
2135         def _check(children):
2136             self.failIfIn(name, children)
2137         d.addCallback(_check)
2138         return d
2139
2140     def failUnlessChildContentsAre(self, node, name, expected_contents):
2141         assert isinstance(name, unicode)
2142         d = node.get_child_at_path(name)
2143         d.addCallback(lambda node: download_to_data(node))
2144         def _check(contents):
2145             self.failUnlessReallyEqual(contents, expected_contents)
2146         d.addCallback(_check)
2147         return d
2148
2149     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
2150         assert isinstance(name, unicode)
2151         d = node.get_child_at_path(name)
2152         d.addCallback(lambda node: node.download_best_version())
2153         def _check(contents):
2154             self.failUnlessReallyEqual(contents, expected_contents)
2155         d.addCallback(_check)
2156         return d
2157
2158     def failUnlessRWChildURIIs(self, node, name, expected_uri):
2159         assert isinstance(name, unicode)
2160         d = node.get_child_at_path(name)
2161         def _check(child):
2162             self.failUnless(child.is_unknown() or not child.is_readonly())
2163             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
2164             self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
2165             expected_ro_uri = self._make_readonly(expected_uri)
2166             if expected_ro_uri:
2167                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
2168         d.addCallback(_check)
2169         return d
2170
2171     def failUnlessROChildURIIs(self, node, name, expected_uri):
2172         assert isinstance(name, unicode)
2173         d = node.get_child_at_path(name)
2174         def _check(child):
2175             self.failUnless(child.is_unknown() or child.is_readonly())
2176             self.failUnlessReallyEqual(child.get_write_uri(), None)
2177             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
2178             self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
2179         d.addCallback(_check)
2180         return d
2181
2182     def failUnlessURIMatchesRWChild(self, got_uri, node, name):
2183         assert isinstance(name, unicode)
2184         d = node.get_child_at_path(name)
2185         def _check(child):
2186             self.failUnless(child.is_unknown() or not child.is_readonly())
2187             self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
2188             self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
2189             expected_ro_uri = self._make_readonly(got_uri)
2190             if expected_ro_uri:
2191                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
2192         d.addCallback(_check)
2193         return d
2194
2195     def failUnlessURIMatchesROChild(self, got_uri, node, name):
2196         assert isinstance(name, unicode)
2197         d = node.get_child_at_path(name)
2198         def _check(child):
2199             self.failUnless(child.is_unknown() or child.is_readonly())
2200             self.failUnlessReallyEqual(child.get_write_uri(), None)
2201             self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
2202             self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
2203         d.addCallback(_check)
2204         return d
2205
2206     def failUnlessCHKURIHasContents(self, got_uri, contents):
2207         self.failUnless(self.get_all_contents()[got_uri] == contents)
2208
2209     def test_POST_upload(self):
2210         d = self.POST(self.public_url + "/foo", t="upload",
2211                       file=("new.txt", self.NEWFILE_CONTENTS))
2212         fn = self._foo_node
2213         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2214         d.addCallback(lambda res:
2215                       self.failUnlessChildContentsAre(fn, u"new.txt",
2216                                                       self.NEWFILE_CONTENTS))
2217         return d
2218
2219     def test_POST_upload_unicode(self):
2220         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2221         d = self.POST(self.public_url + "/foo", t="upload",
2222                       file=(filename, self.NEWFILE_CONTENTS))
2223         fn = self._foo_node
2224         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2225         d.addCallback(lambda res:
2226                       self.failUnlessChildContentsAre(fn, filename,
2227                                                       self.NEWFILE_CONTENTS))
2228         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2229         d.addCallback(lambda res: self.GET(target_url))
2230         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2231                                                                   self.NEWFILE_CONTENTS,
2232                                                                   contents))
2233         return d
2234
2235     def test_POST_upload_unicode_named(self):
2236         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2237         d = self.POST(self.public_url + "/foo", t="upload",
2238                       name=filename,
2239                       file=("overridden", self.NEWFILE_CONTENTS))
2240         fn = self._foo_node
2241         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2242         d.addCallback(lambda res:
2243                       self.failUnlessChildContentsAre(fn, filename,
2244                                                       self.NEWFILE_CONTENTS))
2245         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2246         d.addCallback(lambda res: self.GET(target_url))
2247         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2248                                                                   self.NEWFILE_CONTENTS,
2249                                                                   contents))
2250         return d
2251
2252     def test_POST_upload_no_link(self):
2253         d = self.POST("/uri", t="upload",
2254                       file=("new.txt", self.NEWFILE_CONTENTS))
2255         def _check_upload_results(page):
2256             # this should be a page which describes the results of the upload
2257             # that just finished.
2258             self.failUnlessIn("Upload Results:", page)
2259             self.failUnlessIn("URI:", page)
2260             uri_re = re.compile("URI: <tt><span>(.*)</span>")
2261             mo = uri_re.search(page)
2262             self.failUnless(mo, page)
2263             new_uri = mo.group(1)
2264             return new_uri
2265         d.addCallback(_check_upload_results)
2266         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
2267         return d
2268
2269     def test_POST_upload_no_link_whendone(self):
2270         d = self.POST("/uri", t="upload", when_done="/",
2271                       file=("new.txt", self.NEWFILE_CONTENTS))
2272         d.addBoth(self.shouldRedirect, "/")
2273         return d
2274
2275     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
2276         d = defer.maybeDeferred(callable, *args, **kwargs)
2277         def done(res):
2278             if isinstance(res, failure.Failure):
2279                 res.trap(error.PageRedirect)
2280                 statuscode = res.value.status
2281                 target = res.value.location
2282                 return checker(statuscode, target)
2283             self.fail("%s: callable was supposed to redirect, not return '%s'"
2284                       % (which, res))
2285         d.addBoth(done)
2286         return d
2287
2288     def test_POST_upload_no_link_whendone_results(self):
2289         def check(statuscode, target):
2290             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2291             self.failUnless(target.startswith(self.webish_url), target)
2292             return client.getPage(target, method="GET")
2293         # We encode "uri" as "%75ri" to exercise a case affected by ticket #1860.
2294         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
2295                                  check,
2296                                  self.POST, "/uri", t="upload",
2297                                  when_done="/%75ri/%(uri)s",
2298                                  file=("new.txt", self.NEWFILE_CONTENTS))
2299         d.addCallback(lambda res:
2300                       self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
2301         return d
2302
2303     def test_POST_upload_no_link_mutable(self):
2304         d = self.POST("/uri", t="upload", mutable="true",
2305                       file=("new.txt", self.NEWFILE_CONTENTS))
2306         def _check(filecap):
2307             filecap = filecap.strip()
2308             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2309             self.filecap = filecap
2310             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2311             self.failUnlessIn(u.get_storage_index(), self.get_all_contents())
2312             n = self.s.create_node_from_uri(filecap)
2313             return n.download_best_version()
2314         d.addCallback(_check)
2315         def _check2(data):
2316             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2317             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2318         d.addCallback(_check2)
2319         def _check3(data):
2320             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2321             return self.GET("/file/%s" % urllib.quote(self.filecap))
2322         d.addCallback(_check3)
2323         def _check4(data):
2324             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2325         d.addCallback(_check4)
2326         return d
2327
2328     def test_POST_upload_no_link_mutable_toobig(self):
2329         # The SDMF size limit is no longer in place, so we should be
2330         # able to upload mutable files that are as large as we want them
2331         # to be.
2332         d = self.POST("/uri", t="upload", mutable="true",
2333                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2334         return d
2335
2336
2337     def test_POST_upload_format_unlinked(self):
2338         def _check_upload_unlinked(ign, format, uri_prefix):
2339             filename = format + ".txt"
2340             d = self.POST("/uri?t=upload&format=" + format,
2341                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2342             def _got_results(results):
2343                 if format.upper() in ("SDMF", "MDMF"):
2344                     # webapi.rst says this returns a filecap
2345                     filecap = results
2346                 else:
2347                     # for immutable, it returns an "upload results page", and
2348                     # the filecap is buried inside
2349                     line = [l for l in results.split("\n") if "URI: " in l][0]
2350                     mo = re.search(r'<span>([^<]+)</span>', line)
2351                     filecap = mo.group(1)
2352                 self.failUnless(filecap.startswith(uri_prefix),
2353                                 (uri_prefix, filecap))
2354                 return self.GET("/uri/%s?t=json" % filecap)
2355             d.addCallback(_got_results)
2356             def _got_json(json):
2357                 data = simplejson.loads(json)
2358                 data = data[1]
2359                 self.failUnlessIn("format", data)
2360                 self.failUnlessEqual(data["format"], format.upper())
2361             d.addCallback(_got_json)
2362             return d
2363         d = defer.succeed(None)
2364         d.addCallback(_check_upload_unlinked, "chk", "URI:CHK")
2365         d.addCallback(_check_upload_unlinked, "CHK", "URI:CHK")
2366         d.addCallback(_check_upload_unlinked, "sdmf", "URI:SSK")
2367         d.addCallback(_check_upload_unlinked, "mdmf", "URI:MDMF")
2368         return d
2369
2370     def test_POST_upload_bad_format_unlinked(self):
2371         return self.shouldHTTPError("POST_upload_bad_format_unlinked",
2372                                     400, "Bad Request", "Unknown format: foo",
2373                                     self.POST,
2374                                     "/uri?t=upload&format=foo",
2375                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2376
2377     def test_POST_upload_format(self):
2378         def _check_upload(ign, format, uri_prefix, fn=None):
2379             filename = format + ".txt"
2380             d = self.POST(self.public_url +
2381                           "/foo?t=upload&format=" + format,
2382                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2383             def _got_filecap(filecap):
2384                 if fn is not None:
2385                     filenameu = unicode(filename)
2386                     self.failUnlessURIMatchesRWChild(filecap, fn, filenameu)
2387                 self.failUnless(filecap.startswith(uri_prefix))
2388                 return self.GET(self.public_url + "/foo/%s?t=json" % filename)
2389             d.addCallback(_got_filecap)
2390             def _got_json(json):
2391                 data = simplejson.loads(json)
2392                 data = data[1]
2393                 self.failUnlessIn("format", data)
2394                 self.failUnlessEqual(data["format"], format.upper())
2395             d.addCallback(_got_json)
2396             return d
2397
2398         d = defer.succeed(None)
2399         d.addCallback(_check_upload, "chk", "URI:CHK")
2400         d.addCallback(_check_upload, "sdmf", "URI:SSK", self._foo_node)
2401         d.addCallback(_check_upload, "mdmf", "URI:MDMF")
2402         d.addCallback(_check_upload, "MDMF", "URI:MDMF")
2403         return d
2404
2405     def test_POST_upload_bad_format(self):
2406         return self.shouldHTTPError("POST_upload_bad_format",
2407                                     400, "Bad Request", "Unknown format: foo",
2408                                     self.POST, self.public_url + \
2409                                     "/foo?t=upload&format=foo",
2410                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2411
2412     def test_POST_upload_mutable(self):
2413         # this creates a mutable file
2414         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
2415                       file=("new.txt", self.NEWFILE_CONTENTS))
2416         fn = self._foo_node
2417         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2418         d.addCallback(lambda res:
2419                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2420                                                              self.NEWFILE_CONTENTS))
2421         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2422         def _got(newnode):
2423             self.failUnless(IMutableFileNode.providedBy(newnode))
2424             self.failUnless(newnode.is_mutable())
2425             self.failIf(newnode.is_readonly())
2426             self._mutable_node = newnode
2427             self._mutable_uri = newnode.get_uri()
2428         d.addCallback(_got)
2429
2430         # now upload it again and make sure that the URI doesn't change
2431         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
2432         d.addCallback(lambda res:
2433                       self.POST(self.public_url + "/foo", t="upload",
2434                                 mutable="true",
2435                                 file=("new.txt", NEWER_CONTENTS)))
2436         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2437         d.addCallback(lambda res:
2438                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2439                                                              NEWER_CONTENTS))
2440         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2441         def _got2(newnode):
2442             self.failUnless(IMutableFileNode.providedBy(newnode))
2443             self.failUnless(newnode.is_mutable())
2444             self.failIf(newnode.is_readonly())
2445             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2446         d.addCallback(_got2)
2447
2448         # upload a second time, using PUT instead of POST
2449         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
2450         d.addCallback(lambda res:
2451                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
2452         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2453         d.addCallback(lambda res:
2454                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2455                                                              NEW2_CONTENTS))
2456
2457         # finally list the directory, since mutable files are displayed
2458         # slightly differently
2459
2460         d.addCallback(lambda res:
2461                       self.GET(self.public_url + "/foo/",
2462                                followRedirect=True))
2463         def _check_page(res):
2464             # TODO: assert more about the contents
2465             self.failUnlessIn("SSK", res)
2466             return res
2467         d.addCallback(_check_page)
2468
2469         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2470         def _got3(newnode):
2471             self.failUnless(IMutableFileNode.providedBy(newnode))
2472             self.failUnless(newnode.is_mutable())
2473             self.failIf(newnode.is_readonly())
2474             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2475         d.addCallback(_got3)
2476
2477         # look at the JSON form of the enclosing directory
2478         d.addCallback(lambda res:
2479                       self.GET(self.public_url + "/foo/?t=json",
2480                                followRedirect=True))
2481         def _check_page_json(res):
2482             parsed = simplejson.loads(res)
2483             self.failUnlessEqual(parsed[0], "dirnode")
2484             children = dict( [(unicode(name),value)
2485                               for (name,value)
2486                               in parsed[1]["children"].iteritems()] )
2487             self.failUnlessIn(u"new.txt", children)
2488             new_json = children[u"new.txt"]
2489             self.failUnlessEqual(new_json[0], "filenode")
2490             self.failUnless(new_json[1]["mutable"])
2491             self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri)
2492             ro_uri = self._mutable_node.get_readonly().to_string()
2493             self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri)
2494         d.addCallback(_check_page_json)
2495
2496         # and the JSON form of the file
2497         d.addCallback(lambda res:
2498                       self.GET(self.public_url + "/foo/new.txt?t=json"))
2499         def _check_file_json(res):
2500             parsed = simplejson.loads(res)
2501             self.failUnlessEqual(parsed[0], "filenode")
2502             self.failUnless(parsed[1]["mutable"])
2503             self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri)
2504             ro_uri = self._mutable_node.get_readonly().to_string()
2505             self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri)
2506         d.addCallback(_check_file_json)
2507
2508         # and look at t=uri and t=readonly-uri
2509         d.addCallback(lambda res:
2510                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
2511         d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
2512         d.addCallback(lambda res:
2513                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
2514         def _check_ro_uri(res):
2515             ro_uri = self._mutable_node.get_readonly().to_string()
2516             self.failUnlessReallyEqual(res, ro_uri)
2517         d.addCallback(_check_ro_uri)
2518
2519         # make sure we can get to it from /uri/URI
2520         d.addCallback(lambda res:
2521                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
2522         d.addCallback(lambda res:
2523                       self.failUnlessReallyEqual(res, NEW2_CONTENTS))
2524
2525         # and that HEAD computes the size correctly
2526         d.addCallback(lambda res:
2527                       self.HEAD(self.public_url + "/foo/new.txt",
2528                                 return_response=True))
2529         def _got_headers((res, status, headers)):
2530             self.failUnlessReallyEqual(res, "")
2531             self.failUnlessReallyEqual(headers["content-length"][0],
2532                                        str(len(NEW2_CONTENTS)))
2533             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
2534         d.addCallback(_got_headers)
2535
2536         # make sure that outdated size limits aren't enforced anymore.
2537         d.addCallback(lambda ignored:
2538             self.POST(self.public_url + "/foo", t="upload",
2539                       mutable="true",
2540                       file=("new.txt",
2541                             "b" * (self.s.MUTABLE_SIZELIMIT+1))))
2542         d.addErrback(self.dump_error)
2543         return d
2544
2545     def test_POST_upload_mutable_toobig(self):
2546         # SDMF had a size limti that was removed a while ago. MDMF has
2547         # never had a size limit. Test to make sure that we do not
2548         # encounter errors when trying to upload large mutable files,
2549         # since there should be no coded prohibitions regarding large
2550         # mutable files.
2551         d = self.POST(self.public_url + "/foo",
2552                       t="upload", mutable="true",
2553                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2554         return d
2555
2556     def dump_error(self, f):
2557         # if the web server returns an error code (like 400 Bad Request),
2558         # web.client.getPage puts the HTTP response body into the .response
2559         # attribute of the exception object that it gives back. It does not
2560         # appear in the Failure's repr(), so the ERROR that trial displays
2561         # will be rather terse and unhelpful. addErrback this method to the
2562         # end of your chain to get more information out of these errors.
2563         if f.check(error.Error):
2564             print "web.error.Error:"
2565             print f
2566             print f.value.response
2567         return f
2568
2569     def test_POST_upload_replace(self):
2570         d = self.POST(self.public_url + "/foo", t="upload",
2571                       file=("bar.txt", self.NEWFILE_CONTENTS))
2572         fn = self._foo_node
2573         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
2574         d.addCallback(lambda res:
2575                       self.failUnlessChildContentsAre(fn, u"bar.txt",
2576                                                       self.NEWFILE_CONTENTS))
2577         return d
2578
2579     def test_POST_upload_no_replace_ok(self):
2580         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2581                       file=("new.txt", self.NEWFILE_CONTENTS))
2582         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
2583         d.addCallback(lambda res: self.failUnlessReallyEqual(res,
2584                                                              self.NEWFILE_CONTENTS))
2585         return d
2586
2587     def test_POST_upload_no_replace_queryarg(self):
2588         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2589                       file=("bar.txt", self.NEWFILE_CONTENTS))
2590         d.addBoth(self.shouldFail, error.Error,
2591                   "POST_upload_no_replace_queryarg",
2592                   "409 Conflict",
2593                   "There was already a child by that name, and you asked me "
2594                   "to not replace it")
2595         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2596         d.addCallback(self.failUnlessIsBarDotTxt)
2597         return d
2598
2599     def test_POST_upload_no_replace_field(self):
2600         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
2601                       file=("bar.txt", self.NEWFILE_CONTENTS))
2602         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
2603                   "409 Conflict",
2604                   "There was already a child by that name, and you asked me "
2605                   "to not replace it")
2606         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2607         d.addCallback(self.failUnlessIsBarDotTxt)
2608         return d
2609
2610     def test_POST_upload_whendone(self):
2611         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
2612                       file=("new.txt", self.NEWFILE_CONTENTS))
2613         d.addBoth(self.shouldRedirect, "/THERE")
2614         fn = self._foo_node
2615         d.addCallback(lambda res:
2616                       self.failUnlessChildContentsAre(fn, u"new.txt",
2617                                                       self.NEWFILE_CONTENTS))
2618         return d
2619
2620     def test_POST_upload_named(self):
2621         fn = self._foo_node
2622         d = self.POST(self.public_url + "/foo", t="upload",
2623                       name="new.txt", file=self.NEWFILE_CONTENTS)
2624         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2625         d.addCallback(lambda res:
2626                       self.failUnlessChildContentsAre(fn, u"new.txt",
2627                                                       self.NEWFILE_CONTENTS))
2628         return d
2629
2630     def test_POST_upload_named_badfilename(self):
2631         d = self.POST(self.public_url + "/foo", t="upload",
2632                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
2633         d.addBoth(self.shouldFail, error.Error,
2634                   "test_POST_upload_named_badfilename",
2635                   "400 Bad Request",
2636                   "name= may not contain a slash",
2637                   )
2638         # make sure that nothing was added
2639         d.addCallback(lambda res:
2640                       self.failUnlessNodeKeysAre(self._foo_node,
2641                                                  [self._htmlname_unicode,
2642                                                   u"bar.txt", u"baz.txt", u"blockingfile",
2643                                                   u"empty", u"n\u00fc.txt", u"quux.txt",
2644                                                   u"sub"]))
2645         return d
2646
2647     def test_POST_FILEURL_check(self):
2648         bar_url = self.public_url + "/foo/bar.txt"
2649         d = self.POST(bar_url, t="check")
2650         def _check(res):
2651             self.failUnlessIn("Healthy :", res)
2652         d.addCallback(_check)
2653         redir_url = "http://allmydata.org/TARGET"
2654         def _check2(statuscode, target):
2655             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2656             self.failUnlessReallyEqual(target, redir_url)
2657         d.addCallback(lambda res:
2658                       self.shouldRedirect2("test_POST_FILEURL_check",
2659                                            _check2,
2660                                            self.POST, bar_url,
2661                                            t="check",
2662                                            when_done=redir_url))
2663         d.addCallback(lambda res:
2664                       self.POST(bar_url, t="check", return_to=redir_url))
2665         def _check3(res):
2666             self.failUnlessIn("Healthy :", res)
2667             self.failUnlessIn("Return to file", res)
2668             self.failUnlessIn(redir_url, res)
2669         d.addCallback(_check3)
2670
2671         d.addCallback(lambda res:
2672                       self.POST(bar_url, t="check", output="JSON"))
2673         def _check_json(res):
2674             data = simplejson.loads(res)
2675             self.failUnlessIn("storage-index", data)
2676             self.failUnless(data["results"]["healthy"])
2677         d.addCallback(_check_json)
2678
2679         return d
2680
2681     def test_POST_FILEURL_check_and_repair(self):
2682         bar_url = self.public_url + "/foo/bar.txt"
2683         d = self.POST(bar_url, t="check", repair="true")
2684         def _check(res):
2685             self.failUnlessIn("Healthy :", res)
2686         d.addCallback(_check)
2687         redir_url = "http://allmydata.org/TARGET"
2688         def _check2(statuscode, target):
2689             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2690             self.failUnlessReallyEqual(target, redir_url)
2691         d.addCallback(lambda res:
2692                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
2693                                            _check2,
2694                                            self.POST, bar_url,
2695                                            t="check", repair="true",
2696                                            when_done=redir_url))
2697         d.addCallback(lambda res:
2698                       self.POST(bar_url, t="check", return_to=redir_url))
2699         def _check3(res):
2700             self.failUnlessIn("Healthy :", res)
2701             self.failUnlessIn("Return to file", res)
2702             self.failUnlessIn(redir_url, res)
2703         d.addCallback(_check3)
2704         return d
2705
2706     def test_POST_DIRURL_check(self):
2707         foo_url = self.public_url + "/foo/"
2708         d = self.POST(foo_url, t="check")
2709         def _check(res):
2710             self.failUnlessIn("Healthy :", res)
2711         d.addCallback(_check)
2712         redir_url = "http://allmydata.org/TARGET"
2713         def _check2(statuscode, target):
2714             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2715             self.failUnlessReallyEqual(target, redir_url)
2716         d.addCallback(lambda res:
2717                       self.shouldRedirect2("test_POST_DIRURL_check",
2718                                            _check2,
2719                                            self.POST, foo_url,
2720                                            t="check",
2721                                            when_done=redir_url))
2722         d.addCallback(lambda res:
2723                       self.POST(foo_url, t="check", return_to=redir_url))
2724         def _check3(res):
2725             self.failUnlessIn("Healthy :", res)
2726             self.failUnlessIn("Return to file/directory", res)
2727             self.failUnlessIn(redir_url, res)
2728         d.addCallback(_check3)
2729
2730         d.addCallback(lambda res:
2731                       self.POST(foo_url, t="check", output="JSON"))
2732         def _check_json(res):
2733             data = simplejson.loads(res)
2734             self.failUnlessIn("storage-index", data)
2735             self.failUnless(data["results"]["healthy"])
2736         d.addCallback(_check_json)
2737
2738         return d
2739
2740     def test_POST_DIRURL_check_and_repair(self):
2741         foo_url = self.public_url + "/foo/"
2742         d = self.POST(foo_url, t="check", repair="true")
2743         def _check(res):
2744             self.failUnlessIn("Healthy :", res)
2745         d.addCallback(_check)
2746         redir_url = "http://allmydata.org/TARGET"
2747         def _check2(statuscode, target):
2748             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2749             self.failUnlessReallyEqual(target, redir_url)
2750         d.addCallback(lambda res:
2751                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2752                                            _check2,
2753                                            self.POST, foo_url,
2754                                            t="check", repair="true",
2755                                            when_done=redir_url))
2756         d.addCallback(lambda res:
2757                       self.POST(foo_url, t="check", return_to=redir_url))
2758         def _check3(res):
2759             self.failUnlessIn("Healthy :", res)
2760             self.failUnlessIn("Return to file/directory", res)
2761             self.failUnlessIn(redir_url, res)
2762         d.addCallback(_check3)
2763         return d
2764
2765     def test_POST_FILEURL_mdmf_check(self):
2766         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2767         d = self.POST(quux_url, t="check")
2768         def _check(res):
2769             self.failUnlessIn("Healthy", res)
2770         d.addCallback(_check)
2771         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2772         d.addCallback(lambda ignored:
2773                       self.POST(quux_extension_url, t="check"))
2774         d.addCallback(_check)
2775         return d
2776
2777     def test_POST_FILEURL_mdmf_check_and_repair(self):
2778         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2779         d = self.POST(quux_url, t="check", repair="true")
2780         def _check(res):
2781             self.failUnlessIn("Healthy", res)
2782         d.addCallback(_check)
2783         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2784         d.addCallback(lambda ignored:
2785                       self.POST(quux_extension_url, t="check", repair="true"))
2786         d.addCallback(_check)
2787         return d
2788
2789     def wait_for_operation(self, ignored, ophandle):
2790         url = "/operations/" + ophandle
2791         url += "?t=status&output=JSON"
2792         d = self.GET(url)
2793         def _got(res):
2794             data = simplejson.loads(res)
2795             if not data["finished"]:
2796                 d = self.stall(delay=1.0)
2797                 d.addCallback(self.wait_for_operation, ophandle)
2798                 return d
2799             return data
2800         d.addCallback(_got)
2801         return d
2802
2803     def get_operation_results(self, ignored, ophandle, output=None):
2804         url = "/operations/" + ophandle
2805         url += "?t=status"
2806         if output:
2807             url += "&output=" + output
2808         d = self.GET(url)
2809         def _got(res):
2810             if output and output.lower() == "json":
2811                 return simplejson.loads(res)
2812             return res
2813         d.addCallback(_got)
2814         return d
2815
2816     def test_POST_DIRURL_deepcheck_no_ophandle(self):
2817         d = self.shouldFail2(error.Error,
2818                              "test_POST_DIRURL_deepcheck_no_ophandle",
2819                              "400 Bad Request",
2820                              "slow operation requires ophandle=",
2821                              self.POST, self.public_url, t="start-deep-check")
2822         return d
2823
2824     def test_POST_DIRURL_deepcheck(self):
2825         def _check_redirect(statuscode, target):
2826             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2827             self.failUnless(target.endswith("/operations/123"))
2828         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2829                                  self.POST, self.public_url,
2830                                  t="start-deep-check", ophandle="123")
2831         d.addCallback(self.wait_for_operation, "123")
2832         def _check_json(data):
2833             self.failUnlessReallyEqual(data["finished"], True)
2834             self.failUnlessReallyEqual(data["count-objects-checked"], 11)
2835             self.failUnlessReallyEqual(data["count-objects-healthy"], 11)
2836         d.addCallback(_check_json)
2837         d.addCallback(self.get_operation_results, "123", "html")
2838         def _check_html(res):
2839             self.failUnlessIn("Objects Checked: <span>11</span>", res)
2840             self.failUnlessIn("Objects Healthy: <span>11</span>", res)
2841             self.failUnlessIn(FAVICON_MARKUP, res)
2842         d.addCallback(_check_html)
2843
2844         d.addCallback(lambda res:
2845                       self.GET("/operations/123/"))
2846         d.addCallback(_check_html) # should be the same as without the slash
2847
2848         d.addCallback(lambda res:
2849                       self.shouldFail2(error.Error, "one", "404 Not Found",
2850                                        "No detailed results for SI bogus",
2851                                        self.GET, "/operations/123/bogus"))
2852
2853         foo_si = self._foo_node.get_storage_index()
2854         foo_si_s = base32.b2a(foo_si)
2855         d.addCallback(lambda res:
2856                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2857         def _check_foo_json(res):
2858             data = simplejson.loads(res)
2859             self.failUnlessEqual(data["storage-index"], foo_si_s)
2860             self.failUnless(data["results"]["healthy"])
2861         d.addCallback(_check_foo_json)
2862         return d
2863
2864     def test_POST_DIRURL_deepcheck_and_repair(self):
2865         d = self.POST(self.public_url, t="start-deep-check", repair="true",
2866                       ophandle="124", output="json", followRedirect=True)
2867         d.addCallback(self.wait_for_operation, "124")
2868         def _check_json(data):
2869             self.failUnlessReallyEqual(data["finished"], True)
2870             self.failUnlessReallyEqual(data["count-objects-checked"], 11)
2871             self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 11)
2872             self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2873             self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2874             self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2875             self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2876             self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2877             self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 11)
2878             self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2879             self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2880         d.addCallback(_check_json)
2881         d.addCallback(self.get_operation_results, "124", "html")
2882         def _check_html(res):
2883             self.failUnlessIn("Objects Checked: <span>11</span>", res)
2884
2885             self.failUnlessIn("Objects Healthy (before repair): <span>11</span>", res)
2886             self.failUnlessIn("Objects Unhealthy (before repair): <span>0</span>", res)
2887             self.failUnlessIn("Corrupt Shares (before repair): <span>0</span>", res)
2888
2889             self.failUnlessIn("Repairs Attempted: <span>0</span>", res)
2890             self.failUnlessIn("Repairs Successful: <span>0</span>", res)
2891             self.failUnlessIn("Repairs Unsuccessful: <span>0</span>", res)
2892
2893             self.failUnlessIn("Objects Healthy (after repair): <span>11</span>", res)
2894             self.failUnlessIn("Objects Unhealthy (after repair): <span>0</span>", res)
2895             self.failUnlessIn("Corrupt Shares (after repair): <span>0</span>", res)
2896
2897             self.failUnlessIn(FAVICON_MARKUP, res)
2898         d.addCallback(_check_html)
2899         return d
2900
2901     def test_POST_FILEURL_bad_t(self):
2902         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2903                              "POST to file: bad t=bogus",
2904                              self.POST, self.public_url + "/foo/bar.txt",
2905                              t="bogus")
2906         return d
2907
2908     def test_POST_mkdir(self): # return value?
2909         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2910         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2911         d.addCallback(self.failUnlessNodeKeysAre, [])
2912         return d
2913
2914     def test_POST_mkdir_mdmf(self):
2915         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=mdmf")
2916         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2917         d.addCallback(lambda node:
2918             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2919         return d
2920
2921     def test_POST_mkdir_sdmf(self):
2922         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=sdmf")
2923         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2924         d.addCallback(lambda node:
2925             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2926         return d
2927
2928     def test_POST_mkdir_bad_format(self):
2929         return self.shouldHTTPError("POST_mkdir_bad_format",
2930                                     400, "Bad Request", "Unknown format: foo",
2931                                     self.POST, self.public_url +
2932                                     "/foo?t=mkdir&name=newdir&format=foo")
2933
2934     def test_POST_mkdir_initial_children(self):
2935         (newkids, caps) = self._create_initial_children()
2936         d = self.POST2(self.public_url +
2937                        "/foo?t=mkdir-with-children&name=newdir",
2938                        simplejson.dumps(newkids))
2939         d.addCallback(lambda res:
2940                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2941         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2942         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2943         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2944         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2945         return d
2946
2947     def test_POST_mkdir_initial_children_mdmf(self):
2948         (newkids, caps) = self._create_initial_children()
2949         d = self.POST2(self.public_url +
2950                        "/foo?t=mkdir-with-children&name=newdir&format=mdmf",
2951                        simplejson.dumps(newkids))
2952         d.addCallback(lambda res:
2953                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2954         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2955         d.addCallback(lambda node:
2956             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2957         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2958         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2959                        caps['filecap1'])
2960         return d
2961
2962     # XXX: Duplication.
2963     def test_POST_mkdir_initial_children_sdmf(self):
2964         (newkids, caps) = self._create_initial_children()
2965         d = self.POST2(self.public_url +
2966                        "/foo?t=mkdir-with-children&name=newdir&format=sdmf",
2967                        simplejson.dumps(newkids))
2968         d.addCallback(lambda res:
2969                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2970         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2971         d.addCallback(lambda node:
2972             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2973         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2974         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2975                        caps['filecap1'])
2976         return d
2977
2978     def test_POST_mkdir_initial_children_bad_format(self):
2979         (newkids, caps) = self._create_initial_children()
2980         return self.shouldHTTPError("POST_mkdir_initial_children_bad_format",
2981                                     400, "Bad Request", "Unknown format: foo",
2982                                     self.POST, self.public_url + \
2983                                     "/foo?t=mkdir-with-children&name=newdir&format=foo",
2984                                     simplejson.dumps(newkids))
2985
2986     def test_POST_mkdir_immutable(self):
2987         (newkids, caps) = self._create_immutable_children()
2988         d = self.POST2(self.public_url +
2989                        "/foo?t=mkdir-immutable&name=newdir",
2990                        simplejson.dumps(newkids))
2991         d.addCallback(lambda res:
2992                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2993         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2994         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2995         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2996         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2997         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2998         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2999         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3000         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
3001         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3002         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
3003         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3004         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
3005         return d
3006
3007     def test_POST_mkdir_immutable_bad(self):
3008         (newkids, caps) = self._create_initial_children()
3009         d = self.shouldFail2(error.Error, "POST_mkdir_immutable_bad",
3010                              "400 Bad Request",
3011                              "needed to be immutable but was not",
3012                              self.POST2,
3013                              self.public_url +
3014                              "/foo?t=mkdir-immutable&name=newdir",
3015                              simplejson.dumps(newkids))
3016         return d
3017
3018     def test_POST_mkdir_2(self):
3019         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
3020         d.addCallback(lambda res:
3021                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
3022         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3023         d.addCallback(self.failUnlessNodeKeysAre, [])
3024         return d
3025
3026     def test_POST_mkdirs_2(self):
3027         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
3028         d.addCallback(lambda res:
3029                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
3030         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
3031         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
3032         d.addCallback(self.failUnlessNodeKeysAre, [])
3033         return d
3034
3035     def test_POST_mkdir_no_parentdir_noredirect(self):
3036         d = self.POST("/uri?t=mkdir")
3037         def _after_mkdir(res):
3038             uri.DirectoryURI.init_from_string(res)
3039         d.addCallback(_after_mkdir)
3040         return d
3041
3042     def test_POST_mkdir_no_parentdir_noredirect_mdmf(self):
3043         d = self.POST("/uri?t=mkdir&format=mdmf")
3044         def _after_mkdir(res):
3045             u = uri.from_string(res)
3046             # Check that this is an MDMF writecap
3047             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
3048         d.addCallback(_after_mkdir)
3049         return d
3050
3051     def test_POST_mkdir_no_parentdir_noredirect_sdmf(self):
3052         d = self.POST("/uri?t=mkdir&format=sdmf")
3053         def _after_mkdir(res):
3054             u = uri.from_string(res)
3055             self.failUnlessIsInstance(u, uri.DirectoryURI)
3056         d.addCallback(_after_mkdir)
3057         return d
3058
3059     def test_POST_mkdir_no_parentdir_noredirect_bad_format(self):
3060         return self.shouldHTTPError("POST_mkdir_no_parentdir_noredirect_bad_format",
3061                                     400, "Bad Request", "Unknown format: foo",
3062                                     self.POST, self.public_url +
3063                                     "/uri?t=mkdir&format=foo")
3064
3065     def test_POST_mkdir_no_parentdir_noredirect2(self):
3066         # make sure form-based arguments (as on the welcome page) still work
3067         d = self.POST("/uri", t="mkdir")
3068         def _after_mkdir(res):
3069             uri.DirectoryURI.init_from_string(res)
3070         d.addCallback(_after_mkdir)
3071         d.addErrback(self.explain_web_error)
3072         return d
3073
3074     def test_POST_mkdir_no_parentdir_redirect(self):
3075         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
3076         d.addBoth(self.shouldRedirect, None, statuscode='303')
3077         def _check_target(target):
3078             target = urllib.unquote(target)
3079             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
3080         d.addCallback(_check_target)
3081         return d
3082
3083     def test_POST_mkdir_no_parentdir_redirect2(self):
3084         d = self.POST("/uri", t="mkdir", redirect_to_result="true")
3085         d.addBoth(self.shouldRedirect, None, statuscode='303')
3086         def _check_target(target):
3087             target = urllib.unquote(target)
3088             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
3089         d.addCallback(_check_target)
3090         d.addErrback(self.explain_web_error)
3091         return d
3092
3093     def _make_readonly(self, u):
3094         ro_uri = uri.from_string(u).get_readonly()
3095         if ro_uri is None:
3096             return None
3097         return ro_uri.to_string()
3098
3099     def _create_initial_children(self):
3100         contents, n, filecap1 = self.makefile(12)
3101         md1 = {"metakey1": "metavalue1"}
3102         filecap2 = make_mutable_file_uri()
3103         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
3104         filecap3 = node3.get_readonly_uri()
3105         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
3106         dircap = DirectoryNode(node4, None, None).get_uri()
3107         mdmfcap = make_mutable_file_uri(mdmf=True)
3108         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
3109         emptydircap = "URI:DIR2-LIT:"
3110         newkids = {u"child-imm":        ["filenode", {"rw_uri": filecap1,
3111                                                       "ro_uri": self._make_readonly(filecap1),
3112                                                       "metadata": md1, }],
3113                    u"child-mutable":    ["filenode", {"rw_uri": filecap2,
3114                                                       "ro_uri": self._make_readonly(filecap2)}],
3115                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
3116                    u"unknownchild-rw":  ["unknown",  {"rw_uri": unknown_rwcap,
3117                                                       "ro_uri": unknown_rocap}],
3118                    u"unknownchild-ro":  ["unknown",  {"ro_uri": unknown_rocap}],
3119                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
3120                    u"dirchild":         ["dirnode",  {"rw_uri": dircap,
3121                                                       "ro_uri": self._make_readonly(dircap)}],
3122                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
3123                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
3124                    u"child-mutable-mdmf": ["filenode", {"rw_uri": mdmfcap,
3125                                                         "ro_uri": self._make_readonly(mdmfcap)}],
3126                    }
3127         return newkids, {'filecap1': filecap1,
3128                          'filecap2': filecap2,
3129                          'filecap3': filecap3,
3130                          'unknown_rwcap': unknown_rwcap,
3131                          'unknown_rocap': unknown_rocap,
3132                          'unknown_immcap': unknown_immcap,
3133                          'dircap': dircap,
3134                          'litdircap': litdircap,
3135                          'emptydircap': emptydircap,
3136                          'mdmfcap': mdmfcap}
3137
3138     def _create_immutable_children(self):
3139         contents, n, filecap1 = self.makefile(12)
3140         md1 = {"metakey1": "metavalue1"}
3141         tnode = create_chk_filenode("immutable directory contents\n"*10,
3142                                     self.get_all_contents())
3143         dnode = DirectoryNode(tnode, None, None)
3144         assert not dnode.is_mutable()
3145         immdircap = dnode.get_uri()
3146         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
3147         emptydircap = "URI:DIR2-LIT:"
3148         newkids = {u"child-imm":        ["filenode", {"ro_uri": filecap1,
3149                                                       "metadata": md1, }],
3150                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
3151                    u"dirchild-imm":     ["dirnode",  {"ro_uri": immdircap}],
3152                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
3153                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
3154                    }
3155         return newkids, {'filecap1': filecap1,
3156                          'unknown_immcap': unknown_immcap,
3157                          'immdircap': immdircap,
3158                          'litdircap': litdircap,
3159                          'emptydircap': emptydircap}
3160
3161     def test_POST_mkdir_no_parentdir_initial_children(self):
3162         (newkids, caps) = self._create_initial_children()
3163         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
3164         def _after_mkdir(res):
3165             self.failUnless(res.startswith("URI:DIR"), res)
3166             n = self.s.create_node_from_uri(res)
3167             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3168             d2.addCallback(lambda ign:
3169                            self.failUnlessROChildURIIs(n, u"child-imm",
3170                                                        caps['filecap1']))
3171             d2.addCallback(lambda ign:
3172                            self.failUnlessRWChildURIIs(n, u"child-mutable",
3173                                                        caps['filecap2']))
3174             d2.addCallback(lambda ign:
3175                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
3176                                                        caps['filecap3']))
3177             d2.addCallback(lambda ign:
3178                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
3179                                                        caps['unknown_rwcap']))
3180             d2.addCallback(lambda ign:
3181                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
3182                                                        caps['unknown_rocap']))
3183             d2.addCallback(lambda ign:
3184                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3185                                                        caps['unknown_immcap']))
3186             d2.addCallback(lambda ign:
3187                            self.failUnlessRWChildURIIs(n, u"dirchild",
3188                                                        caps['dircap']))
3189             return d2
3190         d.addCallback(_after_mkdir)
3191         return d
3192
3193     def test_POST_mkdir_no_parentdir_unexpected_children(self):
3194         # the regular /uri?t=mkdir operation is specified to ignore its body.
3195         # Only t=mkdir-with-children pays attention to it.
3196         (newkids, caps) = self._create_initial_children()
3197         d = self.shouldHTTPError("POST_mkdir_no_parentdir_unexpected_children",
3198                                  400, "Bad Request",
3199                                  "t=mkdir does not accept children=, "
3200                                  "try t=mkdir-with-children instead",
3201                                  self.POST2, "/uri?t=mkdir", # without children
3202                                  simplejson.dumps(newkids))
3203         return d
3204
3205     def test_POST_noparent_bad(self):
3206         d = self.shouldHTTPError("POST_noparent_bad",
3207                                  400, "Bad Request",
3208                                  "/uri accepts only PUT, PUT?t=mkdir, "
3209                                  "POST?t=upload, and POST?t=mkdir",
3210                                  self.POST, "/uri?t=bogus")
3211         return d
3212
3213     def test_POST_mkdir_no_parentdir_immutable(self):
3214         (newkids, caps) = self._create_immutable_children()
3215         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
3216         def _after_mkdir(res):
3217             self.failUnless(res.startswith("URI:DIR"), res)
3218             n = self.s.create_node_from_uri(res)
3219             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3220             d2.addCallback(lambda ign:
3221                            self.failUnlessROChildURIIs(n, u"child-imm",
3222                                                           caps['filecap1']))
3223             d2.addCallback(lambda ign:
3224                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3225                                                           caps['unknown_immcap']))
3226             d2.addCallback(lambda ign:
3227                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
3228                                                           caps['immdircap']))
3229             d2.addCallback(lambda ign:
3230                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
3231                                                           caps['litdircap']))
3232             d2.addCallback(lambda ign:
3233                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
3234                                                           caps['emptydircap']))
3235             return d2
3236         d.addCallback(_after_mkdir)
3237         return d
3238
3239     def test_POST_mkdir_no_parentdir_immutable_bad(self):
3240         (newkids, caps) = self._create_initial_children()
3241         d = self.shouldFail2(error.Error,
3242                              "test_POST_mkdir_no_parentdir_immutable_bad",
3243                              "400 Bad Request",
3244                              "needed to be immutable but was not",
3245                              self.POST2,
3246                              "/uri?t=mkdir-immutable",
3247                              simplejson.dumps(newkids))
3248         return d
3249
3250     def test_welcome_page_mkdir_button(self):
3251         # Fetch the welcome page.
3252         d = self.GET("/")
3253         def _after_get_welcome_page(res):
3254             MKDIR_BUTTON_RE = re.compile(
3255                 '<form action="([^"]*)" method="post".*'
3256                 '<input type="hidden" name="t" value="([^"]*)" />[ ]*'
3257                 '<input type="hidden" name="([^"]*)" value="([^"]*)" />[ ]*'
3258                 '<input type="submit" class="btn" value="Create a directory[^"]*" />')
3259             html = res.replace('\n', ' ')
3260             mo = MKDIR_BUTTON_RE.search(html)
3261             self.failUnless(mo, html)
3262             formaction = mo.group(1)
3263             formt = mo.group(2)
3264             formaname = mo.group(3)
3265             formavalue = mo.group(4)
3266             return (formaction, formt, formaname, formavalue)
3267         d.addCallback(_after_get_welcome_page)
3268         def _after_parse_form(res):
3269             (formaction, formt, formaname, formavalue) = res
3270             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
3271         d.addCallback(_after_parse_form)
3272         d.addBoth(self.shouldRedirect, None, statuscode='303')
3273         return d
3274
3275     def test_POST_mkdir_replace(self): # return value?
3276         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
3277         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3278         d.addCallback(self.failUnlessNodeKeysAre, [])
3279         return d
3280
3281     def test_POST_mkdir_no_replace_queryarg(self): # return value?
3282         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
3283         d.addBoth(self.shouldFail, error.Error,
3284                   "POST_mkdir_no_replace_queryarg",
3285                   "409 Conflict",
3286                   "There was already a child by that name, and you asked me "
3287                   "to not replace it")
3288         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3289         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3290         return d
3291
3292     def test_POST_mkdir_no_replace_field(self): # return value?
3293         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
3294                       replace="false")
3295         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
3296                   "409 Conflict",
3297                   "There was already a child by that name, and you asked me "
3298                   "to not replace it")
3299         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3300         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3301         return d
3302
3303     def test_POST_mkdir_whendone_field(self):
3304         d = self.POST(self.public_url + "/foo",
3305                       t="mkdir", name="newdir", when_done="/THERE")
3306         d.addBoth(self.shouldRedirect, "/THERE")
3307         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3308         d.addCallback(self.failUnlessNodeKeysAre, [])
3309         return d
3310
3311     def test_POST_mkdir_whendone_queryarg(self):
3312         d = self.POST(self.public_url + "/foo?when_done=/THERE",
3313                       t="mkdir", name="newdir")
3314         d.addBoth(self.shouldRedirect, "/THERE")
3315         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3316         d.addCallback(self.failUnlessNodeKeysAre, [])
3317         return d
3318
3319     def test_POST_bad_t(self):
3320         d = self.shouldFail2(error.Error, "POST_bad_t",
3321                              "400 Bad Request",
3322                              "POST to a directory with bad t=BOGUS",
3323                              self.POST, self.public_url + "/foo", t="BOGUS")
3324         return d
3325
3326     def test_POST_set_children(self, command_name="set_children"):
3327         contents9, n9, newuri9 = self.makefile(9)
3328         contents10, n10, newuri10 = self.makefile(10)
3329         contents11, n11, newuri11 = self.makefile(11)
3330
3331         reqbody = """{
3332                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
3333                                                 "size": 0,
3334                                                 "metadata": {
3335                                                   "ctime": 1002777696.7564139,
3336                                                   "mtime": 1002777696.7564139
3337                                                  }
3338                                                } ],
3339                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
3340                                                 "size": 1,
3341                                                 "metadata": {
3342                                                   "ctime": 1002777696.7564139,
3343                                                   "mtime": 1002777696.7564139
3344                                                  }
3345                                                } ],
3346                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
3347                                                 "size": 2,
3348                                                 "metadata": {
3349                                                   "ctime": 1002777696.7564139,
3350                                                   "mtime": 1002777696.7564139
3351                                                  }
3352                                                } ]
3353                     }""" % (newuri9, newuri10, newuri11)
3354
3355         url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
3356
3357         d = client.getPage(url, method="POST", postdata=reqbody)
3358         def _then(res):
3359             self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
3360             self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
3361             self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
3362
3363         d.addCallback(_then)
3364         d.addErrback(self.dump_error)
3365         return d
3366
3367     def test_POST_set_children_with_hyphen(self):
3368         return self.test_POST_set_children(command_name="set-children")
3369
3370     def test_POST_link_uri(self):
3371         contents, n, newuri = self.makefile(8)
3372         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
3373         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
3374         d.addCallback(lambda res:
3375                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3376                                                       contents))
3377         return d
3378
3379     def test_POST_link_uri_replace(self):
3380         contents, n, newuri = self.makefile(8)
3381         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
3382         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
3383         d.addCallback(lambda res:
3384                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3385                                                       contents))
3386         return d
3387
3388     def test_POST_link_uri_unknown_bad(self):
3389         d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
3390         d.addBoth(self.shouldFail, error.Error,
3391                   "POST_link_uri_unknown_bad",
3392                   "400 Bad Request",
3393                   "unknown cap in a write slot")
3394         return d
3395
3396     def test_POST_link_uri_unknown_ro_good(self):
3397         d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
3398         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
3399         return d
3400
3401     def test_POST_link_uri_unknown_imm_good(self):
3402         d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
3403         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
3404         return d
3405
3406     def test_POST_link_uri_no_replace_queryarg(self):
3407         contents, n, newuri = self.makefile(8)
3408         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
3409                       name="bar.txt", uri=newuri)
3410         d.addBoth(self.shouldFail, error.Error,
3411                   "POST_link_uri_no_replace_queryarg",
3412                   "409 Conflict",
3413                   "There was already a child by that name, and you asked me "
3414                   "to not replace it")
3415         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3416         d.addCallback(self.failUnlessIsBarDotTxt)
3417         return d
3418
3419     def test_POST_link_uri_no_replace_field(self):
3420         contents, n, newuri = self.makefile(8)
3421         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
3422                       name="bar.txt", uri=newuri)
3423         d.addBoth(self.shouldFail, error.Error,
3424                   "POST_link_uri_no_replace_field",
3425                   "409 Conflict",
3426                   "There was already a child by that name, and you asked me "
3427                   "to not replace it")
3428         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3429         d.addCallback(self.failUnlessIsBarDotTxt)
3430         return d
3431
3432     def test_POST_delete(self, command_name='delete'):
3433         d = self._foo_node.list()
3434         def _check_before(children):
3435             self.failUnlessIn(u"bar.txt", children)
3436         d.addCallback(_check_before)
3437         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t=command_name, name="bar.txt"))
3438         d.addCallback(lambda res: self._foo_node.list())
3439         def _check_after(children):
3440             self.failIfIn(u"bar.txt", children)
3441         d.addCallback(_check_after)
3442         return d
3443
3444     def test_POST_unlink(self):
3445         return self.test_POST_delete(command_name='unlink')
3446
3447     def test_POST_rename_file(self):
3448         d = self.POST(self.public_url + "/foo", t="rename",
3449                       from_name="bar.txt", to_name='wibble.txt')
3450         d.addCallback(lambda res:
3451                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3452         d.addCallback(lambda res:
3453                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
3454         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
3455         d.addCallback(self.failUnlessIsBarDotTxt)
3456         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
3457         d.addCallback(self.failUnlessIsBarJSON)
3458         return d
3459
3460     def test_POST_rename_file_redundant(self):
3461         d = self.POST(self.public_url + "/foo", t="rename",
3462                       from_name="bar.txt", to_name='bar.txt')
3463         d.addCallback(lambda res:
3464                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3465         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3466         d.addCallback(self.failUnlessIsBarDotTxt)
3467         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3468         d.addCallback(self.failUnlessIsBarJSON)
3469         return d
3470
3471     def test_POST_rename_file_replace(self):
3472         # rename a file and replace a directory with it
3473         d = self.POST(self.public_url + "/foo", t="rename",
3474                       from_name="bar.txt", to_name='empty')
3475         d.addCallback(lambda res:
3476                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3477         d.addCallback(lambda res:
3478                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
3479         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
3480         d.addCallback(self.failUnlessIsBarDotTxt)
3481         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3482         d.addCallback(self.failUnlessIsBarJSON)
3483         return d
3484
3485     def test_POST_rename_file_no_replace_queryarg(self):
3486         # rename a file and replace a directory with it
3487         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
3488                       from_name="bar.txt", to_name='empty')
3489         d.addBoth(self.shouldFail, error.Error,
3490                   "POST_rename_file_no_replace_queryarg",
3491                   "409 Conflict",
3492                   "There was already a child by that name, and you asked me "
3493                   "to not replace it")
3494         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3495         d.addCallback(self.failUnlessIsEmptyJSON)
3496         return d
3497
3498     def test_POST_rename_file_no_replace_field(self):
3499         # rename a file and replace a directory with it
3500         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
3501                       from_name="bar.txt", to_name='empty')
3502         d.addBoth(self.shouldFail, error.Error,
3503                   "POST_rename_file_no_replace_field",
3504                   "409 Conflict",
3505                   "There was already a child by that name, and you asked me "
3506                   "to not replace it")
3507         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3508         d.addCallback(self.failUnlessIsEmptyJSON)
3509         return d
3510
3511     def test_POST_rename_file_no_replace_same_link(self):
3512         d = self.POST(self.public_url + "/foo", t="rename",
3513                       replace="false", from_name="bar.txt", to_name="bar.txt")
3514         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3515         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3516         d.addCallback(self.failUnlessIsBarDotTxt)
3517         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3518         d.addCallback(self.failUnlessIsBarJSON)
3519         return d
3520
3521     def test_POST_rename_file_replace_only_files(self):
3522         d = self.POST(self.public_url + "/foo", t="rename",
3523                       replace="only-files", from_name="bar.txt",
3524                       to_name="baz.txt")
3525         d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3526         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3527         d.addCallback(self.failUnlessIsBarDotTxt)
3528         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt?t=json"))
3529         d.addCallback(self.failUnlessIsBarJSON)
3530         return d
3531
3532     def test_POST_rename_file_replace_only_files_conflict(self):
3533         d = self.shouldFail2(error.Error, "POST_relink_file_replace_only_files_conflict",
3534                              "409 Conflict",
3535                              "There was already a child by that name, and you asked me to not replace it.",
3536                              self.POST, self.public_url + "/foo", t="relink",
3537                              replace="only-files", from_name="bar.txt",
3538                              to_name="empty")
3539         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3540         d.addCallback(self.failUnlessIsBarDotTxt)
3541         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3542         d.addCallback(self.failUnlessIsBarJSON)
3543         return d
3544
3545     def failUnlessIsEmptyJSON(self, res):
3546         data = simplejson.loads(res)
3547         self.failUnlessEqual(data[0], "dirnode", data)
3548         self.failUnlessReallyEqual(len(data[1]["children"]), 0)
3549
3550     def test_POST_rename_file_to_slash_fail(self):
3551         d = self.POST(self.public_url + "/foo", t="rename",
3552                       from_name="bar.txt", to_name='kirk/spock.txt')
3553         d.addBoth(self.shouldFail, error.Error,
3554                   "test_POST_rename_file_to_slash_fail",
3555                   "400 Bad Request",
3556                   "to_name= may not contain a slash",
3557                   )
3558         d.addCallback(lambda res:
3559                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3560         return d
3561
3562     def test_POST_rename_file_from_slash_fail(self):
3563         d = self.POST(self.public_url + "/foo", t="rename",
3564                       from_name="sub/bar.txt", to_name='spock.txt')
3565         d.addBoth(self.shouldFail, error.Error,
3566                   "test_POST_rename_from_file_slash_fail",
3567                   "400 Bad Request",
3568                   "from_name= may not contain a slash",
3569                   )
3570         d.addCallback(lambda res:
3571                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3572         return d
3573
3574     def test_POST_rename_dir(self):
3575         d = self.POST(self.public_url, t="rename",
3576                       from_name="foo", to_name='plunk')
3577         d.addCallback(lambda res:
3578                       self.failIfNodeHasChild(self.public_root, u"foo"))
3579         d.addCallback(lambda res:
3580                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
3581         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
3582         d.addCallback(self.failUnlessIsFooJSON)
3583         return d
3584
3585     def test_POST_relink_file(self):
3586         d = self.POST(self.public_url + "/foo", t="relink",
3587                       from_name="bar.txt",
3588                       to_dir=self.public_root.get_uri() + "/foo/sub")
3589         d.addCallback(lambda res:
3590                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3591         d.addCallback(lambda res:
3592                       self.failUnlessNodeHasChild(self._sub_node, u"bar.txt"))
3593         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3594         d.addCallback(self.failUnlessIsBarDotTxt)
3595         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3596         d.addCallback(self.failUnlessIsBarJSON)
3597         return d
3598
3599     def test_POST_relink_file_new_name(self):
3600         d = self.POST(self.public_url + "/foo", t="relink",
3601                       from_name="bar.txt",
3602                       to_name="wibble.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3603         d.addCallback(lambda res:
3604                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3605         d.addCallback(lambda res:
3606                       self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3607         d.addCallback(lambda res:
3608                       self.failUnlessNodeHasChild(self._sub_node, u"wibble.txt"))
3609         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt"))
3610         d.addCallback(self.failUnlessIsBarDotTxt)
3611         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt?t=json"))
3612         d.addCallback(self.failUnlessIsBarJSON)
3613         return d
3614
3615     def test_POST_relink_file_replace(self):
3616         d = self.POST(self.public_url + "/foo", t="relink",
3617                       from_name="bar.txt",
3618                       to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3619         d.addCallback(lambda res:
3620                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3621         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3622         d.addCallback(self.failUnlessIsBarDotTxt)
3623         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt?t=json"))
3624         d.addCallback(self.failUnlessIsBarJSON)
3625         return d
3626
3627     def test_POST_relink_file_no_replace(self):
3628         d = self.shouldFail2(error.Error, "POST_relink_file_no_replace",
3629                              "409 Conflict",
3630                              "There was already a child by that name, and you asked me to not replace it",
3631                              self.POST, self.public_url + "/foo", t="relink",
3632                              replace="false", from_name="bar.txt",
3633                              to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3634         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3635         d.addCallback(self.failUnlessIsBarDotTxt)
3636         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3637         d.addCallback(self.failUnlessIsBarJSON)
3638         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3639         d.addCallback(self.failUnlessIsSubBazDotTxt)
3640         return d
3641
3642     def test_POST_relink_file_no_replace_explicitly_same_link(self):
3643         d = self.POST(self.public_url + "/foo", t="relink",
3644                       replace="false", from_name="bar.txt",
3645                       to_name="bar.txt", to_dir=self.public_root.get_uri() + "/foo")
3646         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3647         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3648         d.addCallback(self.failUnlessIsBarDotTxt)
3649         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3650         d.addCallback(self.failUnlessIsBarJSON)
3651         return d
3652
3653     def test_POST_relink_file_replace_only_files(self):
3654         d = self.POST(self.public_url + "/foo", t="relink",
3655                       replace="only-files", from_name="bar.txt",
3656                       to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3657         d.addCallback(lambda res:
3658                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3659         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3660         d.addCallback(self.failUnlessIsBarDotTxt)
3661         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt?t=json"))
3662         d.addCallback(self.failUnlessIsBarJSON)
3663         return d
3664
3665     def test_POST_relink_file_replace_only_files_conflict(self):
3666         d = self.shouldFail2(error.Error, "POST_relink_file_replace_only_files_conflict",
3667                              "409 Conflict",
3668                              "There was already a child by that name, and you asked me to not replace it.",
3669                              self.POST, self.public_url + "/foo", t="relink",
3670                              replace="only-files", from_name="bar.txt",
3671                              to_name="sub", to_dir=self.public_root.get_uri() + "/foo")
3672         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3673         d.addCallback(self.failUnlessIsBarDotTxt)
3674         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3675         d.addCallback(self.failUnlessIsBarJSON)
3676         return d
3677
3678     def test_POST_relink_file_to_slash_fail(self):
3679         d = self.shouldFail2(error.Error, "test_POST_rename_file_slash_fail",
3680                              "400 Bad Request",
3681                              "to_name= may not contain a slash",
3682                              self.POST, self.public_url + "/foo", t="relink",
3683                              from_name="bar.txt",
3684                              to_name="slash/fail.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3685         d.addCallback(lambda res:
3686                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3687         d.addCallback(lambda res:
3688                       self.failIfNodeHasChild(self._sub_node, u"slash/fail.txt"))
3689         d.addCallback(lambda ign:
3690                       self.shouldFail2(error.Error,
3691                                        "test_POST_rename_file_slash_fail2",
3692                                        "400 Bad Request",
3693                                        "from_name= may not contain a slash",
3694                                        self.POST, self.public_url + "/foo",
3695                                        t="relink",
3696                                        from_name="nope/bar.txt",
3697                                        to_name="fail.txt",
3698                                        to_dir=self.public_root.get_uri() + "/foo/sub"))
3699         return d
3700
3701     def test_POST_relink_file_explicitly_same_link(self):
3702         d = self.POST(self.public_url + "/foo", t="relink",
3703                       from_name="bar.txt",
3704                       to_name="bar.txt", to_dir=self.public_root.get_uri() + "/foo")
3705         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3706         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3707         d.addCallback(self.failUnlessIsBarDotTxt)
3708         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3709         d.addCallback(self.failUnlessIsBarJSON)
3710         return d
3711
3712     def test_POST_relink_file_implicitly_same_link(self):
3713         d = self.POST(self.public_url + "/foo", t="relink",
3714                       from_name="bar.txt")
3715         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3716         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3717         d.addCallback(self.failUnlessIsBarDotTxt)
3718         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3719         d.addCallback(self.failUnlessIsBarJSON)
3720         return d
3721
3722     def test_POST_relink_file_same_dir(self):
3723         d = self.POST(self.public_url + "/foo", t="relink",
3724                       from_name="bar.txt",
3725                       to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo")
3726         d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3727         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._sub_node, u"baz.txt"))
3728         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3729         d.addCallback(self.failUnlessIsBarDotTxt)
3730         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt?t=json"))
3731         d.addCallback(self.failUnlessIsBarJSON)
3732         return d
3733
3734     def test_POST_relink_file_bad_replace(self):
3735         d = self.shouldFail2(error.Error, "test_POST_relink_file_bad_replace",
3736                              "400 Bad Request", "invalid replace= argument: 'boogabooga'",
3737                              self.POST,
3738                              self.public_url + "/foo", t="relink",
3739                              replace="boogabooga", from_name="bar.txt",
3740                              to_dir=self.public_root.get_uri() + "/foo/sub")
3741         return d
3742
3743     def test_POST_relink_file_multi_level(self):
3744         d = self.POST(self.public_url + "/foo/sub/level2?t=mkdir", "")
3745         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t="relink",
3746                       from_name="bar.txt", to_dir=self.public_root.get_uri() + "/foo/sub/level2"))
3747         d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3748         d.addCallback(lambda res: self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3749         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt"))
3750         d.addCallback(self.failUnlessIsBarDotTxt)
3751         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt?t=json"))
3752         d.addCallback(self.failUnlessIsBarJSON)
3753         return d
3754
3755     def test_POST_relink_file_to_uri(self):
3756         d = self.POST(self.public_url + "/foo", t="relink", target_type="uri",
3757                       from_name="bar.txt", to_dir=self._sub_uri)
3758         d.addCallback(lambda res:
3759                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3760         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3761         d.addCallback(self.failUnlessIsBarDotTxt)
3762         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3763         d.addCallback(self.failUnlessIsBarJSON)
3764         return d
3765
3766     def test_POST_relink_file_to_nonexistent_dir(self):
3767         d = self.shouldFail2(error.Error, "POST_relink_file_to_nonexistent_dir",
3768                             "404 Not Found", "No such child: nopechucktesta",
3769                             self.POST, self.public_url + "/foo", t="relink",
3770                             from_name="bar.txt",
3771                             to_dir=self.public_root.get_uri() + "/nopechucktesta")
3772         return d
3773
3774     def test_POST_relink_file_into_file(self):
3775         d = self.shouldFail2(error.Error, "POST_relink_file_into_file",
3776                              "400 Bad Request", "to_dir is not a directory",
3777                              self.POST, self.public_url + "/foo", t="relink",
3778                              from_name="bar.txt",
3779                              to_dir=self.public_root.get_uri() + "/foo/baz.txt")
3780         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3781         d.addCallback(self.failUnlessIsBazDotTxt)
3782         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3783         d.addCallback(self.failUnlessIsBarDotTxt)
3784         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3785         d.addCallback(self.failUnlessIsBarJSON)
3786         return d
3787
3788     def test_POST_relink_file_to_bad_uri(self):
3789         d =  self.shouldFail2(error.Error, "POST_relink_file_to_bad_uri",
3790                               "400 Bad Request", "to_dir is not a directory",
3791                               self.POST, self.public_url + "/foo", t="relink",
3792                               from_name="bar.txt",
3793                               to_dir="URI:DIR2:mn5jlyjnrjeuydyswlzyui72i:rmneifcj6k6sycjljjhj3f6majsq2zqffydnnul5hfa4j577arma")
3794         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3795         d.addCallback(self.failUnlessIsBarDotTxt)
3796         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3797         d.addCallback(self.failUnlessIsBarJSON)
3798         return d
3799
3800     def test_POST_relink_dir(self):
3801         d = self.POST(self.public_url + "/foo", t="relink",
3802                       from_name="bar.txt",
3803                       to_dir=self.public_root.get_uri() + "/foo/empty")
3804         d.addCallback(lambda res: self.POST(self.public_url + "/foo",
3805                       t="relink", from_name="empty",
3806                       to_dir=self.public_root.get_uri() + "/foo/sub"))
3807         d.addCallback(lambda res:
3808                       self.failIfNodeHasChild(self._foo_node, u"empty"))
3809         d.addCallback(lambda res:
3810                       self.failUnlessNodeHasChild(self._sub_node, u"empty"))
3811         d.addCallback(lambda res:
3812                       self._sub_node.get_child_at_path(u"empty"))
3813         d.addCallback(lambda node:
3814                       self.failUnlessNodeHasChild(node, u"bar.txt"))
3815         d.addCallback(lambda res:
3816                       self.GET(self.public_url + "/foo/sub/empty/bar.txt"))
3817         d.addCallback(self.failUnlessIsBarDotTxt)
3818         return d
3819
3820     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
3821         """ If target is not None then the redirection has to go to target.  If
3822         statuscode is not None then the redirection has to be accomplished with
3823         that HTTP status code."""
3824         if not isinstance(res, failure.Failure):
3825             to_where = (target is None) and "somewhere" or ("to " + target)
3826             self.fail("%s: we were expecting to get redirected %s, not get an"
3827                       " actual page: %s" % (which, to_where, res))
3828         res.trap(error.PageRedirect)
3829         if statuscode is not None:
3830             self.failUnlessReallyEqual(res.value.status, statuscode,
3831                                        "%s: not a redirect" % which)
3832         if target is not None:
3833             # the PageRedirect does not seem to capture the uri= query arg
3834             # properly, so we can't check for it.
3835             realtarget = self.webish_url + target
3836             self.failUnlessReallyEqual(res.value.location, realtarget,
3837                                        "%s: wrong target" % which)
3838         return res.value.location
3839
3840     def test_GET_URI_form(self):
3841         base = "/uri?uri=%s" % self._bar_txt_uri
3842         # this is supposed to give us a redirect to /uri/$URI, plus arguments
3843         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
3844         d = self.GET(base)
3845         d.addBoth(self.shouldRedirect, targetbase)
3846         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
3847         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
3848         d.addCallback(lambda res: self.GET(base+"&t=json"))
3849         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
3850         d.addCallback(self.log, "about to get file by uri")
3851         d.addCallback(lambda res: self.GET(base, followRedirect=True))
3852         d.addCallback(self.failUnlessIsBarDotTxt)
3853         d.addCallback(self.log, "got file by uri, about to get dir by uri")
3854         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
3855                                            followRedirect=True))
3856         d.addCallback(self.failUnlessIsFooJSON)
3857         d.addCallback(self.log, "got dir by uri")
3858
3859         return d
3860
3861     def test_GET_URI_form_bad(self):
3862         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
3863                              "400 Bad Request", "GET /uri requires uri=",
3864                              self.GET, "/uri")
3865         return d
3866
3867     def test_GET_rename_form(self):
3868         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
3869                      followRedirect=True)
3870         def _check(res):
3871             self.failUnlessIn('name="when_done" value="."', res)
3872             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
3873             self.failUnlessIn(FAVICON_MARKUP, res)
3874         d.addCallback(_check)
3875         return d
3876
3877     def log(self, res, msg):
3878         #print "MSG: %s  RES: %s" % (msg, res)
3879         log.msg(msg)
3880         return res
3881
3882     def test_GET_URI_URL(self):
3883         base = "/uri/%s" % self._bar_txt_uri
3884         d = self.GET(base)
3885         d.addCallback(self.failUnlessIsBarDotTxt)
3886         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
3887         d.addCallback(self.failUnlessIsBarDotTxt)
3888         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
3889         d.addCallback(self.failUnlessIsBarDotTxt)
3890         return d
3891
3892     def test_GET_URI_URL_dir(self):
3893         base = "/uri/%s?t=json" % self._foo_uri
3894         d = self.GET(base)
3895         d.addCallback(self.failUnlessIsFooJSON)
3896         return d
3897
3898     def test_GET_URI_URL_missing(self):
3899         base = "/uri/%s" % self._bad_file_uri
3900         d = self.shouldHTTPError("test_GET_URI_URL_missing",
3901                                  http.GONE, None, "NotEnoughSharesError",
3902                                  self.GET, base)
3903         # TODO: how can we exercise both sides of WebDownloadTarget.fail
3904         # here? we must arrange for a download to fail after target.open()
3905         # has been called, and then inspect the response to see that it is
3906         # shorter than we expected.
3907         return d
3908
3909     def test_PUT_DIRURL_uri(self):
3910         d = self.s.create_dirnode()
3911         def _made_dir(dn):
3912             new_uri = dn.get_uri()
3913             # replace /foo with a new (empty) directory
3914             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
3915             d.addCallback(lambda res:
3916                           self.failUnlessReallyEqual(res.strip(), new_uri))
3917             d.addCallback(lambda res:
3918                           self.failUnlessRWChildURIIs(self.public_root,
3919                                                       u"foo",
3920                                                       new_uri))
3921             return d
3922         d.addCallback(_made_dir)
3923         return d
3924
3925     def test_PUT_DIRURL_uri_noreplace(self):
3926         d = self.s.create_dirnode()
3927         def _made_dir(dn):
3928             new_uri = dn.get_uri()
3929             # replace /foo with a new (empty) directory, but ask that
3930             # replace=false, so it should fail
3931             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
3932                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
3933                                  self.PUT,
3934                                  self.public_url + "/foo?t=uri&replace=false",
3935                                  new_uri)
3936             d.addCallback(lambda res:
3937                           self.failUnlessRWChildURIIs(self.public_root,
3938                                                       u"foo",
3939                                                       self._foo_uri))
3940             return d
3941         d.addCallback(_made_dir)
3942         return d
3943
3944     def test_PUT_DIRURL_bad_t(self):
3945         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
3946                              "400 Bad Request", "PUT to a directory",
3947                              self.PUT, self.public_url + "/foo?t=BOGUS", "")
3948         d.addCallback(lambda res:
3949                       self.failUnlessRWChildURIIs(self.public_root,
3950                                                   u"foo",
3951                                                   self._foo_uri))
3952         return d
3953
3954     def test_PUT_NEWFILEURL_uri(self):
3955         contents, n, new_uri = self.makefile(8)
3956         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
3957         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3958         d.addCallback(lambda res:
3959                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3960                                                       contents))
3961         return d
3962
3963     def test_PUT_NEWFILEURL_mdmf(self):
3964         new_contents = self.NEWFILE_CONTENTS * 300000
3965         d = self.PUT(self.public_url + \
3966                      "/foo/mdmf.txt?format=mdmf",
3967                      new_contents)
3968         d.addCallback(lambda ignored:
3969             self.GET(self.public_url + "/foo/mdmf.txt?t=json"))
3970         def _got_json(json):
3971             data = simplejson.loads(json)
3972             data = data[1]
3973             self.failUnlessIn("format", data)
3974             self.failUnlessEqual(data["format"], "MDMF")
3975             self.failUnless(data['rw_uri'].startswith("URI:MDMF"))
3976             self.failUnless(data['ro_uri'].startswith("URI:MDMF"))
3977         d.addCallback(_got_json)
3978         return d
3979
3980     def test_PUT_NEWFILEURL_sdmf(self):
3981         new_contents = self.NEWFILE_CONTENTS * 300000
3982         d = self.PUT(self.public_url + \
3983                      "/foo/sdmf.txt?format=sdmf",
3984                      new_contents)
3985         d.addCallback(lambda ignored:
3986             self.GET(self.public_url + "/foo/sdmf.txt?t=json"))
3987         def _got_json(json):
3988             data = simplejson.loads(json)
3989             data = data[1]
3990             self.failUnlessIn("format", data)
3991             self.failUnlessEqual(data["format"], "SDMF")
3992         d.addCallback(_got_json)
3993         return d
3994
3995     def test_PUT_NEWFILEURL_bad_format(self):
3996         new_contents = self.NEWFILE_CONTENTS * 300000
3997         return self.shouldHTTPError("PUT_NEWFILEURL_bad_format",
3998                                     400, "Bad Request", "Unknown format: foo",
3999                                     self.PUT, self.public_url + \
4000                                     "/foo/foo.txt?format=foo",
4001                                     new_contents)
4002
4003     def test_PUT_NEWFILEURL_uri_replace(self):
4004         contents, n, new_uri = self.makefile(8)
4005         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
4006         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
4007         d.addCallback(lambda res:
4008                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
4009                                                       contents))
4010         return d
4011
4012     def test_PUT_NEWFILEURL_uri_no_replace(self):
4013         contents, n, new_uri = self.makefile(8)
4014         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
4015         d.addBoth(self.shouldFail, error.Error,
4016                   "PUT_NEWFILEURL_uri_no_replace",
4017                   "409 Conflict",
4018                   "There was already a child by that name, and you asked me "
4019                   "to not replace it")
4020         return d
4021
4022     def test_PUT_NEWFILEURL_uri_unknown_bad(self):
4023         d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
4024         d.addBoth(self.shouldFail, error.Error,
4025                   "POST_put_uri_unknown_bad",
4026                   "400 Bad Request",
4027                   "unknown cap in a write slot")
4028         return d
4029
4030     def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
4031         d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
4032         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
4033                       u"put-future-ro.txt")
4034         return d
4035
4036     def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
4037         d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
4038         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
4039                       u"put-future-imm.txt")
4040         return d
4041
4042     def test_PUT_NEWFILE_URI(self):
4043         file_contents = "New file contents here\n"
4044         d = self.PUT("/uri", file_contents)
4045         def _check(uri):
4046             assert isinstance(uri, str), uri
4047             self.failUnlessIn(uri, self.get_all_contents())
4048             self.failUnlessReallyEqual(self.get_all_contents()[uri],
4049                                        file_contents)
4050             return self.GET("/uri/%s" % uri)
4051         d.addCallback(_check)
4052         def _check2(res):
4053             self.failUnlessReallyEqual(res, file_contents)
4054         d.addCallback(_check2)
4055         return d
4056
4057     def test_PUT_NEWFILE_URI_not_mutable(self):
4058         file_contents = "New file contents here\n"
4059         d = self.PUT("/uri?mutable=false", file_contents)
4060         def _check(uri):
4061             assert isinstance(uri, str), uri
4062             self.failUnlessIn(uri, self.get_all_contents())
4063             self.failUnlessReallyEqual(self.get_all_contents()[uri],
4064                                        file_contents)
4065             return self.GET("/uri/%s" % uri)
4066         d.addCallback(_check)
4067         def _check2(res):
4068             self.failUnlessReallyEqual(res, file_contents)
4069         d.addCallback(_check2)
4070         return d
4071
4072     def test_PUT_NEWFILE_URI_only_PUT(self):
4073         d = self.PUT("/uri?t=bogus", "")
4074         d.addBoth(self.shouldFail, error.Error,
4075                   "PUT_NEWFILE_URI_only_PUT",
4076                   "400 Bad Request",
4077                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
4078         return d
4079
4080     def test_PUT_NEWFILE_URI_mutable(self):
4081         file_contents = "New file contents here\n"
4082         d = self.PUT("/uri?mutable=true", file_contents)
4083         def _check1(filecap):
4084             filecap = filecap.strip()
4085             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
4086             self.filecap = filecap
4087             u = uri.WriteableSSKFileURI.init_from_string(filecap)
4088             self.failUnlessIn(u.get_storage_index(), self.get_all_contents())
4089             n = self.s.create_node_from_uri(filecap)
4090             return n.download_best_version()
4091         d.addCallback(_check1)
4092         def _check2(data):
4093             self.failUnlessReallyEqual(data, file_contents)
4094             return self.GET("/uri/%s" % urllib.quote(self.filecap))
4095         d.addCallback(_check2)
4096         def _check3(res):
4097             self.failUnlessReallyEqual(res, file_contents)
4098         d.addCallback(_check3)
4099         return d
4100
4101     def test_PUT_mkdir(self):
4102         d = self.PUT("/uri?t=mkdir", "")
4103         def _check(uri):
4104             n = self.s.create_node_from_uri(uri.strip())
4105             d2 = self.failUnlessNodeKeysAre(n, [])
4106             d2.addCallback(lambda res:
4107                            self.GET("/uri/%s?t=json" % uri))
4108             return d2
4109         d.addCallback(_check)
4110         d.addCallback(self.failUnlessIsEmptyJSON)
4111         return d
4112
4113     def test_PUT_mkdir_mdmf(self):
4114         d = self.PUT("/uri?t=mkdir&format=mdmf", "")
4115         def _got(res):
4116             u = uri.from_string(res)
4117             # Check that this is an MDMF writecap
4118             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
4119         d.addCallback(_got)
4120         return d
4121
4122     def test_PUT_mkdir_sdmf(self):
4123         d = self.PUT("/uri?t=mkdir&format=sdmf", "")
4124         def _got(res):
4125             u = uri.from_string(res)
4126             self.failUnlessIsInstance(u, uri.DirectoryURI)
4127         d.addCallback(_got)
4128         return d
4129
4130     def test_PUT_mkdir_bad_format(self):
4131         return self.shouldHTTPError("PUT_mkdir_bad_format",
4132                                     400, "Bad Request", "Unknown format: foo",
4133                                     self.PUT, "/uri?t=mkdir&format=foo",
4134                                     "")
4135
4136     def test_POST_check(self):
4137         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
4138         def _done(res):
4139             # this returns a string form of the results, which are probably
4140             # None since we're using fake filenodes.
4141             # TODO: verify that the check actually happened, by changing
4142             # FakeCHKFileNode to count how many times .check() has been
4143             # called.
4144             pass
4145         d.addCallback(_done)
4146         return d
4147
4148
4149     def test_PUT_update_at_offset(self):
4150         file_contents = "test file" * 100000 # about 900 KiB
4151         d = self.PUT("/uri?mutable=true", file_contents)
4152         def _then(filecap):
4153             self.filecap = filecap
4154             new_data = file_contents[:100]
4155             new = "replaced and so on"
4156             new_data += new
4157             new_data += file_contents[len(new_data):]
4158             assert len(new_data) == len(file_contents)
4159             self.new_data = new_data
4160         d.addCallback(_then)
4161         d.addCallback(lambda ignored:
4162             self.PUT("/uri/%s?replace=True&offset=100" % self.filecap,
4163                      "replaced and so on"))
4164         def _get_data(filecap):
4165             n = self.s.create_node_from_uri(filecap)
4166             return n.download_best_version()
4167         d.addCallback(_get_data)
4168         d.addCallback(lambda results:
4169             self.failUnlessEqual(results, self.new_data))
4170         # Now try appending things to the file
4171         d.addCallback(lambda ignored:
4172             self.PUT("/uri/%s?offset=%d" % (self.filecap, len(self.new_data)),
4173                      "puppies" * 100))
4174         d.addCallback(_get_data)
4175         d.addCallback(lambda results:
4176             self.failUnlessEqual(results, self.new_data + ("puppies" * 100)))
4177         # and try replacing the beginning of the file
4178         d.addCallback(lambda ignored:
4179             self.PUT("/uri/%s?offset=0" % self.filecap, "begin"))
4180         d.addCallback(_get_data)
4181         d.addCallback(lambda results:
4182             self.failUnlessEqual(results, "begin"+self.new_data[len("begin"):]+("puppies"*100)))
4183         return d
4184
4185     def test_PUT_update_at_invalid_offset(self):
4186         file_contents = "test file" * 100000 # about 900 KiB
4187         d = self.PUT("/uri?mutable=true", file_contents)
4188         def _then(filecap):
4189             self.filecap = filecap
4190         d.addCallback(_then)
4191         # Negative offsets should cause an error.
4192         d.addCallback(lambda ignored:
4193             self.shouldHTTPError("PUT_update_at_invalid_offset",
4194                                  400, "Bad Request",
4195                                  "Invalid offset",
4196                                  self.PUT,
4197                                  "/uri/%s?offset=-1" % self.filecap,
4198                                  "foo"))
4199         return d
4200
4201     def test_PUT_update_at_offset_immutable(self):
4202         file_contents = "Test file" * 100000
4203         d = self.PUT("/uri", file_contents)
4204         def _then(filecap):
4205             self.filecap = filecap
4206         d.addCallback(_then)
4207         d.addCallback(lambda ignored:
4208             self.shouldHTTPError("PUT_update_at_offset_immutable",
4209                                  400, "Bad Request",
4210                                  "immutable",
4211                                  self.PUT,
4212                                  "/uri/%s?offset=50" % self.filecap,
4213                                  "foo"))
4214         return d
4215
4216
4217     def test_bad_method(self):
4218         url = self.webish_url + self.public_url + "/foo/bar.txt"
4219         d = self.shouldHTTPError("bad_method",
4220                                  501, "Not Implemented",
4221                                  "I don't know how to treat a BOGUS request.",
4222                                  client.getPage, url, method="BOGUS")
4223         return d
4224
4225     def test_short_url(self):
4226         url = self.webish_url + "/uri"
4227         d = self.shouldHTTPError("short_url", 501, "Not Implemented",
4228                                  "I don't know how to treat a DELETE request.",
4229                                  client.getPage, url, method="DELETE")
4230         return d
4231
4232     def test_ophandle_bad(self):
4233         url = self.webish_url + "/operations/bogus?t=status"
4234         d = self.shouldHTTPError("ophandle_bad", 404, "404 Not Found",
4235                                  "unknown/expired handle 'bogus'",
4236                                  client.getPage, url)
4237         return d
4238
4239     def test_ophandle_cancel(self):
4240         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
4241                       followRedirect=True)
4242         d.addCallback(lambda ignored:
4243                       self.GET("/operations/128?t=status&output=JSON"))
4244         def _check1(res):
4245             data = simplejson.loads(res)
4246             self.failUnless("finished" in data, res)
4247             monitor = self.ws.root.child_operations.handles["128"][0]
4248             d = self.POST("/operations/128?t=cancel&output=JSON")
4249             def _check2(res):
4250                 data = simplejson.loads(res)
4251                 self.failUnless("finished" in data, res)
4252                 # t=cancel causes the handle to be forgotten
4253                 self.failUnless(monitor.is_cancelled())
4254             d.addCallback(_check2)
4255             return d
4256         d.addCallback(_check1)
4257         d.addCallback(lambda ignored:
4258                       self.shouldHTTPError("ophandle_cancel",
4259                                            404, "404 Not Found",
4260                                            "unknown/expired handle '128'",
4261                                            self.GET,
4262                                            "/operations/128?t=status&output=JSON"))
4263         return d
4264
4265     def test_ophandle_retainfor(self):
4266         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
4267                       followRedirect=True)
4268         d.addCallback(lambda ignored:
4269                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
4270         def _check1(res):
4271             data = simplejson.loads(res)
4272             self.failUnless("finished" in data, res)
4273         d.addCallback(_check1)
4274         # the retain-for=0 will cause the handle to be expired very soon
4275         d.addCallback(lambda ign:
4276             self.clock.advance(2.0))
4277         d.addCallback(lambda ignored:
4278                       self.shouldHTTPError("ophandle_retainfor",
4279                                            404, "404 Not Found",
4280                                            "unknown/expired handle '129'",
4281                                            self.GET,
4282                                            "/operations/129?t=status&output=JSON"))
4283         return d
4284
4285     def test_ophandle_release_after_complete(self):
4286         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
4287                       followRedirect=True)
4288         d.addCallback(self.wait_for_operation, "130")
4289         d.addCallback(lambda ignored:
4290                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
4291         # the release-after-complete=true will cause the handle to be expired
4292         d.addCallback(lambda ignored:
4293                       self.shouldHTTPError("ophandle_release_after_complete",
4294                                            404, "404 Not Found",
4295                                            "unknown/expired handle '130'",
4296                                            self.GET,
4297                                            "/operations/130?t=status&output=JSON"))
4298         return d
4299
4300     def test_uncollected_ophandle_expiration(self):
4301         # uncollected ophandles should expire after 4 days
4302         def _make_uncollected_ophandle(ophandle):
4303             d = self.POST(self.public_url +
4304                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
4305                           followRedirect=False)
4306             # When we start the operation, the webapi server will want
4307             # to redirect us to the page for the ophandle, so we get
4308             # confirmation that the operation has started. If the
4309             # manifest operation has finished by the time we get there,
4310             # following that redirect (by setting followRedirect=True
4311             # above) has the side effect of collecting the ophandle that
4312             # we've just created, which means that we can't use the
4313             # ophandle to test the uncollected timeout anymore. So,
4314             # instead, catch the 302 here and don't follow it.
4315             d.addBoth(self.should302, "uncollected_ophandle_creation")
4316             return d
4317         # Create an ophandle, don't collect it, then advance the clock by
4318         # 4 days - 1 second and make sure that the ophandle is still there.
4319         d = _make_uncollected_ophandle(131)
4320         d.addCallback(lambda ign:
4321             self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
4322         d.addCallback(lambda ign:
4323             self.GET("/operations/131?t=status&output=JSON"))
4324         def _check1(res):
4325             data = simplejson.loads(res)
4326             self.failUnless("finished" in data, res)
4327         d.addCallback(_check1)
4328         # Create an ophandle, don't collect it, then try to collect it
4329         # after 4 days. It should be gone.
4330         d.addCallback(lambda ign:
4331             _make_uncollected_ophandle(132))
4332         d.addCallback(lambda ign:
4333             self.clock.advance(96*60*60))
4334         d.addCallback(lambda ign:
4335             self.shouldHTTPError("uncollected_ophandle_expired_after_100_hours",
4336                                  404, "404 Not Found",
4337                                  "unknown/expired handle '132'",
4338                                  self.GET,
4339                                  "/operations/132?t=status&output=JSON"))
4340         return d
4341
4342     def test_collected_ophandle_expiration(self):
4343         # collected ophandles should expire after 1 day
4344         def _make_collected_ophandle(ophandle):
4345             d = self.POST(self.public_url +
4346                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
4347                           followRedirect=True)
4348             # By following the initial redirect, we collect the ophandle
4349             # we've just created.
4350             return d
4351         # Create a collected ophandle, then collect it after 23 hours
4352         # and 59 seconds to make sure that it is still there.
4353         d = _make_collected_ophandle(133)
4354         d.addCallback(lambda ign:
4355             self.clock.advance((24*60*60) - 1))
4356         d.addCallback(lambda ign:
4357             self.GET("/operations/133?t=status&output=JSON"))
4358         def _check1(res):
4359             data = simplejson.loads(res)
4360             self.failUnless("finished" in data, res)
4361         d.addCallback(_check1)
4362         # Create another uncollected ophandle, then try to collect it
4363         # after 24 hours to make sure that it is gone.
4364         d.addCallback(lambda ign:
4365             _make_collected_ophandle(134))
4366         d.addCallback(lambda ign:
4367             self.clock.advance(24*60*60))
4368         d.addCallback(lambda ign:
4369             self.shouldHTTPError("collected_ophandle_expired_after_1_day",
4370                                  404, "404 Not Found",
4371                                  "unknown/expired handle '134'",
4372                                  self.GET,
4373                                  "/operations/134?t=status&output=JSON"))
4374         return d
4375
4376     def test_incident(self):
4377         d = self.POST("/report_incident", details="eek")
4378         def _done(res):
4379             self.failIfIn("<html>", res)
4380             self.failUnlessIn("An incident report has been saved", res)
4381         d.addCallback(_done)
4382         return d
4383
4384     def test_static(self):
4385         webdir = os.path.join(self.staticdir, "subdir")
4386         fileutil.make_dirs(webdir)
4387         f = open(os.path.join(webdir, "hello.txt"), "wb")
4388         f.write("hello")
4389         f.close()
4390
4391         d = self.GET("/static/subdir/hello.txt")
4392         def _check(res):
4393             self.failUnlessReallyEqual(res, "hello")
4394         d.addCallback(_check)
4395         return d
4396
4397
4398 class IntroducerWeb(unittest.TestCase):
4399     def setUp(self):
4400         self.node = None
4401
4402     def tearDown(self):
4403         d = defer.succeed(None)
4404         if self.node:
4405             d.addCallback(lambda ign: self.node.stopService())
4406         d.addCallback(flushEventualQueue)
4407         return d
4408
4409     def test_welcome(self):
4410         basedir = "web.IntroducerWeb.test_welcome"
4411         os.mkdir(basedir)
4412         fileutil.write(os.path.join(basedir, "tahoe.cfg"), "[node]\nweb.port = tcp:0\n")
4413         self.node = IntroducerNode(basedir)
4414         self.ws = self.node.getServiceNamed("webish")
4415
4416         d = fireEventually(None)
4417         d.addCallback(lambda ign: self.node.startService())
4418         d.addCallback(lambda ign: self.node.when_tub_ready())
4419
4420         d.addCallback(lambda ign: self.GET("/"))
4421         def _check(res):
4422             self.failUnlessIn('Welcome to the Tahoe-LAFS Introducer', res)
4423             self.failUnlessIn(FAVICON_MARKUP, res)
4424         d.addCallback(_check)
4425         return d
4426
4427     def GET(self, urlpath, followRedirect=False, return_response=False,
4428             **kwargs):
4429         # if return_response=True, this fires with (data, statuscode,
4430         # respheaders) instead of just data.
4431         assert not isinstance(urlpath, unicode)
4432         url = self.ws.getURL().rstrip('/') + urlpath
4433         factory = HTTPClientGETFactory(url, method="GET",
4434                                        followRedirect=followRedirect, **kwargs)
4435         reactor.connectTCP("localhost", self.ws.getPortnum(), factory)
4436         d = factory.deferred
4437         def _got_data(data):
4438             return (data, factory.status, factory.response_headers)
4439         if return_response:
4440             d.addCallback(_got_data)
4441         return factory.deferred
4442
4443
4444 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4445     def test_load_file(self):
4446         # This will raise an exception unless a well-formed XML file is found under that name.
4447         common.getxmlfile('directory.xhtml').load()
4448
4449     def test_parse_replace_arg(self):
4450         self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
4451         self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
4452         self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
4453                                    "only-files")
4454         self.failUnlessRaises(common.WebError, common.parse_replace_arg, "only_fles")
4455
4456     def test_abbreviate_time(self):
4457         self.failUnlessReallyEqual(common.abbreviate_time(None), "")
4458         self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
4459         self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
4460         self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
4461         self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
4462         self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
4463
4464     def test_compute_rate(self):
4465         self.failUnlessReallyEqual(common.compute_rate(None, None), None)
4466         self.failUnlessReallyEqual(common.compute_rate(None, 1), None)
4467         self.failUnlessReallyEqual(common.compute_rate(250000, None), None)
4468         self.failUnlessReallyEqual(common.compute_rate(250000, 0), None)
4469         self.failUnlessReallyEqual(common.compute_rate(250000, 10), 25000.0)
4470         self.failUnlessReallyEqual(common.compute_rate(0, 10), 0.0)
4471         self.shouldFail(AssertionError, "test_compute_rate", "",
4472                         common.compute_rate, -100, 10)
4473         self.shouldFail(AssertionError, "test_compute_rate", "",
4474                         common.compute_rate, 100, -10)
4475
4476         # Sanity check
4477         rate = common.compute_rate(10*1000*1000, 1)
4478         self.failUnlessReallyEqual(common.abbreviate_rate(rate), "10.00MBps")
4479
4480     def test_abbreviate_rate(self):
4481         self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
4482         self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
4483         self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
4484         self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
4485
4486     def test_abbreviate_size(self):
4487         self.failUnlessReallyEqual(common.abbreviate_size(None), "")
4488         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
4489         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
4490         self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
4491         self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
4492
4493     def test_plural(self):
4494         def convert(s):
4495             return "%d second%s" % (s, status.plural(s))
4496         self.failUnlessReallyEqual(convert(0), "0 seconds")
4497         self.failUnlessReallyEqual(convert(1), "1 second")
4498         self.failUnlessReallyEqual(convert(2), "2 seconds")
4499         def convert2(s):
4500             return "has share%s: %s" % (status.plural(s), ",".join(s))
4501         self.failUnlessReallyEqual(convert2([]), "has shares: ")
4502         self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
4503         self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
4504
4505
4506 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4507
4508     def CHECK(self, ign, which, args, clientnum=0):
4509         fileurl = self.fileurls[which]
4510         url = fileurl + "?" + args
4511         return self.GET(url, method="POST", clientnum=clientnum)
4512
4513     def test_filecheck(self):
4514         self.basedir = "web/Grid/filecheck"
4515         self.set_up_grid()
4516         c0 = self.g.clients[0]
4517         self.uris = {}
4518         DATA = "data" * 100
4519         d = c0.upload(upload.Data(DATA, convergence=""))
4520         def _stash_uri(ur, which):
4521             self.uris[which] = ur.get_uri()
4522         d.addCallback(_stash_uri, "good")
4523         d.addCallback(lambda ign:
4524                       c0.upload(upload.Data(DATA+"1", convergence="")))
4525         d.addCallback(_stash_uri, "sick")
4526         d.addCallback(lambda ign:
4527                       c0.upload(upload.Data(DATA+"2", convergence="")))
4528         d.addCallback(_stash_uri, "dead")
4529         def _stash_mutable_uri(n, which):
4530             self.uris[which] = n.get_uri()
4531             assert isinstance(self.uris[which], str)
4532         d.addCallback(lambda ign:
4533             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4534         d.addCallback(_stash_mutable_uri, "corrupt")
4535         d.addCallback(lambda ign:
4536                       c0.upload(upload.Data("literal", convergence="")))
4537         d.addCallback(_stash_uri, "small")
4538         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
4539         d.addCallback(_stash_mutable_uri, "smalldir")
4540
4541         def _compute_fileurls(ignored):
4542             self.fileurls = {}
4543             for which in self.uris:
4544                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4545         d.addCallback(_compute_fileurls)
4546
4547         def _clobber_shares(ignored):
4548             good_shares = self.find_uri_shares(self.uris["good"])
4549             self.failUnlessReallyEqual(len(good_shares), 10)
4550             sick_shares = self.find_uri_shares(self.uris["sick"])
4551             os.unlink(sick_shares[0][2])
4552             dead_shares = self.find_uri_shares(self.uris["dead"])
4553             for i in range(1, 10):
4554                 os.unlink(dead_shares[i][2])
4555             c_shares = self.find_uri_shares(self.uris["corrupt"])
4556             cso = CorruptShareOptions()
4557             cso.stdout = StringIO()
4558             cso.parseOptions([c_shares[0][2]])
4559             corrupt_share(cso)
4560         d.addCallback(_clobber_shares)
4561
4562         d.addCallback(self.CHECK, "good", "t=check")
4563         def _got_html_good(res):
4564             self.failUnlessIn("Healthy", res)
4565             self.failIfIn("Not Healthy", res)
4566             self.failUnlessIn(FAVICON_MARKUP, res)
4567         d.addCallback(_got_html_good)
4568         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
4569         def _got_html_good_return_to(res):
4570             self.failUnlessIn("Healthy", res)
4571             self.failIfIn("Not Healthy", res)
4572             self.failUnlessIn('<a href="somewhere">Return to file', res)
4573         d.addCallback(_got_html_good_return_to)
4574         d.addCallback(self.CHECK, "good", "t=check&output=json")
4575         def _got_json_good(res):
4576             r = simplejson.loads(res)
4577             self.failUnlessEqual(r["summary"], "Healthy")
4578             self.failUnless(r["results"]["healthy"])
4579             self.failIf(r["results"]["needs-rebalancing"])
4580             self.failUnless(r["results"]["recoverable"])
4581         d.addCallback(_got_json_good)
4582
4583         d.addCallback(self.CHECK, "small", "t=check")
4584         def _got_html_small(res):
4585             self.failUnlessIn("Literal files are always healthy", res)
4586             self.failIfIn("Not Healthy", res)
4587         d.addCallback(_got_html_small)
4588         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
4589         def _got_html_small_return_to(res):
4590             self.failUnlessIn("Literal files are always healthy", res)
4591             self.failIfIn("Not Healthy", res)
4592             self.failUnlessIn('<a href="somewhere">Return to file', res)
4593         d.addCallback(_got_html_small_return_to)
4594         d.addCallback(self.CHECK, "small", "t=check&output=json")
4595         def _got_json_small(res):
4596             r = simplejson.loads(res)
4597             self.failUnlessEqual(r["storage-index"], "")
4598             self.failUnless(r["results"]["healthy"])
4599         d.addCallback(_got_json_small)
4600
4601         d.addCallback(self.CHECK, "smalldir", "t=check")
4602         def _got_html_smalldir(res):
4603             self.failUnlessIn("Literal files are always healthy", res)
4604             self.failIfIn("Not Healthy", res)
4605         d.addCallback(_got_html_smalldir)
4606         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
4607         def _got_json_smalldir(res):
4608             r = simplejson.loads(res)
4609             self.failUnlessEqual(r["storage-index"], "")
4610             self.failUnless(r["results"]["healthy"])
4611         d.addCallback(_got_json_smalldir)
4612
4613         d.addCallback(self.CHECK, "sick", "t=check")
4614         def _got_html_sick(res):
4615             self.failUnlessIn("Not Healthy", res)
4616         d.addCallback(_got_html_sick)
4617         d.addCallback(self.CHECK, "sick", "t=check&output=json")
4618         def _got_json_sick(res):
4619             r = simplejson.loads(res)
4620             self.failUnlessEqual(r["summary"],
4621                                  "Not Healthy: 9 shares (enc 3-of-10)")
4622             self.failIf(r["results"]["healthy"])
4623             self.failIf(r["results"]["needs-rebalancing"])
4624             self.failUnless(r["results"]["recoverable"])
4625         d.addCallback(_got_json_sick)
4626
4627         d.addCallback(self.CHECK, "dead", "t=check")
4628         def _got_html_dead(res):
4629             self.failUnlessIn("Not Healthy", res)
4630         d.addCallback(_got_html_dead)
4631         d.addCallback(self.CHECK, "dead", "t=check&output=json")
4632         def _got_json_dead(res):
4633             r = simplejson.loads(res)
4634             self.failUnlessEqual(r["summary"],
4635                                  "Not Healthy: 1 shares (enc 3-of-10)")
4636             self.failIf(r["results"]["healthy"])
4637             self.failIf(r["results"]["needs-rebalancing"])
4638             self.failIf(r["results"]["recoverable"])
4639         d.addCallback(_got_json_dead)
4640
4641         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
4642         def _got_html_corrupt(res):
4643             self.failUnlessIn("Not Healthy! : Unhealthy", res)
4644         d.addCallback(_got_html_corrupt)
4645         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
4646         def _got_json_corrupt(res):
4647             r = simplejson.loads(res)
4648             self.failUnlessIn("Unhealthy: 9 shares (enc 3-of-10)", r["summary"])
4649             self.failIf(r["results"]["healthy"])
4650             self.failUnless(r["results"]["recoverable"])
4651             self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
4652             self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
4653         d.addCallback(_got_json_corrupt)
4654
4655         d.addErrback(self.explain_web_error)
4656         return d
4657
4658     def test_repair_html(self):
4659         self.basedir = "web/Grid/repair_html"
4660         self.set_up_grid()
4661         c0 = self.g.clients[0]
4662         self.uris = {}
4663         DATA = "data" * 100
4664         d = c0.upload(upload.Data(DATA, convergence=""))
4665         def _stash_uri(ur, which):
4666             self.uris[which] = ur.get_uri()
4667         d.addCallback(_stash_uri, "good")
4668         d.addCallback(lambda ign:
4669                       c0.upload(upload.Data(DATA+"1", convergence="")))
4670         d.addCallback(_stash_uri, "sick")
4671         d.addCallback(lambda ign:
4672                       c0.upload(upload.Data(DATA+"2", convergence="")))
4673         d.addCallback(_stash_uri, "dead")
4674         def _stash_mutable_uri(n, which):
4675             self.uris[which] = n.get_uri()
4676             assert isinstance(self.uris[which], str)
4677         d.addCallback(lambda ign:
4678             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4679         d.addCallback(_stash_mutable_uri, "corrupt")
4680
4681         def _compute_fileurls(ignored):
4682             self.fileurls = {}
4683             for which in self.uris:
4684                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4685         d.addCallback(_compute_fileurls)
4686
4687         def _clobber_shares(ignored):
4688             good_shares = self.find_uri_shares(self.uris["good"])
4689             self.failUnlessReallyEqual(len(good_shares), 10)
4690             sick_shares = self.find_uri_shares(self.uris["sick"])
4691             os.unlink(sick_shares[0][2])
4692             dead_shares = self.find_uri_shares(self.uris["dead"])
4693             for i in range(1, 10):
4694                 os.unlink(dead_shares[i][2])
4695             c_shares = self.find_uri_shares(self.uris["corrupt"])
4696             cso = CorruptShareOptions()
4697             cso.stdout = StringIO()
4698             cso.parseOptions([c_shares[0][2]])
4699             corrupt_share(cso)
4700         d.addCallback(_clobber_shares)
4701
4702         d.addCallback(self.CHECK, "good", "t=check&repair=true")
4703         def _got_html_good(res):
4704             self.failUnlessIn("Healthy", res)
4705             self.failIfIn("Not Healthy", res)
4706             self.failUnlessIn("No repair necessary", res)
4707             self.failUnlessIn(FAVICON_MARKUP, res)
4708         d.addCallback(_got_html_good)
4709
4710         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
4711         def _got_html_sick(res):
4712             self.failUnlessIn("Healthy : healthy", res)
4713             self.failIfIn("Not Healthy", res)
4714             self.failUnlessIn("Repair successful", res)
4715         d.addCallback(_got_html_sick)
4716
4717         # repair of a dead file will fail, of course, but it isn't yet
4718         # clear how this should be reported. Right now it shows up as
4719         # a "410 Gone".
4720         #
4721         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
4722         #def _got_html_dead(res):
4723         #    print res
4724         #    self.failUnlessIn("Healthy : healthy", res)
4725         #    self.failIfIn("Not Healthy", res)
4726         #    self.failUnlessIn("No repair necessary", res)
4727         #d.addCallback(_got_html_dead)
4728
4729         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
4730         def _got_html_corrupt(res):
4731             self.failUnlessIn("Healthy : Healthy", res)
4732             self.failIfIn("Not Healthy", res)
4733             self.failUnlessIn("Repair successful", res)
4734         d.addCallback(_got_html_corrupt)
4735
4736         d.addErrback(self.explain_web_error)
4737         return d
4738
4739     def test_repair_json(self):
4740         self.basedir = "web/Grid/repair_json"
4741         self.set_up_grid()
4742         c0 = self.g.clients[0]
4743         self.uris = {}
4744         DATA = "data" * 100
4745         d = c0.upload(upload.Data(DATA+"1", convergence=""))
4746         def _stash_uri(ur, which):
4747             self.uris[which] = ur.get_uri()
4748         d.addCallback(_stash_uri, "sick")
4749
4750         def _compute_fileurls(ignored):
4751             self.fileurls = {}
4752             for which in self.uris:
4753                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4754         d.addCallback(_compute_fileurls)
4755
4756         def _clobber_shares(ignored):
4757             sick_shares = self.find_uri_shares(self.uris["sick"])
4758             os.unlink(sick_shares[0][2])
4759         d.addCallback(_clobber_shares)
4760
4761         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
4762         def _got_json_sick(res):
4763             r = simplejson.loads(res)
4764             self.failUnlessReallyEqual(r["repair-attempted"], True)
4765             self.failUnlessReallyEqual(r["repair-successful"], True)
4766             self.failUnlessEqual(r["pre-repair-results"]["summary"],
4767                                  "Not Healthy: 9 shares (enc 3-of-10)")
4768             self.failIf(r["pre-repair-results"]["results"]["healthy"])
4769             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
4770             self.failUnless(r["post-repair-results"]["results"]["healthy"])
4771         d.addCallback(_got_json_sick)
4772
4773         d.addErrback(self.explain_web_error)
4774         return d
4775
4776     def test_unknown(self, immutable=False):
4777         self.basedir = "web/Grid/unknown"
4778         if immutable:
4779             self.basedir = "web/Grid/unknown-immutable"
4780
4781         self.set_up_grid()
4782         c0 = self.g.clients[0]
4783         self.uris = {}
4784         self.fileurls = {}
4785
4786         # the future cap format may contain slashes, which must be tolerated
4787         expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
4788                                                            safe="")
4789
4790         if immutable:
4791             name = u"future-imm"
4792             future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
4793             d = c0.create_immutable_dirnode({name: (future_node, {})})
4794         else:
4795             name = u"future"
4796             future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4797             d = c0.create_dirnode()
4798
4799         def _stash_root_and_create_file(n):
4800             self.rootnode = n
4801             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
4802             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
4803             if not immutable:
4804                 return self.rootnode.set_node(name, future_node)
4805         d.addCallback(_stash_root_and_create_file)
4806
4807         # make sure directory listing tolerates unknown nodes
4808         d.addCallback(lambda ign: self.GET(self.rooturl))
4809         def _check_directory_html(res, expected_type_suffix):
4810             pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
4811                                   '<td>%s</td>' % (expected_type_suffix, str(name)),
4812                                  re.DOTALL)
4813             self.failUnless(re.search(pattern, res), res)
4814             # find the More Info link for name, should be relative
4815             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4816             info_url = mo.group(1)
4817             self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
4818         if immutable:
4819             d.addCallback(_check_directory_html, "-IMM")
4820         else:
4821             d.addCallback(_check_directory_html, "")
4822
4823         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4824         def _check_directory_json(res, expect_rw_uri):
4825             data = simplejson.loads(res)
4826             self.failUnlessEqual(data[0], "dirnode")
4827             f = data[1]["children"][name]
4828             self.failUnlessEqual(f[0], "unknown")
4829             if expect_rw_uri:
4830                 self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data)
4831             else:
4832                 self.failIfIn("rw_uri", f[1])
4833             if immutable:
4834                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
4835             else:
4836                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
4837             self.failUnlessIn("metadata", f[1])
4838         d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
4839
4840         def _check_info(res, expect_rw_uri, expect_ro_uri):
4841             self.failUnlessIn("Object Type: <span>unknown</span>", res)
4842             if expect_rw_uri:
4843                 self.failUnlessIn(unknown_rwcap, res)
4844             if expect_ro_uri:
4845                 if immutable:
4846                     self.failUnlessIn(unknown_immcap, res)
4847                 else:
4848                     self.failUnlessIn(unknown_rocap, res)
4849             else:
4850                 self.failIfIn(unknown_rocap, res)
4851             self.failIfIn("Raw data as", res)
4852             self.failIfIn("Directory writecap", res)
4853             self.failIfIn("Checker Operations", res)
4854             self.failIfIn("Mutable File Operations", res)
4855             self.failIfIn("Directory Operations", res)
4856
4857         # FIXME: these should have expect_rw_uri=not immutable; I don't know
4858         # why they fail. Possibly related to ticket #922.
4859
4860         d.addCallback(lambda ign: self.GET(expected_info_url))
4861         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
4862         d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
4863         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
4864
4865         def _check_json(res, expect_rw_uri):
4866             data = simplejson.loads(res)
4867             self.failUnlessEqual(data[0], "unknown")
4868             if expect_rw_uri:
4869                 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data)
4870             else:
4871                 self.failIfIn("rw_uri", data[1])
4872
4873             if immutable:
4874                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data)
4875                 self.failUnlessReallyEqual(data[1]["mutable"], False)
4876             elif expect_rw_uri:
4877                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4878                 self.failUnlessReallyEqual(data[1]["mutable"], True)
4879             else:
4880                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4881                 self.failIfIn("mutable", data[1])
4882
4883             # TODO: check metadata contents
4884             self.failUnlessIn("metadata", data[1])
4885
4886         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
4887         d.addCallback(_check_json, expect_rw_uri=not immutable)
4888
4889         # and make sure that a read-only version of the directory can be
4890         # rendered too. This version will not have unknown_rwcap, whether
4891         # or not future_node was immutable.
4892         d.addCallback(lambda ign: self.GET(self.rourl))
4893         if immutable:
4894             d.addCallback(_check_directory_html, "-IMM")
4895         else:
4896             d.addCallback(_check_directory_html, "-RO")
4897
4898         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
4899         d.addCallback(_check_directory_json, expect_rw_uri=False)
4900
4901         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
4902         d.addCallback(_check_json, expect_rw_uri=False)
4903
4904         # TODO: check that getting t=info from the Info link in the ro directory
4905         # works, and does not include the writecap URI.
4906         return d
4907
4908     def test_immutable_unknown(self):
4909         return self.test_unknown(immutable=True)
4910
4911     def test_mutant_dirnodes_are_omitted(self):
4912         self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
4913
4914         self.set_up_grid()
4915         c = self.g.clients[0]
4916         nm = c.nodemaker
4917         self.uris = {}
4918         self.fileurls = {}
4919
4920         lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
4921         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
4922         mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
4923
4924         # This method tests mainly dirnode, but we'd have to duplicate code in order to
4925         # test the dirnode and web layers separately.
4926
4927         # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
4928         # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
4929         # When the directory is read, the mutants should be silently disposed of, leaving
4930         # their lonely sibling.
4931         # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
4932         # because immutable directories don't have a writecap and therefore that field
4933         # isn't (and can't be) decrypted.
4934         # TODO: The field still exists in the netstring. Technically we should check what
4935         # happens if something is put there (_unpack_contents should raise ValueError),
4936         # but that can wait.
4937
4938         lonely_child = nm.create_from_cap(lonely_uri)
4939         mutant_ro_child = nm.create_from_cap(mut_read_uri)
4940         mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
4941
4942         def _by_hook_or_by_crook():
4943             return True
4944         for n in [mutant_ro_child, mutant_write_in_ro_child]:
4945             n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
4946
4947         mutant_write_in_ro_child.get_write_uri    = lambda: None
4948         mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
4949
4950         kids = {u"lonely":      (lonely_child, {}),
4951                 u"ro":          (mutant_ro_child, {}),
4952                 u"write-in-ro": (mutant_write_in_ro_child, {}),
4953                 }
4954         d = c.create_immutable_dirnode(kids)
4955
4956         def _created(dn):
4957             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
4958             self.failIf(dn.is_mutable())
4959             self.failUnless(dn.is_readonly())
4960             # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
4961             self.failIf(hasattr(dn._node, 'get_writekey'))
4962             rep = str(dn)
4963             self.failUnlessIn("RO-IMM", rep)
4964             cap = dn.get_cap()
4965             self.failUnlessIn("CHK", cap.to_string())
4966             self.cap = cap
4967             self.rootnode = dn
4968             self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
4969             return download_to_data(dn._node)
4970         d.addCallback(_created)
4971
4972         def _check_data(data):
4973             # Decode the netstring representation of the directory to check that all children
4974             # are present. This is a bit of an abstraction violation, but there's not really
4975             # any other way to do it given that the real DirectoryNode._unpack_contents would
4976             # strip the mutant children out (which is what we're trying to test, later).
4977             position = 0
4978             numkids = 0
4979             while position < len(data):
4980                 entries, position = split_netstring(data, 1, position)
4981                 entry = entries[0]
4982                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
4983                 name = name_utf8.decode("utf-8")
4984                 self.failUnlessEqual(rwcapdata, "")
4985                 self.failUnlessIn(name, kids)
4986                 (expected_child, ign) = kids[name]
4987                 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
4988                 numkids += 1
4989
4990             self.failUnlessReallyEqual(numkids, 3)
4991             return self.rootnode.list()
4992         d.addCallback(_check_data)
4993
4994         # Now when we use the real directory listing code, the mutants should be absent.
4995         def _check_kids(children):
4996             self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
4997             lonely_node, lonely_metadata = children[u"lonely"]
4998
4999             self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
5000             self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
5001         d.addCallback(_check_kids)
5002
5003         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
5004         d.addCallback(lambda n: n.list())
5005         d.addCallback(_check_kids)  # again with dirnode recreated from cap
5006
5007         # Make sure the lonely child can be listed in HTML...
5008         d.addCallback(lambda ign: self.GET(self.rooturl))
5009         def _check_html(res):
5010             self.failIfIn("URI:SSK", res)
5011             get_lonely = "".join([r'<td>FILE</td>',
5012                                   r'\s+<td>',
5013                                   r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
5014                                   r'</td>',
5015                                   r'\s+<td align="right">%d</td>' % len("one"),
5016                                   ])
5017             self.failUnless(re.search(get_lonely, res), res)
5018
5019             # find the More Info link for name, should be relative
5020             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
5021             info_url = mo.group(1)
5022             self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
5023         d.addCallback(_check_html)
5024
5025         # ... and in JSON.
5026         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
5027         def _check_json(res):
5028             data = simplejson.loads(res)
5029             self.failUnlessEqual(data[0], "dirnode")
5030             listed_children = data[1]["children"]
5031             self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
5032             ll_type, ll_data = listed_children[u"lonely"]
5033             self.failUnlessEqual(ll_type, "filenode")
5034             self.failIfIn("rw_uri", ll_data)
5035             self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
5036         d.addCallback(_check_json)
5037         return d
5038
5039     def test_deep_check(self):
5040         self.basedir = "web/Grid/deep_check"
5041         self.set_up_grid()
5042         c0 = self.g.clients[0]
5043         self.uris = {}
5044         self.fileurls = {}
5045         DATA = "data" * 100
5046         d = c0.create_dirnode()
5047         def _stash_root_and_create_file(n):
5048             self.rootnode = n
5049             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5050             return n.add_file(u"good", upload.Data(DATA, convergence=""))
5051         d.addCallback(_stash_root_and_create_file)
5052         def _stash_uri(fn, which):
5053             self.uris[which] = fn.get_uri()
5054             return fn
5055         d.addCallback(_stash_uri, "good")
5056         d.addCallback(lambda ign:
5057                       self.rootnode.add_file(u"small",
5058                                              upload.Data("literal",
5059                                                         convergence="")))
5060         d.addCallback(_stash_uri, "small")
5061         d.addCallback(lambda ign:
5062                       self.rootnode.add_file(u"sick",
5063                                              upload.Data(DATA+"1",
5064                                                         convergence="")))
5065         d.addCallback(_stash_uri, "sick")
5066
5067         # this tests that deep-check and stream-manifest will ignore
5068         # UnknownNode instances. Hopefully this will also cover deep-stats.
5069         future_node = UnknownNode(unknown_rwcap, unknown_rocap)
5070         d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
5071
5072         def _clobber_shares(ignored):
5073             self.delete_shares_numbered(self.uris["sick"], [0,1])
5074         d.addCallback(_clobber_shares)
5075
5076         # root
5077         # root/good
5078         # root/small
5079         # root/sick
5080         # root/future
5081
5082         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
5083         def _done(res):
5084             try:
5085                 units = [simplejson.loads(line)
5086                          for line in res.splitlines()
5087                          if line]
5088             except ValueError:
5089                 print "response is:", res
5090                 print "undecodeable line was '%s'" % line
5091                 raise
5092             self.failUnlessReallyEqual(len(units), 5+1)
5093             # should be parent-first
5094             u0 = units[0]
5095             self.failUnlessEqual(u0["path"], [])
5096             self.failUnlessEqual(u0["type"], "directory")
5097             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
5098             u0cr = u0["check-results"]
5099             self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
5100
5101             ugood = [u for u in units
5102                      if u["type"] == "file" and u["path"] == [u"good"]][0]
5103             self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
5104             ugoodcr = ugood["check-results"]
5105             self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
5106
5107             stats = units[-1]
5108             self.failUnlessEqual(stats["type"], "stats")
5109             s = stats["stats"]
5110             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
5111             self.failUnlessReallyEqual(s["count-literal-files"], 1)
5112             self.failUnlessReallyEqual(s["count-directories"], 1)
5113             self.failUnlessReallyEqual(s["count-unknown"], 1)
5114         d.addCallback(_done)
5115
5116         d.addCallback(self.CHECK, "root", "t=stream-manifest")
5117         def _check_manifest(res):
5118             self.failUnless(res.endswith("\n"))
5119             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
5120             self.failUnlessReallyEqual(len(units), 5+1)
5121             self.failUnlessEqual(units[-1]["type"], "stats")
5122             first = units[0]
5123             self.failUnlessEqual(first["path"], [])
5124             self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri())
5125             self.failUnlessEqual(first["type"], "directory")
5126             stats = units[-1]["stats"]
5127             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
5128             self.failUnlessReallyEqual(stats["count-literal-files"], 1)
5129             self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
5130             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
5131             self.failUnlessReallyEqual(stats["count-unknown"], 1)
5132         d.addCallback(_check_manifest)
5133
5134         # now add root/subdir and root/subdir/grandchild, then make subdir
5135         # unrecoverable, then see what happens
5136
5137         d.addCallback(lambda ign:
5138                       self.rootnode.create_subdirectory(u"subdir"))
5139         d.addCallback(_stash_uri, "subdir")
5140         d.addCallback(lambda subdir_node:
5141                       subdir_node.add_file(u"grandchild",
5142                                            upload.Data(DATA+"2",
5143                                                        convergence="")))
5144         d.addCallback(_stash_uri, "grandchild")
5145
5146         d.addCallback(lambda ign:
5147                       self.delete_shares_numbered(self.uris["subdir"],
5148                                                   range(1, 10)))
5149
5150         # root
5151         # root/good
5152         # root/small
5153         # root/sick
5154         # root/future
5155         # root/subdir [unrecoverable]
5156         # root/subdir/grandchild
5157
5158         # how should a streaming-JSON API indicate fatal error?
5159         # answer: emit ERROR: instead of a JSON string
5160
5161         d.addCallback(self.CHECK, "root", "t=stream-manifest")
5162         def _check_broken_manifest(res):
5163             lines = res.splitlines()
5164             error_lines = [i
5165                            for (i,line) in enumerate(lines)
5166                            if line.startswith("ERROR:")]
5167             if not error_lines:
5168                 self.fail("no ERROR: in output: %s" % (res,))
5169             first_error = error_lines[0]
5170             error_line = lines[first_error]
5171             error_msg = lines[first_error+1:]
5172             error_msg_s = "\n".join(error_msg) + "\n"
5173             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
5174                               error_line)
5175             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
5176             units = [simplejson.loads(line) for line in lines[:first_error]]
5177             self.failUnlessReallyEqual(len(units), 6) # includes subdir
5178             last_unit = units[-1]
5179             self.failUnlessEqual(last_unit["path"], ["subdir"])
5180         d.addCallback(_check_broken_manifest)
5181
5182         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
5183         def _check_broken_deepcheck(res):
5184             lines = res.splitlines()
5185             error_lines = [i
5186                            for (i,line) in enumerate(lines)
5187                            if line.startswith("ERROR:")]
5188             if not error_lines:
5189                 self.fail("no ERROR: in output: %s" % (res,))
5190             first_error = error_lines[0]
5191             error_line = lines[first_error]
5192             error_msg = lines[first_error+1:]
5193             error_msg_s = "\n".join(error_msg) + "\n"
5194             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
5195                               error_line)
5196             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
5197             units = [simplejson.loads(line) for line in lines[:first_error]]
5198             self.failUnlessReallyEqual(len(units), 6) # includes subdir
5199             last_unit = units[-1]
5200             self.failUnlessEqual(last_unit["path"], ["subdir"])
5201             r = last_unit["check-results"]["results"]
5202             self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
5203             self.failUnlessReallyEqual(r["count-shares-good"], 1)
5204             self.failUnlessReallyEqual(r["recoverable"], False)
5205         d.addCallback(_check_broken_deepcheck)
5206
5207         d.addErrback(self.explain_web_error)
5208         return d
5209
5210     def test_deep_check_and_repair(self):
5211         self.basedir = "web/Grid/deep_check_and_repair"
5212         self.set_up_grid()
5213         c0 = self.g.clients[0]
5214         self.uris = {}
5215         self.fileurls = {}
5216         DATA = "data" * 100
5217         d = c0.create_dirnode()
5218         def _stash_root_and_create_file(n):
5219             self.rootnode = n
5220             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5221             return n.add_file(u"good", upload.Data(DATA, convergence=""))
5222         d.addCallback(_stash_root_and_create_file)
5223         def _stash_uri(fn, which):
5224             self.uris[which] = fn.get_uri()
5225         d.addCallback(_stash_uri, "good")
5226         d.addCallback(lambda ign:
5227                       self.rootnode.add_file(u"small",
5228                                              upload.Data("literal",
5229                                                         convergence="")))
5230         d.addCallback(_stash_uri, "small")
5231         d.addCallback(lambda ign:
5232                       self.rootnode.add_file(u"sick",
5233                                              upload.Data(DATA+"1",
5234                                                         convergence="")))
5235         d.addCallback(_stash_uri, "sick")
5236         #d.addCallback(lambda ign:
5237         #              self.rootnode.add_file(u"dead",
5238         #                                     upload.Data(DATA+"2",
5239         #                                                convergence="")))
5240         #d.addCallback(_stash_uri, "dead")
5241
5242         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
5243         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
5244         #d.addCallback(_stash_uri, "corrupt")
5245
5246         def _clobber_shares(ignored):
5247             good_shares = self.find_uri_shares(self.uris["good"])
5248             self.failUnlessReallyEqual(len(good_shares), 10)
5249             sick_shares = self.find_uri_shares(self.uris["sick"])
5250             os.unlink(sick_shares[0][2])
5251             #dead_shares = self.find_uri_shares(self.uris["dead"])
5252             #for i in range(1, 10):
5253             #    os.unlink(dead_shares[i][2])
5254
5255             #c_shares = self.find_uri_shares(self.uris["corrupt"])
5256             #cso = CorruptShareOptions()
5257             #cso.stdout = StringIO()
5258             #cso.parseOptions([c_shares[0][2]])
5259             #corrupt_share(cso)
5260         d.addCallback(_clobber_shares)
5261
5262         # root
5263         # root/good   CHK, 10 shares
5264         # root/small  LIT
5265         # root/sick   CHK, 9 shares
5266
5267         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
5268         def _done(res):
5269             units = [simplejson.loads(line)
5270                      for line in res.splitlines()
5271                      if line]
5272             self.failUnlessReallyEqual(len(units), 4+1)
5273             # should be parent-first
5274             u0 = units[0]
5275             self.failUnlessEqual(u0["path"], [])
5276             self.failUnlessEqual(u0["type"], "directory")
5277             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
5278             u0crr = u0["check-and-repair-results"]
5279             self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
5280             self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
5281
5282             ugood = [u for u in units
5283                      if u["type"] == "file" and u["path"] == [u"good"]][0]
5284             self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
5285             ugoodcrr = ugood["check-and-repair-results"]
5286             self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
5287             self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
5288
5289             usick = [u for u in units
5290                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
5291             self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"])
5292             usickcrr = usick["check-and-repair-results"]
5293             self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
5294             self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
5295             self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
5296             self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
5297
5298             stats = units[-1]
5299             self.failUnlessEqual(stats["type"], "stats")
5300             s = stats["stats"]
5301             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
5302             self.failUnlessReallyEqual(s["count-literal-files"], 1)
5303             self.failUnlessReallyEqual(s["count-directories"], 1)
5304         d.addCallback(_done)
5305
5306         d.addErrback(self.explain_web_error)
5307         return d
5308
5309     def _count_leases(self, ignored, which):
5310         u = self.uris[which]
5311         shares = self.find_uri_shares(u)
5312         lease_counts = []
5313         for shnum, serverid, fn in shares:
5314             sf = get_share_file(fn)
5315             num_leases = len(list(sf.get_leases()))
5316             lease_counts.append( (fn, num_leases) )
5317         return lease_counts
5318
5319     def _assert_leasecount(self, lease_counts, expected):
5320         for (fn, num_leases) in lease_counts:
5321             if num_leases != expected:
5322                 self.fail("expected %d leases, have %d, on %s" %
5323                           (expected, num_leases, fn))
5324
5325     def test_add_lease(self):
5326         self.basedir = "web/Grid/add_lease"
5327         self.set_up_grid(num_clients=2)
5328         c0 = self.g.clients[0]
5329         self.uris = {}
5330         DATA = "data" * 100
5331         d = c0.upload(upload.Data(DATA, convergence=""))
5332         def _stash_uri(ur, which):
5333             self.uris[which] = ur.get_uri()
5334         d.addCallback(_stash_uri, "one")
5335         d.addCallback(lambda ign:
5336                       c0.upload(upload.Data(DATA+"1", convergence="")))
5337         d.addCallback(_stash_uri, "two")
5338         def _stash_mutable_uri(n, which):
5339             self.uris[which] = n.get_uri()
5340             assert isinstance(self.uris[which], str)
5341         d.addCallback(lambda ign:
5342             c0.create_mutable_file(publish.MutableData(DATA+"2")))
5343         d.addCallback(_stash_mutable_uri, "mutable")
5344
5345         def _compute_fileurls(ignored):
5346             self.fileurls = {}
5347             for which in self.uris:
5348                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
5349         d.addCallback(_compute_fileurls)
5350
5351         d.addCallback(self._count_leases, "one")
5352         d.addCallback(self._assert_leasecount, 1)
5353         d.addCallback(self._count_leases, "two")
5354         d.addCallback(self._assert_leasecount, 1)
5355         d.addCallback(self._count_leases, "mutable")
5356         d.addCallback(self._assert_leasecount, 1)
5357
5358         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
5359         def _got_html_good(res):
5360             self.failUnlessIn("Healthy", res)
5361             self.failIfIn("Not Healthy", res)
5362         d.addCallback(_got_html_good)
5363
5364         d.addCallback(self._count_leases, "one")
5365         d.addCallback(self._assert_leasecount, 1)
5366         d.addCallback(self._count_leases, "two")
5367         d.addCallback(self._assert_leasecount, 1)
5368         d.addCallback(self._count_leases, "mutable")
5369         d.addCallback(self._assert_leasecount, 1)
5370
5371         # this CHECK uses the original client, which uses the same
5372         # lease-secrets, so it will just renew the original lease
5373         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
5374         d.addCallback(_got_html_good)
5375
5376         d.addCallback(self._count_leases, "one")
5377         d.addCallback(self._assert_leasecount, 1)
5378         d.addCallback(self._count_leases, "two")
5379         d.addCallback(self._assert_leasecount, 1)
5380         d.addCallback(self._count_leases, "mutable")
5381         d.addCallback(self._assert_leasecount, 1)
5382
5383         # this CHECK uses an alternate client, which adds a second lease
5384         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
5385         d.addCallback(_got_html_good)
5386
5387         d.addCallback(self._count_leases, "one")
5388         d.addCallback(self._assert_leasecount, 2)
5389         d.addCallback(self._count_leases, "two")
5390         d.addCallback(self._assert_leasecount, 1)
5391         d.addCallback(self._count_leases, "mutable")
5392         d.addCallback(self._assert_leasecount, 1)
5393
5394         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
5395         d.addCallback(_got_html_good)
5396
5397         d.addCallback(self._count_leases, "one")
5398         d.addCallback(self._assert_leasecount, 2)
5399         d.addCallback(self._count_leases, "two")
5400         d.addCallback(self._assert_leasecount, 1)
5401         d.addCallback(self._count_leases, "mutable")
5402         d.addCallback(self._assert_leasecount, 1)
5403
5404         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
5405                       clientnum=1)
5406         d.addCallback(_got_html_good)
5407
5408         d.addCallback(self._count_leases, "one")
5409         d.addCallback(self._assert_leasecount, 2)
5410         d.addCallback(self._count_leases, "two")
5411         d.addCallback(self._assert_leasecount, 1)
5412         d.addCallback(self._count_leases, "mutable")
5413         d.addCallback(self._assert_leasecount, 2)
5414
5415         d.addErrback(self.explain_web_error)
5416         return d
5417
5418     def test_deep_add_lease(self):
5419         self.basedir = "web/Grid/deep_add_lease"
5420         self.set_up_grid(num_clients=2)
5421         c0 = self.g.clients[0]
5422         self.uris = {}
5423         self.fileurls = {}
5424         DATA = "data" * 100
5425         d = c0.create_dirnode()
5426         def _stash_root_and_create_file(n):
5427             self.rootnode = n
5428             self.uris["root"] = n.get_uri()
5429             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5430             return n.add_file(u"one", upload.Data(DATA, convergence=""))
5431         d.addCallback(_stash_root_and_create_file)
5432         def _stash_uri(fn, which):
5433             self.uris[which] = fn.get_uri()
5434         d.addCallback(_stash_uri, "one")
5435         d.addCallback(lambda ign:
5436                       self.rootnode.add_file(u"small",
5437                                              upload.Data("literal",
5438                                                         convergence="")))
5439         d.addCallback(_stash_uri, "small")
5440
5441         d.addCallback(lambda ign:
5442             c0.create_mutable_file(publish.MutableData("mutable")))
5443         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
5444         d.addCallback(_stash_uri, "mutable")
5445
5446         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
5447         def _done(res):
5448             units = [simplejson.loads(line)
5449                      for line in res.splitlines()
5450                      if line]
5451             # root, one, small, mutable,   stats
5452             self.failUnlessReallyEqual(len(units), 4+1)
5453         d.addCallback(_done)
5454
5455         d.addCallback(self._count_leases, "root")
5456         d.addCallback(self._assert_leasecount, 1)
5457         d.addCallback(self._count_leases, "one")
5458         d.addCallback(self._assert_leasecount, 1)
5459         d.addCallback(self._count_leases, "mutable")
5460         d.addCallback(self._assert_leasecount, 1)
5461
5462         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
5463         d.addCallback(_done)
5464
5465         d.addCallback(self._count_leases, "root")
5466         d.addCallback(self._assert_leasecount, 1)
5467         d.addCallback(self._count_leases, "one")
5468         d.addCallback(self._assert_leasecount, 1)
5469         d.addCallback(self._count_leases, "mutable")
5470         d.addCallback(self._assert_leasecount, 1)
5471
5472         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
5473                       clientnum=1)
5474         d.addCallback(_done)
5475
5476         d.addCallback(self._count_leases, "root")
5477         d.addCallback(self._assert_leasecount, 2)
5478         d.addCallback(self._count_leases, "one")
5479         d.addCallback(self._assert_leasecount, 2)
5480         d.addCallback(self._count_leases, "mutable")
5481         d.addCallback(self._assert_leasecount, 2)
5482
5483         d.addErrback(self.explain_web_error)
5484         return d
5485
5486
5487     def test_exceptions(self):
5488         self.basedir = "web/Grid/exceptions"
5489         self.set_up_grid(num_clients=1, num_servers=2)
5490         c0 = self.g.clients[0]
5491         c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 2
5492         self.fileurls = {}
5493         DATA = "data" * 100
5494         d = c0.create_dirnode()
5495         def _stash_root(n):
5496             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5497             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
5498             return n
5499         d.addCallback(_stash_root)
5500         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
5501         def _stash_bad(ur):
5502             self.fileurls["1share"] = "uri/" + urllib.quote(ur.get_uri())
5503             self.delete_shares_numbered(ur.get_uri(), range(1,10))
5504
5505             u = uri.from_string(ur.get_uri())
5506             u.key = testutil.flip_bit(u.key, 0)
5507             baduri = u.to_string()
5508             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
5509         d.addCallback(_stash_bad)
5510         d.addCallback(lambda ign: c0.create_dirnode())
5511         def _mangle_dirnode_1share(n):
5512             u = n.get_uri()
5513             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
5514             self.fileurls["dir-1share-json"] = url + "?t=json"
5515             self.delete_shares_numbered(u, range(1,10))
5516         d.addCallback(_mangle_dirnode_1share)
5517         d.addCallback(lambda ign: c0.create_dirnode())
5518         def _mangle_dirnode_0share(n):
5519             u = n.get_uri()
5520             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
5521             self.fileurls["dir-0share-json"] = url + "?t=json"
5522             self.delete_shares_numbered(u, range(0,10))
5523         d.addCallback(_mangle_dirnode_0share)
5524
5525         # NotEnoughSharesError should be reported sensibly, with a
5526         # text/plain explanation of the problem, and perhaps some
5527         # information on which shares *could* be found.
5528
5529         d.addCallback(lambda ignored:
5530                       self.shouldHTTPError("GET unrecoverable",
5531                                            410, "Gone", "NoSharesError",
5532                                            self.GET, self.fileurls["0shares"]))
5533         def _check_zero_shares(body):
5534             self.failIfIn("<html>", body)
5535             body = " ".join(body.strip().split())
5536             exp = ("NoSharesError: no shares could be found. "
5537                    "Zero shares usually indicates a corrupt URI, or that "
5538                    "no servers were connected, but it might also indicate "
5539                    "severe corruption. You should perform a filecheck on "
5540                    "this object to learn more. The full error message is: "
5541                    "no shares (need 3). Last failure: None")
5542             self.failUnlessReallyEqual(exp, body)
5543         d.addCallback(_check_zero_shares)
5544
5545
5546         d.addCallback(lambda ignored:
5547                       self.shouldHTTPError("GET 1share",
5548                                            410, "Gone", "NotEnoughSharesError",
5549                                            self.GET, self.fileurls["1share"]))
5550         def _check_one_share(body):
5551             self.failIfIn("<html>", body)
5552             body = " ".join(body.strip().split())
5553             msgbase = ("NotEnoughSharesError: This indicates that some "
5554                        "servers were unavailable, or that shares have been "
5555                        "lost to server departure, hard drive failure, or disk "
5556                        "corruption. You should perform a filecheck on "
5557                        "this object to learn more. The full error message is:"
5558                        )
5559             msg1 = msgbase + (" ran out of shares:"
5560                               " complete=sh0"
5561                               " pending="
5562                               " overdue= unused= need 3. Last failure: None")
5563             msg2 = msgbase + (" ran out of shares:"
5564                               " complete="
5565                               " pending=Share(sh0-on-xgru5)"
5566                               " overdue= unused= need 3. Last failure: None")
5567             self.failUnless(body == msg1 or body == msg2, body)
5568         d.addCallback(_check_one_share)
5569
5570         d.addCallback(lambda ignored:
5571                       self.shouldHTTPError("GET imaginary",
5572                                            404, "Not Found", None,
5573                                            self.GET, self.fileurls["imaginary"]))
5574         def _missing_child(body):
5575             self.failUnlessIn("No such child: imaginary", body)
5576         d.addCallback(_missing_child)
5577
5578         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
5579         def _check_0shares_dir_html(body):
5580             self.failUnlessIn("<html>", body)
5581             # we should see the regular page, but without the child table or
5582             # the dirops forms
5583             body = " ".join(body.strip().split())
5584             self.failUnlessIn('href="?t=info">More info on this directory',
5585                               body)
5586             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5587                    "could not be retrieved, because there were insufficient "
5588                    "good shares. This might indicate that no servers were "
5589                    "connected, insufficient servers were connected, the URI "
5590                    "was corrupt, or that shares have been lost due to server "
5591                    "departure, hard drive failure, or disk corruption. You "
5592                    "should perform a filecheck on this object to learn more.")
5593             self.failUnlessIn(exp, body)
5594             self.failUnlessIn("No upload forms: directory is unreadable", body)
5595         d.addCallback(_check_0shares_dir_html)
5596
5597         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
5598         def _check_1shares_dir_html(body):
5599             # at some point, we'll split UnrecoverableFileError into 0-shares
5600             # and some-shares like we did for immutable files (since there
5601             # are different sorts of advice to offer in each case). For now,
5602             # they present the same way.
5603             self.failUnlessIn("<html>", body)
5604             body = " ".join(body.strip().split())
5605             self.failUnlessIn('href="?t=info">More info on this directory',
5606                               body)
5607             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5608                    "could not be retrieved, because there were insufficient "
5609                    "good shares. This might indicate that no servers were "
5610                    "connected, insufficient servers were connected, the URI "
5611                    "was corrupt, or that shares have been lost due to server "
5612                    "departure, hard drive failure, or disk corruption. You "
5613                    "should perform a filecheck on this object to learn more.")
5614             self.failUnlessIn(exp, body)
5615             self.failUnlessIn("No upload forms: directory is unreadable", body)
5616         d.addCallback(_check_1shares_dir_html)
5617
5618         d.addCallback(lambda ignored:
5619                       self.shouldHTTPError("GET dir-0share-json",
5620                                            410, "Gone", "UnrecoverableFileError",
5621                                            self.GET,
5622                                            self.fileurls["dir-0share-json"]))
5623         def _check_unrecoverable_file(body):
5624             self.failIfIn("<html>", body)
5625             body = " ".join(body.strip().split())
5626             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5627                    "could not be retrieved, because there were insufficient "
5628                    "good shares. This might indicate that no servers were "
5629                    "connected, insufficient servers were connected, the URI "
5630                    "was corrupt, or that shares have been lost due to server "
5631                    "departure, hard drive failure, or disk corruption. You "
5632                    "should perform a filecheck on this object to learn more.")
5633             self.failUnlessReallyEqual(exp, body)
5634         d.addCallback(_check_unrecoverable_file)
5635
5636         d.addCallback(lambda ignored:
5637                       self.shouldHTTPError("GET dir-1share-json",
5638                                            410, "Gone", "UnrecoverableFileError",
5639                                            self.GET,
5640                                            self.fileurls["dir-1share-json"]))
5641         d.addCallback(_check_unrecoverable_file)
5642
5643         d.addCallback(lambda ignored:
5644                       self.shouldHTTPError("GET imaginary",
5645                                            404, "Not Found", None,
5646                                            self.GET, self.fileurls["imaginary"]))
5647
5648         # attach a webapi child that throws a random error, to test how it
5649         # gets rendered.
5650         w = c0.getServiceNamed("webish")
5651         w.root.putChild("ERRORBOOM", ErrorBoom())
5652
5653         # "Accept: */*" :        should get a text/html stack trace
5654         # "Accept: text/plain" : should get a text/plain stack trace
5655         # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
5656         # no Accept header:      should get a text/html stack trace
5657
5658         d.addCallback(lambda ignored:
5659                       self.shouldHTTPError("GET errorboom_html",
5660                                            500, "Internal Server Error", None,
5661                                            self.GET, "ERRORBOOM",
5662                                            headers={"accept": "*/*"}))
5663         def _internal_error_html1(body):
5664             self.failUnlessIn("<html>", "expected HTML, not '%s'" % body)
5665         d.addCallback(_internal_error_html1)
5666
5667         d.addCallback(lambda ignored:
5668                       self.shouldHTTPError("GET errorboom_text",
5669                                            500, "Internal Server Error", None,
5670                                            self.GET, "ERRORBOOM",
5671                                            headers={"accept": "text/plain"}))
5672         def _internal_error_text2(body):
5673             self.failIfIn("<html>", body)
5674             self.failUnless(body.startswith("Traceback "), body)
5675         d.addCallback(_internal_error_text2)
5676
5677         CLI_accepts = "text/plain, application/octet-stream"
5678         d.addCallback(lambda ignored:
5679                       self.shouldHTTPError("GET errorboom_text",
5680                                            500, "Internal Server Error", None,
5681                                            self.GET, "ERRORBOOM",
5682                                            headers={"accept": CLI_accepts}))
5683         def _internal_error_text3(body):
5684             self.failIfIn("<html>", body)
5685             self.failUnless(body.startswith("Traceback "), body)
5686         d.addCallback(_internal_error_text3)
5687
5688         d.addCallback(lambda ignored:
5689                       self.shouldHTTPError("GET errorboom_text",
5690                                            500, "Internal Server Error", None,
5691                                            self.GET, "ERRORBOOM"))
5692         def _internal_error_html4(body):
5693             self.failUnlessIn("<html>", body)
5694         d.addCallback(_internal_error_html4)
5695
5696         def _flush_errors(res):
5697             # Trial: please ignore the CompletelyUnhandledError in the logs
5698             self.flushLoggedErrors(CompletelyUnhandledError)
5699             return res
5700         d.addBoth(_flush_errors)
5701
5702         return d
5703
5704     def test_blacklist(self):
5705         # download from a blacklisted URI, get an error
5706         self.basedir = "web/Grid/blacklist"
5707         self.set_up_grid()
5708         c0 = self.g.clients[0]
5709         c0_basedir = c0.basedir
5710         fn = os.path.join(c0_basedir, "access.blacklist")
5711         self.uris = {}
5712         DATA = "off-limits " * 50
5713
5714         d = c0.upload(upload.Data(DATA, convergence=""))
5715         def _stash_uri_and_create_dir(ur):
5716             self.uri = ur.get_uri()
5717             self.url = "uri/"+self.uri
5718             u = uri.from_string_filenode(self.uri)
5719             self.si = u.get_storage_index()
5720             childnode = c0.create_node_from_uri(self.uri, None)
5721             return c0.create_dirnode({u"blacklisted.txt": (childnode,{}) })
5722         d.addCallback(_stash_uri_and_create_dir)
5723         def _stash_dir(node):
5724             self.dir_node = node
5725             self.dir_uri = node.get_uri()
5726             self.dir_url = "uri/"+self.dir_uri
5727         d.addCallback(_stash_dir)
5728         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5729         def _check_dir_html(body):
5730             self.failUnlessIn("<html>", body)
5731             self.failUnlessIn("blacklisted.txt</a>", body)
5732         d.addCallback(_check_dir_html)
5733         d.addCallback(lambda ign: self.GET(self.url))
5734         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5735
5736         def _blacklist(ign):
5737             f = open(fn, "w")
5738             f.write(" # this is a comment\n")
5739             f.write(" \n")
5740             f.write("\n") # also exercise blank lines
5741             f.write("%s %s\n" % (base32.b2a(self.si), "off-limits to you"))
5742             f.close()
5743             # clients should be checking the blacklist each time, so we don't
5744             # need to restart the client
5745         d.addCallback(_blacklist)
5746         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_uri",
5747                                                        403, "Forbidden",
5748                                                        "Access Prohibited: off-limits",
5749                                                        self.GET, self.url))
5750
5751         # We should still be able to list the parent directory, in HTML...
5752         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5753         def _check_dir_html2(body):
5754             self.failUnlessIn("<html>", body)
5755             self.failUnlessIn("blacklisted.txt</strike>", body)
5756         d.addCallback(_check_dir_html2)
5757
5758         # ... and in JSON (used by CLI).
5759         d.addCallback(lambda ign: self.GET(self.dir_url+"?t=json", followRedirect=True))
5760         def _check_dir_json(res):
5761             data = simplejson.loads(res)
5762             self.failUnless(isinstance(data, list), data)
5763             self.failUnlessEqual(data[0], "dirnode")
5764             self.failUnless(isinstance(data[1], dict), data)
5765             self.failUnlessIn("children", data[1])
5766             self.failUnlessIn("blacklisted.txt", data[1]["children"])
5767             childdata = data[1]["children"]["blacklisted.txt"]
5768             self.failUnless(isinstance(childdata, list), data)
5769             self.failUnlessEqual(childdata[0], "filenode")
5770             self.failUnless(isinstance(childdata[1], dict), data)
5771         d.addCallback(_check_dir_json)
5772
5773         def _unblacklist(ign):
5774             open(fn, "w").close()
5775             # the Blacklist object watches mtime to tell when the file has
5776             # changed, but on windows this test will run faster than the
5777             # filesystem's mtime resolution. So we edit Blacklist.last_mtime
5778             # to force a reload.
5779             self.g.clients[0].blacklist.last_mtime -= 2.0
5780         d.addCallback(_unblacklist)
5781
5782         # now a read should work
5783         d.addCallback(lambda ign: self.GET(self.url))
5784         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5785
5786         # read again to exercise the blacklist-is-unchanged logic
5787         d.addCallback(lambda ign: self.GET(self.url))
5788         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5789
5790         # now add a blacklisted directory, and make sure files under it are
5791         # refused too
5792         def _add_dir(ign):
5793             childnode = c0.create_node_from_uri(self.uri, None)
5794             return c0.create_dirnode({u"child": (childnode,{}) })
5795         d.addCallback(_add_dir)
5796         def _get_dircap(dn):
5797             self.dir_si_b32 = base32.b2a(dn.get_storage_index())
5798             self.dir_url_base = "uri/"+dn.get_write_uri()
5799             self.dir_url_json1 = "uri/"+dn.get_write_uri()+"?t=json"
5800             self.dir_url_json2 = "uri/"+dn.get_write_uri()+"/?t=json"
5801             self.dir_url_json_ro = "uri/"+dn.get_readonly_uri()+"/?t=json"
5802             self.child_url = "uri/"+dn.get_readonly_uri()+"/child"
5803         d.addCallback(_get_dircap)
5804         d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True))
5805         d.addCallback(lambda body: self.failUnlessIn("<html>", body))
5806         d.addCallback(lambda ign: self.GET(self.dir_url_json1))
5807         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5808         d.addCallback(lambda ign: self.GET(self.dir_url_json2))
5809         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5810         d.addCallback(lambda ign: self.GET(self.dir_url_json_ro))
5811         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5812         d.addCallback(lambda ign: self.GET(self.child_url))
5813         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5814
5815         def _block_dir(ign):
5816             f = open(fn, "w")
5817             f.write("%s %s\n" % (self.dir_si_b32, "dir-off-limits to you"))
5818             f.close()
5819             self.g.clients[0].blacklist.last_mtime -= 2.0
5820         d.addCallback(_block_dir)
5821         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir base",
5822                                                        403, "Forbidden",
5823                                                        "Access Prohibited: dir-off-limits",
5824                                                        self.GET, self.dir_url_base))
5825         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json1",
5826                                                        403, "Forbidden",
5827                                                        "Access Prohibited: dir-off-limits",
5828                                                        self.GET, self.dir_url_json1))
5829         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json2",
5830                                                        403, "Forbidden",
5831                                                        "Access Prohibited: dir-off-limits",
5832                                                        self.GET, self.dir_url_json2))
5833         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json_ro",
5834                                                        403, "Forbidden",
5835                                                        "Access Prohibited: dir-off-limits",
5836                                                        self.GET, self.dir_url_json_ro))
5837         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir child",
5838                                                        403, "Forbidden",
5839                                                        "Access Prohibited: dir-off-limits",
5840                                                        self.GET, self.child_url))
5841         return d
5842
5843
5844 class CompletelyUnhandledError(Exception):
5845     pass
5846 class ErrorBoom(rend.Page):
5847     def beforeRender(self, ctx):
5848         raise CompletelyUnhandledError("whoops")