src/allmydata/test/test_web.py

   1 import os.path, re, urllib, time, cgi
   2 import simplejson
   3 from StringIO import StringIO
   4
   5 from twisted.application import service
   6 from twisted.trial import unittest
   7 from twisted.internet import defer, reactor
   8 from twisted.internet.task import Clock
   9 from twisted.web import client, error, http
  10 from twisted.python import failure, log
  11
  12 from foolscap.api import fireEventually, flushEventualQueue
  13
  14 from nevow.util import escapeToXML
  15 from nevow import rend
  16
  17 from allmydata import interfaces, uri, webish, dirnode
  18 from allmydata.storage.shares import get_share_file
  19 from allmydata.storage_client import StorageFarmBroker, StubServer
  20 from allmydata.immutable import upload
  21 from allmydata.immutable.downloader.status import DownloadStatus
  22 from allmydata.dirnode import DirectoryNode
  23 from allmydata.nodemaker import NodeMaker
  24 from allmydata.unknown import UnknownNode
  25 from allmydata.web import status, common
  26 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  27 from allmydata.util import fileutil, base32, hashutil
  28 from allmydata.util.consumer import download_to_data
  29 from allmydata.util.netstring import split_netstring
  30 from allmydata.util.encodingutil import to_str
  31 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  32      create_chk_filenode, WebErrorMixin, ShouldFailMixin, \
  33      make_mutable_file_uri, create_mutable_filenode
  34 from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
  35 from allmydata.mutable import servermap, publish, retrieve
  36 import allmydata.test.common_util as testutil
  37 from allmydata.test.no_network import GridTestMixin
  38 from allmydata.test.common_web import HTTPClientGETFactory, \
  39      HTTPClientHEADFactory
  40 from allmydata.client import Client, SecretHolder
  41 from allmydata.introducer import IntroducerNode
  42
  43 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  44 # create a webserver that works against them
  45
  46 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  47
  48 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
  49 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
  50 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
  51
  52 FAVICON_MARKUP = '<link href="/icon.png" rel="shortcut icon" />'
  53 DIR_HTML_TAG = '<html lang="en">'
  54
  55 class FakeStatsProvider:
  56     def get_stats(self):
  57         stats = {'stats': {}, 'counters': {}}
  58         return stats
  59
  60 class FakeNodeMaker(NodeMaker):
  61     encoding_params = {
  62         'k': 3,
  63         'n': 10,
  64         'happy': 7,
  65         'max_segment_size':128*1024 # 1024=KiB
  66     }
  67     def _create_lit(self, cap):
  68         return FakeCHKFileNode(cap, self.all_contents)
  69     def _create_immutable(self, cap):
  70         return FakeCHKFileNode(cap, self.all_contents)
  71     def _create_mutable(self, cap):
  72         return FakeMutableFileNode(None, None,
  73                                    self.encoding_params, None,
  74                                    self.all_contents).init_from_cap(cap)
  75     def create_mutable_file(self, contents="", keysize=None,
  76                             version=SDMF_VERSION):
  77         n = FakeMutableFileNode(None, None, self.encoding_params, None,
  78                                 self.all_contents)
  79         return n.create(contents, version=version)
  80
  81 class FakeUploader(service.Service):
  82     name = "uploader"
  83     helper_furl = None
  84     helper_connected = False
  85
  86     def upload(self, uploadable):
  87         d = uploadable.get_size()
  88         d.addCallback(lambda size: uploadable.read(size))
  89         def _got_data(datav):
  90             data = "".join(datav)
  91             n = create_chk_filenode(data, self.all_contents)
  92             ur = upload.UploadResults(file_size=len(data),
  93                                       ciphertext_fetched=0,
  94                                       preexisting_shares=0,
  95                                       pushed_shares=10,
  96                                       sharemap={},
  97                                       servermap={},
  98                                       timings={},
  99                                       uri_extension_data={},
 100                                       uri_extension_hash="fake",
 101                                       verifycapstr="fakevcap")
 102             ur.set_uri(n.get_uri())
 103             return ur
 104         d.addCallback(_got_data)
 105         return d
 106
 107     def get_helper_info(self):
 108         return (self.helper_furl, self.helper_connected)
 109
 110
 111 def build_one_ds():
 112     ds = DownloadStatus("storage_index", 1234)
 113     now = time.time()
 114
 115     serverA = StubServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
 116     serverB = StubServer(hashutil.tagged_hash("foo", "serverid_b")[:20])
 117     storage_index = hashutil.storage_index_hash("SI")
 118     e0 = ds.add_segment_request(0, now)
 119     e0.activate(now+0.5)
 120     e0.deliver(now+1, 0, 100, 0.5) # when, start,len, decodetime
 121     e1 = ds.add_segment_request(1, now+2)
 122     e1.error(now+3)
 123     # two outstanding requests
 124     e2 = ds.add_segment_request(2, now+4)
 125     e3 = ds.add_segment_request(3, now+5)
 126     del e2,e3 # hush pyflakes
 127
 128     # simulate a segment which gets delivered faster than a system clock tick (ticket #1166)
 129     e = ds.add_segment_request(4, now)
 130     e.activate(now)
 131     e.deliver(now, 0, 140, 0.5)
 132
 133     e = ds.add_dyhb_request(serverA, now)
 134     e.finished([1,2], now+1)
 135     e = ds.add_dyhb_request(serverB, now+2) # left unfinished
 136
 137     e = ds.add_read_event(0, 120, now)
 138     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 139     e.finished(now+1)
 140     e = ds.add_read_event(120, 30, now+2) # left unfinished
 141
 142     e = ds.add_block_request(serverA, 1, 100, 20, now)
 143     e.finished(20, now+1)
 144     e = ds.add_block_request(serverB, 1, 120, 30, now+1) # left unfinished
 145
 146     # make sure that add_read_event() can come first too
 147     ds1 = DownloadStatus(storage_index, 1234)
 148     e = ds1.add_read_event(0, 120, now)
 149     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 150     e.finished(now+1)
 151
 152     return ds
 153
 154 class FakeHistory:
 155     _all_upload_status = [upload.UploadStatus()]
 156     _all_download_status = [build_one_ds()]
 157     _all_mapupdate_statuses = [servermap.UpdateStatus()]
 158     _all_publish_statuses = [publish.PublishStatus()]
 159     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
 160
 161     def list_all_upload_statuses(self):
 162         return self._all_upload_status
 163     def list_all_download_statuses(self):
 164         return self._all_download_status
 165     def list_all_mapupdate_statuses(self):
 166         return self._all_mapupdate_statuses
 167     def list_all_publish_statuses(self):
 168         return self._all_publish_statuses
 169     def list_all_retrieve_statuses(self):
 170         return self._all_retrieve_statuses
 171     def list_all_helper_statuses(self):
 172         return []
 173
 174 class FakeDisplayableServer(StubServer):
 175     def __init__(self, serverid, nickname, connected,
 176                  last_connect_time, last_loss_time, last_rx_time):
 177         StubServer.__init__(self, serverid)
 178         self.announcement = {"my-version": "allmydata-tahoe-fake",
 179                              "service-name": "storage",
 180                              "nickname": nickname}
 181         self.connected = connected
 182         self.last_loss_time = last_loss_time
 183         self.last_rx_time = last_rx_time
 184         self.last_connect_time = last_connect_time
 185     def is_connected(self):
 186         return self.connected
 187     def get_permutation_seed(self):
 188         return ""
 189     def get_remote_host(self):
 190         return ""
 191     def get_last_loss_time(self):
 192         return self.last_loss_time
 193     def get_last_received_data_time(self):
 194         return self.last_rx_time
 195     def get_last_connect_time(self):
 196         return self.last_connect_time
 197     def get_announcement(self):
 198         return self.announcement
 199     def get_nickname(self):
 200         return self.announcement["nickname"]
 201     def get_available_space(self):
 202         return 123456
 203
 204 class FakeBucketCounter(object):
 205     def get_state(self):
 206         return {"last-complete-bucket-count": 0}
 207     def get_progress(self):
 208         return {"estimated-time-per-cycle": 0,
 209                 "cycle-in-progress": False,
 210                 "remaining-wait-time": 0}
 211
 212 class FakeLeaseChecker(object):
 213     def __init__(self):
 214         self.expiration_enabled = False
 215         self.mode = "age"
 216         self.override_lease_duration = None
 217         self.sharetypes_to_expire = {}
 218     def get_state(self):
 219         return {"history": None}
 220     def get_progress(self):
 221         return {"estimated-time-per-cycle": 0,
 222                 "cycle-in-progress": False,
 223                 "remaining-wait-time": 0}
 224
 225 class FakeStorageServer(service.MultiService):
 226     name = 'storage'
 227     def __init__(self, nodeid, nickname):
 228         service.MultiService.__init__(self)
 229         self.my_nodeid = nodeid
 230         self.nickname = nickname
 231         self.bucket_counter = FakeBucketCounter()
 232         self.lease_checker = FakeLeaseChecker()
 233     def get_stats(self):
 234         return {"storage_server.accepting_immutable_shares": False}
 235
 236 class FakeClient(Client):
 237     def __init__(self):
 238         # don't upcall to Client.__init__, since we only want to initialize a
 239         # minimal subset
 240         service.MultiService.__init__(self)
 241         self.all_contents = {}
 242         self.nodeid = "fake_nodeid"
 243         self.nickname = u"fake_nickname \u263A"
 244         self.introducer_furl = "None"
 245         self.stats_provider = FakeStatsProvider()
 246         self._secret_holder = SecretHolder("lease secret", "convergence secret")
 247         self.helper = None
 248         self.convergence = "some random string"
 249         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 250         # fake knowledge of another server
 251         self.storage_broker.test_add_server("other_nodeid",
 252             FakeDisplayableServer(
 253                 serverid="other_nodeid", nickname=u"other_nickname \u263B", connected = True,
 254                 last_connect_time = 10, last_loss_time = 20, last_rx_time = 30))
 255         self.storage_broker.test_add_server("disconnected_nodeid",
 256             FakeDisplayableServer(
 257                 serverid="other_nodeid", nickname=u"disconnected_nickname \u263B", connected = False,
 258                 last_connect_time = 15, last_loss_time = 25, last_rx_time = 35))
 259         self.introducer_client = None
 260         self.history = FakeHistory()
 261         self.uploader = FakeUploader()
 262         self.uploader.all_contents = self.all_contents
 263         self.uploader.setServiceParent(self)
 264         self.blacklist = None
 265         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 266                                        self.uploader, None,
 267                                        None, None, None)
 268         self.nodemaker.all_contents = self.all_contents
 269         self.mutable_file_default = SDMF_VERSION
 270         self.addService(FakeStorageServer(self.nodeid, self.nickname))
 271
 272     def get_long_nodeid(self):
 273         return "v0-nodeid"
 274     def get_long_tubid(self):
 275         return "tubid"
 276
 277     def startService(self):
 278         return service.MultiService.startService(self)
 279     def stopService(self):
 280         return service.MultiService.stopService(self)
 281
 282     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 283
 284 class WebMixin(testutil.TimezoneMixin):
 285     def setUp(self):
 286         self.setTimezone('UTC-13:00')
 287         self.s = FakeClient()
 288         self.s.startService()
 289         self.staticdir = self.mktemp()
 290         self.clock = Clock()
 291         self.fakeTime = 86460 # 1d 0h 1m 0s
 292         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
 293                                       clock=self.clock, now_fn=lambda:self.fakeTime)
 294         self.ws.setServiceParent(self.s)
 295         self.webish_port = self.ws.getPortnum()
 296         self.webish_url = self.ws.getURL()
 297         assert self.webish_url.endswith("/")
 298         self.webish_url = self.webish_url[:-1] # these tests add their own /
 299
 300         l = [ self.s.create_dirnode() for x in range(6) ]
 301         d = defer.DeferredList(l)
 302         def _then(res):
 303             self.public_root = res[0][1]
 304             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 305             self.public_url = "/uri/" + self.public_root.get_uri()
 306             self.private_root = res[1][1]
 307
 308             foo = res[2][1]
 309             self._foo_node = foo
 310             self._foo_uri = foo.get_uri()
 311             self._foo_readonly_uri = foo.get_readonly_uri()
 312             self._foo_verifycap = foo.get_verify_cap().to_string()
 313             # NOTE: we ignore the deferred on all set_uri() calls, because we
 314             # know the fake nodes do these synchronously
 315             self.public_root.set_uri(u"foo", foo.get_uri(),
 316                                      foo.get_readonly_uri())
 317
 318             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 319             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 320             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 321
 322             # sdmf
 323             # XXX: Do we ever use this?
 324             self.BAZ_CONTENTS, n, self._baz_txt_uri, self._baz_txt_readonly_uri = self.makefile_mutable(0)
 325
 326             foo.set_uri(u"baz.txt", self._baz_txt_uri, self._baz_txt_readonly_uri)
 327
 328             # mdmf
 329             self.QUUX_CONTENTS, n, self._quux_txt_uri, self._quux_txt_readonly_uri = self.makefile_mutable(0, mdmf=True)
 330             assert self._quux_txt_uri.startswith("URI:MDMF")
 331             foo.set_uri(u"quux.txt", self._quux_txt_uri, self._quux_txt_readonly_uri)
 332
 333             foo.set_uri(u"empty", res[3][1].get_uri(),
 334                         res[3][1].get_readonly_uri())
 335             sub_uri = res[4][1].get_uri()
 336             self._sub_uri = sub_uri
 337             foo.set_uri(u"sub", sub_uri, sub_uri)
 338             sub = self.s.create_node_from_uri(sub_uri)
 339             self._sub_node = sub
 340
 341             _ign, n, blocking_uri = self.makefile(1)
 342             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 343
 344             # filenode to test for html encoding issues
 345             self._htmlname_unicode = u"<&weirdly'named\"file>>>_<iframe />.txt"
 346             self._htmlname_raw = self._htmlname_unicode.encode('utf-8')
 347             self._htmlname_urlencoded = urllib.quote(self._htmlname_raw, '')
 348             self._htmlname_escaped = escapeToXML(self._htmlname_raw)
 349             self._htmlname_escaped_attr = cgi.escape(self._htmlname_raw, quote=True)
 350             self._htmlname_escaped_double = escapeToXML(cgi.escape(self._htmlname_raw, quote=True))
 351             self.HTMLNAME_CONTENTS, n, self._htmlname_txt_uri = self.makefile(0)
 352             foo.set_uri(self._htmlname_unicode, self._htmlname_txt_uri, self._htmlname_txt_uri)
 353
 354             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 355             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 356             # still think of it as an umlaut
 357             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 358
 359             self.SUBBAZ_CONTENTS, n, baz_file = self.makefile(2)
 360             self._baz_file_uri = baz_file
 361             sub.set_uri(u"baz.txt", baz_file, baz_file)
 362
 363             _ign, n, self._bad_file_uri = self.makefile(3)
 364             # this uri should not be downloadable
 365             del self.s.all_contents[self._bad_file_uri]
 366
 367             rodir = res[5][1]
 368             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 369                                      rodir.get_readonly_uri())
 370             rodir.set_uri(u"nor", baz_file, baz_file)
 371
 372             # public/
 373             # public/foo/
 374             # public/foo/bar.txt
 375             # public/foo/baz.txt
 376             # public/foo/quux.txt
 377             # public/foo/blockingfile
 378             # public/foo/<&weirdly'named\"file>>>_<iframe />.txt
 379             # public/foo/empty/
 380             # public/foo/sub/
 381             # public/foo/sub/baz.txt
 382             # public/reedownlee/
 383             # public/reedownlee/nor
 384             self.NEWFILE_CONTENTS = "newfile contents\n"
 385
 386             return foo.get_metadata_for(u"bar.txt")
 387         d.addCallback(_then)
 388         def _got_metadata(metadata):
 389             self._bar_txt_metadata = metadata
 390         d.addCallback(_got_metadata)
 391         return d
 392
 393     def get_all_contents(self):
 394         return self.s.all_contents
 395
 396     def makefile(self, number):
 397         contents = "contents of file %s\n" % number
 398         n = create_chk_filenode(contents, self.get_all_contents())
 399         return contents, n, n.get_uri()
 400
 401     def makefile_mutable(self, number, mdmf=False):
 402         contents = "contents of mutable file %s\n" % number
 403         n = create_mutable_filenode(contents, mdmf, self.s.all_contents)
 404         return contents, n, n.get_uri(), n.get_readonly_uri()
 405
 406     def tearDown(self):
 407         return self.s.stopService()
 408
 409     def failUnlessIsBarDotTxt(self, res):
 410         self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
 411
 412     def failUnlessIsQuuxDotTxt(self, res):
 413         self.failUnlessReallyEqual(res, self.QUUX_CONTENTS, res)
 414
 415     def failUnlessIsBazDotTxt(self, res):
 416         self.failUnlessReallyEqual(res, self.BAZ_CONTENTS, res)
 417
 418     def failUnlessIsSubBazDotTxt(self, res):
 419         self.failUnlessReallyEqual(res, self.SUBBAZ_CONTENTS, res)
 420
 421     def failUnlessIsBarJSON(self, res):
 422         data = simplejson.loads(res)
 423         self.failUnless(isinstance(data, list))
 424         self.failUnlessEqual(data[0], "filenode")
 425         self.failUnless(isinstance(data[1], dict))
 426         self.failIf(data[1]["mutable"])
 427         self.failIfIn("rw_uri", data[1]) # immutable
 428         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
 429         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
 430         self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
 431
 432     def failUnlessIsQuuxJSON(self, res, readonly=False):
 433         data = simplejson.loads(res)
 434         self.failUnless(isinstance(data, list))
 435         self.failUnlessEqual(data[0], "filenode")
 436         self.failUnless(isinstance(data[1], dict))
 437         metadata = data[1]
 438         return self.failUnlessIsQuuxDotTxtMetadata(metadata, readonly)
 439
 440     def failUnlessIsQuuxDotTxtMetadata(self, metadata, readonly):
 441         self.failUnless(metadata['mutable'])
 442         if readonly:
 443             self.failIfIn("rw_uri", metadata)
 444         else:
 445             self.failUnlessIn("rw_uri", metadata)
 446             self.failUnlessEqual(metadata['rw_uri'], self._quux_txt_uri)
 447         self.failUnlessIn("ro_uri", metadata)
 448         self.failUnlessEqual(metadata['ro_uri'], self._quux_txt_readonly_uri)
 449         self.failUnlessReallyEqual(metadata['size'], len(self.QUUX_CONTENTS))
 450
 451     def failUnlessIsFooJSON(self, res):
 452         data = simplejson.loads(res)
 453         self.failUnless(isinstance(data, list))
 454         self.failUnlessEqual(data[0], "dirnode", res)
 455         self.failUnless(isinstance(data[1], dict))
 456         self.failUnless(data[1]["mutable"])
 457         self.failUnlessIn("rw_uri", data[1]) # mutable
 458         self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
 459         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
 460         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
 461
 462         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 463         self.failUnlessEqual(kidnames,
 464                              [self._htmlname_unicode, u"bar.txt", u"baz.txt",
 465                               u"blockingfile", u"empty", u"n\u00fc.txt", u"quux.txt", u"sub"])
 466         kids = dict( [(unicode(name),value)
 467                       for (name,value)
 468                       in data[1]["children"].iteritems()] )
 469         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 470         self.failUnlessIn("metadata", kids[u"sub"][1])
 471         self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
 472         tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
 473         self.failUnlessIn("linkcrtime", tahoe_md)
 474         self.failUnlessIn("linkmotime", tahoe_md)
 475         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 476         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 477         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri)
 478         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]),
 479                                    self._bar_txt_verifycap)
 480         self.failUnlessIn("metadata", kids[u"bar.txt"][1])
 481         self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
 482         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
 483                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
 484         self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
 485                                    self._bar_txt_uri)
 486         self.failUnlessIn("quux.txt", kids)
 487         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["rw_uri"]),
 488                                    self._quux_txt_uri)
 489         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["ro_uri"]),
 490                                    self._quux_txt_readonly_uri)
 491
 492     def GET(self, urlpath, followRedirect=False, return_response=False,
 493             **kwargs):
 494         # if return_response=True, this fires with (data, statuscode,
 495         # respheaders) instead of just data.
 496         assert not isinstance(urlpath, unicode)
 497         url = self.webish_url + urlpath
 498         factory = HTTPClientGETFactory(url, method="GET",
 499                                        followRedirect=followRedirect, **kwargs)
 500         reactor.connectTCP("localhost", self.webish_port, factory)
 501         d = factory.deferred
 502         def _got_data(data):
 503             return (data, factory.status, factory.response_headers)
 504         if return_response:
 505             d.addCallback(_got_data)
 506         return factory.deferred
 507
 508     def HEAD(self, urlpath, return_response=False, **kwargs):
 509         # this requires some surgery, because twisted.web.client doesn't want
 510         # to give us back the response headers.
 511         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 512         reactor.connectTCP("localhost", self.webish_port, factory)
 513         d = factory.deferred
 514         def _got_data(data):
 515             return (data, factory.status, factory.response_headers)
 516         if return_response:
 517             d.addCallback(_got_data)
 518         return factory.deferred
 519
 520     def PUT(self, urlpath, data, **kwargs):
 521         url = self.webish_url + urlpath
 522         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 523
 524     def DELETE(self, urlpath):
 525         url = self.webish_url + urlpath
 526         return client.getPage(url, method="DELETE")
 527
 528     def POST(self, urlpath, followRedirect=False, **fields):
 529         sepbase = "boogabooga"
 530         sep = "--" + sepbase
 531         form = []
 532         form.append(sep)
 533         form.append('Content-Disposition: form-data; name="_charset"')
 534         form.append('')
 535         form.append('UTF-8')
 536         form.append(sep)
 537         for name, value in fields.iteritems():
 538             if isinstance(value, tuple):
 539                 filename, value = value
 540                 form.append('Content-Disposition: form-data; name="%s"; '
 541                             'filename="%s"' % (name, filename.encode("utf-8")))
 542             else:
 543                 form.append('Content-Disposition: form-data; name="%s"' % name)
 544             form.append('')
 545             if isinstance(value, unicode):
 546                 value = value.encode("utf-8")
 547             else:
 548                 value = str(value)
 549             assert isinstance(value, str)
 550             form.append(value)
 551             form.append(sep)
 552         form[-1] += "--"
 553         body = ""
 554         headers = {}
 555         if fields:
 556             body = "\r\n".join(form) + "\r\n"
 557             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 558         return self.POST2(urlpath, body, headers, followRedirect)
 559
 560     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 561         url = self.webish_url + urlpath
 562         return client.getPage(url, method="POST", postdata=body,
 563                               headers=headers, followRedirect=followRedirect)
 564
 565     def shouldFail(self, res, expected_failure, which,
 566                    substring=None, response_substring=None):
 567         if isinstance(res, failure.Failure):
 568             res.trap(expected_failure)
 569             if substring:
 570                 self.failUnlessIn(substring, str(res), which)
 571             if response_substring:
 572                 self.failUnlessIn(response_substring, res.value.response, which)
 573         else:
 574             self.fail("%s was supposed to raise %s, not get '%s'" %
 575                       (which, expected_failure, res))
 576
 577     def shouldFail2(self, expected_failure, which, substring,
 578                     response_substring,
 579                     callable, *args, **kwargs):
 580         assert substring is None or isinstance(substring, str)
 581         assert response_substring is None or isinstance(response_substring, str)
 582         d = defer.maybeDeferred(callable, *args, **kwargs)
 583         def done(res):
 584             if isinstance(res, failure.Failure):
 585                 res.trap(expected_failure)
 586                 if substring:
 587                     self.failUnlessIn(substring, str(res),
 588                                       "'%s' not in '%s' (response is '%s') for test '%s'" % \
 589                                       (substring, str(res),
 590                                        getattr(res.value, "response", ""),
 591                                        which))
 592                 if response_substring:
 593                     self.failUnlessIn(response_substring, res.value.response,
 594                                       "'%s' not in '%s' for test '%s'" % \
 595                                       (response_substring, res.value.response,
 596                                        which))
 597             else:
 598                 self.fail("%s was supposed to raise %s, not get '%s'" %
 599                           (which, expected_failure, res))
 600         d.addBoth(done)
 601         return d
 602
 603     def should404(self, res, which):
 604         if isinstance(res, failure.Failure):
 605             res.trap(error.Error)
 606             self.failUnlessReallyEqual(res.value.status, "404")
 607         else:
 608             self.fail("%s was supposed to Error(404), not get '%s'" %
 609                       (which, res))
 610
 611     def should302(self, res, which):
 612         if isinstance(res, failure.Failure):
 613             res.trap(error.Error)
 614             self.failUnlessReallyEqual(res.value.status, "302")
 615         else:
 616             self.fail("%s was supposed to Error(302), not get '%s'" %
 617                       (which, res))
 618
 619 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
 620     def test_create(self):
 621         pass
 622
 623     def test_welcome(self):
 624         d = self.GET("/")
 625         def _check(res):
 626             self.failUnlessIn('<title>Tahoe-LAFS - Welcome</title>', res)
 627             self.failUnlessIn(FAVICON_MARKUP, res)
 628             self.failUnlessIn('<a href="status">Recent and Active Operations</a>', res)
 629             self.failUnlessIn('<a href="statistics">Operational Statistics</a>', res)
 630             self.failUnless(re.search('<input (type="hidden" |name="t" |value="report-incident" ){3}/>',res), res)
 631             self.failUnlessIn('Page rendered at', res)
 632             self.failUnlessIn('Tahoe-LAFS code imported from:', res)
 633             res_u = res.decode('utf-8')
 634             self.failUnlessIn(u'<td>fake_nickname \u263A</td>', res_u)
 635             self.failUnlessIn(u'<div class="nickname">other_nickname \u263B</div>', res_u)
 636             self.failUnlessIn(u'Connected to <span>1</span>\n              of <span>2</span> known storage servers', res_u)
 637             def timestamp(t):
 638                 return (u'"%s"' % (t,)) if self.have_working_tzset() else u'"[^"]*"'
 639             self.failUnless(re.search(
 640                 u'<div class="status-indicator"><img (src="img/connected-yes.png" |alt="Connected" ){2}/>'
 641                 u'</div>\n                <a( class="timestamp"| title=%s){2}>1d\u00A00h\u00A00m\u00A050s</a>'
 642                 % timestamp(u'1970-01-01 13:00:10'), res_u), repr(res_u))
 643             self.failUnless(re.search(
 644                 u'<div class="status-indicator"><img (src="img/connected-no.png" |alt="Disconnected" ){2}/>'
 645                 u'</div>\n                <a( class="timestamp"| title=%s){2}>1d\u00A00h\u00A00m\u00A035s</a>'
 646                 % timestamp(u'1970-01-01 13:00:25'), res_u), repr(res_u))
 647             self.failUnless(re.search(
 648                 u'<td class="service-last-received-data"><a( class="timestamp"| title=%s){2}>'
 649                 u'1d\u00A00h\u00A00m\u00A030s</a></td>'
 650                 % timestamp(u'1970-01-01 13:00:30'), res_u), repr(res_u))
 651             self.failUnless(re.search(
 652                 u'<td class="service-last-received-data"><a( class="timestamp"| title=%s){2}>'
 653                 u'1d\u00A00h\u00A00m\u00A025s</a></td>'
 654                 % timestamp(u'1970-01-01 13:00:35'), res_u), repr(res_u))
 655
 656             self.failUnlessIn(u'\u00A9 <a href="https://tahoe-lafs.org/">Tahoe-LAFS Software Foundation', res_u)
 657             self.failUnlessIn('<td><h3>Available</h3></td>', res)
 658             self.failUnlessIn('123.5kB', res)
 659
 660             self.s.basedir = 'web/test_welcome'
 661             fileutil.make_dirs("web/test_welcome")
 662             fileutil.make_dirs("web/test_welcome/private")
 663             return self.GET("/")
 664         d.addCallback(_check)
 665         return d
 666
 667     def test_introducer_status(self):
 668         class MockIntroducerClient(object):
 669             def __init__(self, connected):
 670                 self.connected = connected
 671             def connected_to_introducer(self):
 672                 return self.connected
 673
 674         d = defer.succeed(None)
 675
 676         # introducer not connected, unguessable furl
 677         def _set_introducer_not_connected_unguessable(ign):
 678             self.s.introducer_furl = "pb://someIntroducer/secret"
 679             self.s.introducer_client = MockIntroducerClient(False)
 680             return self.GET("/")
 681         d.addCallback(_set_introducer_not_connected_unguessable)
 682         def _check_introducer_not_connected_unguessable(res):
 683             html = res.replace('\n', ' ')
 684             self.failUnlessIn('<div class="furl">pb://someIntroducer/[censored]</div>', html)
 685             self.failIfIn('pb://someIntroducer/secret', html)
 686             self.failUnless(re.search('<img (alt="Disconnected" |src="img/connected-no.png" ){2}/>', html), res)
 687         d.addCallback(_check_introducer_not_connected_unguessable)
 688
 689         # introducer connected, unguessable furl
 690         def _set_introducer_connected_unguessable(ign):
 691             self.s.introducer_furl = "pb://someIntroducer/secret"
 692             self.s.introducer_client = MockIntroducerClient(True)
 693             return self.GET("/")
 694         d.addCallback(_set_introducer_connected_unguessable)
 695         def _check_introducer_connected_unguessable(res):
 696             html = res.replace('\n', ' ')
 697             self.failUnlessIn('<div class="furl">pb://someIntroducer/[censored]</div>', html)
 698             self.failIfIn('pb://someIntroducer/secret', html)
 699             self.failUnless(re.search('<img (src="img/connected-yes.png" |alt="Connected" ){2}/>', html), res)
 700         d.addCallback(_check_introducer_connected_unguessable)
 701
 702         # introducer connected, guessable furl
 703         def _set_introducer_connected_guessable(ign):
 704             self.s.introducer_furl = "pb://someIntroducer/introducer"
 705             self.s.introducer_client = MockIntroducerClient(True)
 706             return self.GET("/")
 707         d.addCallback(_set_introducer_connected_guessable)
 708         def _check_introducer_connected_guessable(res):
 709             html = res.replace('\n', ' ')
 710             self.failUnlessIn('<div class="furl">pb://someIntroducer/introducer</div>', html)
 711             self.failUnless(re.search('<img (src="img/connected-yes.png" |alt="Connected" ){2}/>', html), res)
 712         d.addCallback(_check_introducer_connected_guessable)
 713         return d
 714
 715     def test_helper_status(self):
 716         d = defer.succeed(None)
 717
 718         # set helper furl to None
 719         def _set_no_helper(ign):
 720             self.s.uploader.helper_furl = None
 721             return self.GET("/")
 722         d.addCallback(_set_no_helper)
 723         def _check_no_helper(res):
 724             html = res.replace('\n', ' ')
 725             self.failUnless(re.search('<img (src="img/connected-not-configured.png" |alt="Not Configured" ){2}/>', html), res)
 726         d.addCallback(_check_no_helper)
 727
 728         # enable helper, not connected
 729         def _set_helper_not_connected(ign):
 730             self.s.uploader.helper_furl = "pb://someHelper/secret"
 731             self.s.uploader.helper_connected = False
 732             return self.GET("/")
 733         d.addCallback(_set_helper_not_connected)
 734         def _check_helper_not_connected(res):
 735             html = res.replace('\n', ' ')
 736             self.failUnlessIn('<div class="furl">pb://someHelper/[censored]</div>', html)
 737             self.failIfIn('pb://someHelper/secret', html)
 738             self.failUnless(re.search('<img (src="img/connected-no.png" |alt="Disconnected" ){2}/>', html), res)
 739         d.addCallback(_check_helper_not_connected)
 740
 741         # enable helper, connected
 742         def _set_helper_connected(ign):
 743             self.s.uploader.helper_furl = "pb://someHelper/secret"
 744             self.s.uploader.helper_connected = True
 745             return self.GET("/")
 746         d.addCallback(_set_helper_connected)
 747         def _check_helper_connected(res):
 748             html = res.replace('\n', ' ')
 749             self.failUnlessIn('<div class="furl">pb://someHelper/[censored]</div>', html)
 750             self.failIfIn('pb://someHelper/secret', html)
 751             self.failUnless(re.search('<img (src="img/connected-yes.png" |alt="Connected" ){2}/>', html), res)
 752         d.addCallback(_check_helper_connected)
 753         return d
 754
 755     def test_storage(self):
 756         d = self.GET("/storage")
 757         def _check(res):
 758             self.failUnlessIn('Storage Server Status', res)
 759             self.failUnlessIn(FAVICON_MARKUP, res)
 760             res_u = res.decode('utf-8')
 761             self.failUnlessIn(u'<li>Server Nickname: <span class="nickname mine">fake_nickname \u263A</span></li>', res_u)
 762         d.addCallback(_check)
 763         return d
 764
 765     def test_status(self):
 766         h = self.s.get_history()
 767         dl_num = h.list_all_download_statuses()[0].get_counter()
 768         ul_num = h.list_all_upload_statuses()[0].get_counter()
 769         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 770         pub_num = h.list_all_publish_statuses()[0].get_counter()
 771         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 772         d = self.GET("/status", followRedirect=True)
 773         def _check(res):
 774             self.failUnlessIn('Recent and Active Operations', res)
 775             self.failUnlessIn('"down-%d"' % dl_num, res)
 776             self.failUnlessIn('"up-%d"' % ul_num, res)
 777             self.failUnlessIn('"mapupdate-%d"' % mu_num, res)
 778             self.failUnlessIn('"publish-%d"' % pub_num, res)
 779             self.failUnlessIn('"retrieve-%d"' % ret_num, res)
 780         d.addCallback(_check)
 781         d.addCallback(lambda res: self.GET("/status/?t=json"))
 782         def _check_json(res):
 783             data = simplejson.loads(res)
 784             self.failUnless(isinstance(data, dict))
 785             #active = data["active"]
 786             # TODO: test more. We need a way to fake an active operation
 787             # here.
 788         d.addCallback(_check_json)
 789
 790         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 791         def _check_dl(res):
 792             self.failUnlessIn("File Download Status", res)
 793         d.addCallback(_check_dl)
 794         d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
 795         def _check_dl_json(res):
 796             data = simplejson.loads(res)
 797             self.failUnless(isinstance(data, dict))
 798             self.failUnlessIn("read", data)
 799             self.failUnlessEqual(data["read"][0]["length"], 120)
 800             self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
 801             self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
 802             self.failUnlessEqual(data["segment"][2]["finish_time"], None)
 803             phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
 804             cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
 805             # serverids[] keys are strings, since that's what JSON does, but
 806             # we'd really like them to be ints
 807             self.failUnlessEqual(data["serverids"]["0"], "phwrsjte")
 808             self.failUnless(data["serverids"].has_key("1"),
 809                             str(data["serverids"]))
 810             self.failUnlessEqual(data["serverids"]["1"], "cmpuvkjm",
 811                                  str(data["serverids"]))
 812             self.failUnlessEqual(data["server_info"][phwr_id]["short"],
 813                                  "phwrsjte")
 814             self.failUnlessEqual(data["server_info"][cmpu_id]["short"],
 815                                  "cmpuvkjm")
 816             self.failUnlessIn("dyhb", data)
 817             self.failUnlessIn("misc", data)
 818         d.addCallback(_check_dl_json)
 819         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 820         def _check_ul(res):
 821             self.failUnlessIn("File Upload Status", res)
 822         d.addCallback(_check_ul)
 823         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 824         def _check_mapupdate(res):
 825             self.failUnlessIn("Mutable File Servermap Update Status", res)
 826         d.addCallback(_check_mapupdate)
 827         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 828         def _check_publish(res):
 829             self.failUnlessIn("Mutable File Publish Status", res)
 830         d.addCallback(_check_publish)
 831         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 832         def _check_retrieve(res):
 833             self.failUnlessIn("Mutable File Retrieve Status", res)
 834         d.addCallback(_check_retrieve)
 835
 836         return d
 837
 838     def test_status_numbers(self):
 839         drrm = status.DownloadResultsRendererMixin()
 840         self.failUnlessReallyEqual(drrm.render_time(None, None), "")
 841         self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
 842         self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
 843         self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
 844         self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
 845         self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
 846         self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 847         self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
 848         self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
 849
 850         urrm = status.UploadResultsRendererMixin()
 851         self.failUnlessReallyEqual(urrm.render_time(None, None), "")
 852         self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
 853         self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
 854         self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
 855         self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
 856         self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
 857         self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 858         self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
 859         self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
 860
 861     def test_GET_FILEURL(self):
 862         d = self.GET(self.public_url + "/foo/bar.txt")
 863         d.addCallback(self.failUnlessIsBarDotTxt)
 864         return d
 865
 866     def test_GET_FILEURL_range(self):
 867         headers = {"range": "bytes=1-10"}
 868         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 869                      return_response=True)
 870         def _got((res, status, headers)):
 871             self.failUnlessReallyEqual(int(status), 206)
 872             self.failUnless(headers.has_key("content-range"))
 873             self.failUnlessReallyEqual(headers["content-range"][0],
 874                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 875             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
 876         d.addCallback(_got)
 877         return d
 878
 879     def test_GET_FILEURL_partial_range(self):
 880         headers = {"range": "bytes=5-"}
 881         length  = len(self.BAR_CONTENTS)
 882         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 883                      return_response=True)
 884         def _got((res, status, headers)):
 885             self.failUnlessReallyEqual(int(status), 206)
 886             self.failUnless(headers.has_key("content-range"))
 887             self.failUnlessReallyEqual(headers["content-range"][0],
 888                                        "bytes 5-%d/%d" % (length-1, length))
 889             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
 890         d.addCallback(_got)
 891         return d
 892
 893     def test_GET_FILEURL_partial_end_range(self):
 894         headers = {"range": "bytes=-5"}
 895         length  = len(self.BAR_CONTENTS)
 896         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 897                      return_response=True)
 898         def _got((res, status, headers)):
 899             self.failUnlessReallyEqual(int(status), 206)
 900             self.failUnless(headers.has_key("content-range"))
 901             self.failUnlessReallyEqual(headers["content-range"][0],
 902                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 903             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
 904         d.addCallback(_got)
 905         return d
 906
 907     def test_GET_FILEURL_partial_range_overrun(self):
 908         headers = {"range": "bytes=100-200"}
 909         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
 910                              "416 Requested Range not satisfiable",
 911                              "First beyond end of file",
 912                              self.GET, self.public_url + "/foo/bar.txt",
 913                              headers=headers)
 914         return d
 915
 916     def test_HEAD_FILEURL_range(self):
 917         headers = {"range": "bytes=1-10"}
 918         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 919                      return_response=True)
 920         def _got((res, status, headers)):
 921             self.failUnlessReallyEqual(res, "")
 922             self.failUnlessReallyEqual(int(status), 206)
 923             self.failUnless(headers.has_key("content-range"))
 924             self.failUnlessReallyEqual(headers["content-range"][0],
 925                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 926         d.addCallback(_got)
 927         return d
 928
 929     def test_HEAD_FILEURL_partial_range(self):
 930         headers = {"range": "bytes=5-"}
 931         length  = len(self.BAR_CONTENTS)
 932         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 933                      return_response=True)
 934         def _got((res, status, headers)):
 935             self.failUnlessReallyEqual(int(status), 206)
 936             self.failUnless(headers.has_key("content-range"))
 937             self.failUnlessReallyEqual(headers["content-range"][0],
 938                                        "bytes 5-%d/%d" % (length-1, length))
 939         d.addCallback(_got)
 940         return d
 941
 942     def test_HEAD_FILEURL_partial_end_range(self):
 943         headers = {"range": "bytes=-5"}
 944         length  = len(self.BAR_CONTENTS)
 945         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 946                      return_response=True)
 947         def _got((res, status, headers)):
 948             self.failUnlessReallyEqual(int(status), 206)
 949             self.failUnless(headers.has_key("content-range"))
 950             self.failUnlessReallyEqual(headers["content-range"][0],
 951                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 952         d.addCallback(_got)
 953         return d
 954
 955     def test_HEAD_FILEURL_partial_range_overrun(self):
 956         headers = {"range": "bytes=100-200"}
 957         d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
 958                              "416 Requested Range not satisfiable",
 959                              "",
 960                              self.HEAD, self.public_url + "/foo/bar.txt",
 961                              headers=headers)
 962         return d
 963
 964     def test_GET_FILEURL_range_bad(self):
 965         headers = {"range": "BOGUS=fizbop-quarnak"}
 966         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 967                      return_response=True)
 968         def _got((res, status, headers)):
 969             self.failUnlessReallyEqual(int(status), 200)
 970             self.failUnless(not headers.has_key("content-range"))
 971             self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
 972         d.addCallback(_got)
 973         return d
 974
 975     def test_HEAD_FILEURL(self):
 976         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 977         def _got((res, status, headers)):
 978             self.failUnlessReallyEqual(res, "")
 979             self.failUnlessReallyEqual(headers["content-length"][0],
 980                                        str(len(self.BAR_CONTENTS)))
 981             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
 982         d.addCallback(_got)
 983         return d
 984
 985     def test_GET_FILEURL_named(self):
 986         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 987         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 988         d = self.GET(base + "/@@name=/blah.txt")
 989         d.addCallback(self.failUnlessIsBarDotTxt)
 990         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 991         d.addCallback(self.failUnlessIsBarDotTxt)
 992         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 993         d.addCallback(self.failUnlessIsBarDotTxt)
 994         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 995         d.addCallback(self.failUnlessIsBarDotTxt)
 996         save_url = base + "?save=true&filename=blah.txt"
 997         d.addCallback(lambda res: self.GET(save_url))
 998         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 999         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1000         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
1001         u_url = base + "?save=true&filename=" + u_fn_e
1002         d.addCallback(lambda res: self.GET(u_url))
1003         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
1004         return d
1005
1006     def test_PUT_FILEURL_named_bad(self):
1007         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
1008         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
1009                              "400 Bad Request",
1010                              "/file can only be used with GET or HEAD",
1011                              self.PUT, base + "/@@name=/blah.txt", "")
1012         return d
1013
1014
1015     def test_GET_DIRURL_named_bad(self):
1016         base = "/file/%s" % urllib.quote(self._foo_uri)
1017         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
1018                              "400 Bad Request",
1019                              "is not a file-cap",
1020                              self.GET, base + "/@@name=/blah.txt")
1021         return d
1022
1023     def test_GET_slash_file_bad(self):
1024         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
1025                              "404 Not Found",
1026                              "/file must be followed by a file-cap and a name",
1027                              self.GET, "/file")
1028         return d
1029
1030     def test_GET_unhandled_URI_named(self):
1031         contents, n, newuri = self.makefile(12)
1032         verifier_cap = n.get_verify_cap().to_string()
1033         base = "/file/%s" % urllib.quote(verifier_cap)
1034         # client.create_node_from_uri() can't handle verify-caps
1035         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
1036                              "400 Bad Request", "is not a file-cap",
1037                              self.GET, base)
1038         return d
1039
1040     def test_GET_unhandled_URI(self):
1041         contents, n, newuri = self.makefile(12)
1042         verifier_cap = n.get_verify_cap().to_string()
1043         base = "/uri/%s" % urllib.quote(verifier_cap)
1044         # client.create_node_from_uri() can't handle verify-caps
1045         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
1046                              "400 Bad Request",
1047                              "GET unknown URI type: can only do t=info",
1048                              self.GET, base)
1049         return d
1050
1051     def test_GET_FILE_URI(self):
1052         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
1053         d = self.GET(base)
1054         d.addCallback(self.failUnlessIsBarDotTxt)
1055         return d
1056
1057     def test_GET_FILE_URI_mdmf(self):
1058         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
1059         d = self.GET(base)
1060         d.addCallback(self.failUnlessIsQuuxDotTxt)
1061         return d
1062
1063     def test_GET_FILE_URI_mdmf_extensions(self):
1064         base = "/uri/%s" % urllib.quote("%s:RANDOMSTUFF" % self._quux_txt_uri)
1065         d = self.GET(base)
1066         d.addCallback(self.failUnlessIsQuuxDotTxt)
1067         return d
1068
1069     def test_GET_FILE_URI_mdmf_readonly(self):
1070         base = "/uri/%s" % urllib.quote(self._quux_txt_readonly_uri)
1071         d = self.GET(base)
1072         d.addCallback(self.failUnlessIsQuuxDotTxt)
1073         return d
1074
1075     def test_GET_FILE_URI_badchild(self):
1076         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
1077         errmsg = "Files have no children, certainly not named 'boguschild'"
1078         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
1079                              "400 Bad Request", errmsg,
1080                              self.GET, base)
1081         return d
1082
1083     def test_PUT_FILE_URI_badchild(self):
1084         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
1085         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
1086         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
1087                              "400 Bad Request", errmsg,
1088                              self.PUT, base, "")
1089         return d
1090
1091     def test_PUT_FILE_URI_mdmf(self):
1092         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
1093         self._quux_new_contents = "new_contents"
1094         d = self.GET(base)
1095         d.addCallback(lambda res:
1096             self.failUnlessIsQuuxDotTxt(res))
1097         d.addCallback(lambda ignored:
1098             self.PUT(base, self._quux_new_contents))
1099         d.addCallback(lambda ignored:
1100             self.GET(base))
1101         d.addCallback(lambda res:
1102             self.failUnlessReallyEqual(res, self._quux_new_contents))
1103         return d
1104
1105     def test_PUT_FILE_URI_mdmf_extensions(self):
1106         base = "/uri/%s" % urllib.quote("%s:EXTENSIONSTUFF" % self._quux_txt_uri)
1107         self._quux_new_contents = "new_contents"
1108         d = self.GET(base)
1109         d.addCallback(lambda res: self.failUnlessIsQuuxDotTxt(res))
1110         d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
1111         d.addCallback(lambda ignored: self.GET(base))
1112         d.addCallback(lambda res: self.failUnlessEqual(self._quux_new_contents,
1113                                                        res))
1114         return d
1115
1116     def test_PUT_FILE_URI_mdmf_readonly(self):
1117         # We're not allowed to PUT things to a readonly cap.
1118         base = "/uri/%s" % self._quux_txt_readonly_uri
1119         d = self.GET(base)
1120         d.addCallback(lambda res:
1121             self.failUnlessIsQuuxDotTxt(res))
1122         # What should we get here? We get a 500 error now; that's not right.
1123         d.addCallback(lambda ignored:
1124             self.shouldFail2(error.Error, "test_PUT_FILE_URI_mdmf_readonly",
1125                              "400 Bad Request", "read-only cap",
1126                              self.PUT, base, "new data"))
1127         return d
1128
1129     def test_PUT_FILE_URI_sdmf_readonly(self):
1130         # We're not allowed to put things to a readonly cap.
1131         base = "/uri/%s" % self._baz_txt_readonly_uri
1132         d = self.GET(base)
1133         d.addCallback(lambda res:
1134             self.failUnlessIsBazDotTxt(res))
1135         d.addCallback(lambda ignored:
1136             self.shouldFail2(error.Error, "test_PUT_FILE_URI_sdmf_readonly",
1137                              "400 Bad Request", "read-only cap",
1138                              self.PUT, base, "new_data"))
1139         return d
1140
1141     def test_GET_etags(self):
1142
1143         def _check_etags(uri):
1144             d1 = _get_etag(uri)
1145             d2 = _get_etag(uri, 'json')
1146             d = defer.DeferredList([d1, d2], consumeErrors=True)
1147             def _check(results):
1148                 # All deferred must succeed
1149                 self.failUnless(all([r[0] for r in results]))
1150                 # the etag for the t=json form should be just like the etag
1151                 # fo the default t='' form, but with a 'json' suffix
1152                 self.failUnlessEqual(results[0][1] + 'json', results[1][1])
1153             d.addCallback(_check)
1154             return d
1155
1156         def _get_etag(uri, t=''):
1157             targetbase = "/uri/%s?t=%s" % (urllib.quote(uri.strip()), t)
1158             d = self.GET(targetbase, return_response=True, followRedirect=True)
1159             def _just_the_etag(result):
1160                 data, response, headers = result
1161                 etag = headers['etag'][0]
1162                 if uri.startswith('URI:DIR'):
1163                     self.failUnless(etag.startswith('DIR:'), etag)
1164                 return etag
1165             return d.addCallback(_just_the_etag)
1166
1167         # Check that etags work with immutable directories
1168         (newkids, caps) = self._create_immutable_children()
1169         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1170                       simplejson.dumps(newkids))
1171         def _stash_immdir_uri(uri):
1172             self._immdir_uri = uri
1173             return uri
1174         d.addCallback(_stash_immdir_uri)
1175         d.addCallback(_check_etags)
1176
1177         # Check that etags work with immutable files
1178         d.addCallback(lambda _: _check_etags(self._bar_txt_uri))
1179
1180         # use the ETag on GET
1181         def _check_match(ign):
1182             uri = "/uri/%s" % self._bar_txt_uri
1183             d = self.GET(uri, return_response=True)
1184             # extract the ETag
1185             d.addCallback(lambda (data, code, headers):
1186                           headers['etag'][0])
1187             # do a GET that's supposed to match the ETag
1188             d.addCallback(lambda etag:
1189                           self.GET(uri, return_response=True,
1190                                    headers={"If-None-Match": etag}))
1191             # make sure it short-circuited (304 instead of 200)
1192             d.addCallback(lambda (data, code, headers):
1193                           self.failUnlessEqual(int(code), http.NOT_MODIFIED))
1194             return d
1195         d.addCallback(_check_match)
1196
1197         def _no_etag(uri, t):
1198             target = "/uri/%s?t=%s" % (uri, t)
1199             d = self.GET(target, return_response=True, followRedirect=True)
1200             d.addCallback(lambda (data, code, headers):
1201                           self.failIf("etag" in headers, target))
1202             return d
1203         def _yes_etag(uri, t):
1204             target = "/uri/%s?t=%s" % (uri, t)
1205             d = self.GET(target, return_response=True, followRedirect=True)
1206             d.addCallback(lambda (data, code, headers):
1207                           self.failUnless("etag" in headers, target))
1208             return d
1209
1210         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, ""))
1211         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "json"))
1212         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "uri"))
1213         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "readonly-uri"))
1214         d.addCallback(lambda ign: _no_etag(self._bar_txt_uri, "info"))
1215
1216         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, ""))
1217         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "json"))
1218         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "uri"))
1219         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "readonly-uri"))
1220         d.addCallback(lambda ign: _no_etag(self._immdir_uri, "info"))
1221         d.addCallback(lambda ign: _no_etag(self._immdir_uri, "rename-form"))
1222
1223         return d
1224
1225     # TODO: version of this with a Unicode filename
1226     def test_GET_FILEURL_save(self):
1227         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
1228                      return_response=True)
1229         def _got((res, statuscode, headers)):
1230             content_disposition = headers["content-disposition"][0]
1231             self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
1232             self.failUnlessIsBarDotTxt(res)
1233         d.addCallback(_got)
1234         return d
1235
1236     def test_GET_FILEURL_missing(self):
1237         d = self.GET(self.public_url + "/foo/missing")
1238         d.addBoth(self.should404, "test_GET_FILEURL_missing")
1239         return d
1240
1241     def test_GET_FILEURL_info_mdmf(self):
1242         d = self.GET("/uri/%s?t=info" % self._quux_txt_uri)
1243         def _got(res):
1244             self.failUnlessIn("mutable file (mdmf)", res)
1245             self.failUnlessIn(self._quux_txt_uri, res)
1246             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1247         d.addCallback(_got)
1248         return d
1249
1250     def test_GET_FILEURL_info_mdmf_readonly(self):
1251         d = self.GET("/uri/%s?t=info" % self._quux_txt_readonly_uri)
1252         def _got(res):
1253             self.failUnlessIn("mutable file (mdmf)", res)
1254             self.failIfIn(self._quux_txt_uri, res)
1255             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1256         d.addCallback(_got)
1257         return d
1258
1259     def test_GET_FILEURL_info_sdmf(self):
1260         d = self.GET("/uri/%s?t=info" % self._baz_txt_uri)
1261         def _got(res):
1262             self.failUnlessIn("mutable file (sdmf)", res)
1263             self.failUnlessIn(self._baz_txt_uri, res)
1264         d.addCallback(_got)
1265         return d
1266
1267     def test_GET_FILEURL_info_mdmf_extensions(self):
1268         d = self.GET("/uri/%s:STUFF?t=info" % self._quux_txt_uri)
1269         def _got(res):
1270             self.failUnlessIn("mutable file (mdmf)", res)
1271             self.failUnlessIn(self._quux_txt_uri, res)
1272             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1273         d.addCallback(_got)
1274         return d
1275
1276     def test_PUT_overwrite_only_files(self):
1277         # create a directory, put a file in that directory.
1278         contents, n, filecap = self.makefile(8)
1279         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
1280         d.addCallback(lambda res:
1281             self.PUT(self.public_url + "/foo/dir/file1.txt",
1282                      self.NEWFILE_CONTENTS))
1283         # try to overwrite the file with replace=only-files
1284         # (this should work)
1285         d.addCallback(lambda res:
1286             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
1287                      filecap))
1288         d.addCallback(lambda res:
1289             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
1290                  "There was already a child by that name, and you asked me "
1291                  "to not replace it",
1292                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
1293                  filecap))
1294         return d
1295
1296     def test_PUT_NEWFILEURL(self):
1297         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
1298         # TODO: we lose the response code, so we can't check this
1299         #self.failUnlessReallyEqual(responsecode, 201)
1300         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1301         d.addCallback(lambda res:
1302                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1303                                                       self.NEWFILE_CONTENTS))
1304         return d
1305
1306     def test_PUT_NEWFILEURL_not_mutable(self):
1307         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
1308                      self.NEWFILE_CONTENTS)
1309         # TODO: we lose the response code, so we can't check this
1310         #self.failUnlessReallyEqual(responsecode, 201)
1311         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1312         d.addCallback(lambda res:
1313                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1314                                                       self.NEWFILE_CONTENTS))
1315         return d
1316
1317     def test_PUT_NEWFILEURL_unlinked_mdmf(self):
1318         # this should get us a few segments of an MDMF mutable file,
1319         # which we can then test for.
1320         contents = self.NEWFILE_CONTENTS * 300000
1321         d = self.PUT("/uri?format=mdmf",
1322                      contents)
1323         def _got_filecap(filecap):
1324             self.failUnless(filecap.startswith("URI:MDMF"))
1325             return filecap
1326         d.addCallback(_got_filecap)
1327         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1328         d.addCallback(lambda json: self.failUnlessIn("MDMF", json))
1329         return d
1330
1331     def test_PUT_NEWFILEURL_unlinked_sdmf(self):
1332         contents = self.NEWFILE_CONTENTS * 300000
1333         d = self.PUT("/uri?format=sdmf",
1334                      contents)
1335         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1336         d.addCallback(lambda json: self.failUnlessIn("SDMF", json))
1337         return d
1338
1339     def test_PUT_NEWFILEURL_unlinked_bad_format(self):
1340         contents = self.NEWFILE_CONTENTS * 300000
1341         return self.shouldHTTPError("PUT_NEWFILEURL_unlinked_bad_format",
1342                                     400, "Bad Request", "Unknown format: foo",
1343                                     self.PUT, "/uri?format=foo",
1344                                     contents)
1345
1346     def test_PUT_NEWFILEURL_range_bad(self):
1347         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
1348         target = self.public_url + "/foo/new.txt"
1349         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
1350                              "501 Not Implemented",
1351                              "Content-Range in PUT not yet supported",
1352                              # (and certainly not for immutable files)
1353                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
1354                              headers=headers)
1355         d.addCallback(lambda res:
1356                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
1357         return d
1358
1359     def test_PUT_NEWFILEURL_mutable(self):
1360         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1361                      self.NEWFILE_CONTENTS)
1362         # TODO: we lose the response code, so we can't check this
1363         #self.failUnlessReallyEqual(responsecode, 201)
1364         def _check_uri(res):
1365             u = uri.from_string_mutable_filenode(res)
1366             self.failUnless(u.is_mutable())
1367             self.failIf(u.is_readonly())
1368             return res
1369         d.addCallback(_check_uri)
1370         d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
1371         d.addCallback(lambda res:
1372                       self.failUnlessMutableChildContentsAre(self._foo_node,
1373                                                              u"new.txt",
1374                                                              self.NEWFILE_CONTENTS))
1375         return d
1376
1377     def test_PUT_NEWFILEURL_mutable_toobig(self):
1378         # It is okay to upload large mutable files, so we should be able
1379         # to do that.
1380         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1381                      "b" * (self.s.MUTABLE_SIZELIMIT + 1))
1382         return d
1383
1384     def test_PUT_NEWFILEURL_replace(self):
1385         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
1386         # TODO: we lose the response code, so we can't check this
1387         #self.failUnlessReallyEqual(responsecode, 200)
1388         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
1389         d.addCallback(lambda res:
1390                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1391                                                       self.NEWFILE_CONTENTS))
1392         return d
1393
1394     def test_PUT_NEWFILEURL_bad_t(self):
1395         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
1396                              "PUT to a file: bad t=bogus",
1397                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
1398                              "contents")
1399         return d
1400
1401     def test_PUT_NEWFILEURL_no_replace(self):
1402         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
1403                      self.NEWFILE_CONTENTS)
1404         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
1405                   "409 Conflict",
1406                   "There was already a child by that name, and you asked me "
1407                   "to not replace it")
1408         return d
1409
1410     def test_PUT_NEWFILEURL_mkdirs(self):
1411         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
1412         fn = self._foo_node
1413         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
1414         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
1415         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
1416         d.addCallback(lambda res:
1417                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
1418                                                       self.NEWFILE_CONTENTS))
1419         return d
1420
1421     def test_PUT_NEWFILEURL_blocked(self):
1422         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
1423                      self.NEWFILE_CONTENTS)
1424         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
1425                   "409 Conflict",
1426                   "Unable to create directory 'blockingfile': a file was in the way")
1427         return d
1428
1429     def test_PUT_NEWFILEURL_emptyname(self):
1430         # an empty pathname component (i.e. a double-slash) is disallowed
1431         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
1432                              "400 Bad Request",
1433                              "The webapi does not allow empty pathname components",
1434                              self.PUT, self.public_url + "/foo//new.txt", "")
1435         return d
1436
1437     def test_DELETE_FILEURL(self):
1438         d = self.DELETE(self.public_url + "/foo/bar.txt")
1439         d.addCallback(lambda res:
1440                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1441         return d
1442
1443     def test_DELETE_FILEURL_missing(self):
1444         d = self.DELETE(self.public_url + "/foo/missing")
1445         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
1446         return d
1447
1448     def test_DELETE_FILEURL_missing2(self):
1449         d = self.DELETE(self.public_url + "/missing/missing")
1450         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
1451         return d
1452
1453     def failUnlessHasBarDotTxtMetadata(self, res):
1454         data = simplejson.loads(res)
1455         self.failUnless(isinstance(data, list))
1456         self.failUnlessIn("metadata", data[1])
1457         self.failUnlessIn("tahoe", data[1]["metadata"])
1458         self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
1459         self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
1460         self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
1461                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
1462
1463     def test_GET_FILEURL_json(self):
1464         # twisted.web.http.parse_qs ignores any query args without an '=', so
1465         # I can't do "GET /path?json", I have to do "GET /path/t=json"
1466         # instead. This may make it tricky to emulate the S3 interface
1467         # completely.
1468         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
1469         def _check1(data):
1470             self.failUnlessIsBarJSON(data)
1471             self.failUnlessHasBarDotTxtMetadata(data)
1472             return
1473         d.addCallback(_check1)
1474         return d
1475
1476     def test_GET_FILEURL_json_mutable_type(self):
1477         # The JSON should include format, which says whether the
1478         # file is SDMF or MDMF
1479         d = self.PUT("/uri?format=mdmf",
1480                      self.NEWFILE_CONTENTS * 300000)
1481         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1482         def _got_json(json, version):
1483             data = simplejson.loads(json)
1484             assert "filenode" == data[0]
1485             data = data[1]
1486             assert isinstance(data, dict)
1487
1488             self.failUnlessIn("format", data)
1489             self.failUnlessEqual(data["format"], version)
1490
1491         d.addCallback(_got_json, "MDMF")
1492         # Now make an SDMF file and check that it is reported correctly.
1493         d.addCallback(lambda ignored:
1494             self.PUT("/uri?format=sdmf",
1495                       self.NEWFILE_CONTENTS * 300000))
1496         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1497         d.addCallback(_got_json, "SDMF")
1498         return d
1499
1500     def test_GET_FILEURL_json_mdmf(self):
1501         d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
1502         d.addCallback(self.failUnlessIsQuuxJSON)
1503         return d
1504
1505     def test_GET_FILEURL_json_missing(self):
1506         d = self.GET(self.public_url + "/foo/missing?json")
1507         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
1508         return d
1509
1510     def test_GET_FILEURL_uri(self):
1511         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
1512         def _check(res):
1513             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1514         d.addCallback(_check)
1515         d.addCallback(lambda res:
1516                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
1517         def _check2(res):
1518             # for now, for files, uris and readonly-uris are the same
1519             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1520         d.addCallback(_check2)
1521         return d
1522
1523     def test_GET_FILEURL_badtype(self):
1524         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
1525                                  "bad t=bogus",
1526                                  self.GET,
1527                                  self.public_url + "/foo/bar.txt?t=bogus")
1528         return d
1529
1530     def test_CSS_FILE(self):
1531         d = self.GET("/tahoe.css", followRedirect=True)
1532         def _check(res):
1533             CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
1534             self.failUnless(CSS_STYLE.search(res), res)
1535         d.addCallback(_check)
1536         return d
1537
1538     def test_GET_FILEURL_uri_missing(self):
1539         d = self.GET(self.public_url + "/foo/missing?t=uri")
1540         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
1541         return d
1542
1543     def _check_upload_and_mkdir_forms(self, html):
1544         # We should have a form to create a file, with radio buttons that allow
1545         # the user to toggle whether it is a CHK/LIT (default), SDMF, or MDMF file.
1546         self.failUnless(re.search('<input (name="t" |value="upload" |type="hidden" ){3}/>', html), html)
1547         self.failUnless(re.search('<input [^/]*id="upload-chk"', html), html)
1548         self.failUnless(re.search('<input [^/]*id="upload-sdmf"', html), html)
1549         self.failUnless(re.search('<input [^/]*id="upload-mdmf"', html), html)
1550
1551         # We should also have the ability to create a mutable directory, with
1552         # radio buttons that allow the user to toggle whether it is an SDMF (default)
1553         # or MDMF directory.
1554         self.failUnless(re.search('<input (name="t" |value="mkdir" |type="hidden" ){3}/>', html), html)
1555         self.failUnless(re.search('<input [^/]*id="mkdir-sdmf"', html), html)
1556         self.failUnless(re.search('<input [^/]*id="mkdir-mdmf"', html), html)
1557
1558         self.failUnlessIn(FAVICON_MARKUP, html)
1559
1560     def test_GET_DIRECTORY_html(self):
1561         d = self.GET(self.public_url + "/foo", followRedirect=True)
1562         def _check(html):
1563             self.failUnlessIn('<li class="toolbar-item"><a href="../../..">Return to Welcome page</a></li>', html)
1564             self._check_upload_and_mkdir_forms(html)
1565             self.failUnlessIn("quux", html)
1566         d.addCallback(_check)
1567         return d
1568
1569     def test_GET_DIRECTORY_html_filenode_encoding(self):
1570         d = self.GET(self.public_url + "/foo", followRedirect=True)
1571         def _check(html):
1572             # Check if encoded entries are there
1573             self.failUnlessIn('@@named=/' + self._htmlname_urlencoded + '">'
1574                               + self._htmlname_escaped + '</a>', html)
1575             self.failUnlessIn('value="' + self._htmlname_escaped_attr + '"', html)
1576             self.failIfIn(self._htmlname_escaped_double, html)
1577             # Make sure that Nevow escaping actually works by checking for unsafe characters
1578             # and that '&' is escaped.
1579             for entity in '<>':
1580                 self.failUnlessIn(entity, self._htmlname_raw)
1581                 self.failIfIn(entity, self._htmlname_escaped)
1582             self.failUnlessIn('&', re.sub(r'&(amp|lt|gt|quot|apos);', '', self._htmlname_raw))
1583             self.failIfIn('&', re.sub(r'&(amp|lt|gt|quot|apos);', '', self._htmlname_escaped))
1584         d.addCallback(_check)
1585         return d
1586
1587     def test_GET_root_html(self):
1588         d = self.GET("/")
1589         d.addCallback(self._check_upload_and_mkdir_forms)
1590         return d
1591
1592     def test_GET_DIRURL(self):
1593         # the addSlash means we get a redirect here
1594         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1595         ROOT = "../../.."
1596         d = self.GET(self.public_url + "/foo", followRedirect=True)
1597         def _check(res):
1598             self.failUnlessIn('<a href="%s">Return to Welcome page' % ROOT, res)
1599
1600             # the FILE reference points to a URI, but it should end in bar.txt
1601             bar_url = ("%s/file/%s/@@named=/bar.txt" %
1602                        (ROOT, urllib.quote(self._bar_txt_uri)))
1603             get_bar = "".join([r'<td>FILE</td>',
1604                                r'\s+<td>',
1605                                r'<a href="%s">bar.txt</a>' % bar_url,
1606                                r'</td>',
1607                                r'\s+<td align="right">%d</td>' % len(self.BAR_CONTENTS),
1608                                ])
1609             self.failUnless(re.search(get_bar, res), res)
1610             for label in ['unlink', 'rename/relink']:
1611                 for line in res.split("\n"):
1612                     # find the line that contains the relevant button for bar.txt
1613                     if ("form action" in line and
1614                         ('value="%s"' % (label,)) in line and
1615                         'value="bar.txt"' in line):
1616                         # the form target should use a relative URL
1617                         foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1618                         self.failUnlessIn('action="%s"' % foo_url, line)
1619                         # and the when_done= should too
1620                         #done_url = urllib.quote(???)
1621                         #self.failUnlessIn('name="when_done" value="%s"' % done_url, line)
1622
1623                         # 'unlink' needs to use POST because it directly has a side effect
1624                         if label == 'unlink':
1625                             self.failUnlessIn('method="post"', line)
1626                         break
1627                 else:
1628                     self.fail("unable to find '%s bar.txt' line" % (label,))
1629
1630             # the DIR reference just points to a URI
1631             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1632             get_sub = ((r'<td>DIR</td>')
1633                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1634             self.failUnless(re.search(get_sub, res), res)
1635         d.addCallback(_check)
1636
1637         # look at a readonly directory
1638         d.addCallback(lambda res:
1639                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
1640         def _check2(res):
1641             self.failUnlessIn("(read-only)", res)
1642             self.failIfIn("Upload a file", res)
1643         d.addCallback(_check2)
1644
1645         # and at a directory that contains a readonly directory
1646         d.addCallback(lambda res:
1647                       self.GET(self.public_url, followRedirect=True))
1648         def _check3(res):
1649             self.failUnless(re.search('<td>DIR-RO</td>'
1650                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1651         d.addCallback(_check3)
1652
1653         # and an empty directory
1654         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1655         def _check4(res):
1656             self.failUnlessIn("directory is empty", res)
1657             MKDIR_BUTTON_RE=re.compile('<input (type="hidden" |name="t" |value="mkdir" ){3}/>.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input (type="submit" |class="btn" |value="Create" ){3}/>', re.I)
1658             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1659         d.addCallback(_check4)
1660
1661         # and at a literal directory
1662         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1663         d.addCallback(lambda res:
1664                       self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1665         def _check5(res):
1666             self.failUnlessIn('(immutable)', res)
1667             self.failUnless(re.search('<td>FILE</td>'
1668                                       r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1669         d.addCallback(_check5)
1670         return d
1671
1672     def test_GET_DIRURL_badtype(self):
1673         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1674                                  400, "Bad Request",
1675                                  "bad t=bogus",
1676                                  self.GET,
1677                                  self.public_url + "/foo?t=bogus")
1678         return d
1679
1680     def test_GET_DIRURL_json(self):
1681         d = self.GET(self.public_url + "/foo?t=json")
1682         d.addCallback(self.failUnlessIsFooJSON)
1683         return d
1684
1685     def test_GET_DIRURL_json_format(self):
1686         d = self.PUT(self.public_url + \
1687                      "/foo/sdmf.txt?format=sdmf",
1688                      self.NEWFILE_CONTENTS * 300000)
1689         d.addCallback(lambda ignored:
1690             self.PUT(self.public_url + \
1691                      "/foo/mdmf.txt?format=mdmf",
1692                      self.NEWFILE_CONTENTS * 300000))
1693         # Now we have an MDMF and SDMF file in the directory. If we GET
1694         # its JSON, we should see their encodings.
1695         d.addCallback(lambda ignored:
1696             self.GET(self.public_url + "/foo?t=json"))
1697         def _got_json(json):
1698             data = simplejson.loads(json)
1699             assert data[0] == "dirnode"
1700
1701             data = data[1]
1702             kids = data['children']
1703
1704             mdmf_data = kids['mdmf.txt'][1]
1705             self.failUnlessIn("format", mdmf_data)
1706             self.failUnlessEqual(mdmf_data["format"], "MDMF")
1707
1708             sdmf_data = kids['sdmf.txt'][1]
1709             self.failUnlessIn("format", sdmf_data)
1710             self.failUnlessEqual(sdmf_data["format"], "SDMF")
1711         d.addCallback(_got_json)
1712         return d
1713
1714
1715     def test_POST_DIRURL_manifest_no_ophandle(self):
1716         d = self.shouldFail2(error.Error,
1717                              "test_POST_DIRURL_manifest_no_ophandle",
1718                              "400 Bad Request",
1719                              "slow operation requires ophandle=",
1720                              self.POST, self.public_url, t="start-manifest")
1721         return d
1722
1723     def test_POST_DIRURL_manifest(self):
1724         d = defer.succeed(None)
1725         def getman(ignored, output):
1726             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1727                           followRedirect=True)
1728             d.addCallback(self.wait_for_operation, "125")
1729             d.addCallback(self.get_operation_results, "125", output)
1730             return d
1731         d.addCallback(getman, None)
1732         def _got_html(manifest):
1733             self.failUnlessIn("Manifest of SI=", manifest)
1734             self.failUnlessIn("<td>sub</td>", manifest)
1735             self.failUnlessIn(self._sub_uri, manifest)
1736             self.failUnlessIn("<td>sub/baz.txt</td>", manifest)
1737             self.failUnlessIn(FAVICON_MARKUP, manifest)
1738         d.addCallback(_got_html)
1739
1740         # both t=status and unadorned GET should be identical
1741         d.addCallback(lambda res: self.GET("/operations/125"))
1742         d.addCallback(_got_html)
1743
1744         d.addCallback(getman, "html")
1745         d.addCallback(_got_html)
1746         d.addCallback(getman, "text")
1747         def _got_text(manifest):
1748             self.failUnlessIn("\nsub " + self._sub_uri + "\n", manifest)
1749             self.failUnlessIn("\nsub/baz.txt URI:CHK:", manifest)
1750         d.addCallback(_got_text)
1751         d.addCallback(getman, "JSON")
1752         def _got_json(res):
1753             data = res["manifest"]
1754             got = {}
1755             for (path_list, cap) in data:
1756                 got[tuple(path_list)] = cap
1757             self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
1758             self.failUnlessIn((u"sub", u"baz.txt"), got)
1759             self.failUnlessIn("finished", res)
1760             self.failUnlessIn("origin", res)
1761             self.failUnlessIn("storage-index", res)
1762             self.failUnlessIn("verifycaps", res)
1763             self.failUnlessIn("stats", res)
1764         d.addCallback(_got_json)
1765         return d
1766
1767     def test_POST_DIRURL_deepsize_no_ophandle(self):
1768         d = self.shouldFail2(error.Error,
1769                              "test_POST_DIRURL_deepsize_no_ophandle",
1770                              "400 Bad Request",
1771                              "slow operation requires ophandle=",
1772                              self.POST, self.public_url, t="start-deep-size")
1773         return d
1774
1775     def test_POST_DIRURL_deepsize(self):
1776         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1777                       followRedirect=True)
1778         d.addCallback(self.wait_for_operation, "126")
1779         d.addCallback(self.get_operation_results, "126", "json")
1780         def _got_json(data):
1781             self.failUnlessReallyEqual(data["finished"], True)
1782             size = data["size"]
1783             self.failUnless(size > 1000)
1784         d.addCallback(_got_json)
1785         d.addCallback(self.get_operation_results, "126", "text")
1786         def _got_text(res):
1787             mo = re.search(r'^size: (\d+)$', res, re.M)
1788             self.failUnless(mo, res)
1789             size = int(mo.group(1))
1790             # with directories, the size varies.
1791             self.failUnless(size > 1000)
1792         d.addCallback(_got_text)
1793         return d
1794
1795     def test_POST_DIRURL_deepstats_no_ophandle(self):
1796         d = self.shouldFail2(error.Error,
1797                              "test_POST_DIRURL_deepstats_no_ophandle",
1798                              "400 Bad Request",
1799                              "slow operation requires ophandle=",
1800                              self.POST, self.public_url, t="start-deep-stats")
1801         return d
1802
1803     def test_POST_DIRURL_deepstats(self):
1804         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1805                       followRedirect=True)
1806         d.addCallback(self.wait_for_operation, "127")
1807         d.addCallback(self.get_operation_results, "127", "json")
1808         def _got_json(stats):
1809             expected = {"count-immutable-files": 4,
1810                         "count-mutable-files": 2,
1811                         "count-literal-files": 0,
1812                         "count-files": 6,
1813                         "count-directories": 3,
1814                         "size-immutable-files": 76,
1815                         "size-literal-files": 0,
1816                         #"size-directories": 1912, # varies
1817                         #"largest-directory": 1590,
1818                         "largest-directory-children": 8,
1819                         "largest-immutable-file": 19,
1820                         }
1821             for k,v in expected.iteritems():
1822                 self.failUnlessReallyEqual(stats[k], v,
1823                                            "stats[%s] was %s, not %s" %
1824                                            (k, stats[k], v))
1825             self.failUnlessReallyEqual(stats["size-files-histogram"],
1826                                        [ [11, 31, 4] ])
1827         d.addCallback(_got_json)
1828         return d
1829
1830     def test_POST_DIRURL_stream_manifest(self):
1831         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1832         def _check(res):
1833             self.failUnless(res.endswith("\n"))
1834             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1835             self.failUnlessReallyEqual(len(units), 10)
1836             self.failUnlessEqual(units[-1]["type"], "stats")
1837             first = units[0]
1838             self.failUnlessEqual(first["path"], [])
1839             self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri)
1840             self.failUnlessEqual(first["type"], "directory")
1841             baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0]
1842             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1843             self.failIfEqual(baz["storage-index"], None)
1844             self.failIfEqual(baz["verifycap"], None)
1845             self.failIfEqual(baz["repaircap"], None)
1846             # XXX: Add quux and baz to this test.
1847             return
1848         d.addCallback(_check)
1849         return d
1850
1851     def test_GET_DIRURL_uri(self):
1852         d = self.GET(self.public_url + "/foo?t=uri")
1853         def _check(res):
1854             self.failUnlessReallyEqual(to_str(res), self._foo_uri)
1855         d.addCallback(_check)
1856         return d
1857
1858     def test_GET_DIRURL_readonly_uri(self):
1859         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1860         def _check(res):
1861             self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri)
1862         d.addCallback(_check)
1863         return d
1864
1865     def test_PUT_NEWDIRURL(self):
1866         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1867         d.addCallback(lambda res:
1868                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1869         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1870         d.addCallback(self.failUnlessNodeKeysAre, [])
1871         return d
1872
1873     def test_PUT_NEWDIRURL_mdmf(self):
1874         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1875         d.addCallback(lambda res:
1876                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1877         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1878         d.addCallback(lambda node:
1879             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1880         return d
1881
1882     def test_PUT_NEWDIRURL_sdmf(self):
1883         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=sdmf",
1884                      "")
1885         d.addCallback(lambda res:
1886                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1887         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1888         d.addCallback(lambda node:
1889             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1890         return d
1891
1892     def test_PUT_NEWDIRURL_bad_format(self):
1893         return self.shouldHTTPError("PUT_NEWDIRURL_bad_format",
1894                                     400, "Bad Request", "Unknown format: foo",
1895                                     self.PUT, self.public_url +
1896                                     "/foo/newdir=?t=mkdir&format=foo", "")
1897
1898     def test_POST_NEWDIRURL(self):
1899         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1900         d.addCallback(lambda res:
1901                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1902         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1903         d.addCallback(self.failUnlessNodeKeysAre, [])
1904         return d
1905
1906     def test_POST_NEWDIRURL_mdmf(self):
1907         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1908         d.addCallback(lambda res:
1909                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1910         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1911         d.addCallback(lambda node:
1912             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1913         return d
1914
1915     def test_POST_NEWDIRURL_sdmf(self):
1916         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=sdmf", "")
1917         d.addCallback(lambda res:
1918             self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1919         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1920         d.addCallback(lambda node:
1921             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1922         return d
1923
1924     def test_POST_NEWDIRURL_bad_format(self):
1925         return self.shouldHTTPError("POST_NEWDIRURL_bad_format",
1926                                     400, "Bad Request", "Unknown format: foo",
1927                                     self.POST2, self.public_url + \
1928                                     "/foo/newdir?t=mkdir&format=foo", "")
1929
1930     def test_POST_NEWDIRURL_emptyname(self):
1931         # an empty pathname component (i.e. a double-slash) is disallowed
1932         d = self.shouldFail2(error.Error, "POST_NEWDIRURL_emptyname",
1933                              "400 Bad Request",
1934                              "The webapi does not allow empty pathname components, i.e. a double slash",
1935                              self.POST, self.public_url + "//?t=mkdir")
1936         return d
1937
1938     def _do_POST_NEWDIRURL_initial_children_test(self, version=None):
1939         (newkids, caps) = self._create_initial_children()
1940         query = "/foo/newdir?t=mkdir-with-children"
1941         if version == MDMF_VERSION:
1942             query += "&format=mdmf"
1943         elif version == SDMF_VERSION:
1944             query += "&format=sdmf"
1945         else:
1946             version = SDMF_VERSION # for later
1947         d = self.POST2(self.public_url + query,
1948                        simplejson.dumps(newkids))
1949         def _check(uri):
1950             n = self.s.create_node_from_uri(uri.strip())
1951             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1952             self.failUnlessEqual(n._node.get_version(), version)
1953             d2.addCallback(lambda ign:
1954                            self.failUnlessROChildURIIs(n, u"child-imm",
1955                                                        caps['filecap1']))
1956             d2.addCallback(lambda ign:
1957                            self.failUnlessRWChildURIIs(n, u"child-mutable",
1958                                                        caps['filecap2']))
1959             d2.addCallback(lambda ign:
1960                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1961                                                        caps['filecap3']))
1962             d2.addCallback(lambda ign:
1963                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1964                                                        caps['unknown_rocap']))
1965             d2.addCallback(lambda ign:
1966                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1967                                                        caps['unknown_rwcap']))
1968             d2.addCallback(lambda ign:
1969                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1970                                                        caps['unknown_immcap']))
1971             d2.addCallback(lambda ign:
1972                            self.failUnlessRWChildURIIs(n, u"dirchild",
1973                                                        caps['dircap']))
1974             d2.addCallback(lambda ign:
1975                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1976                                                        caps['litdircap']))
1977             d2.addCallback(lambda ign:
1978                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1979                                                        caps['emptydircap']))
1980             return d2
1981         d.addCallback(_check)
1982         d.addCallback(lambda res:
1983                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1984         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1985         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1986         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1987         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1988         return d
1989
1990     def test_POST_NEWDIRURL_initial_children(self):
1991         return self._do_POST_NEWDIRURL_initial_children_test()
1992
1993     def test_POST_NEWDIRURL_initial_children_mdmf(self):
1994         return self._do_POST_NEWDIRURL_initial_children_test(MDMF_VERSION)
1995
1996     def test_POST_NEWDIRURL_initial_children_sdmf(self):
1997         return self._do_POST_NEWDIRURL_initial_children_test(SDMF_VERSION)
1998
1999     def test_POST_NEWDIRURL_initial_children_bad_format(self):
2000         (newkids, caps) = self._create_initial_children()
2001         return self.shouldHTTPError("POST_NEWDIRURL_initial_children_bad_format",
2002                                     400, "Bad Request", "Unknown format: foo",
2003                                     self.POST2, self.public_url + \
2004                                     "/foo/newdir?t=mkdir-with-children&format=foo",
2005                                     simplejson.dumps(newkids))
2006
2007     def test_POST_NEWDIRURL_immutable(self):
2008         (newkids, caps) = self._create_immutable_children()
2009         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
2010                        simplejson.dumps(newkids))
2011         def _check(uri):
2012             n = self.s.create_node_from_uri(uri.strip())
2013             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2014             d2.addCallback(lambda ign:
2015                            self.failUnlessROChildURIIs(n, u"child-imm",
2016                                                        caps['filecap1']))
2017             d2.addCallback(lambda ign:
2018                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2019                                                        caps['unknown_immcap']))
2020             d2.addCallback(lambda ign:
2021                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
2022                                                        caps['immdircap']))
2023             d2.addCallback(lambda ign:
2024                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
2025                                                        caps['litdircap']))
2026             d2.addCallback(lambda ign:
2027                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
2028                                                        caps['emptydircap']))
2029             return d2
2030         d.addCallback(_check)
2031         d.addCallback(lambda res:
2032                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2033         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2034         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2035         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2036         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2037         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2038         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2039         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2040         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2041         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2042         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2043         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2044         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2045         d.addErrback(self.explain_web_error)
2046         return d
2047
2048     def test_POST_NEWDIRURL_immutable_bad(self):
2049         (newkids, caps) = self._create_initial_children()
2050         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
2051                              "400 Bad Request",
2052                              "needed to be immutable but was not",
2053                              self.POST2,
2054                              self.public_url + "/foo/newdir?t=mkdir-immutable",
2055                              simplejson.dumps(newkids))
2056         return d
2057
2058     def test_PUT_NEWDIRURL_exists(self):
2059         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
2060         d.addCallback(lambda res:
2061                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
2062         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2063         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2064         return d
2065
2066     def test_PUT_NEWDIRURL_blocked(self):
2067         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
2068                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
2069                              self.PUT,
2070                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
2071         d.addCallback(lambda res:
2072                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
2073         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2074         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2075         return d
2076
2077     def test_PUT_NEWDIRURL_mkdirs(self):
2078         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
2079         d.addCallback(lambda res:
2080                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
2081         d.addCallback(lambda res:
2082                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
2083         d.addCallback(lambda res:
2084                       self._foo_node.get_child_at_path(u"subdir/newdir"))
2085         d.addCallback(self.failUnlessNodeKeysAre, [])
2086         return d
2087
2088     def test_PUT_NEWDIRURL_mkdirs_mdmf(self):
2089         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=mdmf", "")
2090         d.addCallback(lambda ignored:
2091             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
2092         d.addCallback(lambda ignored:
2093             self.failIfNodeHasChild(self._foo_node, u"newdir"))
2094         d.addCallback(lambda ignored:
2095             self._foo_node.get_child_at_path(u"subdir"))
2096         def _got_subdir(subdir):
2097             # XXX: What we want?
2098             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
2099             self.failUnlessNodeHasChild(subdir, u"newdir")
2100             return subdir.get_child_at_path(u"newdir")
2101         d.addCallback(_got_subdir)
2102         d.addCallback(lambda newdir:
2103             self.failUnlessEqual(newdir._node.get_version(), MDMF_VERSION))
2104         return d
2105
2106     def test_PUT_NEWDIRURL_mkdirs_sdmf(self):
2107         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=sdmf", "")
2108         d.addCallback(lambda ignored:
2109             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
2110         d.addCallback(lambda ignored:
2111             self.failIfNodeHasChild(self._foo_node, u"newdir"))
2112         d.addCallback(lambda ignored:
2113             self._foo_node.get_child_at_path(u"subdir"))
2114         def _got_subdir(subdir):
2115             # XXX: What we want?
2116             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
2117             self.failUnlessNodeHasChild(subdir, u"newdir")
2118             return subdir.get_child_at_path(u"newdir")
2119         d.addCallback(_got_subdir)
2120         d.addCallback(lambda newdir:
2121             self.failUnlessEqual(newdir._node.get_version(), SDMF_VERSION))
2122         return d
2123
2124     def test_PUT_NEWDIRURL_mkdirs_bad_format(self):
2125         return self.shouldHTTPError("PUT_NEWDIRURL_mkdirs_bad_format",
2126                                     400, "Bad Request", "Unknown format: foo",
2127                                     self.PUT, self.public_url + \
2128                                     "/foo/subdir/newdir?t=mkdir&format=foo",
2129                                     "")
2130
2131     def test_DELETE_DIRURL(self):
2132         d = self.DELETE(self.public_url + "/foo")
2133         d.addCallback(lambda res:
2134                       self.failIfNodeHasChild(self.public_root, u"foo"))
2135         return d
2136
2137     def test_DELETE_DIRURL_missing(self):
2138         d = self.DELETE(self.public_url + "/foo/missing")
2139         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
2140         d.addCallback(lambda res:
2141                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
2142         return d
2143
2144     def test_DELETE_DIRURL_missing2(self):
2145         d = self.DELETE(self.public_url + "/missing")
2146         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
2147         return d
2148
2149     def dump_root(self):
2150         print "NODEWALK"
2151         w = webish.DirnodeWalkerMixin()
2152         def visitor(childpath, childnode, metadata):
2153             print childpath
2154         d = w.walk(self.public_root, visitor)
2155         return d
2156
2157     def failUnlessNodeKeysAre(self, node, expected_keys):
2158         for k in expected_keys:
2159             assert isinstance(k, unicode)
2160         d = node.list()
2161         def _check(children):
2162             self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
2163         d.addCallback(_check)
2164         return d
2165     def failUnlessNodeHasChild(self, node, name):
2166         assert isinstance(name, unicode)
2167         d = node.list()
2168         def _check(children):
2169             self.failUnlessIn(name, children)
2170         d.addCallback(_check)
2171         return d
2172     def failIfNodeHasChild(self, node, name):
2173         assert isinstance(name, unicode)
2174         d = node.list()
2175         def _check(children):
2176             self.failIfIn(name, children)
2177         d.addCallback(_check)
2178         return d
2179
2180     def failUnlessChildContentsAre(self, node, name, expected_contents):
2181         assert isinstance(name, unicode)
2182         d = node.get_child_at_path(name)
2183         d.addCallback(lambda node: download_to_data(node))
2184         def _check(contents):
2185             self.failUnlessReallyEqual(contents, expected_contents)
2186         d.addCallback(_check)
2187         return d
2188
2189     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
2190         assert isinstance(name, unicode)
2191         d = node.get_child_at_path(name)
2192         d.addCallback(lambda node: node.download_best_version())
2193         def _check(contents):
2194             self.failUnlessReallyEqual(contents, expected_contents)
2195         d.addCallback(_check)
2196         return d
2197
2198     def failUnlessRWChildURIIs(self, node, name, expected_uri):
2199         assert isinstance(name, unicode)
2200         d = node.get_child_at_path(name)
2201         def _check(child):
2202             self.failUnless(child.is_unknown() or not child.is_readonly())
2203             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
2204             self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
2205             expected_ro_uri = self._make_readonly(expected_uri)
2206             if expected_ro_uri:
2207                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
2208         d.addCallback(_check)
2209         return d
2210
2211     def failUnlessROChildURIIs(self, node, name, expected_uri):
2212         assert isinstance(name, unicode)
2213         d = node.get_child_at_path(name)
2214         def _check(child):
2215             self.failUnless(child.is_unknown() or child.is_readonly())
2216             self.failUnlessReallyEqual(child.get_write_uri(), None)
2217             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
2218             self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
2219         d.addCallback(_check)
2220         return d
2221
2222     def failUnlessURIMatchesRWChild(self, got_uri, node, name):
2223         assert isinstance(name, unicode)
2224         d = node.get_child_at_path(name)
2225         def _check(child):
2226             self.failUnless(child.is_unknown() or not child.is_readonly())
2227             self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
2228             self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
2229             expected_ro_uri = self._make_readonly(got_uri)
2230             if expected_ro_uri:
2231                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
2232         d.addCallback(_check)
2233         return d
2234
2235     def failUnlessURIMatchesROChild(self, got_uri, node, name):
2236         assert isinstance(name, unicode)
2237         d = node.get_child_at_path(name)
2238         def _check(child):
2239             self.failUnless(child.is_unknown() or child.is_readonly())
2240             self.failUnlessReallyEqual(child.get_write_uri(), None)
2241             self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
2242             self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
2243         d.addCallback(_check)
2244         return d
2245
2246     def failUnlessCHKURIHasContents(self, got_uri, contents):
2247         self.failUnless(self.get_all_contents()[got_uri] == contents)
2248
2249     def test_POST_upload(self):
2250         d = self.POST(self.public_url + "/foo", t="upload",
2251                       file=("new.txt", self.NEWFILE_CONTENTS))
2252         fn = self._foo_node
2253         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2254         d.addCallback(lambda res:
2255                       self.failUnlessChildContentsAre(fn, u"new.txt",
2256                                                       self.NEWFILE_CONTENTS))
2257         return d
2258
2259     def test_POST_upload_unicode(self):
2260         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2261         d = self.POST(self.public_url + "/foo", t="upload",
2262                       file=(filename, self.NEWFILE_CONTENTS))
2263         fn = self._foo_node
2264         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2265         d.addCallback(lambda res:
2266                       self.failUnlessChildContentsAre(fn, filename,
2267                                                       self.NEWFILE_CONTENTS))
2268         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2269         d.addCallback(lambda res: self.GET(target_url))
2270         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2271                                                                   self.NEWFILE_CONTENTS,
2272                                                                   contents))
2273         return d
2274
2275     def test_POST_upload_unicode_named(self):
2276         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2277         d = self.POST(self.public_url + "/foo", t="upload",
2278                       name=filename,
2279                       file=("overridden", self.NEWFILE_CONTENTS))
2280         fn = self._foo_node
2281         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2282         d.addCallback(lambda res:
2283                       self.failUnlessChildContentsAre(fn, filename,
2284                                                       self.NEWFILE_CONTENTS))
2285         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2286         d.addCallback(lambda res: self.GET(target_url))
2287         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2288                                                                   self.NEWFILE_CONTENTS,
2289                                                                   contents))
2290         return d
2291
2292     def test_POST_upload_no_link(self):
2293         d = self.POST("/uri", t="upload",
2294                       file=("new.txt", self.NEWFILE_CONTENTS))
2295         def _check_upload_results(page):
2296             # this should be a page which describes the results of the upload
2297             # that just finished.
2298             self.failUnlessIn("Upload Results:", page)
2299             self.failUnlessIn("URI:", page)
2300             uri_re = re.compile("URI: <tt><span>(.*)</span>")
2301             mo = uri_re.search(page)
2302             self.failUnless(mo, page)
2303             new_uri = mo.group(1)
2304             return new_uri
2305         d.addCallback(_check_upload_results)
2306         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
2307         return d
2308
2309     def test_POST_upload_no_link_whendone(self):
2310         d = self.POST("/uri", t="upload", when_done="/",
2311                       file=("new.txt", self.NEWFILE_CONTENTS))
2312         d.addBoth(self.shouldRedirect, "/")
2313         return d
2314
2315     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
2316         d = defer.maybeDeferred(callable, *args, **kwargs)
2317         def done(res):
2318             if isinstance(res, failure.Failure):
2319                 res.trap(error.PageRedirect)
2320                 statuscode = res.value.status
2321                 target = res.value.location
2322                 return checker(statuscode, target)
2323             self.fail("%s: callable was supposed to redirect, not return '%s'"
2324                       % (which, res))
2325         d.addBoth(done)
2326         return d
2327
2328     def test_POST_upload_no_link_whendone_results(self):
2329         def check(statuscode, target):
2330             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2331             self.failUnless(target.startswith(self.webish_url), target)
2332             return client.getPage(target, method="GET")
2333         # We encode "uri" as "%75ri" to exercise a case affected by ticket #1860.
2334         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
2335                                  check,
2336                                  self.POST, "/uri", t="upload",
2337                                  when_done="/%75ri/%(uri)s",
2338                                  file=("new.txt", self.NEWFILE_CONTENTS))
2339         d.addCallback(lambda res:
2340                       self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
2341         return d
2342
2343     def test_POST_upload_no_link_mutable(self):
2344         d = self.POST("/uri", t="upload", mutable="true",
2345                       file=("new.txt", self.NEWFILE_CONTENTS))
2346         def _check(filecap):
2347             filecap = filecap.strip()
2348             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2349             self.filecap = filecap
2350             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2351             self.failUnlessIn(u.get_storage_index(), self.get_all_contents())
2352             n = self.s.create_node_from_uri(filecap)
2353             return n.download_best_version()
2354         d.addCallback(_check)
2355         def _check2(data):
2356             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2357             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2358         d.addCallback(_check2)
2359         def _check3(data):
2360             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2361             return self.GET("/file/%s" % urllib.quote(self.filecap))
2362         d.addCallback(_check3)
2363         def _check4(data):
2364             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2365         d.addCallback(_check4)
2366         return d
2367
2368     def test_POST_upload_no_link_mutable_toobig(self):
2369         # The SDMF size limit is no longer in place, so we should be
2370         # able to upload mutable files that are as large as we want them
2371         # to be.
2372         d = self.POST("/uri", t="upload", mutable="true",
2373                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2374         return d
2375
2376
2377     def test_POST_upload_format_unlinked(self):
2378         def _check_upload_unlinked(ign, format, uri_prefix):
2379             filename = format + ".txt"
2380             d = self.POST("/uri?t=upload&format=" + format,
2381                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2382             def _got_results(results):
2383                 if format.upper() in ("SDMF", "MDMF"):
2384                     # webapi.rst says this returns a filecap
2385                     filecap = results
2386                 else:
2387                     # for immutable, it returns an "upload results page", and
2388                     # the filecap is buried inside
2389                     line = [l for l in results.split("\n") if "URI: " in l][0]
2390                     mo = re.search(r'<span>([^<]+)</span>', line)
2391                     filecap = mo.group(1)
2392                 self.failUnless(filecap.startswith(uri_prefix),
2393                                 (uri_prefix, filecap))
2394                 return self.GET("/uri/%s?t=json" % filecap)
2395             d.addCallback(_got_results)
2396             def _got_json(json):
2397                 data = simplejson.loads(json)
2398                 data = data[1]
2399                 self.failUnlessIn("format", data)
2400                 self.failUnlessEqual(data["format"], format.upper())
2401             d.addCallback(_got_json)
2402             return d
2403         d = defer.succeed(None)
2404         d.addCallback(_check_upload_unlinked, "chk", "URI:CHK")
2405         d.addCallback(_check_upload_unlinked, "CHK", "URI:CHK")
2406         d.addCallback(_check_upload_unlinked, "sdmf", "URI:SSK")
2407         d.addCallback(_check_upload_unlinked, "mdmf", "URI:MDMF")
2408         return d
2409
2410     def test_POST_upload_bad_format_unlinked(self):
2411         return self.shouldHTTPError("POST_upload_bad_format_unlinked",
2412                                     400, "Bad Request", "Unknown format: foo",
2413                                     self.POST,
2414                                     "/uri?t=upload&format=foo",
2415                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2416
2417     def test_POST_upload_format(self):
2418         def _check_upload(ign, format, uri_prefix, fn=None):
2419             filename = format + ".txt"
2420             d = self.POST(self.public_url +
2421                           "/foo?t=upload&format=" + format,
2422                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2423             def _got_filecap(filecap):
2424                 if fn is not None:
2425                     filenameu = unicode(filename)
2426                     self.failUnlessURIMatchesRWChild(filecap, fn, filenameu)
2427                 self.failUnless(filecap.startswith(uri_prefix))
2428                 return self.GET(self.public_url + "/foo/%s?t=json" % filename)
2429             d.addCallback(_got_filecap)
2430             def _got_json(json):
2431                 data = simplejson.loads(json)
2432                 data = data[1]
2433                 self.failUnlessIn("format", data)
2434                 self.failUnlessEqual(data["format"], format.upper())
2435             d.addCallback(_got_json)
2436             return d
2437
2438         d = defer.succeed(None)
2439         d.addCallback(_check_upload, "chk", "URI:CHK")
2440         d.addCallback(_check_upload, "sdmf", "URI:SSK", self._foo_node)
2441         d.addCallback(_check_upload, "mdmf", "URI:MDMF")
2442         d.addCallback(_check_upload, "MDMF", "URI:MDMF")
2443         return d
2444
2445     def test_POST_upload_bad_format(self):
2446         return self.shouldHTTPError("POST_upload_bad_format",
2447                                     400, "Bad Request", "Unknown format: foo",
2448                                     self.POST, self.public_url + \
2449                                     "/foo?t=upload&format=foo",
2450                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2451
2452     def test_POST_upload_mutable(self):
2453         # this creates a mutable file
2454         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
2455                       file=("new.txt", self.NEWFILE_CONTENTS))
2456         fn = self._foo_node
2457         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2458         d.addCallback(lambda res:
2459                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2460                                                              self.NEWFILE_CONTENTS))
2461         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2462         def _got(newnode):
2463             self.failUnless(IMutableFileNode.providedBy(newnode))
2464             self.failUnless(newnode.is_mutable())
2465             self.failIf(newnode.is_readonly())
2466             self._mutable_node = newnode
2467             self._mutable_uri = newnode.get_uri()
2468         d.addCallback(_got)
2469
2470         # now upload it again and make sure that the URI doesn't change
2471         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
2472         d.addCallback(lambda res:
2473                       self.POST(self.public_url + "/foo", t="upload",
2474                                 mutable="true",
2475                                 file=("new.txt", NEWER_CONTENTS)))
2476         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2477         d.addCallback(lambda res:
2478                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2479                                                              NEWER_CONTENTS))
2480         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2481         def _got2(newnode):
2482             self.failUnless(IMutableFileNode.providedBy(newnode))
2483             self.failUnless(newnode.is_mutable())
2484             self.failIf(newnode.is_readonly())
2485             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2486         d.addCallback(_got2)
2487
2488         # upload a second time, using PUT instead of POST
2489         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
2490         d.addCallback(lambda res:
2491                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
2492         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2493         d.addCallback(lambda res:
2494                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2495                                                              NEW2_CONTENTS))
2496
2497         # finally list the directory, since mutable files are displayed
2498         # slightly differently
2499
2500         d.addCallback(lambda res:
2501                       self.GET(self.public_url + "/foo/",
2502                                followRedirect=True))
2503         def _check_page(res):
2504             # TODO: assert more about the contents
2505             self.failUnlessIn("SSK", res)
2506             return res
2507         d.addCallback(_check_page)
2508
2509         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2510         def _got3(newnode):
2511             self.failUnless(IMutableFileNode.providedBy(newnode))
2512             self.failUnless(newnode.is_mutable())
2513             self.failIf(newnode.is_readonly())
2514             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2515         d.addCallback(_got3)
2516
2517         # look at the JSON form of the enclosing directory
2518         d.addCallback(lambda res:
2519                       self.GET(self.public_url + "/foo/?t=json",
2520                                followRedirect=True))
2521         def _check_page_json(res):
2522             parsed = simplejson.loads(res)
2523             self.failUnlessEqual(parsed[0], "dirnode")
2524             children = dict( [(unicode(name),value)
2525                               for (name,value)
2526                               in parsed[1]["children"].iteritems()] )
2527             self.failUnlessIn(u"new.txt", children)
2528             new_json = children[u"new.txt"]
2529             self.failUnlessEqual(new_json[0], "filenode")
2530             self.failUnless(new_json[1]["mutable"])
2531             self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri)
2532             ro_uri = self._mutable_node.get_readonly().to_string()
2533             self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri)
2534         d.addCallback(_check_page_json)
2535
2536         # and the JSON form of the file
2537         d.addCallback(lambda res:
2538                       self.GET(self.public_url + "/foo/new.txt?t=json"))
2539         def _check_file_json(res):
2540             parsed = simplejson.loads(res)
2541             self.failUnlessEqual(parsed[0], "filenode")
2542             self.failUnless(parsed[1]["mutable"])
2543             self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri)
2544             ro_uri = self._mutable_node.get_readonly().to_string()
2545             self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri)
2546         d.addCallback(_check_file_json)
2547
2548         # and look at t=uri and t=readonly-uri
2549         d.addCallback(lambda res:
2550                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
2551         d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
2552         d.addCallback(lambda res:
2553                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
2554         def _check_ro_uri(res):
2555             ro_uri = self._mutable_node.get_readonly().to_string()
2556             self.failUnlessReallyEqual(res, ro_uri)
2557         d.addCallback(_check_ro_uri)
2558
2559         # make sure we can get to it from /uri/URI
2560         d.addCallback(lambda res:
2561                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
2562         d.addCallback(lambda res:
2563                       self.failUnlessReallyEqual(res, NEW2_CONTENTS))
2564
2565         # and that HEAD computes the size correctly
2566         d.addCallback(lambda res:
2567                       self.HEAD(self.public_url + "/foo/new.txt",
2568                                 return_response=True))
2569         def _got_headers((res, status, headers)):
2570             self.failUnlessReallyEqual(res, "")
2571             self.failUnlessReallyEqual(headers["content-length"][0],
2572                                        str(len(NEW2_CONTENTS)))
2573             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
2574         d.addCallback(_got_headers)
2575
2576         # make sure that outdated size limits aren't enforced anymore.
2577         d.addCallback(lambda ignored:
2578             self.POST(self.public_url + "/foo", t="upload",
2579                       mutable="true",
2580                       file=("new.txt",
2581                             "b" * (self.s.MUTABLE_SIZELIMIT+1))))
2582         d.addErrback(self.dump_error)
2583         return d
2584
2585     def test_POST_upload_mutable_toobig(self):
2586         # SDMF had a size limti that was removed a while ago. MDMF has
2587         # never had a size limit. Test to make sure that we do not
2588         # encounter errors when trying to upload large mutable files,
2589         # since there should be no coded prohibitions regarding large
2590         # mutable files.
2591         d = self.POST(self.public_url + "/foo",
2592                       t="upload", mutable="true",
2593                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2594         return d
2595
2596     def dump_error(self, f):
2597         # if the web server returns an error code (like 400 Bad Request),
2598         # web.client.getPage puts the HTTP response body into the .response
2599         # attribute of the exception object that it gives back. It does not
2600         # appear in the Failure's repr(), so the ERROR that trial displays
2601         # will be rather terse and unhelpful. addErrback this method to the
2602         # end of your chain to get more information out of these errors.
2603         if f.check(error.Error):
2604             print "web.error.Error:"
2605             print f
2606             print f.value.response
2607         return f
2608
2609     def test_POST_upload_replace(self):
2610         d = self.POST(self.public_url + "/foo", t="upload",
2611                       file=("bar.txt", self.NEWFILE_CONTENTS))
2612         fn = self._foo_node
2613         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
2614         d.addCallback(lambda res:
2615                       self.failUnlessChildContentsAre(fn, u"bar.txt",
2616                                                       self.NEWFILE_CONTENTS))
2617         return d
2618
2619     def test_POST_upload_no_replace_ok(self):
2620         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2621                       file=("new.txt", self.NEWFILE_CONTENTS))
2622         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
2623         d.addCallback(lambda res: self.failUnlessReallyEqual(res,
2624                                                              self.NEWFILE_CONTENTS))
2625         return d
2626
2627     def test_POST_upload_no_replace_queryarg(self):
2628         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2629                       file=("bar.txt", self.NEWFILE_CONTENTS))
2630         d.addBoth(self.shouldFail, error.Error,
2631                   "POST_upload_no_replace_queryarg",
2632                   "409 Conflict",
2633                   "There was already a child by that name, and you asked me "
2634                   "to not replace it")
2635         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2636         d.addCallback(self.failUnlessIsBarDotTxt)
2637         return d
2638
2639     def test_POST_upload_no_replace_field(self):
2640         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
2641                       file=("bar.txt", self.NEWFILE_CONTENTS))
2642         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
2643                   "409 Conflict",
2644                   "There was already a child by that name, and you asked me "
2645                   "to not replace it")
2646         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2647         d.addCallback(self.failUnlessIsBarDotTxt)
2648         return d
2649
2650     def test_POST_upload_whendone(self):
2651         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
2652                       file=("new.txt", self.NEWFILE_CONTENTS))
2653         d.addBoth(self.shouldRedirect, "/THERE")
2654         fn = self._foo_node
2655         d.addCallback(lambda res:
2656                       self.failUnlessChildContentsAre(fn, u"new.txt",
2657                                                       self.NEWFILE_CONTENTS))
2658         return d
2659
2660     def test_POST_upload_named(self):
2661         fn = self._foo_node
2662         d = self.POST(self.public_url + "/foo", t="upload",
2663                       name="new.txt", file=self.NEWFILE_CONTENTS)
2664         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2665         d.addCallback(lambda res:
2666                       self.failUnlessChildContentsAre(fn, u"new.txt",
2667                                                       self.NEWFILE_CONTENTS))
2668         return d
2669
2670     def test_POST_upload_named_badfilename(self):
2671         d = self.POST(self.public_url + "/foo", t="upload",
2672                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
2673         d.addBoth(self.shouldFail, error.Error,
2674                   "test_POST_upload_named_badfilename",
2675                   "400 Bad Request",
2676                   "name= may not contain a slash",
2677                   )
2678         # make sure that nothing was added
2679         d.addCallback(lambda res:
2680                       self.failUnlessNodeKeysAre(self._foo_node,
2681                                                  [self._htmlname_unicode,
2682                                                   u"bar.txt", u"baz.txt", u"blockingfile",
2683                                                   u"empty", u"n\u00fc.txt", u"quux.txt",
2684                                                   u"sub"]))
2685         return d
2686
2687     def test_POST_FILEURL_check(self):
2688         bar_url = self.public_url + "/foo/bar.txt"
2689         d = self.POST(bar_url, t="check")
2690         def _check(res):
2691             self.failUnlessIn("Healthy :", res)
2692         d.addCallback(_check)
2693         redir_url = "http://allmydata.org/TARGET"
2694         def _check2(statuscode, target):
2695             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2696             self.failUnlessReallyEqual(target, redir_url)
2697         d.addCallback(lambda res:
2698                       self.shouldRedirect2("test_POST_FILEURL_check",
2699                                            _check2,
2700                                            self.POST, bar_url,
2701                                            t="check",
2702                                            when_done=redir_url))
2703         d.addCallback(lambda res:
2704                       self.POST(bar_url, t="check", return_to=redir_url))
2705         def _check3(res):
2706             self.failUnlessIn("Healthy :", res)
2707             self.failUnlessIn("Return to file", res)
2708             self.failUnlessIn(redir_url, res)
2709         d.addCallback(_check3)
2710
2711         d.addCallback(lambda res:
2712                       self.POST(bar_url, t="check", output="JSON"))
2713         def _check_json(res):
2714             data = simplejson.loads(res)
2715             self.failUnlessIn("storage-index", data)
2716             self.failUnless(data["results"]["healthy"])
2717         d.addCallback(_check_json)
2718
2719         return d
2720
2721     def test_POST_FILEURL_check_and_repair(self):
2722         bar_url = self.public_url + "/foo/bar.txt"
2723         d = self.POST(bar_url, t="check", repair="true")
2724         def _check(res):
2725             self.failUnlessIn("Healthy :", res)
2726         d.addCallback(_check)
2727         redir_url = "http://allmydata.org/TARGET"
2728         def _check2(statuscode, target):
2729             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2730             self.failUnlessReallyEqual(target, redir_url)
2731         d.addCallback(lambda res:
2732                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
2733                                            _check2,
2734                                            self.POST, bar_url,
2735                                            t="check", repair="true",
2736                                            when_done=redir_url))
2737         d.addCallback(lambda res:
2738                       self.POST(bar_url, t="check", return_to=redir_url))
2739         def _check3(res):
2740             self.failUnlessIn("Healthy :", res)
2741             self.failUnlessIn("Return to file", res)
2742             self.failUnlessIn(redir_url, res)
2743         d.addCallback(_check3)
2744         return d
2745
2746     def test_POST_DIRURL_check(self):
2747         foo_url = self.public_url + "/foo/"
2748         d = self.POST(foo_url, t="check")
2749         def _check(res):
2750             self.failUnlessIn("Healthy :", res)
2751         d.addCallback(_check)
2752         redir_url = "http://allmydata.org/TARGET"
2753         def _check2(statuscode, target):
2754             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2755             self.failUnlessReallyEqual(target, redir_url)
2756         d.addCallback(lambda res:
2757                       self.shouldRedirect2("test_POST_DIRURL_check",
2758                                            _check2,
2759                                            self.POST, foo_url,
2760                                            t="check",
2761                                            when_done=redir_url))
2762         d.addCallback(lambda res:
2763                       self.POST(foo_url, t="check", return_to=redir_url))
2764         def _check3(res):
2765             self.failUnlessIn("Healthy :", res)
2766             self.failUnlessIn("Return to file/directory", res)
2767             self.failUnlessIn(redir_url, res)
2768         d.addCallback(_check3)
2769
2770         d.addCallback(lambda res:
2771                       self.POST(foo_url, t="check", output="JSON"))
2772         def _check_json(res):
2773             data = simplejson.loads(res)
2774             self.failUnlessIn("storage-index", data)
2775             self.failUnless(data["results"]["healthy"])
2776         d.addCallback(_check_json)
2777
2778         return d
2779
2780     def test_POST_DIRURL_check_and_repair(self):
2781         foo_url = self.public_url + "/foo/"
2782         d = self.POST(foo_url, t="check", repair="true")
2783         def _check(res):
2784             self.failUnlessIn("Healthy :", res)
2785         d.addCallback(_check)
2786         redir_url = "http://allmydata.org/TARGET"
2787         def _check2(statuscode, target):
2788             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2789             self.failUnlessReallyEqual(target, redir_url)
2790         d.addCallback(lambda res:
2791                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2792                                            _check2,
2793                                            self.POST, foo_url,
2794                                            t="check", repair="true",
2795                                            when_done=redir_url))
2796         d.addCallback(lambda res:
2797                       self.POST(foo_url, t="check", return_to=redir_url))
2798         def _check3(res):
2799             self.failUnlessIn("Healthy :", res)
2800             self.failUnlessIn("Return to file/directory", res)
2801             self.failUnlessIn(redir_url, res)
2802         d.addCallback(_check3)
2803         return d
2804
2805     def test_POST_FILEURL_mdmf_check(self):
2806         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2807         d = self.POST(quux_url, t="check")
2808         def _check(res):
2809             self.failUnlessIn("Healthy", res)
2810         d.addCallback(_check)
2811         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2812         d.addCallback(lambda ignored:
2813                       self.POST(quux_extension_url, t="check"))
2814         d.addCallback(_check)
2815         return d
2816
2817     def test_POST_FILEURL_mdmf_check_and_repair(self):
2818         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2819         d = self.POST(quux_url, t="check", repair="true")
2820         def _check(res):
2821             self.failUnlessIn("Healthy", res)
2822         d.addCallback(_check)
2823         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2824         d.addCallback(lambda ignored:
2825                       self.POST(quux_extension_url, t="check", repair="true"))
2826         d.addCallback(_check)
2827         return d
2828
2829     def wait_for_operation(self, ignored, ophandle):
2830         url = "/operations/" + ophandle
2831         url += "?t=status&output=JSON"
2832         d = self.GET(url)
2833         def _got(res):
2834             data = simplejson.loads(res)
2835             if not data["finished"]:
2836                 d = self.stall(delay=1.0)
2837                 d.addCallback(self.wait_for_operation, ophandle)
2838                 return d
2839             return data
2840         d.addCallback(_got)
2841         return d
2842
2843     def get_operation_results(self, ignored, ophandle, output=None):
2844         url = "/operations/" + ophandle
2845         url += "?t=status"
2846         if output:
2847             url += "&output=" + output
2848         d = self.GET(url)
2849         def _got(res):
2850             if output and output.lower() == "json":
2851                 return simplejson.loads(res)
2852             return res
2853         d.addCallback(_got)
2854         return d
2855
2856     def test_POST_DIRURL_deepcheck_no_ophandle(self):
2857         d = self.shouldFail2(error.Error,
2858                              "test_POST_DIRURL_deepcheck_no_ophandle",
2859                              "400 Bad Request",
2860                              "slow operation requires ophandle=",
2861                              self.POST, self.public_url, t="start-deep-check")
2862         return d
2863
2864     def test_POST_DIRURL_deepcheck(self):
2865         def _check_redirect(statuscode, target):
2866             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2867             self.failUnless(target.endswith("/operations/123"))
2868         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2869                                  self.POST, self.public_url,
2870                                  t="start-deep-check", ophandle="123")
2871         d.addCallback(self.wait_for_operation, "123")
2872         def _check_json(data):
2873             self.failUnlessReallyEqual(data["finished"], True)
2874             self.failUnlessReallyEqual(data["count-objects-checked"], 11)
2875             self.failUnlessReallyEqual(data["count-objects-healthy"], 11)
2876         d.addCallback(_check_json)
2877         d.addCallback(self.get_operation_results, "123", "html")
2878         def _check_html(res):
2879             self.failUnlessIn("Objects Checked: <span>11</span>", res)
2880             self.failUnlessIn("Objects Healthy: <span>11</span>", res)
2881             self.failUnlessIn(FAVICON_MARKUP, res)
2882         d.addCallback(_check_html)
2883
2884         d.addCallback(lambda res:
2885                       self.GET("/operations/123/"))
2886         d.addCallback(_check_html) # should be the same as without the slash
2887
2888         d.addCallback(lambda res:
2889                       self.shouldFail2(error.Error, "one", "404 Not Found",
2890                                        "No detailed results for SI bogus",
2891                                        self.GET, "/operations/123/bogus"))
2892
2893         foo_si = self._foo_node.get_storage_index()
2894         foo_si_s = base32.b2a(foo_si)
2895         d.addCallback(lambda res:
2896                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2897         def _check_foo_json(res):
2898             data = simplejson.loads(res)
2899             self.failUnlessEqual(data["storage-index"], foo_si_s)
2900             self.failUnless(data["results"]["healthy"])
2901         d.addCallback(_check_foo_json)
2902         return d
2903
2904     def test_POST_DIRURL_deepcheck_and_repair(self):
2905         d = self.POST(self.public_url, t="start-deep-check", repair="true",
2906                       ophandle="124", output="json", followRedirect=True)
2907         d.addCallback(self.wait_for_operation, "124")
2908         def _check_json(data):
2909             self.failUnlessReallyEqual(data["finished"], True)
2910             self.failUnlessReallyEqual(data["count-objects-checked"], 11)
2911             self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 11)
2912             self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2913             self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2914             self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2915             self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2916             self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2917             self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 11)
2918             self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2919             self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2920         d.addCallback(_check_json)
2921         d.addCallback(self.get_operation_results, "124", "html")
2922         def _check_html(res):
2923             self.failUnlessIn("Objects Checked: <span>11</span>", res)
2924
2925             self.failUnlessIn("Objects Healthy (before repair): <span>11</span>", res)
2926             self.failUnlessIn("Objects Unhealthy (before repair): <span>0</span>", res)
2927             self.failUnlessIn("Corrupt Shares (before repair): <span>0</span>", res)
2928
2929             self.failUnlessIn("Repairs Attempted: <span>0</span>", res)
2930             self.failUnlessIn("Repairs Successful: <span>0</span>", res)
2931             self.failUnlessIn("Repairs Unsuccessful: <span>0</span>", res)
2932
2933             self.failUnlessIn("Objects Healthy (after repair): <span>11</span>", res)
2934             self.failUnlessIn("Objects Unhealthy (after repair): <span>0</span>", res)
2935             self.failUnlessIn("Corrupt Shares (after repair): <span>0</span>", res)
2936
2937             self.failUnlessIn(FAVICON_MARKUP, res)
2938         d.addCallback(_check_html)
2939         return d
2940
2941     def test_POST_FILEURL_bad_t(self):
2942         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2943                              "POST to file: bad t=bogus",
2944                              self.POST, self.public_url + "/foo/bar.txt",
2945                              t="bogus")
2946         return d
2947
2948     def test_POST_mkdir(self): # return value?
2949         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2950         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2951         d.addCallback(self.failUnlessNodeKeysAre, [])
2952         return d
2953
2954     def test_POST_mkdir_mdmf(self):
2955         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=mdmf")
2956         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2957         d.addCallback(lambda node:
2958             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2959         return d
2960
2961     def test_POST_mkdir_sdmf(self):
2962         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=sdmf")
2963         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2964         d.addCallback(lambda node:
2965             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2966         return d
2967
2968     def test_POST_mkdir_bad_format(self):
2969         return self.shouldHTTPError("POST_mkdir_bad_format",
2970                                     400, "Bad Request", "Unknown format: foo",
2971                                     self.POST, self.public_url +
2972                                     "/foo?t=mkdir&name=newdir&format=foo")
2973
2974     def test_POST_mkdir_initial_children(self):
2975         (newkids, caps) = self._create_initial_children()
2976         d = self.POST2(self.public_url +
2977                        "/foo?t=mkdir-with-children&name=newdir",
2978                        simplejson.dumps(newkids))
2979         d.addCallback(lambda res:
2980                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2981         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2982         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2983         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2984         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2985         return d
2986
2987     def test_POST_mkdir_initial_children_mdmf(self):
2988         (newkids, caps) = self._create_initial_children()
2989         d = self.POST2(self.public_url +
2990                        "/foo?t=mkdir-with-children&name=newdir&format=mdmf",
2991                        simplejson.dumps(newkids))
2992         d.addCallback(lambda res:
2993                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2994         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2995         d.addCallback(lambda node:
2996             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2997         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2998         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2999                        caps['filecap1'])
3000         return d
3001
3002     # XXX: Duplication.
3003     def test_POST_mkdir_initial_children_sdmf(self):
3004         (newkids, caps) = self._create_initial_children()
3005         d = self.POST2(self.public_url +
3006                        "/foo?t=mkdir-with-children&name=newdir&format=sdmf",
3007                        simplejson.dumps(newkids))
3008         d.addCallback(lambda res:
3009                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
3010         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3011         d.addCallback(lambda node:
3012             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
3013         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3014         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
3015                        caps['filecap1'])
3016         return d
3017
3018     def test_POST_mkdir_initial_children_bad_format(self):
3019         (newkids, caps) = self._create_initial_children()
3020         return self.shouldHTTPError("POST_mkdir_initial_children_bad_format",
3021                                     400, "Bad Request", "Unknown format: foo",
3022                                     self.POST, self.public_url + \
3023                                     "/foo?t=mkdir-with-children&name=newdir&format=foo",
3024                                     simplejson.dumps(newkids))
3025
3026     def test_POST_mkdir_immutable(self):
3027         (newkids, caps) = self._create_immutable_children()
3028         d = self.POST2(self.public_url +
3029                        "/foo?t=mkdir-immutable&name=newdir",
3030                        simplejson.dumps(newkids))
3031         d.addCallback(lambda res:
3032                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
3033         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3034         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
3035         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3036         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
3037         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3038         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
3039         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3040         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
3041         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3042         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
3043         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3044         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
3045         return d
3046
3047     def test_POST_mkdir_immutable_bad(self):
3048         (newkids, caps) = self._create_initial_children()
3049         d = self.shouldFail2(error.Error, "POST_mkdir_immutable_bad",
3050                              "400 Bad Request",
3051                              "needed to be immutable but was not",
3052                              self.POST2,
3053                              self.public_url +
3054                              "/foo?t=mkdir-immutable&name=newdir",
3055                              simplejson.dumps(newkids))
3056         return d
3057
3058     def test_POST_mkdir_2(self):
3059         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
3060         d.addCallback(lambda res:
3061                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
3062         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3063         d.addCallback(self.failUnlessNodeKeysAre, [])
3064         return d
3065
3066     def test_POST_mkdirs_2(self):
3067         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
3068         d.addCallback(lambda res:
3069                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
3070         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
3071         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
3072         d.addCallback(self.failUnlessNodeKeysAre, [])
3073         return d
3074
3075     def test_POST_mkdir_no_parentdir_noredirect(self):
3076         d = self.POST("/uri?t=mkdir")
3077         def _after_mkdir(res):
3078             uri.DirectoryURI.init_from_string(res)
3079         d.addCallback(_after_mkdir)
3080         return d
3081
3082     def test_POST_mkdir_no_parentdir_noredirect_mdmf(self):
3083         d = self.POST("/uri?t=mkdir&format=mdmf")
3084         def _after_mkdir(res):
3085             u = uri.from_string(res)
3086             # Check that this is an MDMF writecap
3087             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
3088         d.addCallback(_after_mkdir)
3089         return d
3090
3091     def test_POST_mkdir_no_parentdir_noredirect_sdmf(self):
3092         d = self.POST("/uri?t=mkdir&format=sdmf")
3093         def _after_mkdir(res):
3094             u = uri.from_string(res)
3095             self.failUnlessIsInstance(u, uri.DirectoryURI)
3096         d.addCallback(_after_mkdir)
3097         return d
3098
3099     def test_POST_mkdir_no_parentdir_noredirect_bad_format(self):
3100         return self.shouldHTTPError("POST_mkdir_no_parentdir_noredirect_bad_format",
3101                                     400, "Bad Request", "Unknown format: foo",
3102                                     self.POST, self.public_url +
3103                                     "/uri?t=mkdir&format=foo")
3104
3105     def test_POST_mkdir_no_parentdir_noredirect2(self):
3106         # make sure form-based arguments (as on the welcome page) still work
3107         d = self.POST("/uri", t="mkdir")
3108         def _after_mkdir(res):
3109             uri.DirectoryURI.init_from_string(res)
3110         d.addCallback(_after_mkdir)
3111         d.addErrback(self.explain_web_error)
3112         return d
3113
3114     def test_POST_mkdir_no_parentdir_redirect(self):
3115         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
3116         d.addBoth(self.shouldRedirect, None, statuscode='303')
3117         def _check_target(target):
3118             target = urllib.unquote(target)
3119             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
3120         d.addCallback(_check_target)
3121         return d
3122
3123     def test_POST_mkdir_no_parentdir_redirect2(self):
3124         d = self.POST("/uri", t="mkdir", redirect_to_result="true")
3125         d.addBoth(self.shouldRedirect, None, statuscode='303')
3126         def _check_target(target):
3127             target = urllib.unquote(target)
3128             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
3129         d.addCallback(_check_target)
3130         d.addErrback(self.explain_web_error)
3131         return d
3132
3133     def _make_readonly(self, u):
3134         ro_uri = uri.from_string(u).get_readonly()
3135         if ro_uri is None:
3136             return None
3137         return ro_uri.to_string()
3138
3139     def _create_initial_children(self):
3140         contents, n, filecap1 = self.makefile(12)
3141         md1 = {"metakey1": "metavalue1"}
3142         filecap2 = make_mutable_file_uri()
3143         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
3144         filecap3 = node3.get_readonly_uri()
3145         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
3146         dircap = DirectoryNode(node4, None, None).get_uri()
3147         mdmfcap = make_mutable_file_uri(mdmf=True)
3148         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
3149         emptydircap = "URI:DIR2-LIT:"
3150         newkids = {u"child-imm":        ["filenode", {"rw_uri": filecap1,
3151                                                       "ro_uri": self._make_readonly(filecap1),
3152                                                       "metadata": md1, }],
3153                    u"child-mutable":    ["filenode", {"rw_uri": filecap2,
3154                                                       "ro_uri": self._make_readonly(filecap2)}],
3155                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
3156                    u"unknownchild-rw":  ["unknown",  {"rw_uri": unknown_rwcap,
3157                                                       "ro_uri": unknown_rocap}],
3158                    u"unknownchild-ro":  ["unknown",  {"ro_uri": unknown_rocap}],
3159                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
3160                    u"dirchild":         ["dirnode",  {"rw_uri": dircap,
3161                                                       "ro_uri": self._make_readonly(dircap)}],
3162                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
3163                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
3164                    u"child-mutable-mdmf": ["filenode", {"rw_uri": mdmfcap,
3165                                                         "ro_uri": self._make_readonly(mdmfcap)}],
3166                    }
3167         return newkids, {'filecap1': filecap1,
3168                          'filecap2': filecap2,
3169                          'filecap3': filecap3,
3170                          'unknown_rwcap': unknown_rwcap,
3171                          'unknown_rocap': unknown_rocap,
3172                          'unknown_immcap': unknown_immcap,
3173                          'dircap': dircap,
3174                          'litdircap': litdircap,
3175                          'emptydircap': emptydircap,
3176                          'mdmfcap': mdmfcap}
3177
3178     def _create_immutable_children(self):
3179         contents, n, filecap1 = self.makefile(12)
3180         md1 = {"metakey1": "metavalue1"}
3181         tnode = create_chk_filenode("immutable directory contents\n"*10,
3182                                     self.get_all_contents())
3183         dnode = DirectoryNode(tnode, None, None)
3184         assert not dnode.is_mutable()
3185         immdircap = dnode.get_uri()
3186         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
3187         emptydircap = "URI:DIR2-LIT:"
3188         newkids = {u"child-imm":        ["filenode", {"ro_uri": filecap1,
3189                                                       "metadata": md1, }],
3190                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
3191                    u"dirchild-imm":     ["dirnode",  {"ro_uri": immdircap}],
3192                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
3193                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
3194                    }
3195         return newkids, {'filecap1': filecap1,
3196                          'unknown_immcap': unknown_immcap,
3197                          'immdircap': immdircap,
3198                          'litdircap': litdircap,
3199                          'emptydircap': emptydircap}
3200
3201     def test_POST_mkdir_no_parentdir_initial_children(self):
3202         (newkids, caps) = self._create_initial_children()
3203         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
3204         def _after_mkdir(res):
3205             self.failUnless(res.startswith("URI:DIR"), res)
3206             n = self.s.create_node_from_uri(res)
3207             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3208             d2.addCallback(lambda ign:
3209                            self.failUnlessROChildURIIs(n, u"child-imm",
3210                                                        caps['filecap1']))
3211             d2.addCallback(lambda ign:
3212                            self.failUnlessRWChildURIIs(n, u"child-mutable",
3213                                                        caps['filecap2']))
3214             d2.addCallback(lambda ign:
3215                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
3216                                                        caps['filecap3']))
3217             d2.addCallback(lambda ign:
3218                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
3219                                                        caps['unknown_rwcap']))
3220             d2.addCallback(lambda ign:
3221                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
3222                                                        caps['unknown_rocap']))
3223             d2.addCallback(lambda ign:
3224                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3225                                                        caps['unknown_immcap']))
3226             d2.addCallback(lambda ign:
3227                            self.failUnlessRWChildURIIs(n, u"dirchild",
3228                                                        caps['dircap']))
3229             return d2
3230         d.addCallback(_after_mkdir)
3231         return d
3232
3233     def test_POST_mkdir_no_parentdir_unexpected_children(self):
3234         # the regular /uri?t=mkdir operation is specified to ignore its body.
3235         # Only t=mkdir-with-children pays attention to it.
3236         (newkids, caps) = self._create_initial_children()
3237         d = self.shouldHTTPError("POST_mkdir_no_parentdir_unexpected_children",
3238                                  400, "Bad Request",
3239                                  "t=mkdir does not accept children=, "
3240                                  "try t=mkdir-with-children instead",
3241                                  self.POST2, "/uri?t=mkdir", # without children
3242                                  simplejson.dumps(newkids))
3243         return d
3244
3245     def test_POST_noparent_bad(self):
3246         d = self.shouldHTTPError("POST_noparent_bad",
3247                                  400, "Bad Request",
3248                                  "/uri accepts only PUT, PUT?t=mkdir, "
3249                                  "POST?t=upload, and POST?t=mkdir",
3250                                  self.POST, "/uri?t=bogus")
3251         return d
3252
3253     def test_POST_mkdir_no_parentdir_immutable(self):
3254         (newkids, caps) = self._create_immutable_children()
3255         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
3256         def _after_mkdir(res):
3257             self.failUnless(res.startswith("URI:DIR"), res)
3258             n = self.s.create_node_from_uri(res)
3259             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3260             d2.addCallback(lambda ign:
3261                            self.failUnlessROChildURIIs(n, u"child-imm",
3262                                                           caps['filecap1']))
3263             d2.addCallback(lambda ign:
3264                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3265                                                           caps['unknown_immcap']))
3266             d2.addCallback(lambda ign:
3267                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
3268                                                           caps['immdircap']))
3269             d2.addCallback(lambda ign:
3270                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
3271                                                           caps['litdircap']))
3272             d2.addCallback(lambda ign:
3273                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
3274                                                           caps['emptydircap']))
3275             return d2
3276         d.addCallback(_after_mkdir)
3277         return d
3278
3279     def test_POST_mkdir_no_parentdir_immutable_bad(self):
3280         (newkids, caps) = self._create_initial_children()
3281         d = self.shouldFail2(error.Error,
3282                              "test_POST_mkdir_no_parentdir_immutable_bad",
3283                              "400 Bad Request",
3284                              "needed to be immutable but was not",
3285                              self.POST2,
3286                              "/uri?t=mkdir-immutable",
3287                              simplejson.dumps(newkids))
3288         return d
3289
3290     def test_welcome_page_mkdir_button(self):
3291         # Fetch the welcome page.
3292         d = self.GET("/")
3293         def _after_get_welcome_page(res):
3294             MKDIR_BUTTON_RE = re.compile(
3295                 '<form(?: action="([^"]*)"| method="post"| enctype="multipart/form-data"){3}>.*'
3296                 '<input (?:type="hidden" |name="t" |value="([^"]*?)" ){3}/>[ ]*'
3297                 '<input (?:type="hidden" |name="([^"]*)" |value="([^"]*)" ){3}/>[ ]*'
3298                 '<input (type="submit" |class="btn" |value="Create a directory[^"]*" ){3}/>')
3299             html = res.replace('\n', ' ')
3300             mo = MKDIR_BUTTON_RE.search(html)
3301             self.failUnless(mo, html)
3302             formaction = mo.group(1)
3303             formt = mo.group(2)
3304             formaname = mo.group(3)
3305             formavalue = mo.group(4)
3306             return (formaction, formt, formaname, formavalue)
3307         d.addCallback(_after_get_welcome_page)
3308         def _after_parse_form(res):
3309             (formaction, formt, formaname, formavalue) = res
3310             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
3311         d.addCallback(_after_parse_form)
3312         d.addBoth(self.shouldRedirect, None, statuscode='303')
3313         return d
3314
3315     def test_POST_mkdir_replace(self): # return value?
3316         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
3317         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3318         d.addCallback(self.failUnlessNodeKeysAre, [])
3319         return d
3320
3321     def test_POST_mkdir_no_replace_queryarg(self): # return value?
3322         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
3323         d.addBoth(self.shouldFail, error.Error,
3324                   "POST_mkdir_no_replace_queryarg",
3325                   "409 Conflict",
3326                   "There was already a child by that name, and you asked me "
3327                   "to not replace it")
3328         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3329         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3330         return d
3331
3332     def test_POST_mkdir_no_replace_field(self): # return value?
3333         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
3334                       replace="false")
3335         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
3336                   "409 Conflict",
3337                   "There was already a child by that name, and you asked me "
3338                   "to not replace it")
3339         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3340         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3341         return d
3342
3343     def test_POST_mkdir_whendone_field(self):
3344         d = self.POST(self.public_url + "/foo",
3345                       t="mkdir", name="newdir", when_done="/THERE")
3346         d.addBoth(self.shouldRedirect, "/THERE")
3347         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3348         d.addCallback(self.failUnlessNodeKeysAre, [])
3349         return d
3350
3351     def test_POST_mkdir_whendone_queryarg(self):
3352         d = self.POST(self.public_url + "/foo?when_done=/THERE",
3353                       t="mkdir", name="newdir")
3354         d.addBoth(self.shouldRedirect, "/THERE")
3355         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3356         d.addCallback(self.failUnlessNodeKeysAre, [])
3357         return d
3358
3359     def test_POST_bad_t(self):
3360         d = self.shouldFail2(error.Error, "POST_bad_t",
3361                              "400 Bad Request",
3362                              "POST to a directory with bad t=BOGUS",
3363                              self.POST, self.public_url + "/foo", t="BOGUS")
3364         return d
3365
3366     def test_POST_set_children(self, command_name="set_children"):
3367         contents9, n9, newuri9 = self.makefile(9)
3368         contents10, n10, newuri10 = self.makefile(10)
3369         contents11, n11, newuri11 = self.makefile(11)
3370
3371         reqbody = """{
3372                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
3373                                                 "size": 0,
3374                                                 "metadata": {
3375                                                   "ctime": 1002777696.7564139,
3376                                                   "mtime": 1002777696.7564139
3377                                                  }
3378                                                } ],
3379                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
3380                                                 "size": 1,
3381                                                 "metadata": {
3382                                                   "ctime": 1002777696.7564139,
3383                                                   "mtime": 1002777696.7564139
3384                                                  }
3385                                                } ],
3386                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
3387                                                 "size": 2,
3388                                                 "metadata": {
3389                                                   "ctime": 1002777696.7564139,
3390                                                   "mtime": 1002777696.7564139
3391                                                  }
3392                                                } ]
3393                     }""" % (newuri9, newuri10, newuri11)
3394
3395         url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
3396
3397         d = client.getPage(url, method="POST", postdata=reqbody)
3398         def _then(res):
3399             self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
3400             self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
3401             self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
3402
3403         d.addCallback(_then)
3404         d.addErrback(self.dump_error)
3405         return d
3406
3407     def test_POST_set_children_with_hyphen(self):
3408         return self.test_POST_set_children(command_name="set-children")
3409
3410     def test_POST_link_uri(self):
3411         contents, n, newuri = self.makefile(8)
3412         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
3413         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
3414         d.addCallback(lambda res:
3415                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3416                                                       contents))
3417         return d
3418
3419     def test_POST_link_uri_replace(self):
3420         contents, n, newuri = self.makefile(8)
3421         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
3422         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
3423         d.addCallback(lambda res:
3424                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3425                                                       contents))
3426         return d
3427
3428     def test_POST_link_uri_unknown_bad(self):
3429         d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
3430         d.addBoth(self.shouldFail, error.Error,
3431                   "POST_link_uri_unknown_bad",
3432                   "400 Bad Request",
3433                   "unknown cap in a write slot")
3434         return d
3435
3436     def test_POST_link_uri_unknown_ro_good(self):
3437         d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
3438         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
3439         return d
3440
3441     def test_POST_link_uri_unknown_imm_good(self):
3442         d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
3443         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
3444         return d
3445
3446     def test_POST_link_uri_no_replace_queryarg(self):
3447         contents, n, newuri = self.makefile(8)
3448         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
3449                       name="bar.txt", uri=newuri)
3450         d.addBoth(self.shouldFail, error.Error,
3451                   "POST_link_uri_no_replace_queryarg",
3452                   "409 Conflict",
3453                   "There was already a child by that name, and you asked me "
3454                   "to not replace it")
3455         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3456         d.addCallback(self.failUnlessIsBarDotTxt)
3457         return d
3458
3459     def test_POST_link_uri_no_replace_field(self):
3460         contents, n, newuri = self.makefile(8)
3461         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
3462                       name="bar.txt", uri=newuri)
3463         d.addBoth(self.shouldFail, error.Error,
3464                   "POST_link_uri_no_replace_field",
3465                   "409 Conflict",
3466                   "There was already a child by that name, and you asked me "
3467                   "to not replace it")
3468         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3469         d.addCallback(self.failUnlessIsBarDotTxt)
3470         return d
3471
3472     def test_POST_delete(self, command_name='delete'):
3473         d = self._foo_node.list()
3474         def _check_before(children):
3475             self.failUnlessIn(u"bar.txt", children)
3476         d.addCallback(_check_before)
3477         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t=command_name, name="bar.txt"))
3478         d.addCallback(lambda res: self._foo_node.list())
3479         def _check_after(children):
3480             self.failIfIn(u"bar.txt", children)
3481         d.addCallback(_check_after)
3482         return d
3483
3484     def test_POST_unlink(self):
3485         return self.test_POST_delete(command_name='unlink')
3486
3487     def test_POST_rename_file(self):
3488         d = self.POST(self.public_url + "/foo", t="rename",
3489                       from_name="bar.txt", to_name='wibble.txt')
3490         d.addCallback(lambda res:
3491                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3492         d.addCallback(lambda res:
3493                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
3494         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
3495         d.addCallback(self.failUnlessIsBarDotTxt)
3496         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
3497         d.addCallback(self.failUnlessIsBarJSON)
3498         return d
3499
3500     def test_POST_rename_file_redundant(self):
3501         d = self.POST(self.public_url + "/foo", t="rename",
3502                       from_name="bar.txt", to_name='bar.txt')
3503         d.addCallback(lambda res:
3504                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3505         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3506         d.addCallback(self.failUnlessIsBarDotTxt)
3507         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3508         d.addCallback(self.failUnlessIsBarJSON)
3509         return d
3510
3511     def test_POST_rename_file_replace(self):
3512         # rename a file and replace a directory with it
3513         d = self.POST(self.public_url + "/foo", t="rename",
3514                       from_name="bar.txt", to_name='empty')
3515         d.addCallback(lambda res:
3516                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3517         d.addCallback(lambda res:
3518                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
3519         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
3520         d.addCallback(self.failUnlessIsBarDotTxt)
3521         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3522         d.addCallback(self.failUnlessIsBarJSON)
3523         return d
3524
3525     def test_POST_rename_file_no_replace_queryarg(self):
3526         # rename a file and replace a directory with it
3527         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
3528                       from_name="bar.txt", to_name='empty')
3529         d.addBoth(self.shouldFail, error.Error,
3530                   "POST_rename_file_no_replace_queryarg",
3531                   "409 Conflict",
3532                   "There was already a child by that name, and you asked me "
3533                   "to not replace it")
3534         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3535         d.addCallback(self.failUnlessIsEmptyJSON)
3536         return d
3537
3538     def test_POST_rename_file_no_replace_field(self):
3539         # rename a file and replace a directory with it
3540         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
3541                       from_name="bar.txt", to_name='empty')
3542         d.addBoth(self.shouldFail, error.Error,
3543                   "POST_rename_file_no_replace_field",
3544                   "409 Conflict",
3545                   "There was already a child by that name, and you asked me "
3546                   "to not replace it")
3547         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3548         d.addCallback(self.failUnlessIsEmptyJSON)
3549         return d
3550
3551     def test_POST_rename_file_no_replace_same_link(self):
3552         d = self.POST(self.public_url + "/foo", t="rename",
3553                       replace="false", from_name="bar.txt", to_name="bar.txt")
3554         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3555         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3556         d.addCallback(self.failUnlessIsBarDotTxt)
3557         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3558         d.addCallback(self.failUnlessIsBarJSON)
3559         return d
3560
3561     def test_POST_rename_file_replace_only_files(self):
3562         d = self.POST(self.public_url + "/foo", t="rename",
3563                       replace="only-files", from_name="bar.txt",
3564                       to_name="baz.txt")
3565         d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3566         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3567         d.addCallback(self.failUnlessIsBarDotTxt)
3568         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt?t=json"))
3569         d.addCallback(self.failUnlessIsBarJSON)
3570         return d
3571
3572     def test_POST_rename_file_replace_only_files_conflict(self):
3573         d = self.shouldFail2(error.Error, "POST_relink_file_replace_only_files_conflict",
3574                              "409 Conflict",
3575                              "There was already a child by that name, and you asked me to not replace it.",
3576                              self.POST, self.public_url + "/foo", t="relink",
3577                              replace="only-files", from_name="bar.txt",
3578                              to_name="empty")
3579         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3580         d.addCallback(self.failUnlessIsBarDotTxt)
3581         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3582         d.addCallback(self.failUnlessIsBarJSON)
3583         return d
3584
3585     def failUnlessIsEmptyJSON(self, res):
3586         data = simplejson.loads(res)
3587         self.failUnlessEqual(data[0], "dirnode", data)
3588         self.failUnlessReallyEqual(len(data[1]["children"]), 0)
3589
3590     def test_POST_rename_file_to_slash_fail(self):
3591         d = self.POST(self.public_url + "/foo", t="rename",
3592                       from_name="bar.txt", to_name='kirk/spock.txt')
3593         d.addBoth(self.shouldFail, error.Error,
3594                   "test_POST_rename_file_to_slash_fail",
3595                   "400 Bad Request",
3596                   "to_name= may not contain a slash",
3597                   )
3598         d.addCallback(lambda res:
3599                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3600         return d
3601
3602     def test_POST_rename_file_from_slash_fail(self):
3603         d = self.POST(self.public_url + "/foo", t="rename",
3604                       from_name="sub/bar.txt", to_name='spock.txt')
3605         d.addBoth(self.shouldFail, error.Error,
3606                   "test_POST_rename_from_file_slash_fail",
3607                   "400 Bad Request",
3608                   "from_name= may not contain a slash",
3609                   )
3610         d.addCallback(lambda res:
3611                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3612         return d
3613
3614     def test_POST_rename_dir(self):
3615         d = self.POST(self.public_url, t="rename",
3616                       from_name="foo", to_name='plunk')
3617         d.addCallback(lambda res:
3618                       self.failIfNodeHasChild(self.public_root, u"foo"))
3619         d.addCallback(lambda res:
3620                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
3621         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
3622         d.addCallback(self.failUnlessIsFooJSON)
3623         return d
3624
3625     def test_POST_relink_file(self):
3626         d = self.POST(self.public_url + "/foo", t="relink",
3627                       from_name="bar.txt",
3628                       to_dir=self.public_root.get_uri() + "/foo/sub")
3629         d.addCallback(lambda res:
3630                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3631         d.addCallback(lambda res:
3632                       self.failUnlessNodeHasChild(self._sub_node, u"bar.txt"))
3633         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3634         d.addCallback(self.failUnlessIsBarDotTxt)
3635         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3636         d.addCallback(self.failUnlessIsBarJSON)
3637         return d
3638
3639     def test_POST_relink_file_new_name(self):
3640         d = self.POST(self.public_url + "/foo", t="relink",
3641                       from_name="bar.txt",
3642                       to_name="wibble.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3643         d.addCallback(lambda res:
3644                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3645         d.addCallback(lambda res:
3646                       self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3647         d.addCallback(lambda res:
3648                       self.failUnlessNodeHasChild(self._sub_node, u"wibble.txt"))
3649         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt"))
3650         d.addCallback(self.failUnlessIsBarDotTxt)
3651         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt?t=json"))
3652         d.addCallback(self.failUnlessIsBarJSON)
3653         return d
3654
3655     def test_POST_relink_file_replace(self):
3656         d = self.POST(self.public_url + "/foo", t="relink",
3657                       from_name="bar.txt",
3658                       to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3659         d.addCallback(lambda res:
3660                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3661         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3662         d.addCallback(self.failUnlessIsBarDotTxt)
3663         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt?t=json"))
3664         d.addCallback(self.failUnlessIsBarJSON)
3665         return d
3666
3667     def test_POST_relink_file_no_replace(self):
3668         d = self.shouldFail2(error.Error, "POST_relink_file_no_replace",
3669                              "409 Conflict",
3670                              "There was already a child by that name, and you asked me to not replace it",
3671                              self.POST, self.public_url + "/foo", t="relink",
3672                              replace="false", from_name="bar.txt",
3673                              to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3674         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3675         d.addCallback(self.failUnlessIsBarDotTxt)
3676         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3677         d.addCallback(self.failUnlessIsBarJSON)
3678         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3679         d.addCallback(self.failUnlessIsSubBazDotTxt)
3680         return d
3681
3682     def test_POST_relink_file_no_replace_explicitly_same_link(self):
3683         d = self.POST(self.public_url + "/foo", t="relink",
3684                       replace="false", from_name="bar.txt",
3685                       to_name="bar.txt", to_dir=self.public_root.get_uri() + "/foo")
3686         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3687         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3688         d.addCallback(self.failUnlessIsBarDotTxt)
3689         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3690         d.addCallback(self.failUnlessIsBarJSON)
3691         return d
3692
3693     def test_POST_relink_file_replace_only_files(self):
3694         d = self.POST(self.public_url + "/foo", t="relink",
3695                       replace="only-files", from_name="bar.txt",
3696                       to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3697         d.addCallback(lambda res:
3698                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3699         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3700         d.addCallback(self.failUnlessIsBarDotTxt)
3701         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt?t=json"))
3702         d.addCallback(self.failUnlessIsBarJSON)
3703         return d
3704
3705     def test_POST_relink_file_replace_only_files_conflict(self):
3706         d = self.shouldFail2(error.Error, "POST_relink_file_replace_only_files_conflict",
3707                              "409 Conflict",
3708                              "There was already a child by that name, and you asked me to not replace it.",
3709                              self.POST, self.public_url + "/foo", t="relink",
3710                              replace="only-files", from_name="bar.txt",
3711                              to_name="sub", to_dir=self.public_root.get_uri() + "/foo")
3712         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3713         d.addCallback(self.failUnlessIsBarDotTxt)
3714         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3715         d.addCallback(self.failUnlessIsBarJSON)
3716         return d
3717
3718     def test_POST_relink_file_to_slash_fail(self):
3719         d = self.shouldFail2(error.Error, "test_POST_rename_file_slash_fail",
3720                              "400 Bad Request",
3721                              "to_name= may not contain a slash",
3722                              self.POST, self.public_url + "/foo", t="relink",
3723                              from_name="bar.txt",
3724                              to_name="slash/fail.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3725         d.addCallback(lambda res:
3726                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3727         d.addCallback(lambda res:
3728                       self.failIfNodeHasChild(self._sub_node, u"slash/fail.txt"))
3729         d.addCallback(lambda ign:
3730                       self.shouldFail2(error.Error,
3731                                        "test_POST_rename_file_slash_fail2",
3732                                        "400 Bad Request",
3733                                        "from_name= may not contain a slash",
3734                                        self.POST, self.public_url + "/foo",
3735                                        t="relink",
3736                                        from_name="nope/bar.txt",
3737                                        to_name="fail.txt",
3738                                        to_dir=self.public_root.get_uri() + "/foo/sub"))
3739         return d
3740
3741     def test_POST_relink_file_explicitly_same_link(self):
3742         d = self.POST(self.public_url + "/foo", t="relink",
3743                       from_name="bar.txt",
3744                       to_name="bar.txt", to_dir=self.public_root.get_uri() + "/foo")
3745         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3746         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3747         d.addCallback(self.failUnlessIsBarDotTxt)
3748         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3749         d.addCallback(self.failUnlessIsBarJSON)
3750         return d
3751
3752     def test_POST_relink_file_implicitly_same_link(self):
3753         d = self.POST(self.public_url + "/foo", t="relink",
3754                       from_name="bar.txt")
3755         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3756         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3757         d.addCallback(self.failUnlessIsBarDotTxt)
3758         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3759         d.addCallback(self.failUnlessIsBarJSON)
3760         return d
3761
3762     def test_POST_relink_file_same_dir(self):
3763         d = self.POST(self.public_url + "/foo", t="relink",
3764                       from_name="bar.txt",
3765                       to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo")
3766         d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3767         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._sub_node, u"baz.txt"))
3768         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3769         d.addCallback(self.failUnlessIsBarDotTxt)
3770         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt?t=json"))
3771         d.addCallback(self.failUnlessIsBarJSON)
3772         return d
3773
3774     def test_POST_relink_file_bad_replace(self):
3775         d = self.shouldFail2(error.Error, "test_POST_relink_file_bad_replace",
3776                              "400 Bad Request", "invalid replace= argument: 'boogabooga'",
3777                              self.POST,
3778                              self.public_url + "/foo", t="relink",
3779                              replace="boogabooga", from_name="bar.txt",
3780                              to_dir=self.public_root.get_uri() + "/foo/sub")
3781         return d
3782
3783     def test_POST_relink_file_multi_level(self):
3784         d = self.POST(self.public_url + "/foo/sub/level2?t=mkdir", "")
3785         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t="relink",
3786                       from_name="bar.txt", to_dir=self.public_root.get_uri() + "/foo/sub/level2"))
3787         d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3788         d.addCallback(lambda res: self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3789         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt"))
3790         d.addCallback(self.failUnlessIsBarDotTxt)
3791         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt?t=json"))
3792         d.addCallback(self.failUnlessIsBarJSON)
3793         return d
3794
3795     def test_POST_relink_file_to_uri(self):
3796         d = self.POST(self.public_url + "/foo", t="relink", target_type="uri",
3797                       from_name="bar.txt", to_dir=self._sub_uri)
3798         d.addCallback(lambda res:
3799                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3800         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3801         d.addCallback(self.failUnlessIsBarDotTxt)
3802         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3803         d.addCallback(self.failUnlessIsBarJSON)
3804         return d
3805
3806     def test_POST_relink_file_to_nonexistent_dir(self):
3807         d = self.shouldFail2(error.Error, "POST_relink_file_to_nonexistent_dir",
3808                             "404 Not Found", "No such child: nopechucktesta",
3809                             self.POST, self.public_url + "/foo", t="relink",
3810                             from_name="bar.txt",
3811                             to_dir=self.public_root.get_uri() + "/nopechucktesta")
3812         return d
3813
3814     def test_POST_relink_file_into_file(self):
3815         d = self.shouldFail2(error.Error, "POST_relink_file_into_file",
3816                              "400 Bad Request", "to_dir is not a directory",
3817                              self.POST, self.public_url + "/foo", t="relink",
3818                              from_name="bar.txt",
3819                              to_dir=self.public_root.get_uri() + "/foo/baz.txt")
3820         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3821         d.addCallback(self.failUnlessIsBazDotTxt)
3822         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3823         d.addCallback(self.failUnlessIsBarDotTxt)
3824         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3825         d.addCallback(self.failUnlessIsBarJSON)
3826         return d
3827
3828     def test_POST_relink_file_to_bad_uri(self):
3829         d =  self.shouldFail2(error.Error, "POST_relink_file_to_bad_uri",
3830                               "400 Bad Request", "to_dir is not a directory",
3831                               self.POST, self.public_url + "/foo", t="relink",
3832                               from_name="bar.txt",
3833                               to_dir="URI:DIR2:mn5jlyjnrjeuydyswlzyui72i:rmneifcj6k6sycjljjhj3f6majsq2zqffydnnul5hfa4j577arma")
3834         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3835         d.addCallback(self.failUnlessIsBarDotTxt)
3836         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3837         d.addCallback(self.failUnlessIsBarJSON)
3838         return d
3839
3840     def test_POST_relink_dir(self):
3841         d = self.POST(self.public_url + "/foo", t="relink",
3842                       from_name="bar.txt",
3843                       to_dir=self.public_root.get_uri() + "/foo/empty")
3844         d.addCallback(lambda res: self.POST(self.public_url + "/foo",
3845                       t="relink", from_name="empty",
3846                       to_dir=self.public_root.get_uri() + "/foo/sub"))
3847         d.addCallback(lambda res:
3848                       self.failIfNodeHasChild(self._foo_node, u"empty"))
3849         d.addCallback(lambda res:
3850                       self.failUnlessNodeHasChild(self._sub_node, u"empty"))
3851         d.addCallback(lambda res:
3852                       self._sub_node.get_child_at_path(u"empty"))
3853         d.addCallback(lambda node:
3854                       self.failUnlessNodeHasChild(node, u"bar.txt"))
3855         d.addCallback(lambda res:
3856                       self.GET(self.public_url + "/foo/sub/empty/bar.txt"))
3857         d.addCallback(self.failUnlessIsBarDotTxt)
3858         return d
3859
3860     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
3861         """ If target is not None then the redirection has to go to target.  If
3862         statuscode is not None then the redirection has to be accomplished with
3863         that HTTP status code."""
3864         if not isinstance(res, failure.Failure):
3865             to_where = (target is None) and "somewhere" or ("to " + target)
3866             self.fail("%s: we were expecting to get redirected %s, not get an"
3867                       " actual page: %s" % (which, to_where, res))
3868         res.trap(error.PageRedirect)
3869         if statuscode is not None:
3870             self.failUnlessReallyEqual(res.value.status, statuscode,
3871                                        "%s: not a redirect" % which)
3872         if target is not None:
3873             # the PageRedirect does not seem to capture the uri= query arg
3874             # properly, so we can't check for it.
3875             realtarget = self.webish_url + target
3876             self.failUnlessReallyEqual(res.value.location, realtarget,
3877                                        "%s: wrong target" % which)
3878         return res.value.location
3879
3880     def test_GET_URI_form(self):
3881         base = "/uri?uri=%s" % self._bar_txt_uri
3882         # this is supposed to give us a redirect to /uri/$URI, plus arguments
3883         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
3884         d = self.GET(base)
3885         d.addBoth(self.shouldRedirect, targetbase)
3886         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
3887         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
3888         d.addCallback(lambda res: self.GET(base+"&t=json"))
3889         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
3890         d.addCallback(self.log, "about to get file by uri")
3891         d.addCallback(lambda res: self.GET(base, followRedirect=True))
3892         d.addCallback(self.failUnlessIsBarDotTxt)
3893         d.addCallback(self.log, "got file by uri, about to get dir by uri")
3894         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
3895                                            followRedirect=True))
3896         d.addCallback(self.failUnlessIsFooJSON)
3897         d.addCallback(self.log, "got dir by uri")
3898
3899         return d
3900
3901     def test_GET_URI_form_bad(self):
3902         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
3903                              "400 Bad Request", "GET /uri requires uri=",
3904                              self.GET, "/uri")
3905         return d
3906
3907     def test_GET_rename_form(self):
3908         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
3909                      followRedirect=True)
3910         def _check(res):
3911             self.failUnless(re.search('<input (name="when_done" |value="." |type="hidden" ){3}/>', res), res)
3912             self.failUnless(re.search(r'<input (readonly="true" |type="text" |name="from_name" |value="bar\.txt" ){4}/>', res), res)
3913             self.failUnlessIn(FAVICON_MARKUP, res)
3914         d.addCallback(_check)
3915         return d
3916
3917     def log(self, res, msg):
3918         #print "MSG: %s  RES: %s" % (msg, res)
3919         log.msg(msg)
3920         return res
3921
3922     def test_GET_URI_URL(self):
3923         base = "/uri/%s" % self._bar_txt_uri
3924         d = self.GET(base)
3925         d.addCallback(self.failUnlessIsBarDotTxt)
3926         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
3927         d.addCallback(self.failUnlessIsBarDotTxt)
3928         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
3929         d.addCallback(self.failUnlessIsBarDotTxt)
3930         return d
3931
3932     def test_GET_URI_URL_dir(self):
3933         base = "/uri/%s?t=json" % self._foo_uri
3934         d = self.GET(base)
3935         d.addCallback(self.failUnlessIsFooJSON)
3936         return d
3937
3938     def test_GET_URI_URL_missing(self):
3939         base = "/uri/%s" % self._bad_file_uri
3940         d = self.shouldHTTPError("test_GET_URI_URL_missing",
3941                                  http.GONE, None, "NotEnoughSharesError",
3942                                  self.GET, base)
3943         # TODO: how can we exercise both sides of WebDownloadTarget.fail
3944         # here? we must arrange for a download to fail after target.open()
3945         # has been called, and then inspect the response to see that it is
3946         # shorter than we expected.
3947         return d
3948
3949     def test_PUT_DIRURL_uri(self):
3950         d = self.s.create_dirnode()
3951         def _made_dir(dn):
3952             new_uri = dn.get_uri()
3953             # replace /foo with a new (empty) directory
3954             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
3955             d.addCallback(lambda res:
3956                           self.failUnlessReallyEqual(res.strip(), new_uri))
3957             d.addCallback(lambda res:
3958                           self.failUnlessRWChildURIIs(self.public_root,
3959                                                       u"foo",
3960                                                       new_uri))
3961             return d
3962         d.addCallback(_made_dir)
3963         return d
3964
3965     def test_PUT_DIRURL_uri_noreplace(self):
3966         d = self.s.create_dirnode()
3967         def _made_dir(dn):
3968             new_uri = dn.get_uri()
3969             # replace /foo with a new (empty) directory, but ask that
3970             # replace=false, so it should fail
3971             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
3972                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
3973                                  self.PUT,
3974                                  self.public_url + "/foo?t=uri&replace=false",
3975                                  new_uri)
3976             d.addCallback(lambda res:
3977                           self.failUnlessRWChildURIIs(self.public_root,
3978                                                       u"foo",
3979                                                       self._foo_uri))
3980             return d
3981         d.addCallback(_made_dir)
3982         return d
3983
3984     def test_PUT_DIRURL_bad_t(self):
3985         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
3986                              "400 Bad Request", "PUT to a directory",
3987                              self.PUT, self.public_url + "/foo?t=BOGUS", "")
3988         d.addCallback(lambda res:
3989                       self.failUnlessRWChildURIIs(self.public_root,
3990                                                   u"foo",
3991                                                   self._foo_uri))
3992         return d
3993
3994     def test_PUT_NEWFILEURL_uri(self):
3995         contents, n, new_uri = self.makefile(8)
3996         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
3997         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3998         d.addCallback(lambda res:
3999                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
4000                                                       contents))
4001         return d
4002
4003     def test_PUT_NEWFILEURL_mdmf(self):
4004         new_contents = self.NEWFILE_CONTENTS * 300000
4005         d = self.PUT(self.public_url + \
4006                      "/foo/mdmf.txt?format=mdmf",
4007                      new_contents)
4008         d.addCallback(lambda ignored:
4009             self.GET(self.public_url + "/foo/mdmf.txt?t=json"))
4010         def _got_json(json):
4011             data = simplejson.loads(json)
4012             data = data[1]
4013             self.failUnlessIn("format", data)
4014             self.failUnlessEqual(data["format"], "MDMF")
4015             self.failUnless(data['rw_uri'].startswith("URI:MDMF"))
4016             self.failUnless(data['ro_uri'].startswith("URI:MDMF"))
4017         d.addCallback(_got_json)
4018         return d
4019
4020     def test_PUT_NEWFILEURL_sdmf(self):
4021         new_contents = self.NEWFILE_CONTENTS * 300000
4022         d = self.PUT(self.public_url + \
4023                      "/foo/sdmf.txt?format=sdmf",
4024                      new_contents)
4025         d.addCallback(lambda ignored:
4026             self.GET(self.public_url + "/foo/sdmf.txt?t=json"))
4027         def _got_json(json):
4028             data = simplejson.loads(json)
4029             data = data[1]
4030             self.failUnlessIn("format", data)
4031             self.failUnlessEqual(data["format"], "SDMF")
4032         d.addCallback(_got_json)
4033         return d
4034
4035     def test_PUT_NEWFILEURL_bad_format(self):
4036         new_contents = self.NEWFILE_CONTENTS * 300000
4037         return self.shouldHTTPError("PUT_NEWFILEURL_bad_format",
4038                                     400, "Bad Request", "Unknown format: foo",
4039                                     self.PUT, self.public_url + \
4040                                     "/foo/foo.txt?format=foo",
4041                                     new_contents)
4042
4043     def test_PUT_NEWFILEURL_uri_replace(self):
4044         contents, n, new_uri = self.makefile(8)
4045         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
4046         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
4047         d.addCallback(lambda res:
4048                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
4049                                                       contents))
4050         return d
4051
4052     def test_PUT_NEWFILEURL_uri_no_replace(self):
4053         contents, n, new_uri = self.makefile(8)
4054         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
4055         d.addBoth(self.shouldFail, error.Error,
4056                   "PUT_NEWFILEURL_uri_no_replace",
4057                   "409 Conflict",
4058                   "There was already a child by that name, and you asked me "
4059                   "to not replace it")
4060         return d
4061
4062     def test_PUT_NEWFILEURL_uri_unknown_bad(self):
4063         d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
4064         d.addBoth(self.shouldFail, error.Error,
4065                   "POST_put_uri_unknown_bad",
4066                   "400 Bad Request",
4067                   "unknown cap in a write slot")
4068         return d
4069
4070     def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
4071         d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
4072         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
4073                       u"put-future-ro.txt")
4074         return d
4075
4076     def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
4077         d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
4078         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
4079                       u"put-future-imm.txt")
4080         return d
4081
4082     def test_PUT_NEWFILE_URI(self):
4083         file_contents = "New file contents here\n"
4084         d = self.PUT("/uri", file_contents)
4085         def _check(uri):
4086             assert isinstance(uri, str), uri
4087             self.failUnlessIn(uri, self.get_all_contents())
4088             self.failUnlessReallyEqual(self.get_all_contents()[uri],
4089                                        file_contents)
4090             return self.GET("/uri/%s" % uri)
4091         d.addCallback(_check)
4092         def _check2(res):
4093             self.failUnlessReallyEqual(res, file_contents)
4094         d.addCallback(_check2)
4095         return d
4096
4097     def test_PUT_NEWFILE_URI_not_mutable(self):
4098         file_contents = "New file contents here\n"
4099         d = self.PUT("/uri?mutable=false", file_contents)
4100         def _check(uri):
4101             assert isinstance(uri, str), uri
4102             self.failUnlessIn(uri, self.get_all_contents())
4103             self.failUnlessReallyEqual(self.get_all_contents()[uri],
4104                                        file_contents)
4105             return self.GET("/uri/%s" % uri)
4106         d.addCallback(_check)
4107         def _check2(res):
4108             self.failUnlessReallyEqual(res, file_contents)
4109         d.addCallback(_check2)
4110         return d
4111
4112     def test_PUT_NEWFILE_URI_only_PUT(self):
4113         d = self.PUT("/uri?t=bogus", "")
4114         d.addBoth(self.shouldFail, error.Error,
4115                   "PUT_NEWFILE_URI_only_PUT",
4116                   "400 Bad Request",
4117                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
4118         return d
4119
4120     def test_PUT_NEWFILE_URI_mutable(self):
4121         file_contents = "New file contents here\n"
4122         d = self.PUT("/uri?mutable=true", file_contents)
4123         def _check1(filecap):
4124             filecap = filecap.strip()
4125             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
4126             self.filecap = filecap
4127             u = uri.WriteableSSKFileURI.init_from_string(filecap)
4128             self.failUnlessIn(u.get_storage_index(), self.get_all_contents())
4129             n = self.s.create_node_from_uri(filecap)
4130             return n.download_best_version()
4131         d.addCallback(_check1)
4132         def _check2(data):
4133             self.failUnlessReallyEqual(data, file_contents)
4134             return self.GET("/uri/%s" % urllib.quote(self.filecap))
4135         d.addCallback(_check2)
4136         def _check3(res):
4137             self.failUnlessReallyEqual(res, file_contents)
4138         d.addCallback(_check3)
4139         return d
4140
4141     def test_PUT_mkdir(self):
4142         d = self.PUT("/uri?t=mkdir", "")
4143         def _check(uri):
4144             n = self.s.create_node_from_uri(uri.strip())
4145             d2 = self.failUnlessNodeKeysAre(n, [])
4146             d2.addCallback(lambda res:
4147                            self.GET("/uri/%s?t=json" % uri))
4148             return d2
4149         d.addCallback(_check)
4150         d.addCallback(self.failUnlessIsEmptyJSON)
4151         return d
4152
4153     def test_PUT_mkdir_mdmf(self):
4154         d = self.PUT("/uri?t=mkdir&format=mdmf", "")
4155         def _got(res):
4156             u = uri.from_string(res)
4157             # Check that this is an MDMF writecap
4158             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
4159         d.addCallback(_got)
4160         return d
4161
4162     def test_PUT_mkdir_sdmf(self):
4163         d = self.PUT("/uri?t=mkdir&format=sdmf", "")
4164         def _got(res):
4165             u = uri.from_string(res)
4166             self.failUnlessIsInstance(u, uri.DirectoryURI)
4167         d.addCallback(_got)
4168         return d
4169
4170     def test_PUT_mkdir_bad_format(self):
4171         return self.shouldHTTPError("PUT_mkdir_bad_format",
4172                                     400, "Bad Request", "Unknown format: foo",
4173                                     self.PUT, "/uri?t=mkdir&format=foo",
4174                                     "")
4175
4176     def test_POST_check(self):
4177         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
4178         def _done(res):
4179             # this returns a string form of the results, which are probably
4180             # None since we're using fake filenodes.
4181             # TODO: verify that the check actually happened, by changing
4182             # FakeCHKFileNode to count how many times .check() has been
4183             # called.
4184             pass
4185         d.addCallback(_done)
4186         return d
4187
4188
4189     def test_PUT_update_at_offset(self):
4190         file_contents = "test file" * 100000 # about 900 KiB
4191         d = self.PUT("/uri?mutable=true", file_contents)
4192         def _then(filecap):
4193             self.filecap = filecap
4194             new_data = file_contents[:100]
4195             new = "replaced and so on"
4196             new_data += new
4197             new_data += file_contents[len(new_data):]
4198             assert len(new_data) == len(file_contents)
4199             self.new_data = new_data
4200         d.addCallback(_then)
4201         d.addCallback(lambda ignored:
4202             self.PUT("/uri/%s?replace=True&offset=100" % self.filecap,
4203                      "replaced and so on"))
4204         def _get_data(filecap):
4205             n = self.s.create_node_from_uri(filecap)
4206             return n.download_best_version()
4207         d.addCallback(_get_data)
4208         d.addCallback(lambda results:
4209             self.failUnlessEqual(results, self.new_data))
4210         # Now try appending things to the file
4211         d.addCallback(lambda ignored:
4212             self.PUT("/uri/%s?offset=%d" % (self.filecap, len(self.new_data)),
4213                      "puppies" * 100))
4214         d.addCallback(_get_data)
4215         d.addCallback(lambda results:
4216             self.failUnlessEqual(results, self.new_data + ("puppies" * 100)))
4217         # and try replacing the beginning of the file
4218         d.addCallback(lambda ignored:
4219             self.PUT("/uri/%s?offset=0" % self.filecap, "begin"))
4220         d.addCallback(_get_data)
4221         d.addCallback(lambda results:
4222             self.failUnlessEqual(results, "begin"+self.new_data[len("begin"):]+("puppies"*100)))
4223         return d
4224
4225     def test_PUT_update_at_invalid_offset(self):
4226         file_contents = "test file" * 100000 # about 900 KiB
4227         d = self.PUT("/uri?mutable=true", file_contents)
4228         def _then(filecap):
4229             self.filecap = filecap
4230         d.addCallback(_then)
4231         # Negative offsets should cause an error.
4232         d.addCallback(lambda ignored:
4233             self.shouldHTTPError("PUT_update_at_invalid_offset",
4234                                  400, "Bad Request",
4235                                  "Invalid offset",
4236                                  self.PUT,
4237                                  "/uri/%s?offset=-1" % self.filecap,
4238                                  "foo"))
4239         return d
4240
4241     def test_PUT_update_at_offset_immutable(self):
4242         file_contents = "Test file" * 100000
4243         d = self.PUT("/uri", file_contents)
4244         def _then(filecap):
4245             self.filecap = filecap
4246         d.addCallback(_then)
4247         d.addCallback(lambda ignored:
4248             self.shouldHTTPError("PUT_update_at_offset_immutable",
4249                                  400, "Bad Request",
4250                                  "immutable",
4251                                  self.PUT,
4252                                  "/uri/%s?offset=50" % self.filecap,
4253                                  "foo"))
4254         return d
4255
4256
4257     def test_bad_method(self):
4258         url = self.webish_url + self.public_url + "/foo/bar.txt"
4259         d = self.shouldHTTPError("bad_method",
4260                                  501, "Not Implemented",
4261                                  "I don't know how to treat a BOGUS request.",
4262                                  client.getPage, url, method="BOGUS")
4263         return d
4264
4265     def test_short_url(self):
4266         url = self.webish_url + "/uri"
4267         d = self.shouldHTTPError("short_url", 501, "Not Implemented",
4268                                  "I don't know how to treat a DELETE request.",
4269                                  client.getPage, url, method="DELETE")
4270         return d
4271
4272     def test_ophandle_bad(self):
4273         url = self.webish_url + "/operations/bogus?t=status"
4274         d = self.shouldHTTPError("ophandle_bad", 404, "404 Not Found",
4275                                  "unknown/expired handle 'bogus'",
4276                                  client.getPage, url)
4277         return d
4278
4279     def test_ophandle_cancel(self):
4280         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
4281                       followRedirect=True)
4282         d.addCallback(lambda ignored:
4283                       self.GET("/operations/128?t=status&output=JSON"))
4284         def _check1(res):
4285             data = simplejson.loads(res)
4286             self.failUnless("finished" in data, res)
4287             monitor = self.ws.root.child_operations.handles["128"][0]
4288             d = self.POST("/operations/128?t=cancel&output=JSON")
4289             def _check2(res):
4290                 data = simplejson.loads(res)
4291                 self.failUnless("finished" in data, res)
4292                 # t=cancel causes the handle to be forgotten
4293                 self.failUnless(monitor.is_cancelled())
4294             d.addCallback(_check2)
4295             return d
4296         d.addCallback(_check1)
4297         d.addCallback(lambda ignored:
4298                       self.shouldHTTPError("ophandle_cancel",
4299                                            404, "404 Not Found",
4300                                            "unknown/expired handle '128'",
4301                                            self.GET,
4302                                            "/operations/128?t=status&output=JSON"))
4303         return d
4304
4305     def test_ophandle_retainfor(self):
4306         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
4307                       followRedirect=True)
4308         d.addCallback(lambda ignored:
4309                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
4310         def _check1(res):
4311             data = simplejson.loads(res)
4312             self.failUnless("finished" in data, res)
4313         d.addCallback(_check1)
4314         # the retain-for=0 will cause the handle to be expired very soon
4315         d.addCallback(lambda ign:
4316             self.clock.advance(2.0))
4317         d.addCallback(lambda ignored:
4318                       self.shouldHTTPError("ophandle_retainfor",
4319                                            404, "404 Not Found",
4320                                            "unknown/expired handle '129'",
4321                                            self.GET,
4322                                            "/operations/129?t=status&output=JSON"))
4323         return d
4324
4325     def test_ophandle_release_after_complete(self):
4326         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
4327                       followRedirect=True)
4328         d.addCallback(self.wait_for_operation, "130")
4329         d.addCallback(lambda ignored:
4330                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
4331         # the release-after-complete=true will cause the handle to be expired
4332         d.addCallback(lambda ignored:
4333                       self.shouldHTTPError("ophandle_release_after_complete",
4334                                            404, "404 Not Found",
4335                                            "unknown/expired handle '130'",
4336                                            self.GET,
4337                                            "/operations/130?t=status&output=JSON"))
4338         return d
4339
4340     def test_uncollected_ophandle_expiration(self):
4341         # uncollected ophandles should expire after 4 days
4342         def _make_uncollected_ophandle(ophandle):
4343             d = self.POST(self.public_url +
4344                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
4345                           followRedirect=False)
4346             # When we start the operation, the webapi server will want
4347             # to redirect us to the page for the ophandle, so we get
4348             # confirmation that the operation has started. If the
4349             # manifest operation has finished by the time we get there,
4350             # following that redirect (by setting followRedirect=True
4351             # above) has the side effect of collecting the ophandle that
4352             # we've just created, which means that we can't use the
4353             # ophandle to test the uncollected timeout anymore. So,
4354             # instead, catch the 302 here and don't follow it.
4355             d.addBoth(self.should302, "uncollected_ophandle_creation")
4356             return d
4357         # Create an ophandle, don't collect it, then advance the clock by
4358         # 4 days - 1 second and make sure that the ophandle is still there.
4359         d = _make_uncollected_ophandle(131)
4360         d.addCallback(lambda ign:
4361             self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
4362         d.addCallback(lambda ign:
4363             self.GET("/operations/131?t=status&output=JSON"))
4364         def _check1(res):
4365             data = simplejson.loads(res)
4366             self.failUnless("finished" in data, res)
4367         d.addCallback(_check1)
4368         # Create an ophandle, don't collect it, then try to collect it
4369         # after 4 days. It should be gone.
4370         d.addCallback(lambda ign:
4371             _make_uncollected_ophandle(132))
4372         d.addCallback(lambda ign:
4373             self.clock.advance(96*60*60))
4374         d.addCallback(lambda ign:
4375             self.shouldHTTPError("uncollected_ophandle_expired_after_100_hours",
4376                                  404, "404 Not Found",
4377                                  "unknown/expired handle '132'",
4378                                  self.GET,
4379                                  "/operations/132?t=status&output=JSON"))
4380         return d
4381
4382     def test_collected_ophandle_expiration(self):
4383         # collected ophandles should expire after 1 day
4384         def _make_collected_ophandle(ophandle):
4385             d = self.POST(self.public_url +
4386                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
4387                           followRedirect=True)
4388             # By following the initial redirect, we collect the ophandle
4389             # we've just created.
4390             return d
4391         # Create a collected ophandle, then collect it after 23 hours
4392         # and 59 seconds to make sure that it is still there.
4393         d = _make_collected_ophandle(133)
4394         d.addCallback(lambda ign:
4395             self.clock.advance((24*60*60) - 1))
4396         d.addCallback(lambda ign:
4397             self.GET("/operations/133?t=status&output=JSON"))
4398         def _check1(res):
4399             data = simplejson.loads(res)
4400             self.failUnless("finished" in data, res)
4401         d.addCallback(_check1)
4402         # Create another uncollected ophandle, then try to collect it
4403         # after 24 hours to make sure that it is gone.
4404         d.addCallback(lambda ign:
4405             _make_collected_ophandle(134))
4406         d.addCallback(lambda ign:
4407             self.clock.advance(24*60*60))
4408         d.addCallback(lambda ign:
4409             self.shouldHTTPError("collected_ophandle_expired_after_1_day",
4410                                  404, "404 Not Found",
4411                                  "unknown/expired handle '134'",
4412                                  self.GET,
4413                                  "/operations/134?t=status&output=JSON"))
4414         return d
4415
4416     def test_incident(self):
4417         d = self.POST("/report_incident", details="eek")
4418         def _done(res):
4419             self.failIfIn("<html>", res)
4420             self.failUnlessIn("An incident report has been saved", res)
4421         d.addCallback(_done)
4422         return d
4423
4424     def test_static(self):
4425         webdir = os.path.join(self.staticdir, "subdir")
4426         fileutil.make_dirs(webdir)
4427         f = open(os.path.join(webdir, "hello.txt"), "wb")
4428         f.write("hello")
4429         f.close()
4430
4431         d = self.GET("/static/subdir/hello.txt")
4432         def _check(res):
4433             self.failUnlessReallyEqual(res, "hello")
4434         d.addCallback(_check)
4435         return d
4436
4437
4438 class IntroducerWeb(unittest.TestCase):
4439     def setUp(self):
4440         self.node = None
4441
4442     def tearDown(self):
4443         d = defer.succeed(None)
4444         if self.node:
4445             d.addCallback(lambda ign: self.node.stopService())
4446         d.addCallback(flushEventualQueue)
4447         return d
4448
4449     def test_welcome(self):
4450         basedir = "web.IntroducerWeb.test_welcome"
4451         os.mkdir(basedir)
4452         cfg = "\n".join(["[node]",
4453                          "tub.location = 127.0.0.1:1",
4454                          "web.port = tcp:0",
4455                          ]) + "\n"
4456         fileutil.write(os.path.join(basedir, "tahoe.cfg"), cfg)
4457         self.node = IntroducerNode(basedir)
4458         self.ws = self.node.getServiceNamed("webish")
4459
4460         d = fireEventually(None)
4461         d.addCallback(lambda ign: self.node.startService())
4462         d.addCallback(lambda ign: self.node.when_tub_ready())
4463
4464         d.addCallback(lambda ign: self.GET("/"))
4465         def _check(res):
4466             self.failUnlessIn('Welcome to the Tahoe-LAFS Introducer', res)
4467             self.failUnlessIn(FAVICON_MARKUP, res)
4468             self.failUnlessIn('Page rendered at', res)
4469             self.failUnlessIn('Tahoe-LAFS code imported from:', res)
4470         d.addCallback(_check)
4471         return d
4472
4473     def GET(self, urlpath, followRedirect=False, return_response=False,
4474             **kwargs):
4475         # if return_response=True, this fires with (data, statuscode,
4476         # respheaders) instead of just data.
4477         assert not isinstance(urlpath, unicode)
4478         url = self.ws.getURL().rstrip('/') + urlpath
4479         factory = HTTPClientGETFactory(url, method="GET",
4480                                        followRedirect=followRedirect, **kwargs)
4481         reactor.connectTCP("localhost", self.ws.getPortnum(), factory)
4482         d = factory.deferred
4483         def _got_data(data):
4484             return (data, factory.status, factory.response_headers)
4485         if return_response:
4486             d.addCallback(_got_data)
4487         return factory.deferred
4488
4489
4490 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4491     def test_load_file(self):
4492         # This will raise an exception unless a well-formed XML file is found under that name.
4493         common.getxmlfile('directory.xhtml').load()
4494
4495     def test_parse_replace_arg(self):
4496         self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
4497         self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
4498         self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
4499                                    "only-files")
4500         self.failUnlessRaises(common.WebError, common.parse_replace_arg, "only_fles")
4501
4502     def test_abbreviate_time(self):
4503         self.failUnlessReallyEqual(common.abbreviate_time(None), "")
4504         self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
4505         self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
4506         self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
4507         self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
4508         self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
4509
4510     def test_compute_rate(self):
4511         self.failUnlessReallyEqual(common.compute_rate(None, None), None)
4512         self.failUnlessReallyEqual(common.compute_rate(None, 1), None)
4513         self.failUnlessReallyEqual(common.compute_rate(250000, None), None)
4514         self.failUnlessReallyEqual(common.compute_rate(250000, 0), None)
4515         self.failUnlessReallyEqual(common.compute_rate(250000, 10), 25000.0)
4516         self.failUnlessReallyEqual(common.compute_rate(0, 10), 0.0)
4517         self.shouldFail(AssertionError, "test_compute_rate", "",
4518                         common.compute_rate, -100, 10)
4519         self.shouldFail(AssertionError, "test_compute_rate", "",
4520                         common.compute_rate, 100, -10)
4521
4522         # Sanity check
4523         rate = common.compute_rate(10*1000*1000, 1)
4524         self.failUnlessReallyEqual(common.abbreviate_rate(rate), "10.00MBps")
4525
4526     def test_abbreviate_rate(self):
4527         self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
4528         self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
4529         self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
4530         self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
4531
4532     def test_abbreviate_size(self):
4533         self.failUnlessReallyEqual(common.abbreviate_size(None), "")
4534         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
4535         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
4536         self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
4537         self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
4538
4539     def test_plural(self):
4540         def convert(s):
4541             return "%d second%s" % (s, status.plural(s))
4542         self.failUnlessReallyEqual(convert(0), "0 seconds")
4543         self.failUnlessReallyEqual(convert(1), "1 second")
4544         self.failUnlessReallyEqual(convert(2), "2 seconds")
4545         def convert2(s):
4546             return "has share%s: %s" % (status.plural(s), ",".join(s))
4547         self.failUnlessReallyEqual(convert2([]), "has shares: ")
4548         self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
4549         self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
4550
4551
4552 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4553
4554     def CHECK(self, ign, which, args, clientnum=0):
4555         fileurl = self.fileurls[which]
4556         url = fileurl + "?" + args
4557         return self.GET(url, method="POST", clientnum=clientnum)
4558
4559     def test_filecheck(self):
4560         self.basedir = "web/Grid/filecheck"
4561         self.set_up_grid()
4562         c0 = self.g.clients[0]
4563         self.uris = {}
4564         DATA = "data" * 100
4565         d = c0.upload(upload.Data(DATA, convergence=""))
4566         def _stash_uri(ur, which):
4567             self.uris[which] = ur.get_uri()
4568         d.addCallback(_stash_uri, "good")
4569         d.addCallback(lambda ign:
4570                       c0.upload(upload.Data(DATA+"1", convergence="")))
4571         d.addCallback(_stash_uri, "sick")
4572         d.addCallback(lambda ign:
4573                       c0.upload(upload.Data(DATA+"2", convergence="")))
4574         d.addCallback(_stash_uri, "dead")
4575         def _stash_mutable_uri(n, which):
4576             self.uris[which] = n.get_uri()
4577             assert isinstance(self.uris[which], str)
4578         d.addCallback(lambda ign:
4579             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4580         d.addCallback(_stash_mutable_uri, "corrupt")
4581         d.addCallback(lambda ign:
4582                       c0.upload(upload.Data("literal", convergence="")))
4583         d.addCallback(_stash_uri, "small")
4584         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
4585         d.addCallback(_stash_mutable_uri, "smalldir")
4586
4587         def _compute_fileurls(ignored):
4588             self.fileurls = {}
4589             for which in self.uris:
4590                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4591         d.addCallback(_compute_fileurls)
4592
4593         def _clobber_shares(ignored):
4594             good_shares = self.find_uri_shares(self.uris["good"])
4595             self.failUnlessReallyEqual(len(good_shares), 10)
4596             sick_shares = self.find_uri_shares(self.uris["sick"])
4597             os.unlink(sick_shares[0][2])
4598             dead_shares = self.find_uri_shares(self.uris["dead"])
4599             for i in range(1, 10):
4600                 os.unlink(dead_shares[i][2])
4601             c_shares = self.find_uri_shares(self.uris["corrupt"])
4602             cso = CorruptShareOptions()
4603             cso.stdout = StringIO()
4604             cso.parseOptions([c_shares[0][2]])
4605             corrupt_share(cso)
4606         d.addCallback(_clobber_shares)
4607
4608         d.addCallback(self.CHECK, "good", "t=check")
4609         def _got_html_good(res):
4610             self.failUnlessIn("Healthy", res)
4611             self.failIfIn("Not Healthy", res)
4612             self.failUnlessIn(FAVICON_MARKUP, res)
4613         d.addCallback(_got_html_good)
4614         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
4615         def _got_html_good_return_to(res):
4616             self.failUnlessIn("Healthy", res)
4617             self.failIfIn("Not Healthy", res)
4618             self.failUnlessIn('<a href="somewhere">Return to file', res)
4619         d.addCallback(_got_html_good_return_to)
4620         d.addCallback(self.CHECK, "good", "t=check&output=json")
4621         def _got_json_good(res):
4622             r = simplejson.loads(res)
4623             self.failUnlessEqual(r["summary"], "Healthy")
4624             self.failUnless(r["results"]["healthy"])
4625             self.failIfIn("needs-rebalancing", r["results"])
4626             self.failUnless(r["results"]["recoverable"])
4627         d.addCallback(_got_json_good)
4628
4629         d.addCallback(self.CHECK, "small", "t=check")
4630         def _got_html_small(res):
4631             self.failUnlessIn("Literal files are always healthy", res)
4632             self.failIfIn("Not Healthy", res)
4633         d.addCallback(_got_html_small)
4634         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
4635         def _got_html_small_return_to(res):
4636             self.failUnlessIn("Literal files are always healthy", res)
4637             self.failIfIn("Not Healthy", res)
4638             self.failUnlessIn('<a href="somewhere">Return to file', res)
4639         d.addCallback(_got_html_small_return_to)
4640         d.addCallback(self.CHECK, "small", "t=check&output=json")
4641         def _got_json_small(res):
4642             r = simplejson.loads(res)
4643             self.failUnlessEqual(r["storage-index"], "")
4644             self.failUnless(r["results"]["healthy"])
4645         d.addCallback(_got_json_small)
4646
4647         d.addCallback(self.CHECK, "smalldir", "t=check")
4648         def _got_html_smalldir(res):
4649             self.failUnlessIn("Literal files are always healthy", res)
4650             self.failIfIn("Not Healthy", res)
4651         d.addCallback(_got_html_smalldir)
4652         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
4653         def _got_json_smalldir(res):
4654             r = simplejson.loads(res)
4655             self.failUnlessEqual(r["storage-index"], "")
4656             self.failUnless(r["results"]["healthy"])
4657         d.addCallback(_got_json_smalldir)
4658
4659         d.addCallback(self.CHECK, "sick", "t=check")
4660         def _got_html_sick(res):
4661             self.failUnlessIn("Not Healthy", res)
4662         d.addCallback(_got_html_sick)
4663         d.addCallback(self.CHECK, "sick", "t=check&output=json")
4664         def _got_json_sick(res):
4665             r = simplejson.loads(res)
4666             self.failUnlessEqual(r["summary"],
4667                                  "Not Healthy: 9 shares (enc 3-of-10)")
4668             self.failIf(r["results"]["healthy"])
4669             self.failUnless(r["results"]["recoverable"])
4670             self.failIfIn("needs-rebalancing", r["results"])
4671         d.addCallback(_got_json_sick)
4672
4673         d.addCallback(self.CHECK, "dead", "t=check")
4674         def _got_html_dead(res):
4675             self.failUnlessIn("Not Healthy", res)
4676         d.addCallback(_got_html_dead)
4677         d.addCallback(self.CHECK, "dead", "t=check&output=json")
4678         def _got_json_dead(res):
4679             r = simplejson.loads(res)
4680             self.failUnlessEqual(r["summary"],
4681                                  "Not Healthy: 1 shares (enc 3-of-10)")
4682             self.failIf(r["results"]["healthy"])
4683             self.failIf(r["results"]["recoverable"])
4684             self.failIfIn("needs-rebalancing", r["results"])
4685         d.addCallback(_got_json_dead)
4686
4687         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
4688         def _got_html_corrupt(res):
4689             self.failUnlessIn("Not Healthy! : Unhealthy", res)
4690         d.addCallback(_got_html_corrupt)
4691         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
4692         def _got_json_corrupt(res):
4693             r = simplejson.loads(res)
4694             self.failUnlessIn("Unhealthy: 9 shares (enc 3-of-10)", r["summary"])
4695             self.failIf(r["results"]["healthy"])
4696             self.failUnless(r["results"]["recoverable"])
4697             self.failIfIn("needs-rebalancing", r["results"])
4698             self.failUnlessReallyEqual(r["results"]["count-happiness"], 9)
4699             self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
4700             self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
4701         d.addCallback(_got_json_corrupt)
4702
4703         d.addErrback(self.explain_web_error)
4704         return d
4705
4706     def test_repair_html(self):
4707         self.basedir = "web/Grid/repair_html"
4708         self.set_up_grid()
4709         c0 = self.g.clients[0]
4710         self.uris = {}
4711         DATA = "data" * 100
4712         d = c0.upload(upload.Data(DATA, convergence=""))
4713         def _stash_uri(ur, which):
4714             self.uris[which] = ur.get_uri()
4715         d.addCallback(_stash_uri, "good")
4716         d.addCallback(lambda ign:
4717                       c0.upload(upload.Data(DATA+"1", convergence="")))
4718         d.addCallback(_stash_uri, "sick")
4719         d.addCallback(lambda ign:
4720                       c0.upload(upload.Data(DATA+"2", convergence="")))
4721         d.addCallback(_stash_uri, "dead")
4722         def _stash_mutable_uri(n, which):
4723             self.uris[which] = n.get_uri()
4724             assert isinstance(self.uris[which], str)
4725         d.addCallback(lambda ign:
4726             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4727         d.addCallback(_stash_mutable_uri, "corrupt")
4728
4729         def _compute_fileurls(ignored):
4730             self.fileurls = {}
4731             for which in self.uris:
4732                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4733         d.addCallback(_compute_fileurls)
4734
4735         def _clobber_shares(ignored):
4736             good_shares = self.find_uri_shares(self.uris["good"])
4737             self.failUnlessReallyEqual(len(good_shares), 10)
4738             sick_shares = self.find_uri_shares(self.uris["sick"])
4739             os.unlink(sick_shares[0][2])
4740             dead_shares = self.find_uri_shares(self.uris["dead"])
4741             for i in range(1, 10):
4742                 os.unlink(dead_shares[i][2])
4743             c_shares = self.find_uri_shares(self.uris["corrupt"])
4744             cso = CorruptShareOptions()
4745             cso.stdout = StringIO()
4746             cso.parseOptions([c_shares[0][2]])
4747             corrupt_share(cso)
4748         d.addCallback(_clobber_shares)
4749
4750         d.addCallback(self.CHECK, "good", "t=check&repair=true")
4751         def _got_html_good(res):
4752             self.failUnlessIn("Healthy", res)
4753             self.failIfIn("Not Healthy", res)
4754             self.failUnlessIn("No repair necessary", res)
4755             self.failUnlessIn(FAVICON_MARKUP, res)
4756         d.addCallback(_got_html_good)
4757
4758         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
4759         def _got_html_sick(res):
4760             self.failUnlessIn("Healthy : healthy", res)
4761             self.failIfIn("Not Healthy", res)
4762             self.failUnlessIn("Repair successful", res)
4763         d.addCallback(_got_html_sick)
4764
4765         # repair of a dead file will fail, of course, but it isn't yet
4766         # clear how this should be reported. Right now it shows up as
4767         # a "410 Gone".
4768         #
4769         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
4770         #def _got_html_dead(res):
4771         #    print res
4772         #    self.failUnlessIn("Healthy : healthy", res)
4773         #    self.failIfIn("Not Healthy", res)
4774         #    self.failUnlessIn("No repair necessary", res)
4775         #d.addCallback(_got_html_dead)
4776
4777         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
4778         def _got_html_corrupt(res):
4779             self.failUnlessIn("Healthy : Healthy", res)
4780             self.failIfIn("Not Healthy", res)
4781             self.failUnlessIn("Repair successful", res)
4782         d.addCallback(_got_html_corrupt)
4783
4784         d.addErrback(self.explain_web_error)
4785         return d
4786
4787     def test_repair_json(self):
4788         self.basedir = "web/Grid/repair_json"
4789         self.set_up_grid()
4790         c0 = self.g.clients[0]
4791         self.uris = {}
4792         DATA = "data" * 100
4793         d = c0.upload(upload.Data(DATA+"1", convergence=""))
4794         def _stash_uri(ur, which):
4795             self.uris[which] = ur.get_uri()
4796         d.addCallback(_stash_uri, "sick")
4797
4798         def _compute_fileurls(ignored):
4799             self.fileurls = {}
4800             for which in self.uris:
4801                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4802         d.addCallback(_compute_fileurls)
4803
4804         def _clobber_shares(ignored):
4805             sick_shares = self.find_uri_shares(self.uris["sick"])
4806             os.unlink(sick_shares[0][2])
4807         d.addCallback(_clobber_shares)
4808
4809         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
4810         def _got_json_sick(res):
4811             r = simplejson.loads(res)
4812             self.failUnlessReallyEqual(r["repair-attempted"], True)
4813             self.failUnlessReallyEqual(r["repair-successful"], True)
4814             self.failUnlessEqual(r["pre-repair-results"]["summary"],
4815                                  "Not Healthy: 9 shares (enc 3-of-10)")
4816             self.failIf(r["pre-repair-results"]["results"]["healthy"])
4817             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
4818             self.failUnless(r["post-repair-results"]["results"]["healthy"])
4819         d.addCallback(_got_json_sick)
4820
4821         d.addErrback(self.explain_web_error)
4822         return d
4823
4824     def test_unknown(self, immutable=False):
4825         self.basedir = "web/Grid/unknown"
4826         if immutable:
4827             self.basedir = "web/Grid/unknown-immutable"
4828
4829         self.set_up_grid()
4830         c0 = self.g.clients[0]
4831         self.uris = {}
4832         self.fileurls = {}
4833
4834         # the future cap format may contain slashes, which must be tolerated
4835         expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
4836                                                            safe="")
4837
4838         if immutable:
4839             name = u"future-imm"
4840             future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
4841             d = c0.create_immutable_dirnode({name: (future_node, {})})
4842         else:
4843             name = u"future"
4844             future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4845             d = c0.create_dirnode()
4846
4847         def _stash_root_and_create_file(n):
4848             self.rootnode = n
4849             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
4850             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
4851             if not immutable:
4852                 return self.rootnode.set_node(name, future_node)
4853         d.addCallback(_stash_root_and_create_file)
4854
4855         # make sure directory listing tolerates unknown nodes
4856         d.addCallback(lambda ign: self.GET(self.rooturl))
4857         def _check_directory_html(res, expected_type_suffix):
4858             pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
4859                                   '<td>%s</td>' % (expected_type_suffix, str(name)),
4860                                  re.DOTALL)
4861             self.failUnless(re.search(pattern, res), res)
4862             # find the More Info link for name, should be relative
4863             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4864             info_url = mo.group(1)
4865             self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
4866         if immutable:
4867             d.addCallback(_check_directory_html, "-IMM")
4868         else:
4869             d.addCallback(_check_directory_html, "")
4870
4871         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4872         def _check_directory_json(res, expect_rw_uri):
4873             data = simplejson.loads(res)
4874             self.failUnlessEqual(data[0], "dirnode")
4875             f = data[1]["children"][name]
4876             self.failUnlessEqual(f[0], "unknown")
4877             if expect_rw_uri:
4878                 self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data)
4879             else:
4880                 self.failIfIn("rw_uri", f[1])
4881             if immutable:
4882                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
4883             else:
4884                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
4885             self.failUnlessIn("metadata", f[1])
4886         d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
4887
4888         def _check_info(res, expect_rw_uri, expect_ro_uri):
4889             self.failUnlessIn("Object Type: <span>unknown</span>", res)
4890             if expect_rw_uri:
4891                 self.failUnlessIn(unknown_rwcap, res)
4892             if expect_ro_uri:
4893                 if immutable:
4894                     self.failUnlessIn(unknown_immcap, res)
4895                 else:
4896                     self.failUnlessIn(unknown_rocap, res)
4897             else:
4898                 self.failIfIn(unknown_rocap, res)
4899             self.failIfIn("Raw data as", res)
4900             self.failIfIn("Directory writecap", res)
4901             self.failIfIn("Checker Operations", res)
4902             self.failIfIn("Mutable File Operations", res)
4903             self.failIfIn("Directory Operations", res)
4904
4905         # FIXME: these should have expect_rw_uri=not immutable; I don't know
4906         # why they fail. Possibly related to ticket #922.
4907
4908         d.addCallback(lambda ign: self.GET(expected_info_url))
4909         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
4910         d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
4911         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
4912
4913         def _check_json(res, expect_rw_uri):
4914             data = simplejson.loads(res)
4915             self.failUnlessEqual(data[0], "unknown")
4916             if expect_rw_uri:
4917                 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data)
4918             else:
4919                 self.failIfIn("rw_uri", data[1])
4920
4921             if immutable:
4922                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data)
4923                 self.failUnlessReallyEqual(data[1]["mutable"], False)
4924             elif expect_rw_uri:
4925                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4926                 self.failUnlessReallyEqual(data[1]["mutable"], True)
4927             else:
4928                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4929                 self.failIfIn("mutable", data[1])
4930
4931             # TODO: check metadata contents
4932             self.failUnlessIn("metadata", data[1])
4933
4934         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
4935         d.addCallback(_check_json, expect_rw_uri=not immutable)
4936
4937         # and make sure that a read-only version of the directory can be
4938         # rendered too. This version will not have unknown_rwcap, whether
4939         # or not future_node was immutable.
4940         d.addCallback(lambda ign: self.GET(self.rourl))
4941         if immutable:
4942             d.addCallback(_check_directory_html, "-IMM")
4943         else:
4944             d.addCallback(_check_directory_html, "-RO")
4945
4946         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
4947         d.addCallback(_check_directory_json, expect_rw_uri=False)
4948
4949         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
4950         d.addCallback(_check_json, expect_rw_uri=False)
4951
4952         # TODO: check that getting t=info from the Info link in the ro directory
4953         # works, and does not include the writecap URI.
4954         return d
4955
4956     def test_immutable_unknown(self):
4957         return self.test_unknown(immutable=True)
4958
4959     def test_mutant_dirnodes_are_omitted(self):
4960         self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
4961
4962         self.set_up_grid()
4963         c = self.g.clients[0]
4964         nm = c.nodemaker
4965         self.uris = {}
4966         self.fileurls = {}
4967
4968         lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
4969         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
4970         mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
4971
4972         # This method tests mainly dirnode, but we'd have to duplicate code in order to
4973         # test the dirnode and web layers separately.
4974
4975         # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
4976         # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
4977         # When the directory is read, the mutants should be silently disposed of, leaving
4978         # their lonely sibling.
4979         # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
4980         # because immutable directories don't have a writecap and therefore that field
4981         # isn't (and can't be) decrypted.
4982         # TODO: The field still exists in the netstring. Technically we should check what
4983         # happens if something is put there (_unpack_contents should raise ValueError),
4984         # but that can wait.
4985
4986         lonely_child = nm.create_from_cap(lonely_uri)
4987         mutant_ro_child = nm.create_from_cap(mut_read_uri)
4988         mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
4989
4990         def _by_hook_or_by_crook():
4991             return True
4992         for n in [mutant_ro_child, mutant_write_in_ro_child]:
4993             n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
4994
4995         mutant_write_in_ro_child.get_write_uri    = lambda: None
4996         mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
4997
4998         kids = {u"lonely":      (lonely_child, {}),
4999                 u"ro":          (mutant_ro_child, {}),
5000                 u"write-in-ro": (mutant_write_in_ro_child, {}),
5001                 }
5002         d = c.create_immutable_dirnode(kids)
5003
5004         def _created(dn):
5005             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
5006             self.failIf(dn.is_mutable())
5007             self.failUnless(dn.is_readonly())
5008             # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
5009             self.failIf(hasattr(dn._node, 'get_writekey'))
5010             rep = str(dn)
5011             self.failUnlessIn("RO-IMM", rep)
5012             cap = dn.get_cap()
5013             self.failUnlessIn("CHK", cap.to_string())
5014             self.cap = cap
5015             self.rootnode = dn
5016             self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
5017             return download_to_data(dn._node)
5018         d.addCallback(_created)
5019
5020         def _check_data(data):
5021             # Decode the netstring representation of the directory to check that all children
5022             # are present. This is a bit of an abstraction violation, but there's not really
5023             # any other way to do it given that the real DirectoryNode._unpack_contents would
5024             # strip the mutant children out (which is what we're trying to test, later).
5025             position = 0
5026             numkids = 0
5027             while position < len(data):
5028                 entries, position = split_netstring(data, 1, position)
5029                 entry = entries[0]
5030                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
5031                 name = name_utf8.decode("utf-8")
5032                 self.failUnlessEqual(rwcapdata, "")
5033                 self.failUnlessIn(name, kids)
5034                 (expected_child, ign) = kids[name]
5035                 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
5036                 numkids += 1
5037
5038             self.failUnlessReallyEqual(numkids, 3)
5039             return self.rootnode.list()
5040         d.addCallback(_check_data)
5041
5042         # Now when we use the real directory listing code, the mutants should be absent.
5043         def _check_kids(children):
5044             self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
5045             lonely_node, lonely_metadata = children[u"lonely"]
5046
5047             self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
5048             self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
5049         d.addCallback(_check_kids)
5050
5051         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
5052         d.addCallback(lambda n: n.list())
5053         d.addCallback(_check_kids)  # again with dirnode recreated from cap
5054
5055         # Make sure the lonely child can be listed in HTML...
5056         d.addCallback(lambda ign: self.GET(self.rooturl))
5057         def _check_html(res):
5058             self.failIfIn("URI:SSK", res)
5059             get_lonely = "".join([r'<td>FILE</td>',
5060                                   r'\s+<td>',
5061                                   r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
5062                                   r'</td>',
5063                                   r'\s+<td align="right">%d</td>' % len("one"),
5064                                   ])
5065             self.failUnless(re.search(get_lonely, res), res)
5066
5067             # find the More Info link for name, should be relative
5068             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
5069             info_url = mo.group(1)
5070             self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
5071         d.addCallback(_check_html)
5072
5073         # ... and in JSON.
5074         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
5075         def _check_json(res):
5076             data = simplejson.loads(res)
5077             self.failUnlessEqual(data[0], "dirnode")
5078             listed_children = data[1]["children"]
5079             self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
5080             ll_type, ll_data = listed_children[u"lonely"]
5081             self.failUnlessEqual(ll_type, "filenode")
5082             self.failIfIn("rw_uri", ll_data)
5083             self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
5084         d.addCallback(_check_json)
5085         return d
5086
5087     def test_deep_check(self):
5088         self.basedir = "web/Grid/deep_check"
5089         self.set_up_grid()
5090         c0 = self.g.clients[0]
5091         self.uris = {}
5092         self.fileurls = {}
5093         DATA = "data" * 100
5094         d = c0.create_dirnode()
5095         def _stash_root_and_create_file(n):
5096             self.rootnode = n
5097             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5098             return n.add_file(u"good", upload.Data(DATA, convergence=""))
5099         d.addCallback(_stash_root_and_create_file)
5100         def _stash_uri(fn, which):
5101             self.uris[which] = fn.get_uri()
5102             return fn
5103         d.addCallback(_stash_uri, "good")
5104         d.addCallback(lambda ign:
5105                       self.rootnode.add_file(u"small",
5106                                              upload.Data("literal",
5107                                                         convergence="")))
5108         d.addCallback(_stash_uri, "small")
5109         d.addCallback(lambda ign:
5110                       self.rootnode.add_file(u"sick",
5111                                              upload.Data(DATA+"1",
5112                                                         convergence="")))
5113         d.addCallback(_stash_uri, "sick")
5114
5115         # this tests that deep-check and stream-manifest will ignore
5116         # UnknownNode instances. Hopefully this will also cover deep-stats.
5117         future_node = UnknownNode(unknown_rwcap, unknown_rocap)
5118         d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
5119
5120         def _clobber_shares(ignored):
5121             self.delete_shares_numbered(self.uris["sick"], [0,1])
5122         d.addCallback(_clobber_shares)
5123
5124         # root
5125         # root/good
5126         # root/small
5127         # root/sick
5128         # root/future
5129
5130         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
5131         def _done(res):
5132             try:
5133                 units = [simplejson.loads(line)
5134                          for line in res.splitlines()
5135                          if line]
5136             except ValueError:
5137                 print "response is:", res
5138                 print "undecodeable line was '%s'" % line
5139                 raise
5140             self.failUnlessReallyEqual(len(units), 5+1)
5141             # should be parent-first
5142             u0 = units[0]
5143             self.failUnlessEqual(u0["path"], [])
5144             self.failUnlessEqual(u0["type"], "directory")
5145             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
5146             u0cr = u0["check-results"]
5147             self.failUnlessReallyEqual(u0cr["results"]["count-happiness"], 10)
5148             self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
5149
5150             ugood = [u for u in units
5151                      if u["type"] == "file" and u["path"] == [u"good"]][0]
5152             self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
5153             ugoodcr = ugood["check-results"]
5154             self.failUnlessReallyEqual(ugoodcr["results"]["count-happiness"], 10)
5155             self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
5156
5157             stats = units[-1]
5158             self.failUnlessEqual(stats["type"], "stats")
5159             s = stats["stats"]
5160             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
5161             self.failUnlessReallyEqual(s["count-literal-files"], 1)
5162             self.failUnlessReallyEqual(s["count-directories"], 1)
5163             self.failUnlessReallyEqual(s["count-unknown"], 1)
5164         d.addCallback(_done)
5165
5166         d.addCallback(self.CHECK, "root", "t=stream-manifest")
5167         def _check_manifest(res):
5168             self.failUnless(res.endswith("\n"))
5169             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
5170             self.failUnlessReallyEqual(len(units), 5+1)
5171             self.failUnlessEqual(units[-1]["type"], "stats")
5172             first = units[0]
5173             self.failUnlessEqual(first["path"], [])
5174             self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri())
5175             self.failUnlessEqual(first["type"], "directory")
5176             stats = units[-1]["stats"]
5177             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
5178             self.failUnlessReallyEqual(stats["count-literal-files"], 1)
5179             self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
5180             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
5181             self.failUnlessReallyEqual(stats["count-unknown"], 1)
5182         d.addCallback(_check_manifest)
5183
5184         # now add root/subdir and root/subdir/grandchild, then make subdir
5185         # unrecoverable, then see what happens
5186
5187         d.addCallback(lambda ign:
5188                       self.rootnode.create_subdirectory(u"subdir"))
5189         d.addCallback(_stash_uri, "subdir")
5190         d.addCallback(lambda subdir_node:
5191                       subdir_node.add_file(u"grandchild",
5192                                            upload.Data(DATA+"2",
5193                                                        convergence="")))
5194         d.addCallback(_stash_uri, "grandchild")
5195
5196         d.addCallback(lambda ign:
5197                       self.delete_shares_numbered(self.uris["subdir"],
5198                                                   range(1, 10)))
5199
5200         # root
5201         # root/good
5202         # root/small
5203         # root/sick
5204         # root/future
5205         # root/subdir [unrecoverable]
5206         # root/subdir/grandchild
5207
5208         # how should a streaming-JSON API indicate fatal error?
5209         # answer: emit ERROR: instead of a JSON string
5210
5211         d.addCallback(self.CHECK, "root", "t=stream-manifest")
5212         def _check_broken_manifest(res):
5213             lines = res.splitlines()
5214             error_lines = [i
5215                            for (i,line) in enumerate(lines)
5216                            if line.startswith("ERROR:")]
5217             if not error_lines:
5218                 self.fail("no ERROR: in output: %s" % (res,))
5219             first_error = error_lines[0]
5220             error_line = lines[first_error]
5221             error_msg = lines[first_error+1:]
5222             error_msg_s = "\n".join(error_msg) + "\n"
5223             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
5224                               error_line)
5225             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
5226             units = [simplejson.loads(line) for line in lines[:first_error]]
5227             self.failUnlessReallyEqual(len(units), 6) # includes subdir
5228             last_unit = units[-1]
5229             self.failUnlessEqual(last_unit["path"], ["subdir"])
5230         d.addCallback(_check_broken_manifest)
5231
5232         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
5233         def _check_broken_deepcheck(res):
5234             lines = res.splitlines()
5235             error_lines = [i
5236                            for (i,line) in enumerate(lines)
5237                            if line.startswith("ERROR:")]
5238             if not error_lines:
5239                 self.fail("no ERROR: in output: %s" % (res,))
5240             first_error = error_lines[0]
5241             error_line = lines[first_error]
5242             error_msg = lines[first_error+1:]
5243             error_msg_s = "\n".join(error_msg) + "\n"
5244             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
5245                               error_line)
5246             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
5247             units = [simplejson.loads(line) for line in lines[:first_error]]
5248             self.failUnlessReallyEqual(len(units), 6) # includes subdir
5249             last_unit = units[-1]
5250             self.failUnlessEqual(last_unit["path"], ["subdir"])
5251             r = last_unit["check-results"]["results"]
5252             self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
5253             self.failUnlessReallyEqual(r["count-happiness"], 1)
5254             self.failUnlessReallyEqual(r["count-shares-good"], 1)
5255             self.failUnlessReallyEqual(r["recoverable"], False)
5256         d.addCallback(_check_broken_deepcheck)
5257
5258         d.addErrback(self.explain_web_error)
5259         return d
5260
5261     def test_deep_check_and_repair(self):
5262         self.basedir = "web/Grid/deep_check_and_repair"
5263         self.set_up_grid()
5264         c0 = self.g.clients[0]
5265         self.uris = {}
5266         self.fileurls = {}
5267         DATA = "data" * 100
5268         d = c0.create_dirnode()
5269         def _stash_root_and_create_file(n):
5270             self.rootnode = n
5271             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5272             return n.add_file(u"good", upload.Data(DATA, convergence=""))
5273         d.addCallback(_stash_root_and_create_file)
5274         def _stash_uri(fn, which):
5275             self.uris[which] = fn.get_uri()
5276         d.addCallback(_stash_uri, "good")
5277         d.addCallback(lambda ign:
5278                       self.rootnode.add_file(u"small",
5279                                              upload.Data("literal",
5280                                                         convergence="")))
5281         d.addCallback(_stash_uri, "small")
5282         d.addCallback(lambda ign:
5283                       self.rootnode.add_file(u"sick",
5284                                              upload.Data(DATA+"1",
5285                                                         convergence="")))
5286         d.addCallback(_stash_uri, "sick")
5287         #d.addCallback(lambda ign:
5288         #              self.rootnode.add_file(u"dead",
5289         #                                     upload.Data(DATA+"2",
5290         #                                                convergence="")))
5291         #d.addCallback(_stash_uri, "dead")
5292
5293         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
5294         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
5295         #d.addCallback(_stash_uri, "corrupt")
5296
5297         def _clobber_shares(ignored):
5298             good_shares = self.find_uri_shares(self.uris["good"])
5299             self.failUnlessReallyEqual(len(good_shares), 10)
5300             sick_shares = self.find_uri_shares(self.uris["sick"])
5301             os.unlink(sick_shares[0][2])
5302             #dead_shares = self.find_uri_shares(self.uris["dead"])
5303             #for i in range(1, 10):
5304             #    os.unlink(dead_shares[i][2])
5305
5306             #c_shares = self.find_uri_shares(self.uris["corrupt"])
5307             #cso = CorruptShareOptions()
5308             #cso.stdout = StringIO()
5309             #cso.parseOptions([c_shares[0][2]])
5310             #corrupt_share(cso)
5311         d.addCallback(_clobber_shares)
5312
5313         # root
5314         # root/good   CHK, 10 shares
5315         # root/small  LIT
5316         # root/sick   CHK, 9 shares
5317
5318         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
5319         def _done(res):
5320             units = [simplejson.loads(line)
5321                      for line in res.splitlines()
5322                      if line]
5323             self.failUnlessReallyEqual(len(units), 4+1)
5324             # should be parent-first
5325             u0 = units[0]
5326             self.failUnlessEqual(u0["path"], [])
5327             self.failUnlessEqual(u0["type"], "directory")
5328             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
5329             u0crr = u0["check-and-repair-results"]
5330             self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
5331             self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-happiness"], 10)
5332             self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
5333
5334             ugood = [u for u in units
5335                      if u["type"] == "file" and u["path"] == [u"good"]][0]
5336             self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
5337             ugoodcrr = ugood["check-and-repair-results"]
5338             self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
5339             self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-happiness"], 10)
5340             self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
5341
5342             usick = [u for u in units
5343                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
5344             self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"])
5345             usickcrr = usick["check-and-repair-results"]
5346             self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
5347             self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
5348             self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-happiness"], 9)
5349             self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
5350             self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-happiness"], 10)
5351             self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
5352
5353             stats = units[-1]
5354             self.failUnlessEqual(stats["type"], "stats")
5355             s = stats["stats"]
5356             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
5357             self.failUnlessReallyEqual(s["count-literal-files"], 1)
5358             self.failUnlessReallyEqual(s["count-directories"], 1)
5359         d.addCallback(_done)
5360
5361         d.addErrback(self.explain_web_error)
5362         return d
5363
5364     def _count_leases(self, ignored, which):
5365         u = self.uris[which]
5366         shares = self.find_uri_shares(u)
5367         lease_counts = []
5368         for shnum, serverid, fn in shares:
5369             sf = get_share_file(fn)
5370             num_leases = len(list(sf.get_leases()))
5371             lease_counts.append( (fn, num_leases) )
5372         return lease_counts
5373
5374     def _assert_leasecount(self, lease_counts, expected):
5375         for (fn, num_leases) in lease_counts:
5376             if num_leases != expected:
5377                 self.fail("expected %d leases, have %d, on %s" %
5378                           (expected, num_leases, fn))
5379
5380     def test_add_lease(self):
5381         self.basedir = "web/Grid/add_lease"
5382         self.set_up_grid(num_clients=2)
5383         c0 = self.g.clients[0]
5384         self.uris = {}
5385         DATA = "data" * 100
5386         d = c0.upload(upload.Data(DATA, convergence=""))
5387         def _stash_uri(ur, which):
5388             self.uris[which] = ur.get_uri()
5389         d.addCallback(_stash_uri, "one")
5390         d.addCallback(lambda ign:
5391                       c0.upload(upload.Data(DATA+"1", convergence="")))
5392         d.addCallback(_stash_uri, "two")
5393         def _stash_mutable_uri(n, which):
5394             self.uris[which] = n.get_uri()
5395             assert isinstance(self.uris[which], str)
5396         d.addCallback(lambda ign:
5397             c0.create_mutable_file(publish.MutableData(DATA+"2")))
5398         d.addCallback(_stash_mutable_uri, "mutable")
5399
5400         def _compute_fileurls(ignored):
5401             self.fileurls = {}
5402             for which in self.uris:
5403                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
5404         d.addCallback(_compute_fileurls)
5405
5406         d.addCallback(self._count_leases, "one")
5407         d.addCallback(self._assert_leasecount, 1)
5408         d.addCallback(self._count_leases, "two")
5409         d.addCallback(self._assert_leasecount, 1)
5410         d.addCallback(self._count_leases, "mutable")
5411         d.addCallback(self._assert_leasecount, 1)
5412
5413         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
5414         def _got_html_good(res):
5415             self.failUnlessIn("Healthy", res)
5416             self.failIfIn("Not Healthy", res)
5417         d.addCallback(_got_html_good)
5418
5419         d.addCallback(self._count_leases, "one")
5420         d.addCallback(self._assert_leasecount, 1)
5421         d.addCallback(self._count_leases, "two")
5422         d.addCallback(self._assert_leasecount, 1)
5423         d.addCallback(self._count_leases, "mutable")
5424         d.addCallback(self._assert_leasecount, 1)
5425
5426         # this CHECK uses the original client, which uses the same
5427         # lease-secrets, so it will just renew the original lease
5428         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
5429         d.addCallback(_got_html_good)
5430
5431         d.addCallback(self._count_leases, "one")
5432         d.addCallback(self._assert_leasecount, 1)
5433         d.addCallback(self._count_leases, "two")
5434         d.addCallback(self._assert_leasecount, 1)
5435         d.addCallback(self._count_leases, "mutable")
5436         d.addCallback(self._assert_leasecount, 1)
5437
5438         # this CHECK uses an alternate client, which adds a second lease
5439         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
5440         d.addCallback(_got_html_good)
5441
5442         d.addCallback(self._count_leases, "one")
5443         d.addCallback(self._assert_leasecount, 2)
5444         d.addCallback(self._count_leases, "two")
5445         d.addCallback(self._assert_leasecount, 1)
5446         d.addCallback(self._count_leases, "mutable")
5447         d.addCallback(self._assert_leasecount, 1)
5448
5449         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
5450         d.addCallback(_got_html_good)
5451
5452         d.addCallback(self._count_leases, "one")
5453         d.addCallback(self._assert_leasecount, 2)
5454         d.addCallback(self._count_leases, "two")
5455         d.addCallback(self._assert_leasecount, 1)
5456         d.addCallback(self._count_leases, "mutable")
5457         d.addCallback(self._assert_leasecount, 1)
5458
5459         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
5460                       clientnum=1)
5461         d.addCallback(_got_html_good)
5462
5463         d.addCallback(self._count_leases, "one")
5464         d.addCallback(self._assert_leasecount, 2)
5465         d.addCallback(self._count_leases, "two")
5466         d.addCallback(self._assert_leasecount, 1)
5467         d.addCallback(self._count_leases, "mutable")
5468         d.addCallback(self._assert_leasecount, 2)
5469
5470         d.addErrback(self.explain_web_error)
5471         return d
5472
5473     def test_deep_add_lease(self):
5474         self.basedir = "web/Grid/deep_add_lease"
5475         self.set_up_grid(num_clients=2)
5476         c0 = self.g.clients[0]
5477         self.uris = {}
5478         self.fileurls = {}
5479         DATA = "data" * 100
5480         d = c0.create_dirnode()
5481         def _stash_root_and_create_file(n):
5482             self.rootnode = n
5483             self.uris["root"] = n.get_uri()
5484             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5485             return n.add_file(u"one", upload.Data(DATA, convergence=""))
5486         d.addCallback(_stash_root_and_create_file)
5487         def _stash_uri(fn, which):
5488             self.uris[which] = fn.get_uri()
5489         d.addCallback(_stash_uri, "one")
5490         d.addCallback(lambda ign:
5491                       self.rootnode.add_file(u"small",
5492                                              upload.Data("literal",
5493                                                         convergence="")))
5494         d.addCallback(_stash_uri, "small")
5495
5496         d.addCallback(lambda ign:
5497             c0.create_mutable_file(publish.MutableData("mutable")))
5498         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
5499         d.addCallback(_stash_uri, "mutable")
5500
5501         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
5502         def _done(res):
5503             units = [simplejson.loads(line)
5504                      for line in res.splitlines()
5505                      if line]
5506             # root, one, small, mutable,   stats
5507             self.failUnlessReallyEqual(len(units), 4+1)
5508         d.addCallback(_done)
5509
5510         d.addCallback(self._count_leases, "root")
5511         d.addCallback(self._assert_leasecount, 1)
5512         d.addCallback(self._count_leases, "one")
5513         d.addCallback(self._assert_leasecount, 1)
5514         d.addCallback(self._count_leases, "mutable")
5515         d.addCallback(self._assert_leasecount, 1)
5516
5517         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
5518         d.addCallback(_done)
5519
5520         d.addCallback(self._count_leases, "root")
5521         d.addCallback(self._assert_leasecount, 1)
5522         d.addCallback(self._count_leases, "one")
5523         d.addCallback(self._assert_leasecount, 1)
5524         d.addCallback(self._count_leases, "mutable")
5525         d.addCallback(self._assert_leasecount, 1)
5526
5527         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
5528                       clientnum=1)
5529         d.addCallback(_done)
5530
5531         d.addCallback(self._count_leases, "root")
5532         d.addCallback(self._assert_leasecount, 2)
5533         d.addCallback(self._count_leases, "one")
5534         d.addCallback(self._assert_leasecount, 2)
5535         d.addCallback(self._count_leases, "mutable")
5536         d.addCallback(self._assert_leasecount, 2)
5537
5538         d.addErrback(self.explain_web_error)
5539         return d
5540
5541
5542     def test_exceptions(self):
5543         self.basedir = "web/Grid/exceptions"
5544         self.set_up_grid(num_clients=1, num_servers=2)
5545         c0 = self.g.clients[0]
5546         c0.encoding_params['happy'] = 2
5547         self.fileurls = {}
5548         DATA = "data" * 100
5549         d = c0.create_dirnode()
5550         def _stash_root(n):
5551             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5552             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
5553             return n
5554         d.addCallback(_stash_root)
5555         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
5556         def _stash_bad(ur):
5557             self.fileurls["1share"] = "uri/" + urllib.quote(ur.get_uri())
5558             self.delete_shares_numbered(ur.get_uri(), range(1,10))
5559
5560             u = uri.from_string(ur.get_uri())
5561             u.key = testutil.flip_bit(u.key, 0)
5562             baduri = u.to_string()
5563             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
5564         d.addCallback(_stash_bad)
5565         d.addCallback(lambda ign: c0.create_dirnode())
5566         def _mangle_dirnode_1share(n):
5567             u = n.get_uri()
5568             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
5569             self.fileurls["dir-1share-json"] = url + "?t=json"
5570             self.delete_shares_numbered(u, range(1,10))
5571         d.addCallback(_mangle_dirnode_1share)
5572         d.addCallback(lambda ign: c0.create_dirnode())
5573         def _mangle_dirnode_0share(n):
5574             u = n.get_uri()
5575             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
5576             self.fileurls["dir-0share-json"] = url + "?t=json"
5577             self.delete_shares_numbered(u, range(0,10))
5578         d.addCallback(_mangle_dirnode_0share)
5579
5580         # NotEnoughSharesError should be reported sensibly, with a
5581         # text/plain explanation of the problem, and perhaps some
5582         # information on which shares *could* be found.
5583
5584         d.addCallback(lambda ignored:
5585                       self.shouldHTTPError("GET unrecoverable",
5586                                            410, "Gone", "NoSharesError",
5587                                            self.GET, self.fileurls["0shares"]))
5588         def _check_zero_shares(body):
5589             self.failIfIn("<html>", body)
5590             body = " ".join(body.strip().split())
5591             exp = ("NoSharesError: no shares could be found. "
5592                    "Zero shares usually indicates a corrupt URI, or that "
5593                    "no servers were connected, but it might also indicate "
5594                    "severe corruption. You should perform a filecheck on "
5595                    "this object to learn more. The full error message is: "
5596                    "no shares (need 3). Last failure: None")
5597             self.failUnlessReallyEqual(exp, body)
5598         d.addCallback(_check_zero_shares)
5599
5600
5601         d.addCallback(lambda ignored:
5602                       self.shouldHTTPError("GET 1share",
5603                                            410, "Gone", "NotEnoughSharesError",
5604                                            self.GET, self.fileurls["1share"]))
5605         def _check_one_share(body):
5606             self.failIfIn("<html>", body)
5607             body = " ".join(body.strip().split())
5608             msgbase = ("NotEnoughSharesError: This indicates that some "
5609                        "servers were unavailable, or that shares have been "
5610                        "lost to server departure, hard drive failure, or disk "
5611                        "corruption. You should perform a filecheck on "
5612                        "this object to learn more. The full error message is:"
5613                        )
5614             msg1 = msgbase + (" ran out of shares:"
5615                               " complete=sh0"
5616                               " pending="
5617                               " overdue= unused= need 3. Last failure: None")
5618             msg2 = msgbase + (" ran out of shares:"
5619                               " complete="
5620                               " pending=Share(sh0-on-xgru5)"
5621                               " overdue= unused= need 3. Last failure: None")
5622             self.failUnless(body == msg1 or body == msg2, body)
5623         d.addCallback(_check_one_share)
5624
5625         d.addCallback(lambda ignored:
5626                       self.shouldHTTPError("GET imaginary",
5627                                            404, "Not Found", None,
5628                                            self.GET, self.fileurls["imaginary"]))
5629         def _missing_child(body):
5630             self.failUnlessIn("No such child: imaginary", body)
5631         d.addCallback(_missing_child)
5632
5633         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
5634         def _check_0shares_dir_html(body):
5635             self.failUnlessIn(DIR_HTML_TAG, body)
5636             # we should see the regular page, but without the child table or
5637             # the dirops forms
5638             body = " ".join(body.strip().split())
5639             self.failUnlessIn('href="?t=info">More info on this directory',
5640                               body)
5641             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5642                    "could not be retrieved, because there were insufficient "
5643                    "good shares. This might indicate that no servers were "
5644                    "connected, insufficient servers were connected, the URI "
5645                    "was corrupt, or that shares have been lost due to server "
5646                    "departure, hard drive failure, or disk corruption. You "
5647                    "should perform a filecheck on this object to learn more.")
5648             self.failUnlessIn(exp, body)
5649             self.failUnlessIn("No upload forms: directory is unreadable", body)
5650         d.addCallback(_check_0shares_dir_html)
5651
5652         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
5653         def _check_1shares_dir_html(body):
5654             # at some point, we'll split UnrecoverableFileError into 0-shares
5655             # and some-shares like we did for immutable files (since there
5656             # are different sorts of advice to offer in each case). For now,
5657             # they present the same way.
5658             self.failUnlessIn(DIR_HTML_TAG, body)
5659             body = " ".join(body.strip().split())
5660             self.failUnlessIn('href="?t=info">More info on this directory',
5661                               body)
5662             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5663                    "could not be retrieved, because there were insufficient "
5664                    "good shares. This might indicate that no servers were "
5665                    "connected, insufficient servers were connected, the URI "
5666                    "was corrupt, or that shares have been lost due to server "
5667                    "departure, hard drive failure, or disk corruption. You "
5668                    "should perform a filecheck on this object to learn more.")
5669             self.failUnlessIn(exp, body)
5670             self.failUnlessIn("No upload forms: directory is unreadable", body)
5671         d.addCallback(_check_1shares_dir_html)
5672
5673         d.addCallback(lambda ignored:
5674                       self.shouldHTTPError("GET dir-0share-json",
5675                                            410, "Gone", "UnrecoverableFileError",
5676                                            self.GET,
5677                                            self.fileurls["dir-0share-json"]))
5678         def _check_unrecoverable_file(body):
5679             self.failIfIn("<html>", body)
5680             body = " ".join(body.strip().split())
5681             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5682                    "could not be retrieved, because there were insufficient "
5683                    "good shares. This might indicate that no servers were "
5684                    "connected, insufficient servers were connected, the URI "
5685                    "was corrupt, or that shares have been lost due to server "
5686                    "departure, hard drive failure, or disk corruption. You "
5687                    "should perform a filecheck on this object to learn more.")
5688             self.failUnlessReallyEqual(exp, body)
5689         d.addCallback(_check_unrecoverable_file)
5690
5691         d.addCallback(lambda ignored:
5692                       self.shouldHTTPError("GET dir-1share-json",
5693                                            410, "Gone", "UnrecoverableFileError",
5694                                            self.GET,
5695                                            self.fileurls["dir-1share-json"]))
5696         d.addCallback(_check_unrecoverable_file)
5697
5698         d.addCallback(lambda ignored:
5699                       self.shouldHTTPError("GET imaginary",
5700                                            404, "Not Found", None,
5701                                            self.GET, self.fileurls["imaginary"]))
5702
5703         # attach a webapi child that throws a random error, to test how it
5704         # gets rendered.
5705         w = c0.getServiceNamed("webish")
5706         w.root.putChild("ERRORBOOM", ErrorBoom())
5707
5708         # "Accept: */*" :        should get a text/html stack trace
5709         # "Accept: text/plain" : should get a text/plain stack trace
5710         # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
5711         # no Accept header:      should get a text/html stack trace
5712
5713         d.addCallback(lambda ignored:
5714                       self.shouldHTTPError("GET errorboom_html",
5715                                            500, "Internal Server Error", None,
5716                                            self.GET, "ERRORBOOM",
5717                                            headers={"accept": "*/*"}))
5718         def _internal_error_html1(body):
5719             self.failUnlessIn("<html>", "expected HTML, not '%s'" % body)
5720         d.addCallback(_internal_error_html1)
5721
5722         d.addCallback(lambda ignored:
5723                       self.shouldHTTPError("GET errorboom_text",
5724                                            500, "Internal Server Error", None,
5725                                            self.GET, "ERRORBOOM",
5726                                            headers={"accept": "text/plain"}))
5727         def _internal_error_text2(body):
5728             self.failIfIn("<html>", body)
5729             self.failUnless(body.startswith("Traceback "), body)
5730         d.addCallback(_internal_error_text2)
5731
5732         CLI_accepts = "text/plain, application/octet-stream"
5733         d.addCallback(lambda ignored:
5734                       self.shouldHTTPError("GET errorboom_text",
5735                                            500, "Internal Server Error", None,
5736                                            self.GET, "ERRORBOOM",
5737                                            headers={"accept": CLI_accepts}))
5738         def _internal_error_text3(body):
5739             self.failIfIn("<html>", body)
5740             self.failUnless(body.startswith("Traceback "), body)
5741         d.addCallback(_internal_error_text3)
5742
5743         d.addCallback(lambda ignored:
5744                       self.shouldHTTPError("GET errorboom_text",
5745                                            500, "Internal Server Error", None,
5746                                            self.GET, "ERRORBOOM"))
5747         def _internal_error_html4(body):
5748             self.failUnlessIn("<html>", body)
5749         d.addCallback(_internal_error_html4)
5750
5751         def _flush_errors(res):
5752             # Trial: please ignore the CompletelyUnhandledError in the logs
5753             self.flushLoggedErrors(CompletelyUnhandledError)
5754             return res
5755         d.addBoth(_flush_errors)
5756
5757         return d
5758
5759     def test_blacklist(self):
5760         # download from a blacklisted URI, get an error
5761         self.basedir = "web/Grid/blacklist"
5762         self.set_up_grid()
5763         c0 = self.g.clients[0]
5764         c0_basedir = c0.basedir
5765         fn = os.path.join(c0_basedir, "access.blacklist")
5766         self.uris = {}
5767         DATA = "off-limits " * 50
5768
5769         d = c0.upload(upload.Data(DATA, convergence=""))
5770         def _stash_uri_and_create_dir(ur):
5771             self.uri = ur.get_uri()
5772             self.url = "uri/"+self.uri
5773             u = uri.from_string_filenode(self.uri)
5774             self.si = u.get_storage_index()
5775             childnode = c0.create_node_from_uri(self.uri, None)
5776             return c0.create_dirnode({u"blacklisted.txt": (childnode,{}) })
5777         d.addCallback(_stash_uri_and_create_dir)
5778         def _stash_dir(node):
5779             self.dir_node = node
5780             self.dir_uri = node.get_uri()
5781             self.dir_url = "uri/"+self.dir_uri
5782         d.addCallback(_stash_dir)
5783         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5784         def _check_dir_html(body):
5785             self.failUnlessIn(DIR_HTML_TAG, body)
5786             self.failUnlessIn("blacklisted.txt</a>", body)
5787         d.addCallback(_check_dir_html)
5788         d.addCallback(lambda ign: self.GET(self.url))
5789         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5790
5791         def _blacklist(ign):
5792             f = open(fn, "w")
5793             f.write(" # this is a comment\n")
5794             f.write(" \n")
5795             f.write("\n") # also exercise blank lines
5796             f.write("%s %s\n" % (base32.b2a(self.si), "off-limits to you"))
5797             f.close()
5798             # clients should be checking the blacklist each time, so we don't
5799             # need to restart the client
5800         d.addCallback(_blacklist)
5801         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_uri",
5802                                                        403, "Forbidden",
5803                                                        "Access Prohibited: off-limits",
5804                                                        self.GET, self.url))
5805
5806         # We should still be able to list the parent directory, in HTML...
5807         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5808         def _check_dir_html2(body):
5809             self.failUnlessIn(DIR_HTML_TAG, body)
5810             self.failUnlessIn("blacklisted.txt</strike>", body)
5811         d.addCallback(_check_dir_html2)
5812
5813         # ... and in JSON (used by CLI).
5814         d.addCallback(lambda ign: self.GET(self.dir_url+"?t=json", followRedirect=True))
5815         def _check_dir_json(res):
5816             data = simplejson.loads(res)
5817             self.failUnless(isinstance(data, list), data)
5818             self.failUnlessEqual(data[0], "dirnode")
5819             self.failUnless(isinstance(data[1], dict), data)
5820             self.failUnlessIn("children", data[1])
5821             self.failUnlessIn("blacklisted.txt", data[1]["children"])
5822             childdata = data[1]["children"]["blacklisted.txt"]
5823             self.failUnless(isinstance(childdata, list), data)
5824             self.failUnlessEqual(childdata[0], "filenode")
5825             self.failUnless(isinstance(childdata[1], dict), data)
5826         d.addCallback(_check_dir_json)
5827
5828         def _unblacklist(ign):
5829             open(fn, "w").close()
5830             # the Blacklist object watches mtime to tell when the file has
5831             # changed, but on windows this test will run faster than the
5832             # filesystem's mtime resolution. So we edit Blacklist.last_mtime
5833             # to force a reload.
5834             self.g.clients[0].blacklist.last_mtime -= 2.0
5835         d.addCallback(_unblacklist)
5836
5837         # now a read should work
5838         d.addCallback(lambda ign: self.GET(self.url))
5839         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5840
5841         # read again to exercise the blacklist-is-unchanged logic
5842         d.addCallback(lambda ign: self.GET(self.url))
5843         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5844
5845         # now add a blacklisted directory, and make sure files under it are
5846         # refused too
5847         def _add_dir(ign):
5848             childnode = c0.create_node_from_uri(self.uri, None)
5849             return c0.create_dirnode({u"child": (childnode,{}) })
5850         d.addCallback(_add_dir)
5851         def _get_dircap(dn):
5852             self.dir_si_b32 = base32.b2a(dn.get_storage_index())
5853             self.dir_url_base = "uri/"+dn.get_write_uri()
5854             self.dir_url_json1 = "uri/"+dn.get_write_uri()+"?t=json"
5855             self.dir_url_json2 = "uri/"+dn.get_write_uri()+"/?t=json"
5856             self.dir_url_json_ro = "uri/"+dn.get_readonly_uri()+"/?t=json"
5857             self.child_url = "uri/"+dn.get_readonly_uri()+"/child"
5858         d.addCallback(_get_dircap)
5859         d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True))
5860         d.addCallback(lambda body: self.failUnlessIn(DIR_HTML_TAG, body))
5861         d.addCallback(lambda ign: self.GET(self.dir_url_json1))
5862         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5863         d.addCallback(lambda ign: self.GET(self.dir_url_json2))
5864         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5865         d.addCallback(lambda ign: self.GET(self.dir_url_json_ro))
5866         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5867         d.addCallback(lambda ign: self.GET(self.child_url))
5868         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5869
5870         def _block_dir(ign):
5871             f = open(fn, "w")
5872             f.write("%s %s\n" % (self.dir_si_b32, "dir-off-limits to you"))
5873             f.close()
5874             self.g.clients[0].blacklist.last_mtime -= 2.0
5875         d.addCallback(_block_dir)
5876         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir base",
5877                                                        403, "Forbidden",
5878                                                        "Access Prohibited: dir-off-limits",
5879                                                        self.GET, self.dir_url_base))
5880         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json1",
5881                                                        403, "Forbidden",
5882                                                        "Access Prohibited: dir-off-limits",
5883                                                        self.GET, self.dir_url_json1))
5884         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json2",
5885                                                        403, "Forbidden",
5886                                                        "Access Prohibited: dir-off-limits",
5887                                                        self.GET, self.dir_url_json2))
5888         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json_ro",
5889                                                        403, "Forbidden",
5890                                                        "Access Prohibited: dir-off-limits",
5891                                                        self.GET, self.dir_url_json_ro))
5892         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir child",
5893                                                        403, "Forbidden",
5894                                                        "Access Prohibited: dir-off-limits",
5895                                                        self.GET, self.child_url))
5896         return d
5897
5898
5899 class CompletelyUnhandledError(Exception):
5900     pass
5901 class ErrorBoom(rend.Page):
5902     def beforeRender(self, ctx):
5903         raise CompletelyUnhandledError("whoops")