src/allmydata/test/test_web.py

   1 import os.path, re, urllib, time, cgi
   2 import simplejson
   3 from StringIO import StringIO
   4
   5 from twisted.application import service
   6 from twisted.trial import unittest
   7 from twisted.internet import defer, reactor
   8 from twisted.internet.task import Clock
   9 from twisted.web import client, error, http
  10 from twisted.python import failure, log
  11
  12 from foolscap.api import fireEventually, flushEventualQueue
  13
  14 from nevow.util import escapeToXML
  15 from nevow import rend
  16
  17 from allmydata import interfaces, uri, webish, dirnode
  18 from allmydata.storage.shares import get_share_file
  19 from allmydata.storage_client import StorageFarmBroker, StubServer
  20 from allmydata.immutable import upload
  21 from allmydata.immutable.downloader.status import DownloadStatus
  22 from allmydata.dirnode import DirectoryNode
  23 from allmydata.nodemaker import NodeMaker
  24 from allmydata.unknown import UnknownNode
  25 from allmydata.web import status, common
  26 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  27 from allmydata.util import fileutil, base32, hashutil
  28 from allmydata.util.consumer import download_to_data
  29 from allmydata.util.netstring import split_netstring
  30 from allmydata.util.encodingutil import to_str
  31 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  32      create_chk_filenode, WebErrorMixin, ShouldFailMixin, \
  33      make_mutable_file_uri, create_mutable_filenode
  34 from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
  35 from allmydata.mutable import servermap, publish, retrieve
  36 import allmydata.test.common_util as testutil
  37 from allmydata.test.no_network import GridTestMixin
  38 from allmydata.test.common_web import HTTPClientGETFactory, \
  39      HTTPClientHEADFactory
  40 from allmydata.client import Client, SecretHolder
  41 from allmydata.introducer import IntroducerNode
  42
  43 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  44 # create a webserver that works against them
  45
  46 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  47
  48 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
  49 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
  50 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
  51
  52 FAVICON_MARKUP = '<link href="/icon.png" rel="shortcut icon" />'
  53 DIR_HTML_TAG = '<html lang="en">'
  54
  55 class FakeStatsProvider:
  56     def get_stats(self):
  57         stats = {'stats': {}, 'counters': {}}
  58         return stats
  59
  60 class FakeNodeMaker(NodeMaker):
  61     encoding_params = {
  62         'k': 3,
  63         'n': 10,
  64         'happy': 7,
  65         'max_segment_size':128*1024 # 1024=KiB
  66     }
  67     def _create_lit(self, cap):
  68         return FakeCHKFileNode(cap, self.all_contents)
  69     def _create_immutable(self, cap):
  70         return FakeCHKFileNode(cap, self.all_contents)
  71     def _create_mutable(self, cap):
  72         return FakeMutableFileNode(None, None,
  73                                    self.encoding_params, None,
  74                                    self.all_contents).init_from_cap(cap)
  75     def create_mutable_file(self, contents="", keysize=None,
  76                             version=SDMF_VERSION):
  77         n = FakeMutableFileNode(None, None, self.encoding_params, None,
  78                                 self.all_contents)
  79         return n.create(contents, version=version)
  80
  81 class FakeUploader(service.Service):
  82     name = "uploader"
  83     helper_furl = None
  84     helper_connected = False
  85
  86     def upload(self, uploadable):
  87         d = uploadable.get_size()
  88         d.addCallback(lambda size: uploadable.read(size))
  89         def _got_data(datav):
  90             data = "".join(datav)
  91             n = create_chk_filenode(data, self.all_contents)
  92             ur = upload.UploadResults(file_size=len(data),
  93                                       ciphertext_fetched=0,
  94                                       preexisting_shares=0,
  95                                       pushed_shares=10,
  96                                       sharemap={},
  97                                       servermap={},
  98                                       timings={},
  99                                       uri_extension_data={},
 100                                       uri_extension_hash="fake",
 101                                       verifycapstr="fakevcap")
 102             ur.set_uri(n.get_uri())
 103             return ur
 104         d.addCallback(_got_data)
 105         return d
 106
 107     def get_helper_info(self):
 108         return (self.helper_furl, self.helper_connected)
 109
 110
 111 def build_one_ds():
 112     ds = DownloadStatus("storage_index", 1234)
 113     now = time.time()
 114
 115     serverA = StubServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
 116     serverB = StubServer(hashutil.tagged_hash("foo", "serverid_b")[:20])
 117     storage_index = hashutil.storage_index_hash("SI")
 118     e0 = ds.add_segment_request(0, now)
 119     e0.activate(now+0.5)
 120     e0.deliver(now+1, 0, 100, 0.5) # when, start,len, decodetime
 121     e1 = ds.add_segment_request(1, now+2)
 122     e1.error(now+3)
 123     # two outstanding requests
 124     e2 = ds.add_segment_request(2, now+4)
 125     e3 = ds.add_segment_request(3, now+5)
 126     del e2,e3 # hush pyflakes
 127
 128     # simulate a segment which gets delivered faster than a system clock tick (ticket #1166)
 129     e = ds.add_segment_request(4, now)
 130     e.activate(now)
 131     e.deliver(now, 0, 140, 0.5)
 132
 133     e = ds.add_dyhb_request(serverA, now)
 134     e.finished([1,2], now+1)
 135     e = ds.add_dyhb_request(serverB, now+2) # left unfinished
 136
 137     e = ds.add_read_event(0, 120, now)
 138     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 139     e.finished(now+1)
 140     e = ds.add_read_event(120, 30, now+2) # left unfinished
 141
 142     e = ds.add_block_request(serverA, 1, 100, 20, now)
 143     e.finished(20, now+1)
 144     e = ds.add_block_request(serverB, 1, 120, 30, now+1) # left unfinished
 145
 146     # make sure that add_read_event() can come first too
 147     ds1 = DownloadStatus(storage_index, 1234)
 148     e = ds1.add_read_event(0, 120, now)
 149     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 150     e.finished(now+1)
 151
 152     return ds
 153
 154 class FakeHistory:
 155     _all_upload_status = [upload.UploadStatus()]
 156     _all_download_status = [build_one_ds()]
 157     _all_mapupdate_statuses = [servermap.UpdateStatus()]
 158     _all_publish_statuses = [publish.PublishStatus()]
 159     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
 160
 161     def list_all_upload_statuses(self):
 162         return self._all_upload_status
 163     def list_all_download_statuses(self):
 164         return self._all_download_status
 165     def list_all_mapupdate_statuses(self):
 166         return self._all_mapupdate_statuses
 167     def list_all_publish_statuses(self):
 168         return self._all_publish_statuses
 169     def list_all_retrieve_statuses(self):
 170         return self._all_retrieve_statuses
 171     def list_all_helper_statuses(self):
 172         return []
 173
 174 class FakeDisplayableServer(StubServer):
 175     def __init__(self, serverid, nickname, connected,
 176                  last_connect_time, last_loss_time, last_rx_time):
 177         StubServer.__init__(self, serverid)
 178         self.announcement = {"my-version": "allmydata-tahoe-fake",
 179                              "service-name": "storage",
 180                              "nickname": nickname}
 181         self.connected = connected
 182         self.last_loss_time = last_loss_time
 183         self.last_rx_time = last_rx_time
 184         self.last_connect_time = last_connect_time
 185     def is_connected(self):
 186         return self.connected
 187     def get_permutation_seed(self):
 188         return ""
 189     def get_remote_host(self):
 190         return ""
 191     def get_last_loss_time(self):
 192         return self.last_loss_time
 193     def get_last_received_data_time(self):
 194         return self.last_rx_time
 195     def get_last_connect_time(self):
 196         return self.last_connect_time
 197     def get_announcement(self):
 198         return self.announcement
 199     def get_nickname(self):
 200         return self.announcement["nickname"]
 201     def get_available_space(self):
 202         return 123456
 203
 204 class FakeBucketCounter(object):
 205     def get_state(self):
 206         return {"last-complete-bucket-count": 0}
 207     def get_progress(self):
 208         return {"estimated-time-per-cycle": 0,
 209                 "cycle-in-progress": False,
 210                 "remaining-wait-time": 0}
 211
 212 class FakeLeaseChecker(object):
 213     def __init__(self):
 214         self.expiration_enabled = False
 215         self.mode = "age"
 216         self.override_lease_duration = None
 217         self.sharetypes_to_expire = {}
 218     def get_state(self):
 219         return {"history": None}
 220     def get_progress(self):
 221         return {"estimated-time-per-cycle": 0,
 222                 "cycle-in-progress": False,
 223                 "remaining-wait-time": 0}
 224
 225 class FakeStorageServer(service.MultiService):
 226     name = 'storage'
 227     def __init__(self, nodeid, nickname):
 228         service.MultiService.__init__(self)
 229         self.my_nodeid = nodeid
 230         self.nickname = nickname
 231         self.bucket_counter = FakeBucketCounter()
 232         self.lease_checker = FakeLeaseChecker()
 233     def get_stats(self):
 234         return {"storage_server.accepting_immutable_shares": False}
 235
 236 class FakeClient(Client):
 237     def __init__(self):
 238         # don't upcall to Client.__init__, since we only want to initialize a
 239         # minimal subset
 240         service.MultiService.__init__(self)
 241         self.all_contents = {}
 242         self.nodeid = "fake_nodeid"
 243         self.nickname = u"fake_nickname \u263A"
 244         self.introducer_furl = "None"
 245         self.stats_provider = FakeStatsProvider()
 246         self._secret_holder = SecretHolder("lease secret", "convergence secret")
 247         self.helper = None
 248         self.convergence = "some random string"
 249         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 250         # fake knowledge of another server
 251         self.storage_broker.test_add_server("other_nodeid",
 252             FakeDisplayableServer(
 253                 serverid="other_nodeid", nickname=u"other_nickname \u263B", connected = True,
 254                 last_connect_time = 10, last_loss_time = 20, last_rx_time = 30))
 255         self.storage_broker.test_add_server("disconnected_nodeid",
 256             FakeDisplayableServer(
 257                 serverid="other_nodeid", nickname=u"disconnected_nickname \u263B", connected = False,
 258                 last_connect_time = 15, last_loss_time = 25, last_rx_time = 35))
 259         self.introducer_client = None
 260         self.history = FakeHistory()
 261         self.uploader = FakeUploader()
 262         self.uploader.all_contents = self.all_contents
 263         self.uploader.setServiceParent(self)
 264         self.blacklist = None
 265         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 266                                        self.uploader, None,
 267                                        None, None, None)
 268         self.nodemaker.all_contents = self.all_contents
 269         self.mutable_file_default = SDMF_VERSION
 270         self.addService(FakeStorageServer(self.nodeid, self.nickname))
 271
 272     def get_long_nodeid(self):
 273         return "v0-nodeid"
 274     def get_long_tubid(self):
 275         return "tubid"
 276
 277     def startService(self):
 278         return service.MultiService.startService(self)
 279     def stopService(self):
 280         return service.MultiService.stopService(self)
 281
 282     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 283
 284 class WebMixin(testutil.TimezoneMixin):
 285     def setUp(self):
 286         self.setTimezone('UTC-13:00')
 287         self.s = FakeClient()
 288         self.s.startService()
 289         self.staticdir = self.mktemp()
 290         self.clock = Clock()
 291         self.fakeTime = 86460 # 1d 0h 1m 0s
 292         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
 293                                       clock=self.clock, now_fn=lambda:self.fakeTime)
 294         self.ws.setServiceParent(self.s)
 295         self.webish_port = self.ws.getPortnum()
 296         self.webish_url = self.ws.getURL()
 297         assert self.webish_url.endswith("/")
 298         self.webish_url = self.webish_url[:-1] # these tests add their own /
 299
 300         l = [ self.s.create_dirnode() for x in range(6) ]
 301         d = defer.DeferredList(l)
 302         def _then(res):
 303             self.public_root = res[0][1]
 304             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 305             self.public_url = "/uri/" + self.public_root.get_uri()
 306             self.private_root = res[1][1]
 307
 308             foo = res[2][1]
 309             self._foo_node = foo
 310             self._foo_uri = foo.get_uri()
 311             self._foo_readonly_uri = foo.get_readonly_uri()
 312             self._foo_verifycap = foo.get_verify_cap().to_string()
 313             # NOTE: we ignore the deferred on all set_uri() calls, because we
 314             # know the fake nodes do these synchronously
 315             self.public_root.set_uri(u"foo", foo.get_uri(),
 316                                      foo.get_readonly_uri())
 317
 318             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 319             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 320             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 321
 322             # sdmf
 323             # XXX: Do we ever use this?
 324             self.BAZ_CONTENTS, n, self._baz_txt_uri, self._baz_txt_readonly_uri = self.makefile_mutable(0)
 325
 326             foo.set_uri(u"baz.txt", self._baz_txt_uri, self._baz_txt_readonly_uri)
 327
 328             # mdmf
 329             self.QUUX_CONTENTS, n, self._quux_txt_uri, self._quux_txt_readonly_uri = self.makefile_mutable(0, mdmf=True)
 330             assert self._quux_txt_uri.startswith("URI:MDMF")
 331             foo.set_uri(u"quux.txt", self._quux_txt_uri, self._quux_txt_readonly_uri)
 332
 333             foo.set_uri(u"empty", res[3][1].get_uri(),
 334                         res[3][1].get_readonly_uri())
 335             sub_uri = res[4][1].get_uri()
 336             self._sub_uri = sub_uri
 337             foo.set_uri(u"sub", sub_uri, sub_uri)
 338             sub = self.s.create_node_from_uri(sub_uri)
 339             self._sub_node = sub
 340
 341             _ign, n, blocking_uri = self.makefile(1)
 342             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 343
 344             # filenode to test for html encoding issues
 345             self._htmlname_unicode = u"<&weirdly'named\"file>>>_<iframe />.txt"
 346             self._htmlname_raw = self._htmlname_unicode.encode('utf-8')
 347             self._htmlname_urlencoded = urllib.quote(self._htmlname_raw, '')
 348             self._htmlname_escaped = escapeToXML(self._htmlname_raw)
 349             self._htmlname_escaped_attr = cgi.escape(self._htmlname_raw, quote=True)
 350             self._htmlname_escaped_double = escapeToXML(cgi.escape(self._htmlname_raw, quote=True))
 351             self.HTMLNAME_CONTENTS, n, self._htmlname_txt_uri = self.makefile(0)
 352             foo.set_uri(self._htmlname_unicode, self._htmlname_txt_uri, self._htmlname_txt_uri)
 353
 354             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 355             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 356             # still think of it as an umlaut
 357             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 358
 359             self.SUBBAZ_CONTENTS, n, baz_file = self.makefile(2)
 360             self._baz_file_uri = baz_file
 361             sub.set_uri(u"baz.txt", baz_file, baz_file)
 362
 363             _ign, n, self._bad_file_uri = self.makefile(3)
 364             # this uri should not be downloadable
 365             del self.s.all_contents[self._bad_file_uri]
 366
 367             rodir = res[5][1]
 368             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 369                                      rodir.get_readonly_uri())
 370             rodir.set_uri(u"nor", baz_file, baz_file)
 371
 372             # public/
 373             # public/foo/
 374             # public/foo/bar.txt
 375             # public/foo/baz.txt
 376             # public/foo/quux.txt
 377             # public/foo/blockingfile
 378             # public/foo/<&weirdly'named\"file>>>_<iframe />.txt
 379             # public/foo/empty/
 380             # public/foo/sub/
 381             # public/foo/sub/baz.txt
 382             # public/reedownlee/
 383             # public/reedownlee/nor
 384             self.NEWFILE_CONTENTS = "newfile contents\n"
 385
 386             return foo.get_metadata_for(u"bar.txt")
 387         d.addCallback(_then)
 388         def _got_metadata(metadata):
 389             self._bar_txt_metadata = metadata
 390         d.addCallback(_got_metadata)
 391         return d
 392
 393     def get_all_contents(self):
 394         return self.s.all_contents
 395
 396     def makefile(self, number):
 397         contents = "contents of file %s\n" % number
 398         n = create_chk_filenode(contents, self.get_all_contents())
 399         return contents, n, n.get_uri()
 400
 401     def makefile_mutable(self, number, mdmf=False):
 402         contents = "contents of mutable file %s\n" % number
 403         n = create_mutable_filenode(contents, mdmf, self.s.all_contents)
 404         return contents, n, n.get_uri(), n.get_readonly_uri()
 405
 406     def tearDown(self):
 407         return self.s.stopService()
 408
 409     def failUnlessIsBarDotTxt(self, res):
 410         self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
 411
 412     def failUnlessIsQuuxDotTxt(self, res):
 413         self.failUnlessReallyEqual(res, self.QUUX_CONTENTS, res)
 414
 415     def failUnlessIsBazDotTxt(self, res):
 416         self.failUnlessReallyEqual(res, self.BAZ_CONTENTS, res)
 417
 418     def failUnlessIsSubBazDotTxt(self, res):
 419         self.failUnlessReallyEqual(res, self.SUBBAZ_CONTENTS, res)
 420
 421     def failUnlessIsBarJSON(self, res):
 422         data = simplejson.loads(res)
 423         self.failUnless(isinstance(data, list))
 424         self.failUnlessEqual(data[0], "filenode")
 425         self.failUnless(isinstance(data[1], dict))
 426         self.failIf(data[1]["mutable"])
 427         self.failIfIn("rw_uri", data[1]) # immutable
 428         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
 429         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
 430         self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
 431
 432     def failUnlessIsQuuxJSON(self, res, readonly=False):
 433         data = simplejson.loads(res)
 434         self.failUnless(isinstance(data, list))
 435         self.failUnlessEqual(data[0], "filenode")
 436         self.failUnless(isinstance(data[1], dict))
 437         metadata = data[1]
 438         return self.failUnlessIsQuuxDotTxtMetadata(metadata, readonly)
 439
 440     def failUnlessIsQuuxDotTxtMetadata(self, metadata, readonly):
 441         self.failUnless(metadata['mutable'])
 442         if readonly:
 443             self.failIfIn("rw_uri", metadata)
 444         else:
 445             self.failUnlessIn("rw_uri", metadata)
 446             self.failUnlessEqual(metadata['rw_uri'], self._quux_txt_uri)
 447         self.failUnlessIn("ro_uri", metadata)
 448         self.failUnlessEqual(metadata['ro_uri'], self._quux_txt_readonly_uri)
 449         self.failUnlessReallyEqual(metadata['size'], len(self.QUUX_CONTENTS))
 450
 451     def failUnlessIsFooJSON(self, res):
 452         data = simplejson.loads(res)
 453         self.failUnless(isinstance(data, list))
 454         self.failUnlessEqual(data[0], "dirnode", res)
 455         self.failUnless(isinstance(data[1], dict))
 456         self.failUnless(data[1]["mutable"])
 457         self.failUnlessIn("rw_uri", data[1]) # mutable
 458         self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
 459         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
 460         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
 461
 462         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 463         self.failUnlessEqual(kidnames,
 464                              [self._htmlname_unicode, u"bar.txt", u"baz.txt",
 465                               u"blockingfile", u"empty", u"n\u00fc.txt", u"quux.txt", u"sub"])
 466         kids = dict( [(unicode(name),value)
 467                       for (name,value)
 468                       in data[1]["children"].iteritems()] )
 469         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 470         self.failUnlessIn("metadata", kids[u"sub"][1])
 471         self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
 472         tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
 473         self.failUnlessIn("linkcrtime", tahoe_md)
 474         self.failUnlessIn("linkmotime", tahoe_md)
 475         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 476         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 477         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri)
 478         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]),
 479                                    self._bar_txt_verifycap)
 480         self.failUnlessIn("metadata", kids[u"bar.txt"][1])
 481         self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
 482         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
 483                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
 484         self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
 485                                    self._bar_txt_uri)
 486         self.failUnlessIn("quux.txt", kids)
 487         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["rw_uri"]),
 488                                    self._quux_txt_uri)
 489         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["ro_uri"]),
 490                                    self._quux_txt_readonly_uri)
 491
 492     def GET(self, urlpath, followRedirect=False, return_response=False,
 493             **kwargs):
 494         # if return_response=True, this fires with (data, statuscode,
 495         # respheaders) instead of just data.
 496         assert not isinstance(urlpath, unicode)
 497         url = self.webish_url + urlpath
 498         factory = HTTPClientGETFactory(url, method="GET",
 499                                        followRedirect=followRedirect, **kwargs)
 500         reactor.connectTCP("localhost", self.webish_port, factory)
 501         d = factory.deferred
 502         def _got_data(data):
 503             return (data, factory.status, factory.response_headers)
 504         if return_response:
 505             d.addCallback(_got_data)
 506         return factory.deferred
 507
 508     def HEAD(self, urlpath, return_response=False, **kwargs):
 509         # this requires some surgery, because twisted.web.client doesn't want
 510         # to give us back the response headers.
 511         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 512         reactor.connectTCP("localhost", self.webish_port, factory)
 513         d = factory.deferred
 514         def _got_data(data):
 515             return (data, factory.status, factory.response_headers)
 516         if return_response:
 517             d.addCallback(_got_data)
 518         return factory.deferred
 519
 520     def PUT(self, urlpath, data, **kwargs):
 521         url = self.webish_url + urlpath
 522         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 523
 524     def DELETE(self, urlpath):
 525         url = self.webish_url + urlpath
 526         return client.getPage(url, method="DELETE")
 527
 528     def POST(self, urlpath, followRedirect=False, **fields):
 529         sepbase = "boogabooga"
 530         sep = "--" + sepbase
 531         form = []
 532         form.append(sep)
 533         form.append('Content-Disposition: form-data; name="_charset"')
 534         form.append('')
 535         form.append('UTF-8')
 536         form.append(sep)
 537         for name, value in fields.iteritems():
 538             if isinstance(value, tuple):
 539                 filename, value = value
 540                 form.append('Content-Disposition: form-data; name="%s"; '
 541                             'filename="%s"' % (name, filename.encode("utf-8")))
 542             else:
 543                 form.append('Content-Disposition: form-data; name="%s"' % name)
 544             form.append('')
 545             if isinstance(value, unicode):
 546                 value = value.encode("utf-8")
 547             else:
 548                 value = str(value)
 549             assert isinstance(value, str)
 550             form.append(value)
 551             form.append(sep)
 552         form[-1] += "--"
 553         body = ""
 554         headers = {}
 555         if fields:
 556             body = "\r\n".join(form) + "\r\n"
 557             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 558         return self.POST2(urlpath, body, headers, followRedirect)
 559
 560     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 561         url = self.webish_url + urlpath
 562         return client.getPage(url, method="POST", postdata=body,
 563                               headers=headers, followRedirect=followRedirect)
 564
 565     def shouldFail(self, res, expected_failure, which,
 566                    substring=None, response_substring=None):
 567         if isinstance(res, failure.Failure):
 568             res.trap(expected_failure)
 569             if substring:
 570                 self.failUnlessIn(substring, str(res), which)
 571             if response_substring:
 572                 self.failUnlessIn(response_substring, res.value.response, which)
 573         else:
 574             self.fail("%s was supposed to raise %s, not get '%s'" %
 575                       (which, expected_failure, res))
 576
 577     def shouldFail2(self, expected_failure, which, substring,
 578                     response_substring,
 579                     callable, *args, **kwargs):
 580         assert substring is None or isinstance(substring, str)
 581         assert response_substring is None or isinstance(response_substring, str)
 582         d = defer.maybeDeferred(callable, *args, **kwargs)
 583         def done(res):
 584             if isinstance(res, failure.Failure):
 585                 res.trap(expected_failure)
 586                 if substring:
 587                     self.failUnlessIn(substring, str(res),
 588                                       "'%s' not in '%s' (response is '%s') for test '%s'" % \
 589                                       (substring, str(res),
 590                                        getattr(res.value, "response", ""),
 591                                        which))
 592                 if response_substring:
 593                     self.failUnlessIn(response_substring, res.value.response,
 594                                       "'%s' not in '%s' for test '%s'" % \
 595                                       (response_substring, res.value.response,
 596                                        which))
 597             else:
 598                 self.fail("%s was supposed to raise %s, not get '%s'" %
 599                           (which, expected_failure, res))
 600         d.addBoth(done)
 601         return d
 602
 603     def should404(self, res, which):
 604         if isinstance(res, failure.Failure):
 605             res.trap(error.Error)
 606             self.failUnlessReallyEqual(res.value.status, "404")
 607         else:
 608             self.fail("%s was supposed to Error(404), not get '%s'" %
 609                       (which, res))
 610
 611     def should302(self, res, which):
 612         if isinstance(res, failure.Failure):
 613             res.trap(error.Error)
 614             self.failUnlessReallyEqual(res.value.status, "302")
 615         else:
 616             self.fail("%s was supposed to Error(302), not get '%s'" %
 617                       (which, res))
 618
 619 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
 620     def test_create(self):
 621         pass
 622
 623     def test_welcome(self):
 624         d = self.GET("/")
 625         def _check(res):
 626             self.failUnlessIn('<title>Tahoe-LAFS - Welcome</title>', res)
 627             self.failUnlessIn(FAVICON_MARKUP, res)
 628             self.failUnlessIn('<a href="status">Recent and Active Operations</a>', res)
 629             self.failUnlessIn('<a href="statistics">Operational Statistics</a>', res)
 630             self.failUnless(re.search('<input (type="hidden" |name="t" |value="report-incident" ){3}/>',res), res)
 631             self.failUnlessIn('Page rendered at', res)
 632             self.failUnlessIn('Tahoe-LAFS code imported from:', res)
 633             res_u = res.decode('utf-8')
 634             self.failUnlessIn(u'<td>fake_nickname \u263A</td>', res_u)
 635             self.failUnlessIn(u'<div class="nickname">other_nickname \u263B</div>', res_u)
 636             self.failUnlessIn(u'Connected to <span>1</span>\n              of <span>2</span> known storage servers', res_u)
 637             self.failUnless(re.search(u'<div class="status-indicator"><img (src="img/connected-yes.png" |alt="Connected" ){2}/></div>\n                <a( class="timestamp"| title="1970-01-01 13:00:10"){2}>1d\u00A00h\u00A00m\u00A050s</a>', res_u), res_u)
 638             self.failUnless(re.search(u'<div class="status-indicator"><img (src="img/connected-no.png" |alt="Disconnected" ){2}/></div>\n                <a( class="timestamp"| title="1970-01-01 13:00:25"){2}>1d\u00A00h\u00A00m\u00A035s</a>', res_u), res_u)
 639             self.failUnless(re.search(u'<td class="service-last-received-data"><a( class="timestamp"| title="1970-01-01 13:00:30"){2}>1d\u00A00h\u00A00m\u00A030s</a></td>', res_u), res_u)
 640             self.failUnless(re.search(u'<td class="service-last-received-data"><a( class="timestamp"| title="1970-01-01 13:00:35"){2}>1d\u00A00h\u00A00m\u00A025s</a></td>', res_u), res_u)
 641             self.failUnlessIn(u'\u00A9 <a href="https://tahoe-lafs.org/">Tahoe-LAFS Software Foundation', res_u)
 642             self.failUnlessIn('<td><h3>Available</h3></td>', res)
 643             self.failUnlessIn('123.5kB', res)
 644
 645             self.s.basedir = 'web/test_welcome'
 646             fileutil.make_dirs("web/test_welcome")
 647             fileutil.make_dirs("web/test_welcome/private")
 648             return self.GET("/")
 649         d.addCallback(_check)
 650         return d
 651
 652     def test_introducer_status(self):
 653         class MockIntroducerClient(object):
 654             def __init__(self, connected):
 655                 self.connected = connected
 656             def connected_to_introducer(self):
 657                 return self.connected
 658
 659         d = defer.succeed(None)
 660
 661         # introducer not connected, unguessable furl
 662         def _set_introducer_not_connected_unguessable(ign):
 663             self.s.introducer_furl = "pb://someIntroducer/secret"
 664             self.s.introducer_client = MockIntroducerClient(False)
 665             return self.GET("/")
 666         d.addCallback(_set_introducer_not_connected_unguessable)
 667         def _check_introducer_not_connected_unguessable(res):
 668             html = res.replace('\n', ' ')
 669             self.failUnlessIn('<div class="furl">pb://someIntroducer/[censored]</div>', html)
 670             self.failIfIn('pb://someIntroducer/secret', html)
 671             self.failUnless(re.search('<img (alt="Disconnected" |src="img/connected-no.png" ){2}/>', html), res)
 672         d.addCallback(_check_introducer_not_connected_unguessable)
 673
 674         # introducer connected, unguessable furl
 675         def _set_introducer_connected_unguessable(ign):
 676             self.s.introducer_furl = "pb://someIntroducer/secret"
 677             self.s.introducer_client = MockIntroducerClient(True)
 678             return self.GET("/")
 679         d.addCallback(_set_introducer_connected_unguessable)
 680         def _check_introducer_connected_unguessable(res):
 681             html = res.replace('\n', ' ')
 682             self.failUnlessIn('<div class="furl">pb://someIntroducer/[censored]</div>', html)
 683             self.failIfIn('pb://someIntroducer/secret', html)
 684             self.failUnless(re.search('<img (src="img/connected-yes.png" |alt="Connected" ){2}/>', html), res)
 685         d.addCallback(_check_introducer_connected_unguessable)
 686
 687         # introducer connected, guessable furl
 688         def _set_introducer_connected_guessable(ign):
 689             self.s.introducer_furl = "pb://someIntroducer/introducer"
 690             self.s.introducer_client = MockIntroducerClient(True)
 691             return self.GET("/")
 692         d.addCallback(_set_introducer_connected_guessable)
 693         def _check_introducer_connected_guessable(res):
 694             html = res.replace('\n', ' ')
 695             self.failUnlessIn('<div class="furl">pb://someIntroducer/introducer</div>', html)
 696             self.failUnless(re.search('<img (src="img/connected-yes.png" |alt="Connected" ){2}/>', html), res)
 697         d.addCallback(_check_introducer_connected_guessable)
 698         return d
 699
 700     def test_helper_status(self):
 701         d = defer.succeed(None)
 702
 703         # set helper furl to None
 704         def _set_no_helper(ign):
 705             self.s.uploader.helper_furl = None
 706             return self.GET("/")
 707         d.addCallback(_set_no_helper)
 708         def _check_no_helper(res):
 709             html = res.replace('\n', ' ')
 710             self.failUnless(re.search('<img (src="img/connected-not-configured.png" |alt="Not Configured" ){2}/>', html), res)
 711         d.addCallback(_check_no_helper)
 712
 713         # enable helper, not connected
 714         def _set_helper_not_connected(ign):
 715             self.s.uploader.helper_furl = "pb://someHelper/secret"
 716             self.s.uploader.helper_connected = False
 717             return self.GET("/")
 718         d.addCallback(_set_helper_not_connected)
 719         def _check_helper_not_connected(res):
 720             html = res.replace('\n', ' ')
 721             self.failUnlessIn('<div class="furl">pb://someHelper/[censored]</div>', html)
 722             self.failIfIn('pb://someHelper/secret', html)
 723             self.failUnless(re.search('<img (src="img/connected-no.png" |alt="Disconnected" ){2}/>', html), res)
 724         d.addCallback(_check_helper_not_connected)
 725
 726         # enable helper, connected
 727         def _set_helper_connected(ign):
 728             self.s.uploader.helper_furl = "pb://someHelper/secret"
 729             self.s.uploader.helper_connected = True
 730             return self.GET("/")
 731         d.addCallback(_set_helper_connected)
 732         def _check_helper_connected(res):
 733             html = res.replace('\n', ' ')
 734             self.failUnlessIn('<div class="furl">pb://someHelper/[censored]</div>', html)
 735             self.failIfIn('pb://someHelper/secret', html)
 736             self.failUnless(re.search('<img (src="img/connected-yes.png" |alt="Connected" ){2}/>', html), res)
 737         d.addCallback(_check_helper_connected)
 738         return d
 739
 740     def test_storage(self):
 741         d = self.GET("/storage")
 742         def _check(res):
 743             self.failUnlessIn('Storage Server Status', res)
 744             self.failUnlessIn(FAVICON_MARKUP, res)
 745             res_u = res.decode('utf-8')
 746             self.failUnlessIn(u'<li>Server Nickname: <span class="nickname mine">fake_nickname \u263A</span></li>', res_u)
 747         d.addCallback(_check)
 748         return d
 749
 750     def test_status(self):
 751         h = self.s.get_history()
 752         dl_num = h.list_all_download_statuses()[0].get_counter()
 753         ul_num = h.list_all_upload_statuses()[0].get_counter()
 754         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 755         pub_num = h.list_all_publish_statuses()[0].get_counter()
 756         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 757         d = self.GET("/status", followRedirect=True)
 758         def _check(res):
 759             self.failUnlessIn('Recent and Active Operations', res)
 760             self.failUnlessIn('"down-%d"' % dl_num, res)
 761             self.failUnlessIn('"up-%d"' % ul_num, res)
 762             self.failUnlessIn('"mapupdate-%d"' % mu_num, res)
 763             self.failUnlessIn('"publish-%d"' % pub_num, res)
 764             self.failUnlessIn('"retrieve-%d"' % ret_num, res)
 765         d.addCallback(_check)
 766         d.addCallback(lambda res: self.GET("/status/?t=json"))
 767         def _check_json(res):
 768             data = simplejson.loads(res)
 769             self.failUnless(isinstance(data, dict))
 770             #active = data["active"]
 771             # TODO: test more. We need a way to fake an active operation
 772             # here.
 773         d.addCallback(_check_json)
 774
 775         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 776         def _check_dl(res):
 777             self.failUnlessIn("File Download Status", res)
 778         d.addCallback(_check_dl)
 779         d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
 780         def _check_dl_json(res):
 781             data = simplejson.loads(res)
 782             self.failUnless(isinstance(data, dict))
 783             self.failUnlessIn("read", data)
 784             self.failUnlessEqual(data["read"][0]["length"], 120)
 785             self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
 786             self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
 787             self.failUnlessEqual(data["segment"][2]["finish_time"], None)
 788             phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
 789             cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
 790             # serverids[] keys are strings, since that's what JSON does, but
 791             # we'd really like them to be ints
 792             self.failUnlessEqual(data["serverids"]["0"], "phwrsjte")
 793             self.failUnless(data["serverids"].has_key("1"),
 794                             str(data["serverids"]))
 795             self.failUnlessEqual(data["serverids"]["1"], "cmpuvkjm",
 796                                  str(data["serverids"]))
 797             self.failUnlessEqual(data["server_info"][phwr_id]["short"],
 798                                  "phwrsjte")
 799             self.failUnlessEqual(data["server_info"][cmpu_id]["short"],
 800                                  "cmpuvkjm")
 801             self.failUnlessIn("dyhb", data)
 802             self.failUnlessIn("misc", data)
 803         d.addCallback(_check_dl_json)
 804         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 805         def _check_ul(res):
 806             self.failUnlessIn("File Upload Status", res)
 807         d.addCallback(_check_ul)
 808         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 809         def _check_mapupdate(res):
 810             self.failUnlessIn("Mutable File Servermap Update Status", res)
 811         d.addCallback(_check_mapupdate)
 812         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 813         def _check_publish(res):
 814             self.failUnlessIn("Mutable File Publish Status", res)
 815         d.addCallback(_check_publish)
 816         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 817         def _check_retrieve(res):
 818             self.failUnlessIn("Mutable File Retrieve Status", res)
 819         d.addCallback(_check_retrieve)
 820
 821         return d
 822
 823     def test_status_numbers(self):
 824         drrm = status.DownloadResultsRendererMixin()
 825         self.failUnlessReallyEqual(drrm.render_time(None, None), "")
 826         self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
 827         self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
 828         self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
 829         self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
 830         self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
 831         self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 832         self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
 833         self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
 834
 835         urrm = status.UploadResultsRendererMixin()
 836         self.failUnlessReallyEqual(urrm.render_time(None, None), "")
 837         self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
 838         self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
 839         self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
 840         self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
 841         self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
 842         self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 843         self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
 844         self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
 845
 846     def test_GET_FILEURL(self):
 847         d = self.GET(self.public_url + "/foo/bar.txt")
 848         d.addCallback(self.failUnlessIsBarDotTxt)
 849         return d
 850
 851     def test_GET_FILEURL_range(self):
 852         headers = {"range": "bytes=1-10"}
 853         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 854                      return_response=True)
 855         def _got((res, status, headers)):
 856             self.failUnlessReallyEqual(int(status), 206)
 857             self.failUnless(headers.has_key("content-range"))
 858             self.failUnlessReallyEqual(headers["content-range"][0],
 859                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 860             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
 861         d.addCallback(_got)
 862         return d
 863
 864     def test_GET_FILEURL_partial_range(self):
 865         headers = {"range": "bytes=5-"}
 866         length  = len(self.BAR_CONTENTS)
 867         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 868                      return_response=True)
 869         def _got((res, status, headers)):
 870             self.failUnlessReallyEqual(int(status), 206)
 871             self.failUnless(headers.has_key("content-range"))
 872             self.failUnlessReallyEqual(headers["content-range"][0],
 873                                        "bytes 5-%d/%d" % (length-1, length))
 874             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
 875         d.addCallback(_got)
 876         return d
 877
 878     def test_GET_FILEURL_partial_end_range(self):
 879         headers = {"range": "bytes=-5"}
 880         length  = len(self.BAR_CONTENTS)
 881         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 882                      return_response=True)
 883         def _got((res, status, headers)):
 884             self.failUnlessReallyEqual(int(status), 206)
 885             self.failUnless(headers.has_key("content-range"))
 886             self.failUnlessReallyEqual(headers["content-range"][0],
 887                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 888             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
 889         d.addCallback(_got)
 890         return d
 891
 892     def test_GET_FILEURL_partial_range_overrun(self):
 893         headers = {"range": "bytes=100-200"}
 894         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
 895                              "416 Requested Range not satisfiable",
 896                              "First beyond end of file",
 897                              self.GET, self.public_url + "/foo/bar.txt",
 898                              headers=headers)
 899         return d
 900
 901     def test_HEAD_FILEURL_range(self):
 902         headers = {"range": "bytes=1-10"}
 903         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 904                      return_response=True)
 905         def _got((res, status, headers)):
 906             self.failUnlessReallyEqual(res, "")
 907             self.failUnlessReallyEqual(int(status), 206)
 908             self.failUnless(headers.has_key("content-range"))
 909             self.failUnlessReallyEqual(headers["content-range"][0],
 910                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 911         d.addCallback(_got)
 912         return d
 913
 914     def test_HEAD_FILEURL_partial_range(self):
 915         headers = {"range": "bytes=5-"}
 916         length  = len(self.BAR_CONTENTS)
 917         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 918                      return_response=True)
 919         def _got((res, status, headers)):
 920             self.failUnlessReallyEqual(int(status), 206)
 921             self.failUnless(headers.has_key("content-range"))
 922             self.failUnlessReallyEqual(headers["content-range"][0],
 923                                        "bytes 5-%d/%d" % (length-1, length))
 924         d.addCallback(_got)
 925         return d
 926
 927     def test_HEAD_FILEURL_partial_end_range(self):
 928         headers = {"range": "bytes=-5"}
 929         length  = len(self.BAR_CONTENTS)
 930         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 931                      return_response=True)
 932         def _got((res, status, headers)):
 933             self.failUnlessReallyEqual(int(status), 206)
 934             self.failUnless(headers.has_key("content-range"))
 935             self.failUnlessReallyEqual(headers["content-range"][0],
 936                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 937         d.addCallback(_got)
 938         return d
 939
 940     def test_HEAD_FILEURL_partial_range_overrun(self):
 941         headers = {"range": "bytes=100-200"}
 942         d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
 943                              "416 Requested Range not satisfiable",
 944                              "",
 945                              self.HEAD, self.public_url + "/foo/bar.txt",
 946                              headers=headers)
 947         return d
 948
 949     def test_GET_FILEURL_range_bad(self):
 950         headers = {"range": "BOGUS=fizbop-quarnak"}
 951         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 952                      return_response=True)
 953         def _got((res, status, headers)):
 954             self.failUnlessReallyEqual(int(status), 200)
 955             self.failUnless(not headers.has_key("content-range"))
 956             self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
 957         d.addCallback(_got)
 958         return d
 959
 960     def test_HEAD_FILEURL(self):
 961         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 962         def _got((res, status, headers)):
 963             self.failUnlessReallyEqual(res, "")
 964             self.failUnlessReallyEqual(headers["content-length"][0],
 965                                        str(len(self.BAR_CONTENTS)))
 966             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
 967         d.addCallback(_got)
 968         return d
 969
 970     def test_GET_FILEURL_named(self):
 971         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 972         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 973         d = self.GET(base + "/@@name=/blah.txt")
 974         d.addCallback(self.failUnlessIsBarDotTxt)
 975         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 976         d.addCallback(self.failUnlessIsBarDotTxt)
 977         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 978         d.addCallback(self.failUnlessIsBarDotTxt)
 979         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 980         d.addCallback(self.failUnlessIsBarDotTxt)
 981         save_url = base + "?save=true&filename=blah.txt"
 982         d.addCallback(lambda res: self.GET(save_url))
 983         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 984         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 985         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 986         u_url = base + "?save=true&filename=" + u_fn_e
 987         d.addCallback(lambda res: self.GET(u_url))
 988         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 989         return d
 990
 991     def test_PUT_FILEURL_named_bad(self):
 992         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 993         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 994                              "400 Bad Request",
 995                              "/file can only be used with GET or HEAD",
 996                              self.PUT, base + "/@@name=/blah.txt", "")
 997         return d
 998
 999
1000     def test_GET_DIRURL_named_bad(self):
1001         base = "/file/%s" % urllib.quote(self._foo_uri)
1002         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
1003                              "400 Bad Request",
1004                              "is not a file-cap",
1005                              self.GET, base + "/@@name=/blah.txt")
1006         return d
1007
1008     def test_GET_slash_file_bad(self):
1009         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
1010                              "404 Not Found",
1011                              "/file must be followed by a file-cap and a name",
1012                              self.GET, "/file")
1013         return d
1014
1015     def test_GET_unhandled_URI_named(self):
1016         contents, n, newuri = self.makefile(12)
1017         verifier_cap = n.get_verify_cap().to_string()
1018         base = "/file/%s" % urllib.quote(verifier_cap)
1019         # client.create_node_from_uri() can't handle verify-caps
1020         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
1021                              "400 Bad Request", "is not a file-cap",
1022                              self.GET, base)
1023         return d
1024
1025     def test_GET_unhandled_URI(self):
1026         contents, n, newuri = self.makefile(12)
1027         verifier_cap = n.get_verify_cap().to_string()
1028         base = "/uri/%s" % urllib.quote(verifier_cap)
1029         # client.create_node_from_uri() can't handle verify-caps
1030         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
1031                              "400 Bad Request",
1032                              "GET unknown URI type: can only do t=info",
1033                              self.GET, base)
1034         return d
1035
1036     def test_GET_FILE_URI(self):
1037         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
1038         d = self.GET(base)
1039         d.addCallback(self.failUnlessIsBarDotTxt)
1040         return d
1041
1042     def test_GET_FILE_URI_mdmf(self):
1043         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
1044         d = self.GET(base)
1045         d.addCallback(self.failUnlessIsQuuxDotTxt)
1046         return d
1047
1048     def test_GET_FILE_URI_mdmf_extensions(self):
1049         base = "/uri/%s" % urllib.quote("%s:RANDOMSTUFF" % self._quux_txt_uri)
1050         d = self.GET(base)
1051         d.addCallback(self.failUnlessIsQuuxDotTxt)
1052         return d
1053
1054     def test_GET_FILE_URI_mdmf_readonly(self):
1055         base = "/uri/%s" % urllib.quote(self._quux_txt_readonly_uri)
1056         d = self.GET(base)
1057         d.addCallback(self.failUnlessIsQuuxDotTxt)
1058         return d
1059
1060     def test_GET_FILE_URI_badchild(self):
1061         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
1062         errmsg = "Files have no children, certainly not named 'boguschild'"
1063         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
1064                              "400 Bad Request", errmsg,
1065                              self.GET, base)
1066         return d
1067
1068     def test_PUT_FILE_URI_badchild(self):
1069         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
1070         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
1071         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
1072                              "400 Bad Request", errmsg,
1073                              self.PUT, base, "")
1074         return d
1075
1076     def test_PUT_FILE_URI_mdmf(self):
1077         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
1078         self._quux_new_contents = "new_contents"
1079         d = self.GET(base)
1080         d.addCallback(lambda res:
1081             self.failUnlessIsQuuxDotTxt(res))
1082         d.addCallback(lambda ignored:
1083             self.PUT(base, self._quux_new_contents))
1084         d.addCallback(lambda ignored:
1085             self.GET(base))
1086         d.addCallback(lambda res:
1087             self.failUnlessReallyEqual(res, self._quux_new_contents))
1088         return d
1089
1090     def test_PUT_FILE_URI_mdmf_extensions(self):
1091         base = "/uri/%s" % urllib.quote("%s:EXTENSIONSTUFF" % self._quux_txt_uri)
1092         self._quux_new_contents = "new_contents"
1093         d = self.GET(base)
1094         d.addCallback(lambda res: self.failUnlessIsQuuxDotTxt(res))
1095         d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
1096         d.addCallback(lambda ignored: self.GET(base))
1097         d.addCallback(lambda res: self.failUnlessEqual(self._quux_new_contents,
1098                                                        res))
1099         return d
1100
1101     def test_PUT_FILE_URI_mdmf_readonly(self):
1102         # We're not allowed to PUT things to a readonly cap.
1103         base = "/uri/%s" % self._quux_txt_readonly_uri
1104         d = self.GET(base)
1105         d.addCallback(lambda res:
1106             self.failUnlessIsQuuxDotTxt(res))
1107         # What should we get here? We get a 500 error now; that's not right.
1108         d.addCallback(lambda ignored:
1109             self.shouldFail2(error.Error, "test_PUT_FILE_URI_mdmf_readonly",
1110                              "400 Bad Request", "read-only cap",
1111                              self.PUT, base, "new data"))
1112         return d
1113
1114     def test_PUT_FILE_URI_sdmf_readonly(self):
1115         # We're not allowed to put things to a readonly cap.
1116         base = "/uri/%s" % self._baz_txt_readonly_uri
1117         d = self.GET(base)
1118         d.addCallback(lambda res:
1119             self.failUnlessIsBazDotTxt(res))
1120         d.addCallback(lambda ignored:
1121             self.shouldFail2(error.Error, "test_PUT_FILE_URI_sdmf_readonly",
1122                              "400 Bad Request", "read-only cap",
1123                              self.PUT, base, "new_data"))
1124         return d
1125
1126     def test_GET_etags(self):
1127
1128         def _check_etags(uri):
1129             d1 = _get_etag(uri)
1130             d2 = _get_etag(uri, 'json')
1131             d = defer.DeferredList([d1, d2], consumeErrors=True)
1132             def _check(results):
1133                 # All deferred must succeed
1134                 self.failUnless(all([r[0] for r in results]))
1135                 # the etag for the t=json form should be just like the etag
1136                 # fo the default t='' form, but with a 'json' suffix
1137                 self.failUnlessEqual(results[0][1] + 'json', results[1][1])
1138             d.addCallback(_check)
1139             return d
1140
1141         def _get_etag(uri, t=''):
1142             targetbase = "/uri/%s?t=%s" % (urllib.quote(uri.strip()), t)
1143             d = self.GET(targetbase, return_response=True, followRedirect=True)
1144             def _just_the_etag(result):
1145                 data, response, headers = result
1146                 etag = headers['etag'][0]
1147                 if uri.startswith('URI:DIR'):
1148                     self.failUnless(etag.startswith('DIR:'), etag)
1149                 return etag
1150             return d.addCallback(_just_the_etag)
1151
1152         # Check that etags work with immutable directories
1153         (newkids, caps) = self._create_immutable_children()
1154         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1155                       simplejson.dumps(newkids))
1156         def _stash_immdir_uri(uri):
1157             self._immdir_uri = uri
1158             return uri
1159         d.addCallback(_stash_immdir_uri)
1160         d.addCallback(_check_etags)
1161
1162         # Check that etags work with immutable files
1163         d.addCallback(lambda _: _check_etags(self._bar_txt_uri))
1164
1165         # use the ETag on GET
1166         def _check_match(ign):
1167             uri = "/uri/%s" % self._bar_txt_uri
1168             d = self.GET(uri, return_response=True)
1169             # extract the ETag
1170             d.addCallback(lambda (data, code, headers):
1171                           headers['etag'][0])
1172             # do a GET that's supposed to match the ETag
1173             d.addCallback(lambda etag:
1174                           self.GET(uri, return_response=True,
1175                                    headers={"If-None-Match": etag}))
1176             # make sure it short-circuited (304 instead of 200)
1177             d.addCallback(lambda (data, code, headers):
1178                           self.failUnlessEqual(int(code), http.NOT_MODIFIED))
1179             return d
1180         d.addCallback(_check_match)
1181
1182         def _no_etag(uri, t):
1183             target = "/uri/%s?t=%s" % (uri, t)
1184             d = self.GET(target, return_response=True, followRedirect=True)
1185             d.addCallback(lambda (data, code, headers):
1186                           self.failIf("etag" in headers, target))
1187             return d
1188         def _yes_etag(uri, t):
1189             target = "/uri/%s?t=%s" % (uri, t)
1190             d = self.GET(target, return_response=True, followRedirect=True)
1191             d.addCallback(lambda (data, code, headers):
1192                           self.failUnless("etag" in headers, target))
1193             return d
1194
1195         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, ""))
1196         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "json"))
1197         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "uri"))
1198         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "readonly-uri"))
1199         d.addCallback(lambda ign: _no_etag(self._bar_txt_uri, "info"))
1200
1201         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, ""))
1202         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "json"))
1203         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "uri"))
1204         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "readonly-uri"))
1205         d.addCallback(lambda ign: _no_etag(self._immdir_uri, "info"))
1206         d.addCallback(lambda ign: _no_etag(self._immdir_uri, "rename-form"))
1207
1208         return d
1209
1210     # TODO: version of this with a Unicode filename
1211     def test_GET_FILEURL_save(self):
1212         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
1213                      return_response=True)
1214         def _got((res, statuscode, headers)):
1215             content_disposition = headers["content-disposition"][0]
1216             self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
1217             self.failUnlessIsBarDotTxt(res)
1218         d.addCallback(_got)
1219         return d
1220
1221     def test_GET_FILEURL_missing(self):
1222         d = self.GET(self.public_url + "/foo/missing")
1223         d.addBoth(self.should404, "test_GET_FILEURL_missing")
1224         return d
1225
1226     def test_GET_FILEURL_info_mdmf(self):
1227         d = self.GET("/uri/%s?t=info" % self._quux_txt_uri)
1228         def _got(res):
1229             self.failUnlessIn("mutable file (mdmf)", res)
1230             self.failUnlessIn(self._quux_txt_uri, res)
1231             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1232         d.addCallback(_got)
1233         return d
1234
1235     def test_GET_FILEURL_info_mdmf_readonly(self):
1236         d = self.GET("/uri/%s?t=info" % self._quux_txt_readonly_uri)
1237         def _got(res):
1238             self.failUnlessIn("mutable file (mdmf)", res)
1239             self.failIfIn(self._quux_txt_uri, res)
1240             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1241         d.addCallback(_got)
1242         return d
1243
1244     def test_GET_FILEURL_info_sdmf(self):
1245         d = self.GET("/uri/%s?t=info" % self._baz_txt_uri)
1246         def _got(res):
1247             self.failUnlessIn("mutable file (sdmf)", res)
1248             self.failUnlessIn(self._baz_txt_uri, res)
1249         d.addCallback(_got)
1250         return d
1251
1252     def test_GET_FILEURL_info_mdmf_extensions(self):
1253         d = self.GET("/uri/%s:STUFF?t=info" % self._quux_txt_uri)
1254         def _got(res):
1255             self.failUnlessIn("mutable file (mdmf)", res)
1256             self.failUnlessIn(self._quux_txt_uri, res)
1257             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1258         d.addCallback(_got)
1259         return d
1260
1261     def test_PUT_overwrite_only_files(self):
1262         # create a directory, put a file in that directory.
1263         contents, n, filecap = self.makefile(8)
1264         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
1265         d.addCallback(lambda res:
1266             self.PUT(self.public_url + "/foo/dir/file1.txt",
1267                      self.NEWFILE_CONTENTS))
1268         # try to overwrite the file with replace=only-files
1269         # (this should work)
1270         d.addCallback(lambda res:
1271             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
1272                      filecap))
1273         d.addCallback(lambda res:
1274             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
1275                  "There was already a child by that name, and you asked me "
1276                  "to not replace it",
1277                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
1278                  filecap))
1279         return d
1280
1281     def test_PUT_NEWFILEURL(self):
1282         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
1283         # TODO: we lose the response code, so we can't check this
1284         #self.failUnlessReallyEqual(responsecode, 201)
1285         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1286         d.addCallback(lambda res:
1287                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1288                                                       self.NEWFILE_CONTENTS))
1289         return d
1290
1291     def test_PUT_NEWFILEURL_not_mutable(self):
1292         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
1293                      self.NEWFILE_CONTENTS)
1294         # TODO: we lose the response code, so we can't check this
1295         #self.failUnlessReallyEqual(responsecode, 201)
1296         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1297         d.addCallback(lambda res:
1298                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1299                                                       self.NEWFILE_CONTENTS))
1300         return d
1301
1302     def test_PUT_NEWFILEURL_unlinked_mdmf(self):
1303         # this should get us a few segments of an MDMF mutable file,
1304         # which we can then test for.
1305         contents = self.NEWFILE_CONTENTS * 300000
1306         d = self.PUT("/uri?format=mdmf",
1307                      contents)
1308         def _got_filecap(filecap):
1309             self.failUnless(filecap.startswith("URI:MDMF"))
1310             return filecap
1311         d.addCallback(_got_filecap)
1312         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1313         d.addCallback(lambda json: self.failUnlessIn("MDMF", json))
1314         return d
1315
1316     def test_PUT_NEWFILEURL_unlinked_sdmf(self):
1317         contents = self.NEWFILE_CONTENTS * 300000
1318         d = self.PUT("/uri?format=sdmf",
1319                      contents)
1320         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1321         d.addCallback(lambda json: self.failUnlessIn("SDMF", json))
1322         return d
1323
1324     def test_PUT_NEWFILEURL_unlinked_bad_format(self):
1325         contents = self.NEWFILE_CONTENTS * 300000
1326         return self.shouldHTTPError("PUT_NEWFILEURL_unlinked_bad_format",
1327                                     400, "Bad Request", "Unknown format: foo",
1328                                     self.PUT, "/uri?format=foo",
1329                                     contents)
1330
1331     def test_PUT_NEWFILEURL_range_bad(self):
1332         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
1333         target = self.public_url + "/foo/new.txt"
1334         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
1335                              "501 Not Implemented",
1336                              "Content-Range in PUT not yet supported",
1337                              # (and certainly not for immutable files)
1338                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
1339                              headers=headers)
1340         d.addCallback(lambda res:
1341                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
1342         return d
1343
1344     def test_PUT_NEWFILEURL_mutable(self):
1345         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1346                      self.NEWFILE_CONTENTS)
1347         # TODO: we lose the response code, so we can't check this
1348         #self.failUnlessReallyEqual(responsecode, 201)
1349         def _check_uri(res):
1350             u = uri.from_string_mutable_filenode(res)
1351             self.failUnless(u.is_mutable())
1352             self.failIf(u.is_readonly())
1353             return res
1354         d.addCallback(_check_uri)
1355         d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
1356         d.addCallback(lambda res:
1357                       self.failUnlessMutableChildContentsAre(self._foo_node,
1358                                                              u"new.txt",
1359                                                              self.NEWFILE_CONTENTS))
1360         return d
1361
1362     def test_PUT_NEWFILEURL_mutable_toobig(self):
1363         # It is okay to upload large mutable files, so we should be able
1364         # to do that.
1365         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1366                      "b" * (self.s.MUTABLE_SIZELIMIT + 1))
1367         return d
1368
1369     def test_PUT_NEWFILEURL_replace(self):
1370         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
1371         # TODO: we lose the response code, so we can't check this
1372         #self.failUnlessReallyEqual(responsecode, 200)
1373         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
1374         d.addCallback(lambda res:
1375                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1376                                                       self.NEWFILE_CONTENTS))
1377         return d
1378
1379     def test_PUT_NEWFILEURL_bad_t(self):
1380         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
1381                              "PUT to a file: bad t=bogus",
1382                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
1383                              "contents")
1384         return d
1385
1386     def test_PUT_NEWFILEURL_no_replace(self):
1387         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
1388                      self.NEWFILE_CONTENTS)
1389         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
1390                   "409 Conflict",
1391                   "There was already a child by that name, and you asked me "
1392                   "to not replace it")
1393         return d
1394
1395     def test_PUT_NEWFILEURL_mkdirs(self):
1396         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
1397         fn = self._foo_node
1398         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
1399         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
1400         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
1401         d.addCallback(lambda res:
1402                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
1403                                                       self.NEWFILE_CONTENTS))
1404         return d
1405
1406     def test_PUT_NEWFILEURL_blocked(self):
1407         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
1408                      self.NEWFILE_CONTENTS)
1409         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
1410                   "409 Conflict",
1411                   "Unable to create directory 'blockingfile': a file was in the way")
1412         return d
1413
1414     def test_PUT_NEWFILEURL_emptyname(self):
1415         # an empty pathname component (i.e. a double-slash) is disallowed
1416         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
1417                              "400 Bad Request",
1418                              "The webapi does not allow empty pathname components",
1419                              self.PUT, self.public_url + "/foo//new.txt", "")
1420         return d
1421
1422     def test_DELETE_FILEURL(self):
1423         d = self.DELETE(self.public_url + "/foo/bar.txt")
1424         d.addCallback(lambda res:
1425                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1426         return d
1427
1428     def test_DELETE_FILEURL_missing(self):
1429         d = self.DELETE(self.public_url + "/foo/missing")
1430         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
1431         return d
1432
1433     def test_DELETE_FILEURL_missing2(self):
1434         d = self.DELETE(self.public_url + "/missing/missing")
1435         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
1436         return d
1437
1438     def failUnlessHasBarDotTxtMetadata(self, res):
1439         data = simplejson.loads(res)
1440         self.failUnless(isinstance(data, list))
1441         self.failUnlessIn("metadata", data[1])
1442         self.failUnlessIn("tahoe", data[1]["metadata"])
1443         self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
1444         self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
1445         self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
1446                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
1447
1448     def test_GET_FILEURL_json(self):
1449         # twisted.web.http.parse_qs ignores any query args without an '=', so
1450         # I can't do "GET /path?json", I have to do "GET /path/t=json"
1451         # instead. This may make it tricky to emulate the S3 interface
1452         # completely.
1453         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
1454         def _check1(data):
1455             self.failUnlessIsBarJSON(data)
1456             self.failUnlessHasBarDotTxtMetadata(data)
1457             return
1458         d.addCallback(_check1)
1459         return d
1460
1461     def test_GET_FILEURL_json_mutable_type(self):
1462         # The JSON should include format, which says whether the
1463         # file is SDMF or MDMF
1464         d = self.PUT("/uri?format=mdmf",
1465                      self.NEWFILE_CONTENTS * 300000)
1466         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1467         def _got_json(json, version):
1468             data = simplejson.loads(json)
1469             assert "filenode" == data[0]
1470             data = data[1]
1471             assert isinstance(data, dict)
1472
1473             self.failUnlessIn("format", data)
1474             self.failUnlessEqual(data["format"], version)
1475
1476         d.addCallback(_got_json, "MDMF")
1477         # Now make an SDMF file and check that it is reported correctly.
1478         d.addCallback(lambda ignored:
1479             self.PUT("/uri?format=sdmf",
1480                       self.NEWFILE_CONTENTS * 300000))
1481         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1482         d.addCallback(_got_json, "SDMF")
1483         return d
1484
1485     def test_GET_FILEURL_json_mdmf(self):
1486         d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
1487         d.addCallback(self.failUnlessIsQuuxJSON)
1488         return d
1489
1490     def test_GET_FILEURL_json_missing(self):
1491         d = self.GET(self.public_url + "/foo/missing?json")
1492         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
1493         return d
1494
1495     def test_GET_FILEURL_uri(self):
1496         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
1497         def _check(res):
1498             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1499         d.addCallback(_check)
1500         d.addCallback(lambda res:
1501                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
1502         def _check2(res):
1503             # for now, for files, uris and readonly-uris are the same
1504             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1505         d.addCallback(_check2)
1506         return d
1507
1508     def test_GET_FILEURL_badtype(self):
1509         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
1510                                  "bad t=bogus",
1511                                  self.GET,
1512                                  self.public_url + "/foo/bar.txt?t=bogus")
1513         return d
1514
1515     def test_CSS_FILE(self):
1516         d = self.GET("/tahoe.css", followRedirect=True)
1517         def _check(res):
1518             CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
1519             self.failUnless(CSS_STYLE.search(res), res)
1520         d.addCallback(_check)
1521         return d
1522
1523     def test_GET_FILEURL_uri_missing(self):
1524         d = self.GET(self.public_url + "/foo/missing?t=uri")
1525         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
1526         return d
1527
1528     def _check_upload_and_mkdir_forms(self, html):
1529         # We should have a form to create a file, with radio buttons that allow
1530         # the user to toggle whether it is a CHK/LIT (default), SDMF, or MDMF file.
1531         self.failUnless(re.search('<input (name="t" |value="upload" |type="hidden" ){3}/>', html), html)
1532         self.failUnless(re.search('<input [^/]*id="upload-chk"', html), html)
1533         self.failUnless(re.search('<input [^/]*id="upload-sdmf"', html), html)
1534         self.failUnless(re.search('<input [^/]*id="upload-mdmf"', html), html)
1535
1536         # We should also have the ability to create a mutable directory, with
1537         # radio buttons that allow the user to toggle whether it is an SDMF (default)
1538         # or MDMF directory.
1539         self.failUnless(re.search('<input (name="t" |value="mkdir" |type="hidden" ){3}/>', html), html)
1540         self.failUnless(re.search('<input [^/]*id="mkdir-sdmf"', html), html)
1541         self.failUnless(re.search('<input [^/]*id="mkdir-mdmf"', html), html)
1542
1543         self.failUnlessIn(FAVICON_MARKUP, html)
1544
1545     def test_GET_DIRECTORY_html(self):
1546         d = self.GET(self.public_url + "/foo", followRedirect=True)
1547         def _check(html):
1548             self.failUnlessIn('<li class="toolbar-item"><a href="../../..">Return to Welcome page</a></li>', html)
1549             self._check_upload_and_mkdir_forms(html)
1550             self.failUnlessIn("quux", html)
1551         d.addCallback(_check)
1552         return d
1553
1554     def test_GET_DIRECTORY_html_filenode_encoding(self):
1555         d = self.GET(self.public_url + "/foo", followRedirect=True)
1556         def _check(html):
1557             # Check if encoded entries are there
1558             self.failUnlessIn('@@named=/' + self._htmlname_urlencoded + '">'
1559                               + self._htmlname_escaped + '</a>', html)
1560             self.failUnlessIn('value="' + self._htmlname_escaped_attr + '"', html)
1561             self.failIfIn(self._htmlname_escaped_double, html)
1562             # Make sure that Nevow escaping actually works by checking for unsafe characters
1563             # and that '&' is escaped.
1564             for entity in '<>':
1565                 self.failUnlessIn(entity, self._htmlname_raw)
1566                 self.failIfIn(entity, self._htmlname_escaped)
1567             self.failUnlessIn('&', re.sub(r'&(amp|lt|gt|quot|apos);', '', self._htmlname_raw))
1568             self.failIfIn('&', re.sub(r'&(amp|lt|gt|quot|apos);', '', self._htmlname_escaped))
1569         d.addCallback(_check)
1570         return d
1571
1572     def test_GET_root_html(self):
1573         d = self.GET("/")
1574         d.addCallback(self._check_upload_and_mkdir_forms)
1575         return d
1576
1577     def test_GET_DIRURL(self):
1578         # the addSlash means we get a redirect here
1579         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1580         ROOT = "../../.."
1581         d = self.GET(self.public_url + "/foo", followRedirect=True)
1582         def _check(res):
1583             self.failUnlessIn('<a href="%s">Return to Welcome page' % ROOT, res)
1584
1585             # the FILE reference points to a URI, but it should end in bar.txt
1586             bar_url = ("%s/file/%s/@@named=/bar.txt" %
1587                        (ROOT, urllib.quote(self._bar_txt_uri)))
1588             get_bar = "".join([r'<td>FILE</td>',
1589                                r'\s+<td>',
1590                                r'<a href="%s">bar.txt</a>' % bar_url,
1591                                r'</td>',
1592                                r'\s+<td align="right">%d</td>' % len(self.BAR_CONTENTS),
1593                                ])
1594             self.failUnless(re.search(get_bar, res), res)
1595             for label in ['unlink', 'rename/relink']:
1596                 for line in res.split("\n"):
1597                     # find the line that contains the relevant button for bar.txt
1598                     if ("form action" in line and
1599                         ('value="%s"' % (label,)) in line and
1600                         'value="bar.txt"' in line):
1601                         # the form target should use a relative URL
1602                         foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1603                         self.failUnlessIn('action="%s"' % foo_url, line)
1604                         # and the when_done= should too
1605                         #done_url = urllib.quote(???)
1606                         #self.failUnlessIn('name="when_done" value="%s"' % done_url, line)
1607
1608                         # 'unlink' needs to use POST because it directly has a side effect
1609                         if label == 'unlink':
1610                             self.failUnlessIn('method="post"', line)
1611                         break
1612                 else:
1613                     self.fail("unable to find '%s bar.txt' line" % (label,))
1614
1615             # the DIR reference just points to a URI
1616             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1617             get_sub = ((r'<td>DIR</td>')
1618                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1619             self.failUnless(re.search(get_sub, res), res)
1620         d.addCallback(_check)
1621
1622         # look at a readonly directory
1623         d.addCallback(lambda res:
1624                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
1625         def _check2(res):
1626             self.failUnlessIn("(read-only)", res)
1627             self.failIfIn("Upload a file", res)
1628         d.addCallback(_check2)
1629
1630         # and at a directory that contains a readonly directory
1631         d.addCallback(lambda res:
1632                       self.GET(self.public_url, followRedirect=True))
1633         def _check3(res):
1634             self.failUnless(re.search('<td>DIR-RO</td>'
1635                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1636         d.addCallback(_check3)
1637
1638         # and an empty directory
1639         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1640         def _check4(res):
1641             self.failUnlessIn("directory is empty", res)
1642             MKDIR_BUTTON_RE=re.compile('<input (type="hidden" |name="t" |value="mkdir" ){3}/>.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input (type="submit" |class="btn" |value="Create" ){3}/>', re.I)
1643             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1644         d.addCallback(_check4)
1645
1646         # and at a literal directory
1647         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1648         d.addCallback(lambda res:
1649                       self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1650         def _check5(res):
1651             self.failUnlessIn('(immutable)', res)
1652             self.failUnless(re.search('<td>FILE</td>'
1653                                       r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1654         d.addCallback(_check5)
1655         return d
1656
1657     def test_GET_DIRURL_badtype(self):
1658         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1659                                  400, "Bad Request",
1660                                  "bad t=bogus",
1661                                  self.GET,
1662                                  self.public_url + "/foo?t=bogus")
1663         return d
1664
1665     def test_GET_DIRURL_json(self):
1666         d = self.GET(self.public_url + "/foo?t=json")
1667         d.addCallback(self.failUnlessIsFooJSON)
1668         return d
1669
1670     def test_GET_DIRURL_json_format(self):
1671         d = self.PUT(self.public_url + \
1672                      "/foo/sdmf.txt?format=sdmf",
1673                      self.NEWFILE_CONTENTS * 300000)
1674         d.addCallback(lambda ignored:
1675             self.PUT(self.public_url + \
1676                      "/foo/mdmf.txt?format=mdmf",
1677                      self.NEWFILE_CONTENTS * 300000))
1678         # Now we have an MDMF and SDMF file in the directory. If we GET
1679         # its JSON, we should see their encodings.
1680         d.addCallback(lambda ignored:
1681             self.GET(self.public_url + "/foo?t=json"))
1682         def _got_json(json):
1683             data = simplejson.loads(json)
1684             assert data[0] == "dirnode"
1685
1686             data = data[1]
1687             kids = data['children']
1688
1689             mdmf_data = kids['mdmf.txt'][1]
1690             self.failUnlessIn("format", mdmf_data)
1691             self.failUnlessEqual(mdmf_data["format"], "MDMF")
1692
1693             sdmf_data = kids['sdmf.txt'][1]
1694             self.failUnlessIn("format", sdmf_data)
1695             self.failUnlessEqual(sdmf_data["format"], "SDMF")
1696         d.addCallback(_got_json)
1697         return d
1698
1699
1700     def test_POST_DIRURL_manifest_no_ophandle(self):
1701         d = self.shouldFail2(error.Error,
1702                              "test_POST_DIRURL_manifest_no_ophandle",
1703                              "400 Bad Request",
1704                              "slow operation requires ophandle=",
1705                              self.POST, self.public_url, t="start-manifest")
1706         return d
1707
1708     def test_POST_DIRURL_manifest(self):
1709         d = defer.succeed(None)
1710         def getman(ignored, output):
1711             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1712                           followRedirect=True)
1713             d.addCallback(self.wait_for_operation, "125")
1714             d.addCallback(self.get_operation_results, "125", output)
1715             return d
1716         d.addCallback(getman, None)
1717         def _got_html(manifest):
1718             self.failUnlessIn("Manifest of SI=", manifest)
1719             self.failUnlessIn("<td>sub</td>", manifest)
1720             self.failUnlessIn(self._sub_uri, manifest)
1721             self.failUnlessIn("<td>sub/baz.txt</td>", manifest)
1722             self.failUnlessIn(FAVICON_MARKUP, manifest)
1723         d.addCallback(_got_html)
1724
1725         # both t=status and unadorned GET should be identical
1726         d.addCallback(lambda res: self.GET("/operations/125"))
1727         d.addCallback(_got_html)
1728
1729         d.addCallback(getman, "html")
1730         d.addCallback(_got_html)
1731         d.addCallback(getman, "text")
1732         def _got_text(manifest):
1733             self.failUnlessIn("\nsub " + self._sub_uri + "\n", manifest)
1734             self.failUnlessIn("\nsub/baz.txt URI:CHK:", manifest)
1735         d.addCallback(_got_text)
1736         d.addCallback(getman, "JSON")
1737         def _got_json(res):
1738             data = res["manifest"]
1739             got = {}
1740             for (path_list, cap) in data:
1741                 got[tuple(path_list)] = cap
1742             self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
1743             self.failUnlessIn((u"sub", u"baz.txt"), got)
1744             self.failUnlessIn("finished", res)
1745             self.failUnlessIn("origin", res)
1746             self.failUnlessIn("storage-index", res)
1747             self.failUnlessIn("verifycaps", res)
1748             self.failUnlessIn("stats", res)
1749         d.addCallback(_got_json)
1750         return d
1751
1752     def test_POST_DIRURL_deepsize_no_ophandle(self):
1753         d = self.shouldFail2(error.Error,
1754                              "test_POST_DIRURL_deepsize_no_ophandle",
1755                              "400 Bad Request",
1756                              "slow operation requires ophandle=",
1757                              self.POST, self.public_url, t="start-deep-size")
1758         return d
1759
1760     def test_POST_DIRURL_deepsize(self):
1761         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1762                       followRedirect=True)
1763         d.addCallback(self.wait_for_operation, "126")
1764         d.addCallback(self.get_operation_results, "126", "json")
1765         def _got_json(data):
1766             self.failUnlessReallyEqual(data["finished"], True)
1767             size = data["size"]
1768             self.failUnless(size > 1000)
1769         d.addCallback(_got_json)
1770         d.addCallback(self.get_operation_results, "126", "text")
1771         def _got_text(res):
1772             mo = re.search(r'^size: (\d+)$', res, re.M)
1773             self.failUnless(mo, res)
1774             size = int(mo.group(1))
1775             # with directories, the size varies.
1776             self.failUnless(size > 1000)
1777         d.addCallback(_got_text)
1778         return d
1779
1780     def test_POST_DIRURL_deepstats_no_ophandle(self):
1781         d = self.shouldFail2(error.Error,
1782                              "test_POST_DIRURL_deepstats_no_ophandle",
1783                              "400 Bad Request",
1784                              "slow operation requires ophandle=",
1785                              self.POST, self.public_url, t="start-deep-stats")
1786         return d
1787
1788     def test_POST_DIRURL_deepstats(self):
1789         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1790                       followRedirect=True)
1791         d.addCallback(self.wait_for_operation, "127")
1792         d.addCallback(self.get_operation_results, "127", "json")
1793         def _got_json(stats):
1794             expected = {"count-immutable-files": 4,
1795                         "count-mutable-files": 2,
1796                         "count-literal-files": 0,
1797                         "count-files": 6,
1798                         "count-directories": 3,
1799                         "size-immutable-files": 76,
1800                         "size-literal-files": 0,
1801                         #"size-directories": 1912, # varies
1802                         #"largest-directory": 1590,
1803                         "largest-directory-children": 8,
1804                         "largest-immutable-file": 19,
1805                         }
1806             for k,v in expected.iteritems():
1807                 self.failUnlessReallyEqual(stats[k], v,
1808                                            "stats[%s] was %s, not %s" %
1809                                            (k, stats[k], v))
1810             self.failUnlessReallyEqual(stats["size-files-histogram"],
1811                                        [ [11, 31, 4] ])
1812         d.addCallback(_got_json)
1813         return d
1814
1815     def test_POST_DIRURL_stream_manifest(self):
1816         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1817         def _check(res):
1818             self.failUnless(res.endswith("\n"))
1819             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1820             self.failUnlessReallyEqual(len(units), 10)
1821             self.failUnlessEqual(units[-1]["type"], "stats")
1822             first = units[0]
1823             self.failUnlessEqual(first["path"], [])
1824             self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri)
1825             self.failUnlessEqual(first["type"], "directory")
1826             baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0]
1827             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1828             self.failIfEqual(baz["storage-index"], None)
1829             self.failIfEqual(baz["verifycap"], None)
1830             self.failIfEqual(baz["repaircap"], None)
1831             # XXX: Add quux and baz to this test.
1832             return
1833         d.addCallback(_check)
1834         return d
1835
1836     def test_GET_DIRURL_uri(self):
1837         d = self.GET(self.public_url + "/foo?t=uri")
1838         def _check(res):
1839             self.failUnlessReallyEqual(to_str(res), self._foo_uri)
1840         d.addCallback(_check)
1841         return d
1842
1843     def test_GET_DIRURL_readonly_uri(self):
1844         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1845         def _check(res):
1846             self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri)
1847         d.addCallback(_check)
1848         return d
1849
1850     def test_PUT_NEWDIRURL(self):
1851         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1852         d.addCallback(lambda res:
1853                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1854         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1855         d.addCallback(self.failUnlessNodeKeysAre, [])
1856         return d
1857
1858     def test_PUT_NEWDIRURL_mdmf(self):
1859         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1860         d.addCallback(lambda res:
1861                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1862         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1863         d.addCallback(lambda node:
1864             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1865         return d
1866
1867     def test_PUT_NEWDIRURL_sdmf(self):
1868         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=sdmf",
1869                      "")
1870         d.addCallback(lambda res:
1871                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1872         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1873         d.addCallback(lambda node:
1874             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1875         return d
1876
1877     def test_PUT_NEWDIRURL_bad_format(self):
1878         return self.shouldHTTPError("PUT_NEWDIRURL_bad_format",
1879                                     400, "Bad Request", "Unknown format: foo",
1880                                     self.PUT, self.public_url +
1881                                     "/foo/newdir=?t=mkdir&format=foo", "")
1882
1883     def test_POST_NEWDIRURL(self):
1884         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1885         d.addCallback(lambda res:
1886                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1887         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1888         d.addCallback(self.failUnlessNodeKeysAre, [])
1889         return d
1890
1891     def test_POST_NEWDIRURL_mdmf(self):
1892         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1893         d.addCallback(lambda res:
1894                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1895         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1896         d.addCallback(lambda node:
1897             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1898         return d
1899
1900     def test_POST_NEWDIRURL_sdmf(self):
1901         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=sdmf", "")
1902         d.addCallback(lambda res:
1903             self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1904         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1905         d.addCallback(lambda node:
1906             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1907         return d
1908
1909     def test_POST_NEWDIRURL_bad_format(self):
1910         return self.shouldHTTPError("POST_NEWDIRURL_bad_format",
1911                                     400, "Bad Request", "Unknown format: foo",
1912                                     self.POST2, self.public_url + \
1913                                     "/foo/newdir?t=mkdir&format=foo", "")
1914
1915     def test_POST_NEWDIRURL_emptyname(self):
1916         # an empty pathname component (i.e. a double-slash) is disallowed
1917         d = self.shouldFail2(error.Error, "POST_NEWDIRURL_emptyname",
1918                              "400 Bad Request",
1919                              "The webapi does not allow empty pathname components, i.e. a double slash",
1920                              self.POST, self.public_url + "//?t=mkdir")
1921         return d
1922
1923     def _do_POST_NEWDIRURL_initial_children_test(self, version=None):
1924         (newkids, caps) = self._create_initial_children()
1925         query = "/foo/newdir?t=mkdir-with-children"
1926         if version == MDMF_VERSION:
1927             query += "&format=mdmf"
1928         elif version == SDMF_VERSION:
1929             query += "&format=sdmf"
1930         else:
1931             version = SDMF_VERSION # for later
1932         d = self.POST2(self.public_url + query,
1933                        simplejson.dumps(newkids))
1934         def _check(uri):
1935             n = self.s.create_node_from_uri(uri.strip())
1936             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1937             self.failUnlessEqual(n._node.get_version(), version)
1938             d2.addCallback(lambda ign:
1939                            self.failUnlessROChildURIIs(n, u"child-imm",
1940                                                        caps['filecap1']))
1941             d2.addCallback(lambda ign:
1942                            self.failUnlessRWChildURIIs(n, u"child-mutable",
1943                                                        caps['filecap2']))
1944             d2.addCallback(lambda ign:
1945                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1946                                                        caps['filecap3']))
1947             d2.addCallback(lambda ign:
1948                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1949                                                        caps['unknown_rocap']))
1950             d2.addCallback(lambda ign:
1951                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1952                                                        caps['unknown_rwcap']))
1953             d2.addCallback(lambda ign:
1954                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1955                                                        caps['unknown_immcap']))
1956             d2.addCallback(lambda ign:
1957                            self.failUnlessRWChildURIIs(n, u"dirchild",
1958                                                        caps['dircap']))
1959             d2.addCallback(lambda ign:
1960                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1961                                                        caps['litdircap']))
1962             d2.addCallback(lambda ign:
1963                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1964                                                        caps['emptydircap']))
1965             return d2
1966         d.addCallback(_check)
1967         d.addCallback(lambda res:
1968                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1969         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1970         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1971         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1972         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1973         return d
1974
1975     def test_POST_NEWDIRURL_initial_children(self):
1976         return self._do_POST_NEWDIRURL_initial_children_test()
1977
1978     def test_POST_NEWDIRURL_initial_children_mdmf(self):
1979         return self._do_POST_NEWDIRURL_initial_children_test(MDMF_VERSION)
1980
1981     def test_POST_NEWDIRURL_initial_children_sdmf(self):
1982         return self._do_POST_NEWDIRURL_initial_children_test(SDMF_VERSION)
1983
1984     def test_POST_NEWDIRURL_initial_children_bad_format(self):
1985         (newkids, caps) = self._create_initial_children()
1986         return self.shouldHTTPError("POST_NEWDIRURL_initial_children_bad_format",
1987                                     400, "Bad Request", "Unknown format: foo",
1988                                     self.POST2, self.public_url + \
1989                                     "/foo/newdir?t=mkdir-with-children&format=foo",
1990                                     simplejson.dumps(newkids))
1991
1992     def test_POST_NEWDIRURL_immutable(self):
1993         (newkids, caps) = self._create_immutable_children()
1994         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1995                        simplejson.dumps(newkids))
1996         def _check(uri):
1997             n = self.s.create_node_from_uri(uri.strip())
1998             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1999             d2.addCallback(lambda ign:
2000                            self.failUnlessROChildURIIs(n, u"child-imm",
2001                                                        caps['filecap1']))
2002             d2.addCallback(lambda ign:
2003                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2004                                                        caps['unknown_immcap']))
2005             d2.addCallback(lambda ign:
2006                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
2007                                                        caps['immdircap']))
2008             d2.addCallback(lambda ign:
2009                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
2010                                                        caps['litdircap']))
2011             d2.addCallback(lambda ign:
2012                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
2013                                                        caps['emptydircap']))
2014             return d2
2015         d.addCallback(_check)
2016         d.addCallback(lambda res:
2017                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2018         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2019         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2020         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2021         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2022         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2023         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2024         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2025         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2026         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2027         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2028         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2029         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2030         d.addErrback(self.explain_web_error)
2031         return d
2032
2033     def test_POST_NEWDIRURL_immutable_bad(self):
2034         (newkids, caps) = self._create_initial_children()
2035         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
2036                              "400 Bad Request",
2037                              "needed to be immutable but was not",
2038                              self.POST2,
2039                              self.public_url + "/foo/newdir?t=mkdir-immutable",
2040                              simplejson.dumps(newkids))
2041         return d
2042
2043     def test_PUT_NEWDIRURL_exists(self):
2044         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
2045         d.addCallback(lambda res:
2046                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
2047         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2048         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2049         return d
2050
2051     def test_PUT_NEWDIRURL_blocked(self):
2052         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
2053                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
2054                              self.PUT,
2055                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
2056         d.addCallback(lambda res:
2057                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
2058         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2059         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2060         return d
2061
2062     def test_PUT_NEWDIRURL_mkdirs(self):
2063         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
2064         d.addCallback(lambda res:
2065                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
2066         d.addCallback(lambda res:
2067                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
2068         d.addCallback(lambda res:
2069                       self._foo_node.get_child_at_path(u"subdir/newdir"))
2070         d.addCallback(self.failUnlessNodeKeysAre, [])
2071         return d
2072
2073     def test_PUT_NEWDIRURL_mkdirs_mdmf(self):
2074         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=mdmf", "")
2075         d.addCallback(lambda ignored:
2076             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
2077         d.addCallback(lambda ignored:
2078             self.failIfNodeHasChild(self._foo_node, u"newdir"))
2079         d.addCallback(lambda ignored:
2080             self._foo_node.get_child_at_path(u"subdir"))
2081         def _got_subdir(subdir):
2082             # XXX: What we want?
2083             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
2084             self.failUnlessNodeHasChild(subdir, u"newdir")
2085             return subdir.get_child_at_path(u"newdir")
2086         d.addCallback(_got_subdir)
2087         d.addCallback(lambda newdir:
2088             self.failUnlessEqual(newdir._node.get_version(), MDMF_VERSION))
2089         return d
2090
2091     def test_PUT_NEWDIRURL_mkdirs_sdmf(self):
2092         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=sdmf", "")
2093         d.addCallback(lambda ignored:
2094             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
2095         d.addCallback(lambda ignored:
2096             self.failIfNodeHasChild(self._foo_node, u"newdir"))
2097         d.addCallback(lambda ignored:
2098             self._foo_node.get_child_at_path(u"subdir"))
2099         def _got_subdir(subdir):
2100             # XXX: What we want?
2101             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
2102             self.failUnlessNodeHasChild(subdir, u"newdir")
2103             return subdir.get_child_at_path(u"newdir")
2104         d.addCallback(_got_subdir)
2105         d.addCallback(lambda newdir:
2106             self.failUnlessEqual(newdir._node.get_version(), SDMF_VERSION))
2107         return d
2108
2109     def test_PUT_NEWDIRURL_mkdirs_bad_format(self):
2110         return self.shouldHTTPError("PUT_NEWDIRURL_mkdirs_bad_format",
2111                                     400, "Bad Request", "Unknown format: foo",
2112                                     self.PUT, self.public_url + \
2113                                     "/foo/subdir/newdir?t=mkdir&format=foo",
2114                                     "")
2115
2116     def test_DELETE_DIRURL(self):
2117         d = self.DELETE(self.public_url + "/foo")
2118         d.addCallback(lambda res:
2119                       self.failIfNodeHasChild(self.public_root, u"foo"))
2120         return d
2121
2122     def test_DELETE_DIRURL_missing(self):
2123         d = self.DELETE(self.public_url + "/foo/missing")
2124         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
2125         d.addCallback(lambda res:
2126                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
2127         return d
2128
2129     def test_DELETE_DIRURL_missing2(self):
2130         d = self.DELETE(self.public_url + "/missing")
2131         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
2132         return d
2133
2134     def dump_root(self):
2135         print "NODEWALK"
2136         w = webish.DirnodeWalkerMixin()
2137         def visitor(childpath, childnode, metadata):
2138             print childpath
2139         d = w.walk(self.public_root, visitor)
2140         return d
2141
2142     def failUnlessNodeKeysAre(self, node, expected_keys):
2143         for k in expected_keys:
2144             assert isinstance(k, unicode)
2145         d = node.list()
2146         def _check(children):
2147             self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
2148         d.addCallback(_check)
2149         return d
2150     def failUnlessNodeHasChild(self, node, name):
2151         assert isinstance(name, unicode)
2152         d = node.list()
2153         def _check(children):
2154             self.failUnlessIn(name, children)
2155         d.addCallback(_check)
2156         return d
2157     def failIfNodeHasChild(self, node, name):
2158         assert isinstance(name, unicode)
2159         d = node.list()
2160         def _check(children):
2161             self.failIfIn(name, children)
2162         d.addCallback(_check)
2163         return d
2164
2165     def failUnlessChildContentsAre(self, node, name, expected_contents):
2166         assert isinstance(name, unicode)
2167         d = node.get_child_at_path(name)
2168         d.addCallback(lambda node: download_to_data(node))
2169         def _check(contents):
2170             self.failUnlessReallyEqual(contents, expected_contents)
2171         d.addCallback(_check)
2172         return d
2173
2174     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
2175         assert isinstance(name, unicode)
2176         d = node.get_child_at_path(name)
2177         d.addCallback(lambda node: node.download_best_version())
2178         def _check(contents):
2179             self.failUnlessReallyEqual(contents, expected_contents)
2180         d.addCallback(_check)
2181         return d
2182
2183     def failUnlessRWChildURIIs(self, node, name, expected_uri):
2184         assert isinstance(name, unicode)
2185         d = node.get_child_at_path(name)
2186         def _check(child):
2187             self.failUnless(child.is_unknown() or not child.is_readonly())
2188             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
2189             self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
2190             expected_ro_uri = self._make_readonly(expected_uri)
2191             if expected_ro_uri:
2192                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
2193         d.addCallback(_check)
2194         return d
2195
2196     def failUnlessROChildURIIs(self, node, name, expected_uri):
2197         assert isinstance(name, unicode)
2198         d = node.get_child_at_path(name)
2199         def _check(child):
2200             self.failUnless(child.is_unknown() or child.is_readonly())
2201             self.failUnlessReallyEqual(child.get_write_uri(), None)
2202             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
2203             self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
2204         d.addCallback(_check)
2205         return d
2206
2207     def failUnlessURIMatchesRWChild(self, got_uri, node, name):
2208         assert isinstance(name, unicode)
2209         d = node.get_child_at_path(name)
2210         def _check(child):
2211             self.failUnless(child.is_unknown() or not child.is_readonly())
2212             self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
2213             self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
2214             expected_ro_uri = self._make_readonly(got_uri)
2215             if expected_ro_uri:
2216                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
2217         d.addCallback(_check)
2218         return d
2219
2220     def failUnlessURIMatchesROChild(self, got_uri, node, name):
2221         assert isinstance(name, unicode)
2222         d = node.get_child_at_path(name)
2223         def _check(child):
2224             self.failUnless(child.is_unknown() or child.is_readonly())
2225             self.failUnlessReallyEqual(child.get_write_uri(), None)
2226             self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
2227             self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
2228         d.addCallback(_check)
2229         return d
2230
2231     def failUnlessCHKURIHasContents(self, got_uri, contents):
2232         self.failUnless(self.get_all_contents()[got_uri] == contents)
2233
2234     def test_POST_upload(self):
2235         d = self.POST(self.public_url + "/foo", t="upload",
2236                       file=("new.txt", self.NEWFILE_CONTENTS))
2237         fn = self._foo_node
2238         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2239         d.addCallback(lambda res:
2240                       self.failUnlessChildContentsAre(fn, u"new.txt",
2241                                                       self.NEWFILE_CONTENTS))
2242         return d
2243
2244     def test_POST_upload_unicode(self):
2245         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2246         d = self.POST(self.public_url + "/foo", t="upload",
2247                       file=(filename, self.NEWFILE_CONTENTS))
2248         fn = self._foo_node
2249         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2250         d.addCallback(lambda res:
2251                       self.failUnlessChildContentsAre(fn, filename,
2252                                                       self.NEWFILE_CONTENTS))
2253         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2254         d.addCallback(lambda res: self.GET(target_url))
2255         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2256                                                                   self.NEWFILE_CONTENTS,
2257                                                                   contents))
2258         return d
2259
2260     def test_POST_upload_unicode_named(self):
2261         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2262         d = self.POST(self.public_url + "/foo", t="upload",
2263                       name=filename,
2264                       file=("overridden", self.NEWFILE_CONTENTS))
2265         fn = self._foo_node
2266         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2267         d.addCallback(lambda res:
2268                       self.failUnlessChildContentsAre(fn, filename,
2269                                                       self.NEWFILE_CONTENTS))
2270         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2271         d.addCallback(lambda res: self.GET(target_url))
2272         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2273                                                                   self.NEWFILE_CONTENTS,
2274                                                                   contents))
2275         return d
2276
2277     def test_POST_upload_no_link(self):
2278         d = self.POST("/uri", t="upload",
2279                       file=("new.txt", self.NEWFILE_CONTENTS))
2280         def _check_upload_results(page):
2281             # this should be a page which describes the results of the upload
2282             # that just finished.
2283             self.failUnlessIn("Upload Results:", page)
2284             self.failUnlessIn("URI:", page)
2285             uri_re = re.compile("URI: <tt><span>(.*)</span>")
2286             mo = uri_re.search(page)
2287             self.failUnless(mo, page)
2288             new_uri = mo.group(1)
2289             return new_uri
2290         d.addCallback(_check_upload_results)
2291         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
2292         return d
2293
2294     def test_POST_upload_no_link_whendone(self):
2295         d = self.POST("/uri", t="upload", when_done="/",
2296                       file=("new.txt", self.NEWFILE_CONTENTS))
2297         d.addBoth(self.shouldRedirect, "/")
2298         return d
2299
2300     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
2301         d = defer.maybeDeferred(callable, *args, **kwargs)
2302         def done(res):
2303             if isinstance(res, failure.Failure):
2304                 res.trap(error.PageRedirect)
2305                 statuscode = res.value.status
2306                 target = res.value.location
2307                 return checker(statuscode, target)
2308             self.fail("%s: callable was supposed to redirect, not return '%s'"
2309                       % (which, res))
2310         d.addBoth(done)
2311         return d
2312
2313     def test_POST_upload_no_link_whendone_results(self):
2314         def check(statuscode, target):
2315             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2316             self.failUnless(target.startswith(self.webish_url), target)
2317             return client.getPage(target, method="GET")
2318         # We encode "uri" as "%75ri" to exercise a case affected by ticket #1860.
2319         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
2320                                  check,
2321                                  self.POST, "/uri", t="upload",
2322                                  when_done="/%75ri/%(uri)s",
2323                                  file=("new.txt", self.NEWFILE_CONTENTS))
2324         d.addCallback(lambda res:
2325                       self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
2326         return d
2327
2328     def test_POST_upload_no_link_mutable(self):
2329         d = self.POST("/uri", t="upload", mutable="true",
2330                       file=("new.txt", self.NEWFILE_CONTENTS))
2331         def _check(filecap):
2332             filecap = filecap.strip()
2333             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2334             self.filecap = filecap
2335             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2336             self.failUnlessIn(u.get_storage_index(), self.get_all_contents())
2337             n = self.s.create_node_from_uri(filecap)
2338             return n.download_best_version()
2339         d.addCallback(_check)
2340         def _check2(data):
2341             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2342             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2343         d.addCallback(_check2)
2344         def _check3(data):
2345             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2346             return self.GET("/file/%s" % urllib.quote(self.filecap))
2347         d.addCallback(_check3)
2348         def _check4(data):
2349             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2350         d.addCallback(_check4)
2351         return d
2352
2353     def test_POST_upload_no_link_mutable_toobig(self):
2354         # The SDMF size limit is no longer in place, so we should be
2355         # able to upload mutable files that are as large as we want them
2356         # to be.
2357         d = self.POST("/uri", t="upload", mutable="true",
2358                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2359         return d
2360
2361
2362     def test_POST_upload_format_unlinked(self):
2363         def _check_upload_unlinked(ign, format, uri_prefix):
2364             filename = format + ".txt"
2365             d = self.POST("/uri?t=upload&format=" + format,
2366                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2367             def _got_results(results):
2368                 if format.upper() in ("SDMF", "MDMF"):
2369                     # webapi.rst says this returns a filecap
2370                     filecap = results
2371                 else:
2372                     # for immutable, it returns an "upload results page", and
2373                     # the filecap is buried inside
2374                     line = [l for l in results.split("\n") if "URI: " in l][0]
2375                     mo = re.search(r'<span>([^<]+)</span>', line)
2376                     filecap = mo.group(1)
2377                 self.failUnless(filecap.startswith(uri_prefix),
2378                                 (uri_prefix, filecap))
2379                 return self.GET("/uri/%s?t=json" % filecap)
2380             d.addCallback(_got_results)
2381             def _got_json(json):
2382                 data = simplejson.loads(json)
2383                 data = data[1]
2384                 self.failUnlessIn("format", data)
2385                 self.failUnlessEqual(data["format"], format.upper())
2386             d.addCallback(_got_json)
2387             return d
2388         d = defer.succeed(None)
2389         d.addCallback(_check_upload_unlinked, "chk", "URI:CHK")
2390         d.addCallback(_check_upload_unlinked, "CHK", "URI:CHK")
2391         d.addCallback(_check_upload_unlinked, "sdmf", "URI:SSK")
2392         d.addCallback(_check_upload_unlinked, "mdmf", "URI:MDMF")
2393         return d
2394
2395     def test_POST_upload_bad_format_unlinked(self):
2396         return self.shouldHTTPError("POST_upload_bad_format_unlinked",
2397                                     400, "Bad Request", "Unknown format: foo",
2398                                     self.POST,
2399                                     "/uri?t=upload&format=foo",
2400                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2401
2402     def test_POST_upload_format(self):
2403         def _check_upload(ign, format, uri_prefix, fn=None):
2404             filename = format + ".txt"
2405             d = self.POST(self.public_url +
2406                           "/foo?t=upload&format=" + format,
2407                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2408             def _got_filecap(filecap):
2409                 if fn is not None:
2410                     filenameu = unicode(filename)
2411                     self.failUnlessURIMatchesRWChild(filecap, fn, filenameu)
2412                 self.failUnless(filecap.startswith(uri_prefix))
2413                 return self.GET(self.public_url + "/foo/%s?t=json" % filename)
2414             d.addCallback(_got_filecap)
2415             def _got_json(json):
2416                 data = simplejson.loads(json)
2417                 data = data[1]
2418                 self.failUnlessIn("format", data)
2419                 self.failUnlessEqual(data["format"], format.upper())
2420             d.addCallback(_got_json)
2421             return d
2422
2423         d = defer.succeed(None)
2424         d.addCallback(_check_upload, "chk", "URI:CHK")
2425         d.addCallback(_check_upload, "sdmf", "URI:SSK", self._foo_node)
2426         d.addCallback(_check_upload, "mdmf", "URI:MDMF")
2427         d.addCallback(_check_upload, "MDMF", "URI:MDMF")
2428         return d
2429
2430     def test_POST_upload_bad_format(self):
2431         return self.shouldHTTPError("POST_upload_bad_format",
2432                                     400, "Bad Request", "Unknown format: foo",
2433                                     self.POST, self.public_url + \
2434                                     "/foo?t=upload&format=foo",
2435                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2436
2437     def test_POST_upload_mutable(self):
2438         # this creates a mutable file
2439         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
2440                       file=("new.txt", self.NEWFILE_CONTENTS))
2441         fn = self._foo_node
2442         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2443         d.addCallback(lambda res:
2444                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2445                                                              self.NEWFILE_CONTENTS))
2446         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2447         def _got(newnode):
2448             self.failUnless(IMutableFileNode.providedBy(newnode))
2449             self.failUnless(newnode.is_mutable())
2450             self.failIf(newnode.is_readonly())
2451             self._mutable_node = newnode
2452             self._mutable_uri = newnode.get_uri()
2453         d.addCallback(_got)
2454
2455         # now upload it again and make sure that the URI doesn't change
2456         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
2457         d.addCallback(lambda res:
2458                       self.POST(self.public_url + "/foo", t="upload",
2459                                 mutable="true",
2460                                 file=("new.txt", NEWER_CONTENTS)))
2461         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2462         d.addCallback(lambda res:
2463                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2464                                                              NEWER_CONTENTS))
2465         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2466         def _got2(newnode):
2467             self.failUnless(IMutableFileNode.providedBy(newnode))
2468             self.failUnless(newnode.is_mutable())
2469             self.failIf(newnode.is_readonly())
2470             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2471         d.addCallback(_got2)
2472
2473         # upload a second time, using PUT instead of POST
2474         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
2475         d.addCallback(lambda res:
2476                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
2477         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2478         d.addCallback(lambda res:
2479                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2480                                                              NEW2_CONTENTS))
2481
2482         # finally list the directory, since mutable files are displayed
2483         # slightly differently
2484
2485         d.addCallback(lambda res:
2486                       self.GET(self.public_url + "/foo/",
2487                                followRedirect=True))
2488         def _check_page(res):
2489             # TODO: assert more about the contents
2490             self.failUnlessIn("SSK", res)
2491             return res
2492         d.addCallback(_check_page)
2493
2494         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2495         def _got3(newnode):
2496             self.failUnless(IMutableFileNode.providedBy(newnode))
2497             self.failUnless(newnode.is_mutable())
2498             self.failIf(newnode.is_readonly())
2499             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2500         d.addCallback(_got3)
2501
2502         # look at the JSON form of the enclosing directory
2503         d.addCallback(lambda res:
2504                       self.GET(self.public_url + "/foo/?t=json",
2505                                followRedirect=True))
2506         def _check_page_json(res):
2507             parsed = simplejson.loads(res)
2508             self.failUnlessEqual(parsed[0], "dirnode")
2509             children = dict( [(unicode(name),value)
2510                               for (name,value)
2511                               in parsed[1]["children"].iteritems()] )
2512             self.failUnlessIn(u"new.txt", children)
2513             new_json = children[u"new.txt"]
2514             self.failUnlessEqual(new_json[0], "filenode")
2515             self.failUnless(new_json[1]["mutable"])
2516             self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri)
2517             ro_uri = self._mutable_node.get_readonly().to_string()
2518             self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri)
2519         d.addCallback(_check_page_json)
2520
2521         # and the JSON form of the file
2522         d.addCallback(lambda res:
2523                       self.GET(self.public_url + "/foo/new.txt?t=json"))
2524         def _check_file_json(res):
2525             parsed = simplejson.loads(res)
2526             self.failUnlessEqual(parsed[0], "filenode")
2527             self.failUnless(parsed[1]["mutable"])
2528             self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri)
2529             ro_uri = self._mutable_node.get_readonly().to_string()
2530             self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri)
2531         d.addCallback(_check_file_json)
2532
2533         # and look at t=uri and t=readonly-uri
2534         d.addCallback(lambda res:
2535                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
2536         d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
2537         d.addCallback(lambda res:
2538                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
2539         def _check_ro_uri(res):
2540             ro_uri = self._mutable_node.get_readonly().to_string()
2541             self.failUnlessReallyEqual(res, ro_uri)
2542         d.addCallback(_check_ro_uri)
2543
2544         # make sure we can get to it from /uri/URI
2545         d.addCallback(lambda res:
2546                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
2547         d.addCallback(lambda res:
2548                       self.failUnlessReallyEqual(res, NEW2_CONTENTS))
2549
2550         # and that HEAD computes the size correctly
2551         d.addCallback(lambda res:
2552                       self.HEAD(self.public_url + "/foo/new.txt",
2553                                 return_response=True))
2554         def _got_headers((res, status, headers)):
2555             self.failUnlessReallyEqual(res, "")
2556             self.failUnlessReallyEqual(headers["content-length"][0],
2557                                        str(len(NEW2_CONTENTS)))
2558             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
2559         d.addCallback(_got_headers)
2560
2561         # make sure that outdated size limits aren't enforced anymore.
2562         d.addCallback(lambda ignored:
2563             self.POST(self.public_url + "/foo", t="upload",
2564                       mutable="true",
2565                       file=("new.txt",
2566                             "b" * (self.s.MUTABLE_SIZELIMIT+1))))
2567         d.addErrback(self.dump_error)
2568         return d
2569
2570     def test_POST_upload_mutable_toobig(self):
2571         # SDMF had a size limti that was removed a while ago. MDMF has
2572         # never had a size limit. Test to make sure that we do not
2573         # encounter errors when trying to upload large mutable files,
2574         # since there should be no coded prohibitions regarding large
2575         # mutable files.
2576         d = self.POST(self.public_url + "/foo",
2577                       t="upload", mutable="true",
2578                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2579         return d
2580
2581     def dump_error(self, f):
2582         # if the web server returns an error code (like 400 Bad Request),
2583         # web.client.getPage puts the HTTP response body into the .response
2584         # attribute of the exception object that it gives back. It does not
2585         # appear in the Failure's repr(), so the ERROR that trial displays
2586         # will be rather terse and unhelpful. addErrback this method to the
2587         # end of your chain to get more information out of these errors.
2588         if f.check(error.Error):
2589             print "web.error.Error:"
2590             print f
2591             print f.value.response
2592         return f
2593
2594     def test_POST_upload_replace(self):
2595         d = self.POST(self.public_url + "/foo", t="upload",
2596                       file=("bar.txt", self.NEWFILE_CONTENTS))
2597         fn = self._foo_node
2598         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
2599         d.addCallback(lambda res:
2600                       self.failUnlessChildContentsAre(fn, u"bar.txt",
2601                                                       self.NEWFILE_CONTENTS))
2602         return d
2603
2604     def test_POST_upload_no_replace_ok(self):
2605         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2606                       file=("new.txt", self.NEWFILE_CONTENTS))
2607         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
2608         d.addCallback(lambda res: self.failUnlessReallyEqual(res,
2609                                                              self.NEWFILE_CONTENTS))
2610         return d
2611
2612     def test_POST_upload_no_replace_queryarg(self):
2613         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2614                       file=("bar.txt", self.NEWFILE_CONTENTS))
2615         d.addBoth(self.shouldFail, error.Error,
2616                   "POST_upload_no_replace_queryarg",
2617                   "409 Conflict",
2618                   "There was already a child by that name, and you asked me "
2619                   "to not replace it")
2620         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2621         d.addCallback(self.failUnlessIsBarDotTxt)
2622         return d
2623
2624     def test_POST_upload_no_replace_field(self):
2625         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
2626                       file=("bar.txt", self.NEWFILE_CONTENTS))
2627         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
2628                   "409 Conflict",
2629                   "There was already a child by that name, and you asked me "
2630                   "to not replace it")
2631         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2632         d.addCallback(self.failUnlessIsBarDotTxt)
2633         return d
2634
2635     def test_POST_upload_whendone(self):
2636         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
2637                       file=("new.txt", self.NEWFILE_CONTENTS))
2638         d.addBoth(self.shouldRedirect, "/THERE")
2639         fn = self._foo_node
2640         d.addCallback(lambda res:
2641                       self.failUnlessChildContentsAre(fn, u"new.txt",
2642                                                       self.NEWFILE_CONTENTS))
2643         return d
2644
2645     def test_POST_upload_named(self):
2646         fn = self._foo_node
2647         d = self.POST(self.public_url + "/foo", t="upload",
2648                       name="new.txt", file=self.NEWFILE_CONTENTS)
2649         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2650         d.addCallback(lambda res:
2651                       self.failUnlessChildContentsAre(fn, u"new.txt",
2652                                                       self.NEWFILE_CONTENTS))
2653         return d
2654
2655     def test_POST_upload_named_badfilename(self):
2656         d = self.POST(self.public_url + "/foo", t="upload",
2657                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
2658         d.addBoth(self.shouldFail, error.Error,
2659                   "test_POST_upload_named_badfilename",
2660                   "400 Bad Request",
2661                   "name= may not contain a slash",
2662                   )
2663         # make sure that nothing was added
2664         d.addCallback(lambda res:
2665                       self.failUnlessNodeKeysAre(self._foo_node,
2666                                                  [self._htmlname_unicode,
2667                                                   u"bar.txt", u"baz.txt", u"blockingfile",
2668                                                   u"empty", u"n\u00fc.txt", u"quux.txt",
2669                                                   u"sub"]))
2670         return d
2671
2672     def test_POST_FILEURL_check(self):
2673         bar_url = self.public_url + "/foo/bar.txt"
2674         d = self.POST(bar_url, t="check")
2675         def _check(res):
2676             self.failUnlessIn("Healthy :", res)
2677         d.addCallback(_check)
2678         redir_url = "http://allmydata.org/TARGET"
2679         def _check2(statuscode, target):
2680             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2681             self.failUnlessReallyEqual(target, redir_url)
2682         d.addCallback(lambda res:
2683                       self.shouldRedirect2("test_POST_FILEURL_check",
2684                                            _check2,
2685                                            self.POST, bar_url,
2686                                            t="check",
2687                                            when_done=redir_url))
2688         d.addCallback(lambda res:
2689                       self.POST(bar_url, t="check", return_to=redir_url))
2690         def _check3(res):
2691             self.failUnlessIn("Healthy :", res)
2692             self.failUnlessIn("Return to file", res)
2693             self.failUnlessIn(redir_url, res)
2694         d.addCallback(_check3)
2695
2696         d.addCallback(lambda res:
2697                       self.POST(bar_url, t="check", output="JSON"))
2698         def _check_json(res):
2699             data = simplejson.loads(res)
2700             self.failUnlessIn("storage-index", data)
2701             self.failUnless(data["results"]["healthy"])
2702         d.addCallback(_check_json)
2703
2704         return d
2705
2706     def test_POST_FILEURL_check_and_repair(self):
2707         bar_url = self.public_url + "/foo/bar.txt"
2708         d = self.POST(bar_url, t="check", repair="true")
2709         def _check(res):
2710             self.failUnlessIn("Healthy :", res)
2711         d.addCallback(_check)
2712         redir_url = "http://allmydata.org/TARGET"
2713         def _check2(statuscode, target):
2714             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2715             self.failUnlessReallyEqual(target, redir_url)
2716         d.addCallback(lambda res:
2717                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
2718                                            _check2,
2719                                            self.POST, bar_url,
2720                                            t="check", repair="true",
2721                                            when_done=redir_url))
2722         d.addCallback(lambda res:
2723                       self.POST(bar_url, t="check", return_to=redir_url))
2724         def _check3(res):
2725             self.failUnlessIn("Healthy :", res)
2726             self.failUnlessIn("Return to file", res)
2727             self.failUnlessIn(redir_url, res)
2728         d.addCallback(_check3)
2729         return d
2730
2731     def test_POST_DIRURL_check(self):
2732         foo_url = self.public_url + "/foo/"
2733         d = self.POST(foo_url, t="check")
2734         def _check(res):
2735             self.failUnlessIn("Healthy :", res)
2736         d.addCallback(_check)
2737         redir_url = "http://allmydata.org/TARGET"
2738         def _check2(statuscode, target):
2739             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2740             self.failUnlessReallyEqual(target, redir_url)
2741         d.addCallback(lambda res:
2742                       self.shouldRedirect2("test_POST_DIRURL_check",
2743                                            _check2,
2744                                            self.POST, foo_url,
2745                                            t="check",
2746                                            when_done=redir_url))
2747         d.addCallback(lambda res:
2748                       self.POST(foo_url, t="check", return_to=redir_url))
2749         def _check3(res):
2750             self.failUnlessIn("Healthy :", res)
2751             self.failUnlessIn("Return to file/directory", res)
2752             self.failUnlessIn(redir_url, res)
2753         d.addCallback(_check3)
2754
2755         d.addCallback(lambda res:
2756                       self.POST(foo_url, t="check", output="JSON"))
2757         def _check_json(res):
2758             data = simplejson.loads(res)
2759             self.failUnlessIn("storage-index", data)
2760             self.failUnless(data["results"]["healthy"])
2761         d.addCallback(_check_json)
2762
2763         return d
2764
2765     def test_POST_DIRURL_check_and_repair(self):
2766         foo_url = self.public_url + "/foo/"
2767         d = self.POST(foo_url, t="check", repair="true")
2768         def _check(res):
2769             self.failUnlessIn("Healthy :", res)
2770         d.addCallback(_check)
2771         redir_url = "http://allmydata.org/TARGET"
2772         def _check2(statuscode, target):
2773             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2774             self.failUnlessReallyEqual(target, redir_url)
2775         d.addCallback(lambda res:
2776                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2777                                            _check2,
2778                                            self.POST, foo_url,
2779                                            t="check", repair="true",
2780                                            when_done=redir_url))
2781         d.addCallback(lambda res:
2782                       self.POST(foo_url, t="check", return_to=redir_url))
2783         def _check3(res):
2784             self.failUnlessIn("Healthy :", res)
2785             self.failUnlessIn("Return to file/directory", res)
2786             self.failUnlessIn(redir_url, res)
2787         d.addCallback(_check3)
2788         return d
2789
2790     def test_POST_FILEURL_mdmf_check(self):
2791         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2792         d = self.POST(quux_url, t="check")
2793         def _check(res):
2794             self.failUnlessIn("Healthy", res)
2795         d.addCallback(_check)
2796         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2797         d.addCallback(lambda ignored:
2798                       self.POST(quux_extension_url, t="check"))
2799         d.addCallback(_check)
2800         return d
2801
2802     def test_POST_FILEURL_mdmf_check_and_repair(self):
2803         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2804         d = self.POST(quux_url, t="check", repair="true")
2805         def _check(res):
2806             self.failUnlessIn("Healthy", res)
2807         d.addCallback(_check)
2808         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2809         d.addCallback(lambda ignored:
2810                       self.POST(quux_extension_url, t="check", repair="true"))
2811         d.addCallback(_check)
2812         return d
2813
2814     def wait_for_operation(self, ignored, ophandle):
2815         url = "/operations/" + ophandle
2816         url += "?t=status&output=JSON"
2817         d = self.GET(url)
2818         def _got(res):
2819             data = simplejson.loads(res)
2820             if not data["finished"]:
2821                 d = self.stall(delay=1.0)
2822                 d.addCallback(self.wait_for_operation, ophandle)
2823                 return d
2824             return data
2825         d.addCallback(_got)
2826         return d
2827
2828     def get_operation_results(self, ignored, ophandle, output=None):
2829         url = "/operations/" + ophandle
2830         url += "?t=status"
2831         if output:
2832             url += "&output=" + output
2833         d = self.GET(url)
2834         def _got(res):
2835             if output and output.lower() == "json":
2836                 return simplejson.loads(res)
2837             return res
2838         d.addCallback(_got)
2839         return d
2840
2841     def test_POST_DIRURL_deepcheck_no_ophandle(self):
2842         d = self.shouldFail2(error.Error,
2843                              "test_POST_DIRURL_deepcheck_no_ophandle",
2844                              "400 Bad Request",
2845                              "slow operation requires ophandle=",
2846                              self.POST, self.public_url, t="start-deep-check")
2847         return d
2848
2849     def test_POST_DIRURL_deepcheck(self):
2850         def _check_redirect(statuscode, target):
2851             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2852             self.failUnless(target.endswith("/operations/123"))
2853         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2854                                  self.POST, self.public_url,
2855                                  t="start-deep-check", ophandle="123")
2856         d.addCallback(self.wait_for_operation, "123")
2857         def _check_json(data):
2858             self.failUnlessReallyEqual(data["finished"], True)
2859             self.failUnlessReallyEqual(data["count-objects-checked"], 11)
2860             self.failUnlessReallyEqual(data["count-objects-healthy"], 11)
2861         d.addCallback(_check_json)
2862         d.addCallback(self.get_operation_results, "123", "html")
2863         def _check_html(res):
2864             self.failUnlessIn("Objects Checked: <span>11</span>", res)
2865             self.failUnlessIn("Objects Healthy: <span>11</span>", res)
2866             self.failUnlessIn(FAVICON_MARKUP, res)
2867         d.addCallback(_check_html)
2868
2869         d.addCallback(lambda res:
2870                       self.GET("/operations/123/"))
2871         d.addCallback(_check_html) # should be the same as without the slash
2872
2873         d.addCallback(lambda res:
2874                       self.shouldFail2(error.Error, "one", "404 Not Found",
2875                                        "No detailed results for SI bogus",
2876                                        self.GET, "/operations/123/bogus"))
2877
2878         foo_si = self._foo_node.get_storage_index()
2879         foo_si_s = base32.b2a(foo_si)
2880         d.addCallback(lambda res:
2881                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2882         def _check_foo_json(res):
2883             data = simplejson.loads(res)
2884             self.failUnlessEqual(data["storage-index"], foo_si_s)
2885             self.failUnless(data["results"]["healthy"])
2886         d.addCallback(_check_foo_json)
2887         return d
2888
2889     def test_POST_DIRURL_deepcheck_and_repair(self):
2890         d = self.POST(self.public_url, t="start-deep-check", repair="true",
2891                       ophandle="124", output="json", followRedirect=True)
2892         d.addCallback(self.wait_for_operation, "124")
2893         def _check_json(data):
2894             self.failUnlessReallyEqual(data["finished"], True)
2895             self.failUnlessReallyEqual(data["count-objects-checked"], 11)
2896             self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 11)
2897             self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2898             self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2899             self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2900             self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2901             self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2902             self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 11)
2903             self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2904             self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2905         d.addCallback(_check_json)
2906         d.addCallback(self.get_operation_results, "124", "html")
2907         def _check_html(res):
2908             self.failUnlessIn("Objects Checked: <span>11</span>", res)
2909
2910             self.failUnlessIn("Objects Healthy (before repair): <span>11</span>", res)
2911             self.failUnlessIn("Objects Unhealthy (before repair): <span>0</span>", res)
2912             self.failUnlessIn("Corrupt Shares (before repair): <span>0</span>", res)
2913
2914             self.failUnlessIn("Repairs Attempted: <span>0</span>", res)
2915             self.failUnlessIn("Repairs Successful: <span>0</span>", res)
2916             self.failUnlessIn("Repairs Unsuccessful: <span>0</span>", res)
2917
2918             self.failUnlessIn("Objects Healthy (after repair): <span>11</span>", res)
2919             self.failUnlessIn("Objects Unhealthy (after repair): <span>0</span>", res)
2920             self.failUnlessIn("Corrupt Shares (after repair): <span>0</span>", res)
2921
2922             self.failUnlessIn(FAVICON_MARKUP, res)
2923         d.addCallback(_check_html)
2924         return d
2925
2926     def test_POST_FILEURL_bad_t(self):
2927         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2928                              "POST to file: bad t=bogus",
2929                              self.POST, self.public_url + "/foo/bar.txt",
2930                              t="bogus")
2931         return d
2932
2933     def test_POST_mkdir(self): # return value?
2934         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2935         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2936         d.addCallback(self.failUnlessNodeKeysAre, [])
2937         return d
2938
2939     def test_POST_mkdir_mdmf(self):
2940         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=mdmf")
2941         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2942         d.addCallback(lambda node:
2943             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2944         return d
2945
2946     def test_POST_mkdir_sdmf(self):
2947         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=sdmf")
2948         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2949         d.addCallback(lambda node:
2950             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2951         return d
2952
2953     def test_POST_mkdir_bad_format(self):
2954         return self.shouldHTTPError("POST_mkdir_bad_format",
2955                                     400, "Bad Request", "Unknown format: foo",
2956                                     self.POST, self.public_url +
2957                                     "/foo?t=mkdir&name=newdir&format=foo")
2958
2959     def test_POST_mkdir_initial_children(self):
2960         (newkids, caps) = self._create_initial_children()
2961         d = self.POST2(self.public_url +
2962                        "/foo?t=mkdir-with-children&name=newdir",
2963                        simplejson.dumps(newkids))
2964         d.addCallback(lambda res:
2965                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2966         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2967         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2968         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2969         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2970         return d
2971
2972     def test_POST_mkdir_initial_children_mdmf(self):
2973         (newkids, caps) = self._create_initial_children()
2974         d = self.POST2(self.public_url +
2975                        "/foo?t=mkdir-with-children&name=newdir&format=mdmf",
2976                        simplejson.dumps(newkids))
2977         d.addCallback(lambda res:
2978                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2979         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2980         d.addCallback(lambda node:
2981             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2982         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2983         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2984                        caps['filecap1'])
2985         return d
2986
2987     # XXX: Duplication.
2988     def test_POST_mkdir_initial_children_sdmf(self):
2989         (newkids, caps) = self._create_initial_children()
2990         d = self.POST2(self.public_url +
2991                        "/foo?t=mkdir-with-children&name=newdir&format=sdmf",
2992                        simplejson.dumps(newkids))
2993         d.addCallback(lambda res:
2994                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2995         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2996         d.addCallback(lambda node:
2997             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2998         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2999         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
3000                        caps['filecap1'])
3001         return d
3002
3003     def test_POST_mkdir_initial_children_bad_format(self):
3004         (newkids, caps) = self._create_initial_children()
3005         return self.shouldHTTPError("POST_mkdir_initial_children_bad_format",
3006                                     400, "Bad Request", "Unknown format: foo",
3007                                     self.POST, self.public_url + \
3008                                     "/foo?t=mkdir-with-children&name=newdir&format=foo",
3009                                     simplejson.dumps(newkids))
3010
3011     def test_POST_mkdir_immutable(self):
3012         (newkids, caps) = self._create_immutable_children()
3013         d = self.POST2(self.public_url +
3014                        "/foo?t=mkdir-immutable&name=newdir",
3015                        simplejson.dumps(newkids))
3016         d.addCallback(lambda res:
3017                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
3018         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3019         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
3020         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3021         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
3022         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3023         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
3024         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3025         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
3026         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3027         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
3028         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3029         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
3030         return d
3031
3032     def test_POST_mkdir_immutable_bad(self):
3033         (newkids, caps) = self._create_initial_children()
3034         d = self.shouldFail2(error.Error, "POST_mkdir_immutable_bad",
3035                              "400 Bad Request",
3036                              "needed to be immutable but was not",
3037                              self.POST2,
3038                              self.public_url +
3039                              "/foo?t=mkdir-immutable&name=newdir",
3040                              simplejson.dumps(newkids))
3041         return d
3042
3043     def test_POST_mkdir_2(self):
3044         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
3045         d.addCallback(lambda res:
3046                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
3047         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3048         d.addCallback(self.failUnlessNodeKeysAre, [])
3049         return d
3050
3051     def test_POST_mkdirs_2(self):
3052         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
3053         d.addCallback(lambda res:
3054                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
3055         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
3056         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
3057         d.addCallback(self.failUnlessNodeKeysAre, [])
3058         return d
3059
3060     def test_POST_mkdir_no_parentdir_noredirect(self):
3061         d = self.POST("/uri?t=mkdir")
3062         def _after_mkdir(res):
3063             uri.DirectoryURI.init_from_string(res)
3064         d.addCallback(_after_mkdir)
3065         return d
3066
3067     def test_POST_mkdir_no_parentdir_noredirect_mdmf(self):
3068         d = self.POST("/uri?t=mkdir&format=mdmf")
3069         def _after_mkdir(res):
3070             u = uri.from_string(res)
3071             # Check that this is an MDMF writecap
3072             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
3073         d.addCallback(_after_mkdir)
3074         return d
3075
3076     def test_POST_mkdir_no_parentdir_noredirect_sdmf(self):
3077         d = self.POST("/uri?t=mkdir&format=sdmf")
3078         def _after_mkdir(res):
3079             u = uri.from_string(res)
3080             self.failUnlessIsInstance(u, uri.DirectoryURI)
3081         d.addCallback(_after_mkdir)
3082         return d
3083
3084     def test_POST_mkdir_no_parentdir_noredirect_bad_format(self):
3085         return self.shouldHTTPError("POST_mkdir_no_parentdir_noredirect_bad_format",
3086                                     400, "Bad Request", "Unknown format: foo",
3087                                     self.POST, self.public_url +
3088                                     "/uri?t=mkdir&format=foo")
3089
3090     def test_POST_mkdir_no_parentdir_noredirect2(self):
3091         # make sure form-based arguments (as on the welcome page) still work
3092         d = self.POST("/uri", t="mkdir")
3093         def _after_mkdir(res):
3094             uri.DirectoryURI.init_from_string(res)
3095         d.addCallback(_after_mkdir)
3096         d.addErrback(self.explain_web_error)
3097         return d
3098
3099     def test_POST_mkdir_no_parentdir_redirect(self):
3100         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
3101         d.addBoth(self.shouldRedirect, None, statuscode='303')
3102         def _check_target(target):
3103             target = urllib.unquote(target)
3104             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
3105         d.addCallback(_check_target)
3106         return d
3107
3108     def test_POST_mkdir_no_parentdir_redirect2(self):
3109         d = self.POST("/uri", t="mkdir", redirect_to_result="true")
3110         d.addBoth(self.shouldRedirect, None, statuscode='303')
3111         def _check_target(target):
3112             target = urllib.unquote(target)
3113             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
3114         d.addCallback(_check_target)
3115         d.addErrback(self.explain_web_error)
3116         return d
3117
3118     def _make_readonly(self, u):
3119         ro_uri = uri.from_string(u).get_readonly()
3120         if ro_uri is None:
3121             return None
3122         return ro_uri.to_string()
3123
3124     def _create_initial_children(self):
3125         contents, n, filecap1 = self.makefile(12)
3126         md1 = {"metakey1": "metavalue1"}
3127         filecap2 = make_mutable_file_uri()
3128         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
3129         filecap3 = node3.get_readonly_uri()
3130         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
3131         dircap = DirectoryNode(node4, None, None).get_uri()
3132         mdmfcap = make_mutable_file_uri(mdmf=True)
3133         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
3134         emptydircap = "URI:DIR2-LIT:"
3135         newkids = {u"child-imm":        ["filenode", {"rw_uri": filecap1,
3136                                                       "ro_uri": self._make_readonly(filecap1),
3137                                                       "metadata": md1, }],
3138                    u"child-mutable":    ["filenode", {"rw_uri": filecap2,
3139                                                       "ro_uri": self._make_readonly(filecap2)}],
3140                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
3141                    u"unknownchild-rw":  ["unknown",  {"rw_uri": unknown_rwcap,
3142                                                       "ro_uri": unknown_rocap}],
3143                    u"unknownchild-ro":  ["unknown",  {"ro_uri": unknown_rocap}],
3144                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
3145                    u"dirchild":         ["dirnode",  {"rw_uri": dircap,
3146                                                       "ro_uri": self._make_readonly(dircap)}],
3147                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
3148                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
3149                    u"child-mutable-mdmf": ["filenode", {"rw_uri": mdmfcap,
3150                                                         "ro_uri": self._make_readonly(mdmfcap)}],
3151                    }
3152         return newkids, {'filecap1': filecap1,
3153                          'filecap2': filecap2,
3154                          'filecap3': filecap3,
3155                          'unknown_rwcap': unknown_rwcap,
3156                          'unknown_rocap': unknown_rocap,
3157                          'unknown_immcap': unknown_immcap,
3158                          'dircap': dircap,
3159                          'litdircap': litdircap,
3160                          'emptydircap': emptydircap,
3161                          'mdmfcap': mdmfcap}
3162
3163     def _create_immutable_children(self):
3164         contents, n, filecap1 = self.makefile(12)
3165         md1 = {"metakey1": "metavalue1"}
3166         tnode = create_chk_filenode("immutable directory contents\n"*10,
3167                                     self.get_all_contents())
3168         dnode = DirectoryNode(tnode, None, None)
3169         assert not dnode.is_mutable()
3170         immdircap = dnode.get_uri()
3171         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
3172         emptydircap = "URI:DIR2-LIT:"
3173         newkids = {u"child-imm":        ["filenode", {"ro_uri": filecap1,
3174                                                       "metadata": md1, }],
3175                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
3176                    u"dirchild-imm":     ["dirnode",  {"ro_uri": immdircap}],
3177                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
3178                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
3179                    }
3180         return newkids, {'filecap1': filecap1,
3181                          'unknown_immcap': unknown_immcap,
3182                          'immdircap': immdircap,
3183                          'litdircap': litdircap,
3184                          'emptydircap': emptydircap}
3185
3186     def test_POST_mkdir_no_parentdir_initial_children(self):
3187         (newkids, caps) = self._create_initial_children()
3188         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
3189         def _after_mkdir(res):
3190             self.failUnless(res.startswith("URI:DIR"), res)
3191             n = self.s.create_node_from_uri(res)
3192             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3193             d2.addCallback(lambda ign:
3194                            self.failUnlessROChildURIIs(n, u"child-imm",
3195                                                        caps['filecap1']))
3196             d2.addCallback(lambda ign:
3197                            self.failUnlessRWChildURIIs(n, u"child-mutable",
3198                                                        caps['filecap2']))
3199             d2.addCallback(lambda ign:
3200                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
3201                                                        caps['filecap3']))
3202             d2.addCallback(lambda ign:
3203                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
3204                                                        caps['unknown_rwcap']))
3205             d2.addCallback(lambda ign:
3206                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
3207                                                        caps['unknown_rocap']))
3208             d2.addCallback(lambda ign:
3209                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3210                                                        caps['unknown_immcap']))
3211             d2.addCallback(lambda ign:
3212                            self.failUnlessRWChildURIIs(n, u"dirchild",
3213                                                        caps['dircap']))
3214             return d2
3215         d.addCallback(_after_mkdir)
3216         return d
3217
3218     def test_POST_mkdir_no_parentdir_unexpected_children(self):
3219         # the regular /uri?t=mkdir operation is specified to ignore its body.
3220         # Only t=mkdir-with-children pays attention to it.
3221         (newkids, caps) = self._create_initial_children()
3222         d = self.shouldHTTPError("POST_mkdir_no_parentdir_unexpected_children",
3223                                  400, "Bad Request",
3224                                  "t=mkdir does not accept children=, "
3225                                  "try t=mkdir-with-children instead",
3226                                  self.POST2, "/uri?t=mkdir", # without children
3227                                  simplejson.dumps(newkids))
3228         return d
3229
3230     def test_POST_noparent_bad(self):
3231         d = self.shouldHTTPError("POST_noparent_bad",
3232                                  400, "Bad Request",
3233                                  "/uri accepts only PUT, PUT?t=mkdir, "
3234                                  "POST?t=upload, and POST?t=mkdir",
3235                                  self.POST, "/uri?t=bogus")
3236         return d
3237
3238     def test_POST_mkdir_no_parentdir_immutable(self):
3239         (newkids, caps) = self._create_immutable_children()
3240         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
3241         def _after_mkdir(res):
3242             self.failUnless(res.startswith("URI:DIR"), res)
3243             n = self.s.create_node_from_uri(res)
3244             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3245             d2.addCallback(lambda ign:
3246                            self.failUnlessROChildURIIs(n, u"child-imm",
3247                                                           caps['filecap1']))
3248             d2.addCallback(lambda ign:
3249                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3250                                                           caps['unknown_immcap']))
3251             d2.addCallback(lambda ign:
3252                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
3253                                                           caps['immdircap']))
3254             d2.addCallback(lambda ign:
3255                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
3256                                                           caps['litdircap']))
3257             d2.addCallback(lambda ign:
3258                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
3259                                                           caps['emptydircap']))
3260             return d2
3261         d.addCallback(_after_mkdir)
3262         return d
3263
3264     def test_POST_mkdir_no_parentdir_immutable_bad(self):
3265         (newkids, caps) = self._create_initial_children()
3266         d = self.shouldFail2(error.Error,
3267                              "test_POST_mkdir_no_parentdir_immutable_bad",
3268                              "400 Bad Request",
3269                              "needed to be immutable but was not",
3270                              self.POST2,
3271                              "/uri?t=mkdir-immutable",
3272                              simplejson.dumps(newkids))
3273         return d
3274
3275     def test_welcome_page_mkdir_button(self):
3276         # Fetch the welcome page.
3277         d = self.GET("/")
3278         def _after_get_welcome_page(res):
3279             MKDIR_BUTTON_RE = re.compile(
3280                 '<form(?: action="([^"]*)"| method="post"| enctype="multipart/form-data"){3}>.*'
3281                 '<input (?:type="hidden" |name="t" |value="([^"]*?)" ){3}/>[ ]*'
3282                 '<input (?:type="hidden" |name="([^"]*)" |value="([^"]*)" ){3}/>[ ]*'
3283                 '<input (type="submit" |class="btn" |value="Create a directory[^"]*" ){3}/>')
3284             html = res.replace('\n', ' ')
3285             mo = MKDIR_BUTTON_RE.search(html)
3286             self.failUnless(mo, html)
3287             formaction = mo.group(1)
3288             formt = mo.group(2)
3289             formaname = mo.group(3)
3290             formavalue = mo.group(4)
3291             return (formaction, formt, formaname, formavalue)
3292         d.addCallback(_after_get_welcome_page)
3293         def _after_parse_form(res):
3294             (formaction, formt, formaname, formavalue) = res
3295             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
3296         d.addCallback(_after_parse_form)
3297         d.addBoth(self.shouldRedirect, None, statuscode='303')
3298         return d
3299
3300     def test_POST_mkdir_replace(self): # return value?
3301         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
3302         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3303         d.addCallback(self.failUnlessNodeKeysAre, [])
3304         return d
3305
3306     def test_POST_mkdir_no_replace_queryarg(self): # return value?
3307         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
3308         d.addBoth(self.shouldFail, error.Error,
3309                   "POST_mkdir_no_replace_queryarg",
3310                   "409 Conflict",
3311                   "There was already a child by that name, and you asked me "
3312                   "to not replace it")
3313         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3314         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3315         return d
3316
3317     def test_POST_mkdir_no_replace_field(self): # return value?
3318         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
3319                       replace="false")
3320         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
3321                   "409 Conflict",
3322                   "There was already a child by that name, and you asked me "
3323                   "to not replace it")
3324         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3325         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3326         return d
3327
3328     def test_POST_mkdir_whendone_field(self):
3329         d = self.POST(self.public_url + "/foo",
3330                       t="mkdir", name="newdir", when_done="/THERE")
3331         d.addBoth(self.shouldRedirect, "/THERE")
3332         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3333         d.addCallback(self.failUnlessNodeKeysAre, [])
3334         return d
3335
3336     def test_POST_mkdir_whendone_queryarg(self):
3337         d = self.POST(self.public_url + "/foo?when_done=/THERE",
3338                       t="mkdir", name="newdir")
3339         d.addBoth(self.shouldRedirect, "/THERE")
3340         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3341         d.addCallback(self.failUnlessNodeKeysAre, [])
3342         return d
3343
3344     def test_POST_bad_t(self):
3345         d = self.shouldFail2(error.Error, "POST_bad_t",
3346                              "400 Bad Request",
3347                              "POST to a directory with bad t=BOGUS",
3348                              self.POST, self.public_url + "/foo", t="BOGUS")
3349         return d
3350
3351     def test_POST_set_children(self, command_name="set_children"):
3352         contents9, n9, newuri9 = self.makefile(9)
3353         contents10, n10, newuri10 = self.makefile(10)
3354         contents11, n11, newuri11 = self.makefile(11)
3355
3356         reqbody = """{
3357                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
3358                                                 "size": 0,
3359                                                 "metadata": {
3360                                                   "ctime": 1002777696.7564139,
3361                                                   "mtime": 1002777696.7564139
3362                                                  }
3363                                                } ],
3364                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
3365                                                 "size": 1,
3366                                                 "metadata": {
3367                                                   "ctime": 1002777696.7564139,
3368                                                   "mtime": 1002777696.7564139
3369                                                  }
3370                                                } ],
3371                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
3372                                                 "size": 2,
3373                                                 "metadata": {
3374                                                   "ctime": 1002777696.7564139,
3375                                                   "mtime": 1002777696.7564139
3376                                                  }
3377                                                } ]
3378                     }""" % (newuri9, newuri10, newuri11)
3379
3380         url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
3381
3382         d = client.getPage(url, method="POST", postdata=reqbody)
3383         def _then(res):
3384             self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
3385             self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
3386             self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
3387
3388         d.addCallback(_then)
3389         d.addErrback(self.dump_error)
3390         return d
3391
3392     def test_POST_set_children_with_hyphen(self):
3393         return self.test_POST_set_children(command_name="set-children")
3394
3395     def test_POST_link_uri(self):
3396         contents, n, newuri = self.makefile(8)
3397         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
3398         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
3399         d.addCallback(lambda res:
3400                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3401                                                       contents))
3402         return d
3403
3404     def test_POST_link_uri_replace(self):
3405         contents, n, newuri = self.makefile(8)
3406         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
3407         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
3408         d.addCallback(lambda res:
3409                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3410                                                       contents))
3411         return d
3412
3413     def test_POST_link_uri_unknown_bad(self):
3414         d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
3415         d.addBoth(self.shouldFail, error.Error,
3416                   "POST_link_uri_unknown_bad",
3417                   "400 Bad Request",
3418                   "unknown cap in a write slot")
3419         return d
3420
3421     def test_POST_link_uri_unknown_ro_good(self):
3422         d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
3423         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
3424         return d
3425
3426     def test_POST_link_uri_unknown_imm_good(self):
3427         d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
3428         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
3429         return d
3430
3431     def test_POST_link_uri_no_replace_queryarg(self):
3432         contents, n, newuri = self.makefile(8)
3433         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
3434                       name="bar.txt", uri=newuri)
3435         d.addBoth(self.shouldFail, error.Error,
3436                   "POST_link_uri_no_replace_queryarg",
3437                   "409 Conflict",
3438                   "There was already a child by that name, and you asked me "
3439                   "to not replace it")
3440         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3441         d.addCallback(self.failUnlessIsBarDotTxt)
3442         return d
3443
3444     def test_POST_link_uri_no_replace_field(self):
3445         contents, n, newuri = self.makefile(8)
3446         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
3447                       name="bar.txt", uri=newuri)
3448         d.addBoth(self.shouldFail, error.Error,
3449                   "POST_link_uri_no_replace_field",
3450                   "409 Conflict",
3451                   "There was already a child by that name, and you asked me "
3452                   "to not replace it")
3453         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3454         d.addCallback(self.failUnlessIsBarDotTxt)
3455         return d
3456
3457     def test_POST_delete(self, command_name='delete'):
3458         d = self._foo_node.list()
3459         def _check_before(children):
3460             self.failUnlessIn(u"bar.txt", children)
3461         d.addCallback(_check_before)
3462         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t=command_name, name="bar.txt"))
3463         d.addCallback(lambda res: self._foo_node.list())
3464         def _check_after(children):
3465             self.failIfIn(u"bar.txt", children)
3466         d.addCallback(_check_after)
3467         return d
3468
3469     def test_POST_unlink(self):
3470         return self.test_POST_delete(command_name='unlink')
3471
3472     def test_POST_rename_file(self):
3473         d = self.POST(self.public_url + "/foo", t="rename",
3474                       from_name="bar.txt", to_name='wibble.txt')
3475         d.addCallback(lambda res:
3476                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3477         d.addCallback(lambda res:
3478                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
3479         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
3480         d.addCallback(self.failUnlessIsBarDotTxt)
3481         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
3482         d.addCallback(self.failUnlessIsBarJSON)
3483         return d
3484
3485     def test_POST_rename_file_redundant(self):
3486         d = self.POST(self.public_url + "/foo", t="rename",
3487                       from_name="bar.txt", to_name='bar.txt')
3488         d.addCallback(lambda res:
3489                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3490         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3491         d.addCallback(self.failUnlessIsBarDotTxt)
3492         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3493         d.addCallback(self.failUnlessIsBarJSON)
3494         return d
3495
3496     def test_POST_rename_file_replace(self):
3497         # rename a file and replace a directory with it
3498         d = self.POST(self.public_url + "/foo", t="rename",
3499                       from_name="bar.txt", to_name='empty')
3500         d.addCallback(lambda res:
3501                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3502         d.addCallback(lambda res:
3503                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
3504         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
3505         d.addCallback(self.failUnlessIsBarDotTxt)
3506         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3507         d.addCallback(self.failUnlessIsBarJSON)
3508         return d
3509
3510     def test_POST_rename_file_no_replace_queryarg(self):
3511         # rename a file and replace a directory with it
3512         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
3513                       from_name="bar.txt", to_name='empty')
3514         d.addBoth(self.shouldFail, error.Error,
3515                   "POST_rename_file_no_replace_queryarg",
3516                   "409 Conflict",
3517                   "There was already a child by that name, and you asked me "
3518                   "to not replace it")
3519         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3520         d.addCallback(self.failUnlessIsEmptyJSON)
3521         return d
3522
3523     def test_POST_rename_file_no_replace_field(self):
3524         # rename a file and replace a directory with it
3525         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
3526                       from_name="bar.txt", to_name='empty')
3527         d.addBoth(self.shouldFail, error.Error,
3528                   "POST_rename_file_no_replace_field",
3529                   "409 Conflict",
3530                   "There was already a child by that name, and you asked me "
3531                   "to not replace it")
3532         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3533         d.addCallback(self.failUnlessIsEmptyJSON)
3534         return d
3535
3536     def test_POST_rename_file_no_replace_same_link(self):
3537         d = self.POST(self.public_url + "/foo", t="rename",
3538                       replace="false", from_name="bar.txt", to_name="bar.txt")
3539         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3540         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3541         d.addCallback(self.failUnlessIsBarDotTxt)
3542         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3543         d.addCallback(self.failUnlessIsBarJSON)
3544         return d
3545
3546     def test_POST_rename_file_replace_only_files(self):
3547         d = self.POST(self.public_url + "/foo", t="rename",
3548                       replace="only-files", from_name="bar.txt",
3549                       to_name="baz.txt")
3550         d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3551         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3552         d.addCallback(self.failUnlessIsBarDotTxt)
3553         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt?t=json"))
3554         d.addCallback(self.failUnlessIsBarJSON)
3555         return d
3556
3557     def test_POST_rename_file_replace_only_files_conflict(self):
3558         d = self.shouldFail2(error.Error, "POST_relink_file_replace_only_files_conflict",
3559                              "409 Conflict",
3560                              "There was already a child by that name, and you asked me to not replace it.",
3561                              self.POST, self.public_url + "/foo", t="relink",
3562                              replace="only-files", from_name="bar.txt",
3563                              to_name="empty")
3564         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3565         d.addCallback(self.failUnlessIsBarDotTxt)
3566         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3567         d.addCallback(self.failUnlessIsBarJSON)
3568         return d
3569
3570     def failUnlessIsEmptyJSON(self, res):
3571         data = simplejson.loads(res)
3572         self.failUnlessEqual(data[0], "dirnode", data)
3573         self.failUnlessReallyEqual(len(data[1]["children"]), 0)
3574
3575     def test_POST_rename_file_to_slash_fail(self):
3576         d = self.POST(self.public_url + "/foo", t="rename",
3577                       from_name="bar.txt", to_name='kirk/spock.txt')
3578         d.addBoth(self.shouldFail, error.Error,
3579                   "test_POST_rename_file_to_slash_fail",
3580                   "400 Bad Request",
3581                   "to_name= may not contain a slash",
3582                   )
3583         d.addCallback(lambda res:
3584                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3585         return d
3586
3587     def test_POST_rename_file_from_slash_fail(self):
3588         d = self.POST(self.public_url + "/foo", t="rename",
3589                       from_name="sub/bar.txt", to_name='spock.txt')
3590         d.addBoth(self.shouldFail, error.Error,
3591                   "test_POST_rename_from_file_slash_fail",
3592                   "400 Bad Request",
3593                   "from_name= may not contain a slash",
3594                   )
3595         d.addCallback(lambda res:
3596                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3597         return d
3598
3599     def test_POST_rename_dir(self):
3600         d = self.POST(self.public_url, t="rename",
3601                       from_name="foo", to_name='plunk')
3602         d.addCallback(lambda res:
3603                       self.failIfNodeHasChild(self.public_root, u"foo"))
3604         d.addCallback(lambda res:
3605                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
3606         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
3607         d.addCallback(self.failUnlessIsFooJSON)
3608         return d
3609
3610     def test_POST_relink_file(self):
3611         d = self.POST(self.public_url + "/foo", t="relink",
3612                       from_name="bar.txt",
3613                       to_dir=self.public_root.get_uri() + "/foo/sub")
3614         d.addCallback(lambda res:
3615                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3616         d.addCallback(lambda res:
3617                       self.failUnlessNodeHasChild(self._sub_node, u"bar.txt"))
3618         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3619         d.addCallback(self.failUnlessIsBarDotTxt)
3620         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3621         d.addCallback(self.failUnlessIsBarJSON)
3622         return d
3623
3624     def test_POST_relink_file_new_name(self):
3625         d = self.POST(self.public_url + "/foo", t="relink",
3626                       from_name="bar.txt",
3627                       to_name="wibble.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3628         d.addCallback(lambda res:
3629                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3630         d.addCallback(lambda res:
3631                       self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3632         d.addCallback(lambda res:
3633                       self.failUnlessNodeHasChild(self._sub_node, u"wibble.txt"))
3634         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt"))
3635         d.addCallback(self.failUnlessIsBarDotTxt)
3636         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt?t=json"))
3637         d.addCallback(self.failUnlessIsBarJSON)
3638         return d
3639
3640     def test_POST_relink_file_replace(self):
3641         d = self.POST(self.public_url + "/foo", t="relink",
3642                       from_name="bar.txt",
3643                       to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3644         d.addCallback(lambda res:
3645                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3646         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3647         d.addCallback(self.failUnlessIsBarDotTxt)
3648         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt?t=json"))
3649         d.addCallback(self.failUnlessIsBarJSON)
3650         return d
3651
3652     def test_POST_relink_file_no_replace(self):
3653         d = self.shouldFail2(error.Error, "POST_relink_file_no_replace",
3654                              "409 Conflict",
3655                              "There was already a child by that name, and you asked me to not replace it",
3656                              self.POST, self.public_url + "/foo", t="relink",
3657                              replace="false", from_name="bar.txt",
3658                              to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3659         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3660         d.addCallback(self.failUnlessIsBarDotTxt)
3661         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3662         d.addCallback(self.failUnlessIsBarJSON)
3663         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3664         d.addCallback(self.failUnlessIsSubBazDotTxt)
3665         return d
3666
3667     def test_POST_relink_file_no_replace_explicitly_same_link(self):
3668         d = self.POST(self.public_url + "/foo", t="relink",
3669                       replace="false", from_name="bar.txt",
3670                       to_name="bar.txt", to_dir=self.public_root.get_uri() + "/foo")
3671         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3672         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3673         d.addCallback(self.failUnlessIsBarDotTxt)
3674         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3675         d.addCallback(self.failUnlessIsBarJSON)
3676         return d
3677
3678     def test_POST_relink_file_replace_only_files(self):
3679         d = self.POST(self.public_url + "/foo", t="relink",
3680                       replace="only-files", from_name="bar.txt",
3681                       to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3682         d.addCallback(lambda res:
3683                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3684         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3685         d.addCallback(self.failUnlessIsBarDotTxt)
3686         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt?t=json"))
3687         d.addCallback(self.failUnlessIsBarJSON)
3688         return d
3689
3690     def test_POST_relink_file_replace_only_files_conflict(self):
3691         d = self.shouldFail2(error.Error, "POST_relink_file_replace_only_files_conflict",
3692                              "409 Conflict",
3693                              "There was already a child by that name, and you asked me to not replace it.",
3694                              self.POST, self.public_url + "/foo", t="relink",
3695                              replace="only-files", from_name="bar.txt",
3696                              to_name="sub", to_dir=self.public_root.get_uri() + "/foo")
3697         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3698         d.addCallback(self.failUnlessIsBarDotTxt)
3699         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3700         d.addCallback(self.failUnlessIsBarJSON)
3701         return d
3702
3703     def test_POST_relink_file_to_slash_fail(self):
3704         d = self.shouldFail2(error.Error, "test_POST_rename_file_slash_fail",
3705                              "400 Bad Request",
3706                              "to_name= may not contain a slash",
3707                              self.POST, self.public_url + "/foo", t="relink",
3708                              from_name="bar.txt",
3709                              to_name="slash/fail.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3710         d.addCallback(lambda res:
3711                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3712         d.addCallback(lambda res:
3713                       self.failIfNodeHasChild(self._sub_node, u"slash/fail.txt"))
3714         d.addCallback(lambda ign:
3715                       self.shouldFail2(error.Error,
3716                                        "test_POST_rename_file_slash_fail2",
3717                                        "400 Bad Request",
3718                                        "from_name= may not contain a slash",
3719                                        self.POST, self.public_url + "/foo",
3720                                        t="relink",
3721                                        from_name="nope/bar.txt",
3722                                        to_name="fail.txt",
3723                                        to_dir=self.public_root.get_uri() + "/foo/sub"))
3724         return d
3725
3726     def test_POST_relink_file_explicitly_same_link(self):
3727         d = self.POST(self.public_url + "/foo", t="relink",
3728                       from_name="bar.txt",
3729                       to_name="bar.txt", to_dir=self.public_root.get_uri() + "/foo")
3730         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3731         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3732         d.addCallback(self.failUnlessIsBarDotTxt)
3733         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3734         d.addCallback(self.failUnlessIsBarJSON)
3735         return d
3736
3737     def test_POST_relink_file_implicitly_same_link(self):
3738         d = self.POST(self.public_url + "/foo", t="relink",
3739                       from_name="bar.txt")
3740         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3741         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3742         d.addCallback(self.failUnlessIsBarDotTxt)
3743         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3744         d.addCallback(self.failUnlessIsBarJSON)
3745         return d
3746
3747     def test_POST_relink_file_same_dir(self):
3748         d = self.POST(self.public_url + "/foo", t="relink",
3749                       from_name="bar.txt",
3750                       to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo")
3751         d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3752         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._sub_node, u"baz.txt"))
3753         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3754         d.addCallback(self.failUnlessIsBarDotTxt)
3755         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt?t=json"))
3756         d.addCallback(self.failUnlessIsBarJSON)
3757         return d
3758
3759     def test_POST_relink_file_bad_replace(self):
3760         d = self.shouldFail2(error.Error, "test_POST_relink_file_bad_replace",
3761                              "400 Bad Request", "invalid replace= argument: 'boogabooga'",
3762                              self.POST,
3763                              self.public_url + "/foo", t="relink",
3764                              replace="boogabooga", from_name="bar.txt",
3765                              to_dir=self.public_root.get_uri() + "/foo/sub")
3766         return d
3767
3768     def test_POST_relink_file_multi_level(self):
3769         d = self.POST(self.public_url + "/foo/sub/level2?t=mkdir", "")
3770         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t="relink",
3771                       from_name="bar.txt", to_dir=self.public_root.get_uri() + "/foo/sub/level2"))
3772         d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3773         d.addCallback(lambda res: self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3774         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt"))
3775         d.addCallback(self.failUnlessIsBarDotTxt)
3776         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt?t=json"))
3777         d.addCallback(self.failUnlessIsBarJSON)
3778         return d
3779
3780     def test_POST_relink_file_to_uri(self):
3781         d = self.POST(self.public_url + "/foo", t="relink", target_type="uri",
3782                       from_name="bar.txt", to_dir=self._sub_uri)
3783         d.addCallback(lambda res:
3784                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3785         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3786         d.addCallback(self.failUnlessIsBarDotTxt)
3787         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3788         d.addCallback(self.failUnlessIsBarJSON)
3789         return d
3790
3791     def test_POST_relink_file_to_nonexistent_dir(self):
3792         d = self.shouldFail2(error.Error, "POST_relink_file_to_nonexistent_dir",
3793                             "404 Not Found", "No such child: nopechucktesta",
3794                             self.POST, self.public_url + "/foo", t="relink",
3795                             from_name="bar.txt",
3796                             to_dir=self.public_root.get_uri() + "/nopechucktesta")
3797         return d
3798
3799     def test_POST_relink_file_into_file(self):
3800         d = self.shouldFail2(error.Error, "POST_relink_file_into_file",
3801                              "400 Bad Request", "to_dir is not a directory",
3802                              self.POST, self.public_url + "/foo", t="relink",
3803                              from_name="bar.txt",
3804                              to_dir=self.public_root.get_uri() + "/foo/baz.txt")
3805         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3806         d.addCallback(self.failUnlessIsBazDotTxt)
3807         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3808         d.addCallback(self.failUnlessIsBarDotTxt)
3809         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3810         d.addCallback(self.failUnlessIsBarJSON)
3811         return d
3812
3813     def test_POST_relink_file_to_bad_uri(self):
3814         d =  self.shouldFail2(error.Error, "POST_relink_file_to_bad_uri",
3815                               "400 Bad Request", "to_dir is not a directory",
3816                               self.POST, self.public_url + "/foo", t="relink",
3817                               from_name="bar.txt",
3818                               to_dir="URI:DIR2:mn5jlyjnrjeuydyswlzyui72i:rmneifcj6k6sycjljjhj3f6majsq2zqffydnnul5hfa4j577arma")
3819         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3820         d.addCallback(self.failUnlessIsBarDotTxt)
3821         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3822         d.addCallback(self.failUnlessIsBarJSON)
3823         return d
3824
3825     def test_POST_relink_dir(self):
3826         d = self.POST(self.public_url + "/foo", t="relink",
3827                       from_name="bar.txt",
3828                       to_dir=self.public_root.get_uri() + "/foo/empty")
3829         d.addCallback(lambda res: self.POST(self.public_url + "/foo",
3830                       t="relink", from_name="empty",
3831                       to_dir=self.public_root.get_uri() + "/foo/sub"))
3832         d.addCallback(lambda res:
3833                       self.failIfNodeHasChild(self._foo_node, u"empty"))
3834         d.addCallback(lambda res:
3835                       self.failUnlessNodeHasChild(self._sub_node, u"empty"))
3836         d.addCallback(lambda res:
3837                       self._sub_node.get_child_at_path(u"empty"))
3838         d.addCallback(lambda node:
3839                       self.failUnlessNodeHasChild(node, u"bar.txt"))
3840         d.addCallback(lambda res:
3841                       self.GET(self.public_url + "/foo/sub/empty/bar.txt"))
3842         d.addCallback(self.failUnlessIsBarDotTxt)
3843         return d
3844
3845     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
3846         """ If target is not None then the redirection has to go to target.  If
3847         statuscode is not None then the redirection has to be accomplished with
3848         that HTTP status code."""
3849         if not isinstance(res, failure.Failure):
3850             to_where = (target is None) and "somewhere" or ("to " + target)
3851             self.fail("%s: we were expecting to get redirected %s, not get an"
3852                       " actual page: %s" % (which, to_where, res))
3853         res.trap(error.PageRedirect)
3854         if statuscode is not None:
3855             self.failUnlessReallyEqual(res.value.status, statuscode,
3856                                        "%s: not a redirect" % which)
3857         if target is not None:
3858             # the PageRedirect does not seem to capture the uri= query arg
3859             # properly, so we can't check for it.
3860             realtarget = self.webish_url + target
3861             self.failUnlessReallyEqual(res.value.location, realtarget,
3862                                        "%s: wrong target" % which)
3863         return res.value.location
3864
3865     def test_GET_URI_form(self):
3866         base = "/uri?uri=%s" % self._bar_txt_uri
3867         # this is supposed to give us a redirect to /uri/$URI, plus arguments
3868         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
3869         d = self.GET(base)
3870         d.addBoth(self.shouldRedirect, targetbase)
3871         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
3872         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
3873         d.addCallback(lambda res: self.GET(base+"&t=json"))
3874         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
3875         d.addCallback(self.log, "about to get file by uri")
3876         d.addCallback(lambda res: self.GET(base, followRedirect=True))
3877         d.addCallback(self.failUnlessIsBarDotTxt)
3878         d.addCallback(self.log, "got file by uri, about to get dir by uri")
3879         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
3880                                            followRedirect=True))
3881         d.addCallback(self.failUnlessIsFooJSON)
3882         d.addCallback(self.log, "got dir by uri")
3883
3884         return d
3885
3886     def test_GET_URI_form_bad(self):
3887         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
3888                              "400 Bad Request", "GET /uri requires uri=",
3889                              self.GET, "/uri")
3890         return d
3891
3892     def test_GET_rename_form(self):
3893         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
3894                      followRedirect=True)
3895         def _check(res):
3896             self.failUnless(re.search('<input (name="when_done" |value="." |type="hidden" ){3}/>', res), res)
3897             self.failUnless(re.search(r'<input (readonly="true" |type="text" |name="from_name" |value="bar\.txt" ){4}/>', res), res)
3898             self.failUnlessIn(FAVICON_MARKUP, res)
3899         d.addCallback(_check)
3900         return d
3901
3902     def log(self, res, msg):
3903         #print "MSG: %s  RES: %s" % (msg, res)
3904         log.msg(msg)
3905         return res
3906
3907     def test_GET_URI_URL(self):
3908         base = "/uri/%s" % self._bar_txt_uri
3909         d = self.GET(base)
3910         d.addCallback(self.failUnlessIsBarDotTxt)
3911         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
3912         d.addCallback(self.failUnlessIsBarDotTxt)
3913         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
3914         d.addCallback(self.failUnlessIsBarDotTxt)
3915         return d
3916
3917     def test_GET_URI_URL_dir(self):
3918         base = "/uri/%s?t=json" % self._foo_uri
3919         d = self.GET(base)
3920         d.addCallback(self.failUnlessIsFooJSON)
3921         return d
3922
3923     def test_GET_URI_URL_missing(self):
3924         base = "/uri/%s" % self._bad_file_uri
3925         d = self.shouldHTTPError("test_GET_URI_URL_missing",
3926                                  http.GONE, None, "NotEnoughSharesError",
3927                                  self.GET, base)
3928         # TODO: how can we exercise both sides of WebDownloadTarget.fail
3929         # here? we must arrange for a download to fail after target.open()
3930         # has been called, and then inspect the response to see that it is
3931         # shorter than we expected.
3932         return d
3933
3934     def test_PUT_DIRURL_uri(self):
3935         d = self.s.create_dirnode()
3936         def _made_dir(dn):
3937             new_uri = dn.get_uri()
3938             # replace /foo with a new (empty) directory
3939             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
3940             d.addCallback(lambda res:
3941                           self.failUnlessReallyEqual(res.strip(), new_uri))
3942             d.addCallback(lambda res:
3943                           self.failUnlessRWChildURIIs(self.public_root,
3944                                                       u"foo",
3945                                                       new_uri))
3946             return d
3947         d.addCallback(_made_dir)
3948         return d
3949
3950     def test_PUT_DIRURL_uri_noreplace(self):
3951         d = self.s.create_dirnode()
3952         def _made_dir(dn):
3953             new_uri = dn.get_uri()
3954             # replace /foo with a new (empty) directory, but ask that
3955             # replace=false, so it should fail
3956             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
3957                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
3958                                  self.PUT,
3959                                  self.public_url + "/foo?t=uri&replace=false",
3960                                  new_uri)
3961             d.addCallback(lambda res:
3962                           self.failUnlessRWChildURIIs(self.public_root,
3963                                                       u"foo",
3964                                                       self._foo_uri))
3965             return d
3966         d.addCallback(_made_dir)
3967         return d
3968
3969     def test_PUT_DIRURL_bad_t(self):
3970         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
3971                              "400 Bad Request", "PUT to a directory",
3972                              self.PUT, self.public_url + "/foo?t=BOGUS", "")
3973         d.addCallback(lambda res:
3974                       self.failUnlessRWChildURIIs(self.public_root,
3975                                                   u"foo",
3976                                                   self._foo_uri))
3977         return d
3978
3979     def test_PUT_NEWFILEURL_uri(self):
3980         contents, n, new_uri = self.makefile(8)
3981         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
3982         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3983         d.addCallback(lambda res:
3984                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3985                                                       contents))
3986         return d
3987
3988     def test_PUT_NEWFILEURL_mdmf(self):
3989         new_contents = self.NEWFILE_CONTENTS * 300000
3990         d = self.PUT(self.public_url + \
3991                      "/foo/mdmf.txt?format=mdmf",
3992                      new_contents)
3993         d.addCallback(lambda ignored:
3994             self.GET(self.public_url + "/foo/mdmf.txt?t=json"))
3995         def _got_json(json):
3996             data = simplejson.loads(json)
3997             data = data[1]
3998             self.failUnlessIn("format", data)
3999             self.failUnlessEqual(data["format"], "MDMF")
4000             self.failUnless(data['rw_uri'].startswith("URI:MDMF"))
4001             self.failUnless(data['ro_uri'].startswith("URI:MDMF"))
4002         d.addCallback(_got_json)
4003         return d
4004
4005     def test_PUT_NEWFILEURL_sdmf(self):
4006         new_contents = self.NEWFILE_CONTENTS * 300000
4007         d = self.PUT(self.public_url + \
4008                      "/foo/sdmf.txt?format=sdmf",
4009                      new_contents)
4010         d.addCallback(lambda ignored:
4011             self.GET(self.public_url + "/foo/sdmf.txt?t=json"))
4012         def _got_json(json):
4013             data = simplejson.loads(json)
4014             data = data[1]
4015             self.failUnlessIn("format", data)
4016             self.failUnlessEqual(data["format"], "SDMF")
4017         d.addCallback(_got_json)
4018         return d
4019
4020     def test_PUT_NEWFILEURL_bad_format(self):
4021         new_contents = self.NEWFILE_CONTENTS * 300000
4022         return self.shouldHTTPError("PUT_NEWFILEURL_bad_format",
4023                                     400, "Bad Request", "Unknown format: foo",
4024                                     self.PUT, self.public_url + \
4025                                     "/foo/foo.txt?format=foo",
4026                                     new_contents)
4027
4028     def test_PUT_NEWFILEURL_uri_replace(self):
4029         contents, n, new_uri = self.makefile(8)
4030         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
4031         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
4032         d.addCallback(lambda res:
4033                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
4034                                                       contents))
4035         return d
4036
4037     def test_PUT_NEWFILEURL_uri_no_replace(self):
4038         contents, n, new_uri = self.makefile(8)
4039         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
4040         d.addBoth(self.shouldFail, error.Error,
4041                   "PUT_NEWFILEURL_uri_no_replace",
4042                   "409 Conflict",
4043                   "There was already a child by that name, and you asked me "
4044                   "to not replace it")
4045         return d
4046
4047     def test_PUT_NEWFILEURL_uri_unknown_bad(self):
4048         d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
4049         d.addBoth(self.shouldFail, error.Error,
4050                   "POST_put_uri_unknown_bad",
4051                   "400 Bad Request",
4052                   "unknown cap in a write slot")
4053         return d
4054
4055     def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
4056         d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
4057         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
4058                       u"put-future-ro.txt")
4059         return d
4060
4061     def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
4062         d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
4063         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
4064                       u"put-future-imm.txt")
4065         return d
4066
4067     def test_PUT_NEWFILE_URI(self):
4068         file_contents = "New file contents here\n"
4069         d = self.PUT("/uri", file_contents)
4070         def _check(uri):
4071             assert isinstance(uri, str), uri
4072             self.failUnlessIn(uri, self.get_all_contents())
4073             self.failUnlessReallyEqual(self.get_all_contents()[uri],
4074                                        file_contents)
4075             return self.GET("/uri/%s" % uri)
4076         d.addCallback(_check)
4077         def _check2(res):
4078             self.failUnlessReallyEqual(res, file_contents)
4079         d.addCallback(_check2)
4080         return d
4081
4082     def test_PUT_NEWFILE_URI_not_mutable(self):
4083         file_contents = "New file contents here\n"
4084         d = self.PUT("/uri?mutable=false", file_contents)
4085         def _check(uri):
4086             assert isinstance(uri, str), uri
4087             self.failUnlessIn(uri, self.get_all_contents())
4088             self.failUnlessReallyEqual(self.get_all_contents()[uri],
4089                                        file_contents)
4090             return self.GET("/uri/%s" % uri)
4091         d.addCallback(_check)
4092         def _check2(res):
4093             self.failUnlessReallyEqual(res, file_contents)
4094         d.addCallback(_check2)
4095         return d
4096
4097     def test_PUT_NEWFILE_URI_only_PUT(self):
4098         d = self.PUT("/uri?t=bogus", "")
4099         d.addBoth(self.shouldFail, error.Error,
4100                   "PUT_NEWFILE_URI_only_PUT",
4101                   "400 Bad Request",
4102                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
4103         return d
4104
4105     def test_PUT_NEWFILE_URI_mutable(self):
4106         file_contents = "New file contents here\n"
4107         d = self.PUT("/uri?mutable=true", file_contents)
4108         def _check1(filecap):
4109             filecap = filecap.strip()
4110             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
4111             self.filecap = filecap
4112             u = uri.WriteableSSKFileURI.init_from_string(filecap)
4113             self.failUnlessIn(u.get_storage_index(), self.get_all_contents())
4114             n = self.s.create_node_from_uri(filecap)
4115             return n.download_best_version()
4116         d.addCallback(_check1)
4117         def _check2(data):
4118             self.failUnlessReallyEqual(data, file_contents)
4119             return self.GET("/uri/%s" % urllib.quote(self.filecap))
4120         d.addCallback(_check2)
4121         def _check3(res):
4122             self.failUnlessReallyEqual(res, file_contents)
4123         d.addCallback(_check3)
4124         return d
4125
4126     def test_PUT_mkdir(self):
4127         d = self.PUT("/uri?t=mkdir", "")
4128         def _check(uri):
4129             n = self.s.create_node_from_uri(uri.strip())
4130             d2 = self.failUnlessNodeKeysAre(n, [])
4131             d2.addCallback(lambda res:
4132                            self.GET("/uri/%s?t=json" % uri))
4133             return d2
4134         d.addCallback(_check)
4135         d.addCallback(self.failUnlessIsEmptyJSON)
4136         return d
4137
4138     def test_PUT_mkdir_mdmf(self):
4139         d = self.PUT("/uri?t=mkdir&format=mdmf", "")
4140         def _got(res):
4141             u = uri.from_string(res)
4142             # Check that this is an MDMF writecap
4143             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
4144         d.addCallback(_got)
4145         return d
4146
4147     def test_PUT_mkdir_sdmf(self):
4148         d = self.PUT("/uri?t=mkdir&format=sdmf", "")
4149         def _got(res):
4150             u = uri.from_string(res)
4151             self.failUnlessIsInstance(u, uri.DirectoryURI)
4152         d.addCallback(_got)
4153         return d
4154
4155     def test_PUT_mkdir_bad_format(self):
4156         return self.shouldHTTPError("PUT_mkdir_bad_format",
4157                                     400, "Bad Request", "Unknown format: foo",
4158                                     self.PUT, "/uri?t=mkdir&format=foo",
4159                                     "")
4160
4161     def test_POST_check(self):
4162         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
4163         def _done(res):
4164             # this returns a string form of the results, which are probably
4165             # None since we're using fake filenodes.
4166             # TODO: verify that the check actually happened, by changing
4167             # FakeCHKFileNode to count how many times .check() has been
4168             # called.
4169             pass
4170         d.addCallback(_done)
4171         return d
4172
4173
4174     def test_PUT_update_at_offset(self):
4175         file_contents = "test file" * 100000 # about 900 KiB
4176         d = self.PUT("/uri?mutable=true", file_contents)
4177         def _then(filecap):
4178             self.filecap = filecap
4179             new_data = file_contents[:100]
4180             new = "replaced and so on"
4181             new_data += new
4182             new_data += file_contents[len(new_data):]
4183             assert len(new_data) == len(file_contents)
4184             self.new_data = new_data
4185         d.addCallback(_then)
4186         d.addCallback(lambda ignored:
4187             self.PUT("/uri/%s?replace=True&offset=100" % self.filecap,
4188                      "replaced and so on"))
4189         def _get_data(filecap):
4190             n = self.s.create_node_from_uri(filecap)
4191             return n.download_best_version()
4192         d.addCallback(_get_data)
4193         d.addCallback(lambda results:
4194             self.failUnlessEqual(results, self.new_data))
4195         # Now try appending things to the file
4196         d.addCallback(lambda ignored:
4197             self.PUT("/uri/%s?offset=%d" % (self.filecap, len(self.new_data)),
4198                      "puppies" * 100))
4199         d.addCallback(_get_data)
4200         d.addCallback(lambda results:
4201             self.failUnlessEqual(results, self.new_data + ("puppies" * 100)))
4202         # and try replacing the beginning of the file
4203         d.addCallback(lambda ignored:
4204             self.PUT("/uri/%s?offset=0" % self.filecap, "begin"))
4205         d.addCallback(_get_data)
4206         d.addCallback(lambda results:
4207             self.failUnlessEqual(results, "begin"+self.new_data[len("begin"):]+("puppies"*100)))
4208         return d
4209
4210     def test_PUT_update_at_invalid_offset(self):
4211         file_contents = "test file" * 100000 # about 900 KiB
4212         d = self.PUT("/uri?mutable=true", file_contents)
4213         def _then(filecap):
4214             self.filecap = filecap
4215         d.addCallback(_then)
4216         # Negative offsets should cause an error.
4217         d.addCallback(lambda ignored:
4218             self.shouldHTTPError("PUT_update_at_invalid_offset",
4219                                  400, "Bad Request",
4220                                  "Invalid offset",
4221                                  self.PUT,
4222                                  "/uri/%s?offset=-1" % self.filecap,
4223                                  "foo"))
4224         return d
4225
4226     def test_PUT_update_at_offset_immutable(self):
4227         file_contents = "Test file" * 100000
4228         d = self.PUT("/uri", file_contents)
4229         def _then(filecap):
4230             self.filecap = filecap
4231         d.addCallback(_then)
4232         d.addCallback(lambda ignored:
4233             self.shouldHTTPError("PUT_update_at_offset_immutable",
4234                                  400, "Bad Request",
4235                                  "immutable",
4236                                  self.PUT,
4237                                  "/uri/%s?offset=50" % self.filecap,
4238                                  "foo"))
4239         return d
4240
4241
4242     def test_bad_method(self):
4243         url = self.webish_url + self.public_url + "/foo/bar.txt"
4244         d = self.shouldHTTPError("bad_method",
4245                                  501, "Not Implemented",
4246                                  "I don't know how to treat a BOGUS request.",
4247                                  client.getPage, url, method="BOGUS")
4248         return d
4249
4250     def test_short_url(self):
4251         url = self.webish_url + "/uri"
4252         d = self.shouldHTTPError("short_url", 501, "Not Implemented",
4253                                  "I don't know how to treat a DELETE request.",
4254                                  client.getPage, url, method="DELETE")
4255         return d
4256
4257     def test_ophandle_bad(self):
4258         url = self.webish_url + "/operations/bogus?t=status"
4259         d = self.shouldHTTPError("ophandle_bad", 404, "404 Not Found",
4260                                  "unknown/expired handle 'bogus'",
4261                                  client.getPage, url)
4262         return d
4263
4264     def test_ophandle_cancel(self):
4265         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
4266                       followRedirect=True)
4267         d.addCallback(lambda ignored:
4268                       self.GET("/operations/128?t=status&output=JSON"))
4269         def _check1(res):
4270             data = simplejson.loads(res)
4271             self.failUnless("finished" in data, res)
4272             monitor = self.ws.root.child_operations.handles["128"][0]
4273             d = self.POST("/operations/128?t=cancel&output=JSON")
4274             def _check2(res):
4275                 data = simplejson.loads(res)
4276                 self.failUnless("finished" in data, res)
4277                 # t=cancel causes the handle to be forgotten
4278                 self.failUnless(monitor.is_cancelled())
4279             d.addCallback(_check2)
4280             return d
4281         d.addCallback(_check1)
4282         d.addCallback(lambda ignored:
4283                       self.shouldHTTPError("ophandle_cancel",
4284                                            404, "404 Not Found",
4285                                            "unknown/expired handle '128'",
4286                                            self.GET,
4287                                            "/operations/128?t=status&output=JSON"))
4288         return d
4289
4290     def test_ophandle_retainfor(self):
4291         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
4292                       followRedirect=True)
4293         d.addCallback(lambda ignored:
4294                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
4295         def _check1(res):
4296             data = simplejson.loads(res)
4297             self.failUnless("finished" in data, res)
4298         d.addCallback(_check1)
4299         # the retain-for=0 will cause the handle to be expired very soon
4300         d.addCallback(lambda ign:
4301             self.clock.advance(2.0))
4302         d.addCallback(lambda ignored:
4303                       self.shouldHTTPError("ophandle_retainfor",
4304                                            404, "404 Not Found",
4305                                            "unknown/expired handle '129'",
4306                                            self.GET,
4307                                            "/operations/129?t=status&output=JSON"))
4308         return d
4309
4310     def test_ophandle_release_after_complete(self):
4311         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
4312                       followRedirect=True)
4313         d.addCallback(self.wait_for_operation, "130")
4314         d.addCallback(lambda ignored:
4315                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
4316         # the release-after-complete=true will cause the handle to be expired
4317         d.addCallback(lambda ignored:
4318                       self.shouldHTTPError("ophandle_release_after_complete",
4319                                            404, "404 Not Found",
4320                                            "unknown/expired handle '130'",
4321                                            self.GET,
4322                                            "/operations/130?t=status&output=JSON"))
4323         return d
4324
4325     def test_uncollected_ophandle_expiration(self):
4326         # uncollected ophandles should expire after 4 days
4327         def _make_uncollected_ophandle(ophandle):
4328             d = self.POST(self.public_url +
4329                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
4330                           followRedirect=False)
4331             # When we start the operation, the webapi server will want
4332             # to redirect us to the page for the ophandle, so we get
4333             # confirmation that the operation has started. If the
4334             # manifest operation has finished by the time we get there,
4335             # following that redirect (by setting followRedirect=True
4336             # above) has the side effect of collecting the ophandle that
4337             # we've just created, which means that we can't use the
4338             # ophandle to test the uncollected timeout anymore. So,
4339             # instead, catch the 302 here and don't follow it.
4340             d.addBoth(self.should302, "uncollected_ophandle_creation")
4341             return d
4342         # Create an ophandle, don't collect it, then advance the clock by
4343         # 4 days - 1 second and make sure that the ophandle is still there.
4344         d = _make_uncollected_ophandle(131)
4345         d.addCallback(lambda ign:
4346             self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
4347         d.addCallback(lambda ign:
4348             self.GET("/operations/131?t=status&output=JSON"))
4349         def _check1(res):
4350             data = simplejson.loads(res)
4351             self.failUnless("finished" in data, res)
4352         d.addCallback(_check1)
4353         # Create an ophandle, don't collect it, then try to collect it
4354         # after 4 days. It should be gone.
4355         d.addCallback(lambda ign:
4356             _make_uncollected_ophandle(132))
4357         d.addCallback(lambda ign:
4358             self.clock.advance(96*60*60))
4359         d.addCallback(lambda ign:
4360             self.shouldHTTPError("uncollected_ophandle_expired_after_100_hours",
4361                                  404, "404 Not Found",
4362                                  "unknown/expired handle '132'",
4363                                  self.GET,
4364                                  "/operations/132?t=status&output=JSON"))
4365         return d
4366
4367     def test_collected_ophandle_expiration(self):
4368         # collected ophandles should expire after 1 day
4369         def _make_collected_ophandle(ophandle):
4370             d = self.POST(self.public_url +
4371                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
4372                           followRedirect=True)
4373             # By following the initial redirect, we collect the ophandle
4374             # we've just created.
4375             return d
4376         # Create a collected ophandle, then collect it after 23 hours
4377         # and 59 seconds to make sure that it is still there.
4378         d = _make_collected_ophandle(133)
4379         d.addCallback(lambda ign:
4380             self.clock.advance((24*60*60) - 1))
4381         d.addCallback(lambda ign:
4382             self.GET("/operations/133?t=status&output=JSON"))
4383         def _check1(res):
4384             data = simplejson.loads(res)
4385             self.failUnless("finished" in data, res)
4386         d.addCallback(_check1)
4387         # Create another uncollected ophandle, then try to collect it
4388         # after 24 hours to make sure that it is gone.
4389         d.addCallback(lambda ign:
4390             _make_collected_ophandle(134))
4391         d.addCallback(lambda ign:
4392             self.clock.advance(24*60*60))
4393         d.addCallback(lambda ign:
4394             self.shouldHTTPError("collected_ophandle_expired_after_1_day",
4395                                  404, "404 Not Found",
4396                                  "unknown/expired handle '134'",
4397                                  self.GET,
4398                                  "/operations/134?t=status&output=JSON"))
4399         return d
4400
4401     def test_incident(self):
4402         d = self.POST("/report_incident", details="eek")
4403         def _done(res):
4404             self.failIfIn("<html>", res)
4405             self.failUnlessIn("An incident report has been saved", res)
4406         d.addCallback(_done)
4407         return d
4408
4409     def test_static(self):
4410         webdir = os.path.join(self.staticdir, "subdir")
4411         fileutil.make_dirs(webdir)
4412         f = open(os.path.join(webdir, "hello.txt"), "wb")
4413         f.write("hello")
4414         f.close()
4415
4416         d = self.GET("/static/subdir/hello.txt")
4417         def _check(res):
4418             self.failUnlessReallyEqual(res, "hello")
4419         d.addCallback(_check)
4420         return d
4421
4422
4423 class IntroducerWeb(unittest.TestCase):
4424     def setUp(self):
4425         self.node = None
4426
4427     def tearDown(self):
4428         d = defer.succeed(None)
4429         if self.node:
4430             d.addCallback(lambda ign: self.node.stopService())
4431         d.addCallback(flushEventualQueue)
4432         return d
4433
4434     def test_welcome(self):
4435         basedir = "web.IntroducerWeb.test_welcome"
4436         os.mkdir(basedir)
4437         cfg = "\n".join(["[node]",
4438                          "tub.location = 127.0.0.1:1",
4439                          "web.port = tcp:0",
4440                          ]) + "\n"
4441         fileutil.write(os.path.join(basedir, "tahoe.cfg"), cfg)
4442         self.node = IntroducerNode(basedir)
4443         self.ws = self.node.getServiceNamed("webish")
4444
4445         d = fireEventually(None)
4446         d.addCallback(lambda ign: self.node.startService())
4447         d.addCallback(lambda ign: self.node.when_tub_ready())
4448
4449         d.addCallback(lambda ign: self.GET("/"))
4450         def _check(res):
4451             self.failUnlessIn('Welcome to the Tahoe-LAFS Introducer', res)
4452             self.failUnlessIn(FAVICON_MARKUP, res)
4453             self.failUnlessIn('Page rendered at', res)
4454             self.failUnlessIn('Tahoe-LAFS code imported from:', res)
4455         d.addCallback(_check)
4456         return d
4457
4458     def GET(self, urlpath, followRedirect=False, return_response=False,
4459             **kwargs):
4460         # if return_response=True, this fires with (data, statuscode,
4461         # respheaders) instead of just data.
4462         assert not isinstance(urlpath, unicode)
4463         url = self.ws.getURL().rstrip('/') + urlpath
4464         factory = HTTPClientGETFactory(url, method="GET",
4465                                        followRedirect=followRedirect, **kwargs)
4466         reactor.connectTCP("localhost", self.ws.getPortnum(), factory)
4467         d = factory.deferred
4468         def _got_data(data):
4469             return (data, factory.status, factory.response_headers)
4470         if return_response:
4471             d.addCallback(_got_data)
4472         return factory.deferred
4473
4474
4475 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4476     def test_load_file(self):
4477         # This will raise an exception unless a well-formed XML file is found under that name.
4478         common.getxmlfile('directory.xhtml').load()
4479
4480     def test_parse_replace_arg(self):
4481         self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
4482         self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
4483         self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
4484                                    "only-files")
4485         self.failUnlessRaises(common.WebError, common.parse_replace_arg, "only_fles")
4486
4487     def test_abbreviate_time(self):
4488         self.failUnlessReallyEqual(common.abbreviate_time(None), "")
4489         self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
4490         self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
4491         self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
4492         self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
4493         self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
4494
4495     def test_compute_rate(self):
4496         self.failUnlessReallyEqual(common.compute_rate(None, None), None)
4497         self.failUnlessReallyEqual(common.compute_rate(None, 1), None)
4498         self.failUnlessReallyEqual(common.compute_rate(250000, None), None)
4499         self.failUnlessReallyEqual(common.compute_rate(250000, 0), None)
4500         self.failUnlessReallyEqual(common.compute_rate(250000, 10), 25000.0)
4501         self.failUnlessReallyEqual(common.compute_rate(0, 10), 0.0)
4502         self.shouldFail(AssertionError, "test_compute_rate", "",
4503                         common.compute_rate, -100, 10)
4504         self.shouldFail(AssertionError, "test_compute_rate", "",
4505                         common.compute_rate, 100, -10)
4506
4507         # Sanity check
4508         rate = common.compute_rate(10*1000*1000, 1)
4509         self.failUnlessReallyEqual(common.abbreviate_rate(rate), "10.00MBps")
4510
4511     def test_abbreviate_rate(self):
4512         self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
4513         self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
4514         self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
4515         self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
4516
4517     def test_abbreviate_size(self):
4518         self.failUnlessReallyEqual(common.abbreviate_size(None), "")
4519         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
4520         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
4521         self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
4522         self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
4523
4524     def test_plural(self):
4525         def convert(s):
4526             return "%d second%s" % (s, status.plural(s))
4527         self.failUnlessReallyEqual(convert(0), "0 seconds")
4528         self.failUnlessReallyEqual(convert(1), "1 second")
4529         self.failUnlessReallyEqual(convert(2), "2 seconds")
4530         def convert2(s):
4531             return "has share%s: %s" % (status.plural(s), ",".join(s))
4532         self.failUnlessReallyEqual(convert2([]), "has shares: ")
4533         self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
4534         self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
4535
4536
4537 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4538
4539     def CHECK(self, ign, which, args, clientnum=0):
4540         fileurl = self.fileurls[which]
4541         url = fileurl + "?" + args
4542         return self.GET(url, method="POST", clientnum=clientnum)
4543
4544     def test_filecheck(self):
4545         self.basedir = "web/Grid/filecheck"
4546         self.set_up_grid()
4547         c0 = self.g.clients[0]
4548         self.uris = {}
4549         DATA = "data" * 100
4550         d = c0.upload(upload.Data(DATA, convergence=""))
4551         def _stash_uri(ur, which):
4552             self.uris[which] = ur.get_uri()
4553         d.addCallback(_stash_uri, "good")
4554         d.addCallback(lambda ign:
4555                       c0.upload(upload.Data(DATA+"1", convergence="")))
4556         d.addCallback(_stash_uri, "sick")
4557         d.addCallback(lambda ign:
4558                       c0.upload(upload.Data(DATA+"2", convergence="")))
4559         d.addCallback(_stash_uri, "dead")
4560         def _stash_mutable_uri(n, which):
4561             self.uris[which] = n.get_uri()
4562             assert isinstance(self.uris[which], str)
4563         d.addCallback(lambda ign:
4564             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4565         d.addCallback(_stash_mutable_uri, "corrupt")
4566         d.addCallback(lambda ign:
4567                       c0.upload(upload.Data("literal", convergence="")))
4568         d.addCallback(_stash_uri, "small")
4569         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
4570         d.addCallback(_stash_mutable_uri, "smalldir")
4571
4572         def _compute_fileurls(ignored):
4573             self.fileurls = {}
4574             for which in self.uris:
4575                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4576         d.addCallback(_compute_fileurls)
4577
4578         def _clobber_shares(ignored):
4579             good_shares = self.find_uri_shares(self.uris["good"])
4580             self.failUnlessReallyEqual(len(good_shares), 10)
4581             sick_shares = self.find_uri_shares(self.uris["sick"])
4582             os.unlink(sick_shares[0][2])
4583             dead_shares = self.find_uri_shares(self.uris["dead"])
4584             for i in range(1, 10):
4585                 os.unlink(dead_shares[i][2])
4586             c_shares = self.find_uri_shares(self.uris["corrupt"])
4587             cso = CorruptShareOptions()
4588             cso.stdout = StringIO()
4589             cso.parseOptions([c_shares[0][2]])
4590             corrupt_share(cso)
4591         d.addCallback(_clobber_shares)
4592
4593         d.addCallback(self.CHECK, "good", "t=check")
4594         def _got_html_good(res):
4595             self.failUnlessIn("Healthy", res)
4596             self.failIfIn("Not Healthy", res)
4597             self.failUnlessIn(FAVICON_MARKUP, res)
4598         d.addCallback(_got_html_good)
4599         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
4600         def _got_html_good_return_to(res):
4601             self.failUnlessIn("Healthy", res)
4602             self.failIfIn("Not Healthy", res)
4603             self.failUnlessIn('<a href="somewhere">Return to file', res)
4604         d.addCallback(_got_html_good_return_to)
4605         d.addCallback(self.CHECK, "good", "t=check&output=json")
4606         def _got_json_good(res):
4607             r = simplejson.loads(res)
4608             self.failUnlessEqual(r["summary"], "Healthy")
4609             self.failUnless(r["results"]["healthy"])
4610             self.failIfIn("needs-rebalancing", r["results"])
4611             self.failUnless(r["results"]["recoverable"])
4612         d.addCallback(_got_json_good)
4613
4614         d.addCallback(self.CHECK, "small", "t=check")
4615         def _got_html_small(res):
4616             self.failUnlessIn("Literal files are always healthy", res)
4617             self.failIfIn("Not Healthy", res)
4618         d.addCallback(_got_html_small)
4619         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
4620         def _got_html_small_return_to(res):
4621             self.failUnlessIn("Literal files are always healthy", res)
4622             self.failIfIn("Not Healthy", res)
4623             self.failUnlessIn('<a href="somewhere">Return to file', res)
4624         d.addCallback(_got_html_small_return_to)
4625         d.addCallback(self.CHECK, "small", "t=check&output=json")
4626         def _got_json_small(res):
4627             r = simplejson.loads(res)
4628             self.failUnlessEqual(r["storage-index"], "")
4629             self.failUnless(r["results"]["healthy"])
4630         d.addCallback(_got_json_small)
4631
4632         d.addCallback(self.CHECK, "smalldir", "t=check")
4633         def _got_html_smalldir(res):
4634             self.failUnlessIn("Literal files are always healthy", res)
4635             self.failIfIn("Not Healthy", res)
4636         d.addCallback(_got_html_smalldir)
4637         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
4638         def _got_json_smalldir(res):
4639             r = simplejson.loads(res)
4640             self.failUnlessEqual(r["storage-index"], "")
4641             self.failUnless(r["results"]["healthy"])
4642         d.addCallback(_got_json_smalldir)
4643
4644         d.addCallback(self.CHECK, "sick", "t=check")
4645         def _got_html_sick(res):
4646             self.failUnlessIn("Not Healthy", res)
4647         d.addCallback(_got_html_sick)
4648         d.addCallback(self.CHECK, "sick", "t=check&output=json")
4649         def _got_json_sick(res):
4650             r = simplejson.loads(res)
4651             self.failUnlessEqual(r["summary"],
4652                                  "Not Healthy: 9 shares (enc 3-of-10)")
4653             self.failIf(r["results"]["healthy"])
4654             self.failUnless(r["results"]["recoverable"])
4655             self.failIfIn("needs-rebalancing", r["results"])
4656         d.addCallback(_got_json_sick)
4657
4658         d.addCallback(self.CHECK, "dead", "t=check")
4659         def _got_html_dead(res):
4660             self.failUnlessIn("Not Healthy", res)
4661         d.addCallback(_got_html_dead)
4662         d.addCallback(self.CHECK, "dead", "t=check&output=json")
4663         def _got_json_dead(res):
4664             r = simplejson.loads(res)
4665             self.failUnlessEqual(r["summary"],
4666                                  "Not Healthy: 1 shares (enc 3-of-10)")
4667             self.failIf(r["results"]["healthy"])
4668             self.failIf(r["results"]["recoverable"])
4669             self.failIfIn("needs-rebalancing", r["results"])
4670         d.addCallback(_got_json_dead)
4671
4672         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
4673         def _got_html_corrupt(res):
4674             self.failUnlessIn("Not Healthy! : Unhealthy", res)
4675         d.addCallback(_got_html_corrupt)
4676         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
4677         def _got_json_corrupt(res):
4678             r = simplejson.loads(res)
4679             self.failUnlessIn("Unhealthy: 9 shares (enc 3-of-10)", r["summary"])
4680             self.failIf(r["results"]["healthy"])
4681             self.failUnless(r["results"]["recoverable"])
4682             self.failIfIn("needs-rebalancing", r["results"])
4683             self.failUnlessReallyEqual(r["results"]["count-happiness"], 9)
4684             self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
4685             self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
4686         d.addCallback(_got_json_corrupt)
4687
4688         d.addErrback(self.explain_web_error)
4689         return d
4690
4691     def test_repair_html(self):
4692         self.basedir = "web/Grid/repair_html"
4693         self.set_up_grid()
4694         c0 = self.g.clients[0]
4695         self.uris = {}
4696         DATA = "data" * 100
4697         d = c0.upload(upload.Data(DATA, convergence=""))
4698         def _stash_uri(ur, which):
4699             self.uris[which] = ur.get_uri()
4700         d.addCallback(_stash_uri, "good")
4701         d.addCallback(lambda ign:
4702                       c0.upload(upload.Data(DATA+"1", convergence="")))
4703         d.addCallback(_stash_uri, "sick")
4704         d.addCallback(lambda ign:
4705                       c0.upload(upload.Data(DATA+"2", convergence="")))
4706         d.addCallback(_stash_uri, "dead")
4707         def _stash_mutable_uri(n, which):
4708             self.uris[which] = n.get_uri()
4709             assert isinstance(self.uris[which], str)
4710         d.addCallback(lambda ign:
4711             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4712         d.addCallback(_stash_mutable_uri, "corrupt")
4713
4714         def _compute_fileurls(ignored):
4715             self.fileurls = {}
4716             for which in self.uris:
4717                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4718         d.addCallback(_compute_fileurls)
4719
4720         def _clobber_shares(ignored):
4721             good_shares = self.find_uri_shares(self.uris["good"])
4722             self.failUnlessReallyEqual(len(good_shares), 10)
4723             sick_shares = self.find_uri_shares(self.uris["sick"])
4724             os.unlink(sick_shares[0][2])
4725             dead_shares = self.find_uri_shares(self.uris["dead"])
4726             for i in range(1, 10):
4727                 os.unlink(dead_shares[i][2])
4728             c_shares = self.find_uri_shares(self.uris["corrupt"])
4729             cso = CorruptShareOptions()
4730             cso.stdout = StringIO()
4731             cso.parseOptions([c_shares[0][2]])
4732             corrupt_share(cso)
4733         d.addCallback(_clobber_shares)
4734
4735         d.addCallback(self.CHECK, "good", "t=check&repair=true")
4736         def _got_html_good(res):
4737             self.failUnlessIn("Healthy", res)
4738             self.failIfIn("Not Healthy", res)
4739             self.failUnlessIn("No repair necessary", res)
4740             self.failUnlessIn(FAVICON_MARKUP, res)
4741         d.addCallback(_got_html_good)
4742
4743         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
4744         def _got_html_sick(res):
4745             self.failUnlessIn("Healthy : healthy", res)
4746             self.failIfIn("Not Healthy", res)
4747             self.failUnlessIn("Repair successful", res)
4748         d.addCallback(_got_html_sick)
4749
4750         # repair of a dead file will fail, of course, but it isn't yet
4751         # clear how this should be reported. Right now it shows up as
4752         # a "410 Gone".
4753         #
4754         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
4755         #def _got_html_dead(res):
4756         #    print res
4757         #    self.failUnlessIn("Healthy : healthy", res)
4758         #    self.failIfIn("Not Healthy", res)
4759         #    self.failUnlessIn("No repair necessary", res)
4760         #d.addCallback(_got_html_dead)
4761
4762         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
4763         def _got_html_corrupt(res):
4764             self.failUnlessIn("Healthy : Healthy", res)
4765             self.failIfIn("Not Healthy", res)
4766             self.failUnlessIn("Repair successful", res)
4767         d.addCallback(_got_html_corrupt)
4768
4769         d.addErrback(self.explain_web_error)
4770         return d
4771
4772     def test_repair_json(self):
4773         self.basedir = "web/Grid/repair_json"
4774         self.set_up_grid()
4775         c0 = self.g.clients[0]
4776         self.uris = {}
4777         DATA = "data" * 100
4778         d = c0.upload(upload.Data(DATA+"1", convergence=""))
4779         def _stash_uri(ur, which):
4780             self.uris[which] = ur.get_uri()
4781         d.addCallback(_stash_uri, "sick")
4782
4783         def _compute_fileurls(ignored):
4784             self.fileurls = {}
4785             for which in self.uris:
4786                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4787         d.addCallback(_compute_fileurls)
4788
4789         def _clobber_shares(ignored):
4790             sick_shares = self.find_uri_shares(self.uris["sick"])
4791             os.unlink(sick_shares[0][2])
4792         d.addCallback(_clobber_shares)
4793
4794         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
4795         def _got_json_sick(res):
4796             r = simplejson.loads(res)
4797             self.failUnlessReallyEqual(r["repair-attempted"], True)
4798             self.failUnlessReallyEqual(r["repair-successful"], True)
4799             self.failUnlessEqual(r["pre-repair-results"]["summary"],
4800                                  "Not Healthy: 9 shares (enc 3-of-10)")
4801             self.failIf(r["pre-repair-results"]["results"]["healthy"])
4802             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
4803             self.failUnless(r["post-repair-results"]["results"]["healthy"])
4804         d.addCallback(_got_json_sick)
4805
4806         d.addErrback(self.explain_web_error)
4807         return d
4808
4809     def test_unknown(self, immutable=False):
4810         self.basedir = "web/Grid/unknown"
4811         if immutable:
4812             self.basedir = "web/Grid/unknown-immutable"
4813
4814         self.set_up_grid()
4815         c0 = self.g.clients[0]
4816         self.uris = {}
4817         self.fileurls = {}
4818
4819         # the future cap format may contain slashes, which must be tolerated
4820         expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
4821                                                            safe="")
4822
4823         if immutable:
4824             name = u"future-imm"
4825             future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
4826             d = c0.create_immutable_dirnode({name: (future_node, {})})
4827         else:
4828             name = u"future"
4829             future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4830             d = c0.create_dirnode()
4831
4832         def _stash_root_and_create_file(n):
4833             self.rootnode = n
4834             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
4835             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
4836             if not immutable:
4837                 return self.rootnode.set_node(name, future_node)
4838         d.addCallback(_stash_root_and_create_file)
4839
4840         # make sure directory listing tolerates unknown nodes
4841         d.addCallback(lambda ign: self.GET(self.rooturl))
4842         def _check_directory_html(res, expected_type_suffix):
4843             pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
4844                                   '<td>%s</td>' % (expected_type_suffix, str(name)),
4845                                  re.DOTALL)
4846             self.failUnless(re.search(pattern, res), res)
4847             # find the More Info link for name, should be relative
4848             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4849             info_url = mo.group(1)
4850             self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
4851         if immutable:
4852             d.addCallback(_check_directory_html, "-IMM")
4853         else:
4854             d.addCallback(_check_directory_html, "")
4855
4856         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4857         def _check_directory_json(res, expect_rw_uri):
4858             data = simplejson.loads(res)
4859             self.failUnlessEqual(data[0], "dirnode")
4860             f = data[1]["children"][name]
4861             self.failUnlessEqual(f[0], "unknown")
4862             if expect_rw_uri:
4863                 self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data)
4864             else:
4865                 self.failIfIn("rw_uri", f[1])
4866             if immutable:
4867                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
4868             else:
4869                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
4870             self.failUnlessIn("metadata", f[1])
4871         d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
4872
4873         def _check_info(res, expect_rw_uri, expect_ro_uri):
4874             self.failUnlessIn("Object Type: <span>unknown</span>", res)
4875             if expect_rw_uri:
4876                 self.failUnlessIn(unknown_rwcap, res)
4877             if expect_ro_uri:
4878                 if immutable:
4879                     self.failUnlessIn(unknown_immcap, res)
4880                 else:
4881                     self.failUnlessIn(unknown_rocap, res)
4882             else:
4883                 self.failIfIn(unknown_rocap, res)
4884             self.failIfIn("Raw data as", res)
4885             self.failIfIn("Directory writecap", res)
4886             self.failIfIn("Checker Operations", res)
4887             self.failIfIn("Mutable File Operations", res)
4888             self.failIfIn("Directory Operations", res)
4889
4890         # FIXME: these should have expect_rw_uri=not immutable; I don't know
4891         # why they fail. Possibly related to ticket #922.
4892
4893         d.addCallback(lambda ign: self.GET(expected_info_url))
4894         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
4895         d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
4896         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
4897
4898         def _check_json(res, expect_rw_uri):
4899             data = simplejson.loads(res)
4900             self.failUnlessEqual(data[0], "unknown")
4901             if expect_rw_uri:
4902                 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data)
4903             else:
4904                 self.failIfIn("rw_uri", data[1])
4905
4906             if immutable:
4907                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data)
4908                 self.failUnlessReallyEqual(data[1]["mutable"], False)
4909             elif expect_rw_uri:
4910                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4911                 self.failUnlessReallyEqual(data[1]["mutable"], True)
4912             else:
4913                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4914                 self.failIfIn("mutable", data[1])
4915
4916             # TODO: check metadata contents
4917             self.failUnlessIn("metadata", data[1])
4918
4919         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
4920         d.addCallback(_check_json, expect_rw_uri=not immutable)
4921
4922         # and make sure that a read-only version of the directory can be
4923         # rendered too. This version will not have unknown_rwcap, whether
4924         # or not future_node was immutable.
4925         d.addCallback(lambda ign: self.GET(self.rourl))
4926         if immutable:
4927             d.addCallback(_check_directory_html, "-IMM")
4928         else:
4929             d.addCallback(_check_directory_html, "-RO")
4930
4931         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
4932         d.addCallback(_check_directory_json, expect_rw_uri=False)
4933
4934         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
4935         d.addCallback(_check_json, expect_rw_uri=False)
4936
4937         # TODO: check that getting t=info from the Info link in the ro directory
4938         # works, and does not include the writecap URI.
4939         return d
4940
4941     def test_immutable_unknown(self):
4942         return self.test_unknown(immutable=True)
4943
4944     def test_mutant_dirnodes_are_omitted(self):
4945         self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
4946
4947         self.set_up_grid()
4948         c = self.g.clients[0]
4949         nm = c.nodemaker
4950         self.uris = {}
4951         self.fileurls = {}
4952
4953         lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
4954         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
4955         mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
4956
4957         # This method tests mainly dirnode, but we'd have to duplicate code in order to
4958         # test the dirnode and web layers separately.
4959
4960         # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
4961         # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
4962         # When the directory is read, the mutants should be silently disposed of, leaving
4963         # their lonely sibling.
4964         # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
4965         # because immutable directories don't have a writecap and therefore that field
4966         # isn't (and can't be) decrypted.
4967         # TODO: The field still exists in the netstring. Technically we should check what
4968         # happens if something is put there (_unpack_contents should raise ValueError),
4969         # but that can wait.
4970
4971         lonely_child = nm.create_from_cap(lonely_uri)
4972         mutant_ro_child = nm.create_from_cap(mut_read_uri)
4973         mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
4974
4975         def _by_hook_or_by_crook():
4976             return True
4977         for n in [mutant_ro_child, mutant_write_in_ro_child]:
4978             n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
4979
4980         mutant_write_in_ro_child.get_write_uri    = lambda: None
4981         mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
4982
4983         kids = {u"lonely":      (lonely_child, {}),
4984                 u"ro":          (mutant_ro_child, {}),
4985                 u"write-in-ro": (mutant_write_in_ro_child, {}),
4986                 }
4987         d = c.create_immutable_dirnode(kids)
4988
4989         def _created(dn):
4990             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
4991             self.failIf(dn.is_mutable())
4992             self.failUnless(dn.is_readonly())
4993             # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
4994             self.failIf(hasattr(dn._node, 'get_writekey'))
4995             rep = str(dn)
4996             self.failUnlessIn("RO-IMM", rep)
4997             cap = dn.get_cap()
4998             self.failUnlessIn("CHK", cap.to_string())
4999             self.cap = cap
5000             self.rootnode = dn
5001             self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
5002             return download_to_data(dn._node)
5003         d.addCallback(_created)
5004
5005         def _check_data(data):
5006             # Decode the netstring representation of the directory to check that all children
5007             # are present. This is a bit of an abstraction violation, but there's not really
5008             # any other way to do it given that the real DirectoryNode._unpack_contents would
5009             # strip the mutant children out (which is what we're trying to test, later).
5010             position = 0
5011             numkids = 0
5012             while position < len(data):
5013                 entries, position = split_netstring(data, 1, position)
5014                 entry = entries[0]
5015                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
5016                 name = name_utf8.decode("utf-8")
5017                 self.failUnlessEqual(rwcapdata, "")
5018                 self.failUnlessIn(name, kids)
5019                 (expected_child, ign) = kids[name]
5020                 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
5021                 numkids += 1
5022
5023             self.failUnlessReallyEqual(numkids, 3)
5024             return self.rootnode.list()
5025         d.addCallback(_check_data)
5026
5027         # Now when we use the real directory listing code, the mutants should be absent.
5028         def _check_kids(children):
5029             self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
5030             lonely_node, lonely_metadata = children[u"lonely"]
5031
5032             self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
5033             self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
5034         d.addCallback(_check_kids)
5035
5036         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
5037         d.addCallback(lambda n: n.list())
5038         d.addCallback(_check_kids)  # again with dirnode recreated from cap
5039
5040         # Make sure the lonely child can be listed in HTML...
5041         d.addCallback(lambda ign: self.GET(self.rooturl))
5042         def _check_html(res):
5043             self.failIfIn("URI:SSK", res)
5044             get_lonely = "".join([r'<td>FILE</td>',
5045                                   r'\s+<td>',
5046                                   r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
5047                                   r'</td>',
5048                                   r'\s+<td align="right">%d</td>' % len("one"),
5049                                   ])
5050             self.failUnless(re.search(get_lonely, res), res)
5051
5052             # find the More Info link for name, should be relative
5053             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
5054             info_url = mo.group(1)
5055             self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
5056         d.addCallback(_check_html)
5057
5058         # ... and in JSON.
5059         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
5060         def _check_json(res):
5061             data = simplejson.loads(res)
5062             self.failUnlessEqual(data[0], "dirnode")
5063             listed_children = data[1]["children"]
5064             self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
5065             ll_type, ll_data = listed_children[u"lonely"]
5066             self.failUnlessEqual(ll_type, "filenode")
5067             self.failIfIn("rw_uri", ll_data)
5068             self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
5069         d.addCallback(_check_json)
5070         return d
5071
5072     def test_deep_check(self):
5073         self.basedir = "web/Grid/deep_check"
5074         self.set_up_grid()
5075         c0 = self.g.clients[0]
5076         self.uris = {}
5077         self.fileurls = {}
5078         DATA = "data" * 100
5079         d = c0.create_dirnode()
5080         def _stash_root_and_create_file(n):
5081             self.rootnode = n
5082             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5083             return n.add_file(u"good", upload.Data(DATA, convergence=""))
5084         d.addCallback(_stash_root_and_create_file)
5085         def _stash_uri(fn, which):
5086             self.uris[which] = fn.get_uri()
5087             return fn
5088         d.addCallback(_stash_uri, "good")
5089         d.addCallback(lambda ign:
5090                       self.rootnode.add_file(u"small",
5091                                              upload.Data("literal",
5092                                                         convergence="")))
5093         d.addCallback(_stash_uri, "small")
5094         d.addCallback(lambda ign:
5095                       self.rootnode.add_file(u"sick",
5096                                              upload.Data(DATA+"1",
5097                                                         convergence="")))
5098         d.addCallback(_stash_uri, "sick")
5099
5100         # this tests that deep-check and stream-manifest will ignore
5101         # UnknownNode instances. Hopefully this will also cover deep-stats.
5102         future_node = UnknownNode(unknown_rwcap, unknown_rocap)
5103         d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
5104
5105         def _clobber_shares(ignored):
5106             self.delete_shares_numbered(self.uris["sick"], [0,1])
5107         d.addCallback(_clobber_shares)
5108
5109         # root
5110         # root/good
5111         # root/small
5112         # root/sick
5113         # root/future
5114
5115         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
5116         def _done(res):
5117             try:
5118                 units = [simplejson.loads(line)
5119                          for line in res.splitlines()
5120                          if line]
5121             except ValueError:
5122                 print "response is:", res
5123                 print "undecodeable line was '%s'" % line
5124                 raise
5125             self.failUnlessReallyEqual(len(units), 5+1)
5126             # should be parent-first
5127             u0 = units[0]
5128             self.failUnlessEqual(u0["path"], [])
5129             self.failUnlessEqual(u0["type"], "directory")
5130             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
5131             u0cr = u0["check-results"]
5132             self.failUnlessReallyEqual(u0cr["results"]["count-happiness"], 10)
5133             self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
5134
5135             ugood = [u for u in units
5136                      if u["type"] == "file" and u["path"] == [u"good"]][0]
5137             self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
5138             ugoodcr = ugood["check-results"]
5139             self.failUnlessReallyEqual(ugoodcr["results"]["count-happiness"], 10)
5140             self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
5141
5142             stats = units[-1]
5143             self.failUnlessEqual(stats["type"], "stats")
5144             s = stats["stats"]
5145             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
5146             self.failUnlessReallyEqual(s["count-literal-files"], 1)
5147             self.failUnlessReallyEqual(s["count-directories"], 1)
5148             self.failUnlessReallyEqual(s["count-unknown"], 1)
5149         d.addCallback(_done)
5150
5151         d.addCallback(self.CHECK, "root", "t=stream-manifest")
5152         def _check_manifest(res):
5153             self.failUnless(res.endswith("\n"))
5154             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
5155             self.failUnlessReallyEqual(len(units), 5+1)
5156             self.failUnlessEqual(units[-1]["type"], "stats")
5157             first = units[0]
5158             self.failUnlessEqual(first["path"], [])
5159             self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri())
5160             self.failUnlessEqual(first["type"], "directory")
5161             stats = units[-1]["stats"]
5162             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
5163             self.failUnlessReallyEqual(stats["count-literal-files"], 1)
5164             self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
5165             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
5166             self.failUnlessReallyEqual(stats["count-unknown"], 1)
5167         d.addCallback(_check_manifest)
5168
5169         # now add root/subdir and root/subdir/grandchild, then make subdir
5170         # unrecoverable, then see what happens
5171
5172         d.addCallback(lambda ign:
5173                       self.rootnode.create_subdirectory(u"subdir"))
5174         d.addCallback(_stash_uri, "subdir")
5175         d.addCallback(lambda subdir_node:
5176                       subdir_node.add_file(u"grandchild",
5177                                            upload.Data(DATA+"2",
5178                                                        convergence="")))
5179         d.addCallback(_stash_uri, "grandchild")
5180
5181         d.addCallback(lambda ign:
5182                       self.delete_shares_numbered(self.uris["subdir"],
5183                                                   range(1, 10)))
5184
5185         # root
5186         # root/good
5187         # root/small
5188         # root/sick
5189         # root/future
5190         # root/subdir [unrecoverable]
5191         # root/subdir/grandchild
5192
5193         # how should a streaming-JSON API indicate fatal error?
5194         # answer: emit ERROR: instead of a JSON string
5195
5196         d.addCallback(self.CHECK, "root", "t=stream-manifest")
5197         def _check_broken_manifest(res):
5198             lines = res.splitlines()
5199             error_lines = [i
5200                            for (i,line) in enumerate(lines)
5201                            if line.startswith("ERROR:")]
5202             if not error_lines:
5203                 self.fail("no ERROR: in output: %s" % (res,))
5204             first_error = error_lines[0]
5205             error_line = lines[first_error]
5206             error_msg = lines[first_error+1:]
5207             error_msg_s = "\n".join(error_msg) + "\n"
5208             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
5209                               error_line)
5210             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
5211             units = [simplejson.loads(line) for line in lines[:first_error]]
5212             self.failUnlessReallyEqual(len(units), 6) # includes subdir
5213             last_unit = units[-1]
5214             self.failUnlessEqual(last_unit["path"], ["subdir"])
5215         d.addCallback(_check_broken_manifest)
5216
5217         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
5218         def _check_broken_deepcheck(res):
5219             lines = res.splitlines()
5220             error_lines = [i
5221                            for (i,line) in enumerate(lines)
5222                            if line.startswith("ERROR:")]
5223             if not error_lines:
5224                 self.fail("no ERROR: in output: %s" % (res,))
5225             first_error = error_lines[0]
5226             error_line = lines[first_error]
5227             error_msg = lines[first_error+1:]
5228             error_msg_s = "\n".join(error_msg) + "\n"
5229             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
5230                               error_line)
5231             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
5232             units = [simplejson.loads(line) for line in lines[:first_error]]
5233             self.failUnlessReallyEqual(len(units), 6) # includes subdir
5234             last_unit = units[-1]
5235             self.failUnlessEqual(last_unit["path"], ["subdir"])
5236             r = last_unit["check-results"]["results"]
5237             self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
5238             self.failUnlessReallyEqual(r["count-happiness"], 1)
5239             self.failUnlessReallyEqual(r["count-shares-good"], 1)
5240             self.failUnlessReallyEqual(r["recoverable"], False)
5241         d.addCallback(_check_broken_deepcheck)
5242
5243         d.addErrback(self.explain_web_error)
5244         return d
5245
5246     def test_deep_check_and_repair(self):
5247         self.basedir = "web/Grid/deep_check_and_repair"
5248         self.set_up_grid()
5249         c0 = self.g.clients[0]
5250         self.uris = {}
5251         self.fileurls = {}
5252         DATA = "data" * 100
5253         d = c0.create_dirnode()
5254         def _stash_root_and_create_file(n):
5255             self.rootnode = n
5256             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5257             return n.add_file(u"good", upload.Data(DATA, convergence=""))
5258         d.addCallback(_stash_root_and_create_file)
5259         def _stash_uri(fn, which):
5260             self.uris[which] = fn.get_uri()
5261         d.addCallback(_stash_uri, "good")
5262         d.addCallback(lambda ign:
5263                       self.rootnode.add_file(u"small",
5264                                              upload.Data("literal",
5265                                                         convergence="")))
5266         d.addCallback(_stash_uri, "small")
5267         d.addCallback(lambda ign:
5268                       self.rootnode.add_file(u"sick",
5269                                              upload.Data(DATA+"1",
5270                                                         convergence="")))
5271         d.addCallback(_stash_uri, "sick")
5272         #d.addCallback(lambda ign:
5273         #              self.rootnode.add_file(u"dead",
5274         #                                     upload.Data(DATA+"2",
5275         #                                                convergence="")))
5276         #d.addCallback(_stash_uri, "dead")
5277
5278         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
5279         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
5280         #d.addCallback(_stash_uri, "corrupt")
5281
5282         def _clobber_shares(ignored):
5283             good_shares = self.find_uri_shares(self.uris["good"])
5284             self.failUnlessReallyEqual(len(good_shares), 10)
5285             sick_shares = self.find_uri_shares(self.uris["sick"])
5286             os.unlink(sick_shares[0][2])
5287             #dead_shares = self.find_uri_shares(self.uris["dead"])
5288             #for i in range(1, 10):
5289             #    os.unlink(dead_shares[i][2])
5290
5291             #c_shares = self.find_uri_shares(self.uris["corrupt"])
5292             #cso = CorruptShareOptions()
5293             #cso.stdout = StringIO()
5294             #cso.parseOptions([c_shares[0][2]])
5295             #corrupt_share(cso)
5296         d.addCallback(_clobber_shares)
5297
5298         # root
5299         # root/good   CHK, 10 shares
5300         # root/small  LIT
5301         # root/sick   CHK, 9 shares
5302
5303         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
5304         def _done(res):
5305             units = [simplejson.loads(line)
5306                      for line in res.splitlines()
5307                      if line]
5308             self.failUnlessReallyEqual(len(units), 4+1)
5309             # should be parent-first
5310             u0 = units[0]
5311             self.failUnlessEqual(u0["path"], [])
5312             self.failUnlessEqual(u0["type"], "directory")
5313             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
5314             u0crr = u0["check-and-repair-results"]
5315             self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
5316             self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-happiness"], 10)
5317             self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
5318
5319             ugood = [u for u in units
5320                      if u["type"] == "file" and u["path"] == [u"good"]][0]
5321             self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
5322             ugoodcrr = ugood["check-and-repair-results"]
5323             self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
5324             self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-happiness"], 10)
5325             self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
5326
5327             usick = [u for u in units
5328                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
5329             self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"])
5330             usickcrr = usick["check-and-repair-results"]
5331             self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
5332             self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
5333             self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-happiness"], 9)
5334             self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
5335             self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-happiness"], 10)
5336             self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
5337
5338             stats = units[-1]
5339             self.failUnlessEqual(stats["type"], "stats")
5340             s = stats["stats"]
5341             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
5342             self.failUnlessReallyEqual(s["count-literal-files"], 1)
5343             self.failUnlessReallyEqual(s["count-directories"], 1)
5344         d.addCallback(_done)
5345
5346         d.addErrback(self.explain_web_error)
5347         return d
5348
5349     def _count_leases(self, ignored, which):
5350         u = self.uris[which]
5351         shares = self.find_uri_shares(u)
5352         lease_counts = []
5353         for shnum, serverid, fn in shares:
5354             sf = get_share_file(fn)
5355             num_leases = len(list(sf.get_leases()))
5356             lease_counts.append( (fn, num_leases) )
5357         return lease_counts
5358
5359     def _assert_leasecount(self, lease_counts, expected):
5360         for (fn, num_leases) in lease_counts:
5361             if num_leases != expected:
5362                 self.fail("expected %d leases, have %d, on %s" %
5363                           (expected, num_leases, fn))
5364
5365     def test_add_lease(self):
5366         self.basedir = "web/Grid/add_lease"
5367         self.set_up_grid(num_clients=2)
5368         c0 = self.g.clients[0]
5369         self.uris = {}
5370         DATA = "data" * 100
5371         d = c0.upload(upload.Data(DATA, convergence=""))
5372         def _stash_uri(ur, which):
5373             self.uris[which] = ur.get_uri()
5374         d.addCallback(_stash_uri, "one")
5375         d.addCallback(lambda ign:
5376                       c0.upload(upload.Data(DATA+"1", convergence="")))
5377         d.addCallback(_stash_uri, "two")
5378         def _stash_mutable_uri(n, which):
5379             self.uris[which] = n.get_uri()
5380             assert isinstance(self.uris[which], str)
5381         d.addCallback(lambda ign:
5382             c0.create_mutable_file(publish.MutableData(DATA+"2")))
5383         d.addCallback(_stash_mutable_uri, "mutable")
5384
5385         def _compute_fileurls(ignored):
5386             self.fileurls = {}
5387             for which in self.uris:
5388                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
5389         d.addCallback(_compute_fileurls)
5390
5391         d.addCallback(self._count_leases, "one")
5392         d.addCallback(self._assert_leasecount, 1)
5393         d.addCallback(self._count_leases, "two")
5394         d.addCallback(self._assert_leasecount, 1)
5395         d.addCallback(self._count_leases, "mutable")
5396         d.addCallback(self._assert_leasecount, 1)
5397
5398         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
5399         def _got_html_good(res):
5400             self.failUnlessIn("Healthy", res)
5401             self.failIfIn("Not Healthy", res)
5402         d.addCallback(_got_html_good)
5403
5404         d.addCallback(self._count_leases, "one")
5405         d.addCallback(self._assert_leasecount, 1)
5406         d.addCallback(self._count_leases, "two")
5407         d.addCallback(self._assert_leasecount, 1)
5408         d.addCallback(self._count_leases, "mutable")
5409         d.addCallback(self._assert_leasecount, 1)
5410
5411         # this CHECK uses the original client, which uses the same
5412         # lease-secrets, so it will just renew the original lease
5413         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
5414         d.addCallback(_got_html_good)
5415
5416         d.addCallback(self._count_leases, "one")
5417         d.addCallback(self._assert_leasecount, 1)
5418         d.addCallback(self._count_leases, "two")
5419         d.addCallback(self._assert_leasecount, 1)
5420         d.addCallback(self._count_leases, "mutable")
5421         d.addCallback(self._assert_leasecount, 1)
5422
5423         # this CHECK uses an alternate client, which adds a second lease
5424         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
5425         d.addCallback(_got_html_good)
5426
5427         d.addCallback(self._count_leases, "one")
5428         d.addCallback(self._assert_leasecount, 2)
5429         d.addCallback(self._count_leases, "two")
5430         d.addCallback(self._assert_leasecount, 1)
5431         d.addCallback(self._count_leases, "mutable")
5432         d.addCallback(self._assert_leasecount, 1)
5433
5434         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
5435         d.addCallback(_got_html_good)
5436
5437         d.addCallback(self._count_leases, "one")
5438         d.addCallback(self._assert_leasecount, 2)
5439         d.addCallback(self._count_leases, "two")
5440         d.addCallback(self._assert_leasecount, 1)
5441         d.addCallback(self._count_leases, "mutable")
5442         d.addCallback(self._assert_leasecount, 1)
5443
5444         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
5445                       clientnum=1)
5446         d.addCallback(_got_html_good)
5447
5448         d.addCallback(self._count_leases, "one")
5449         d.addCallback(self._assert_leasecount, 2)
5450         d.addCallback(self._count_leases, "two")
5451         d.addCallback(self._assert_leasecount, 1)
5452         d.addCallback(self._count_leases, "mutable")
5453         d.addCallback(self._assert_leasecount, 2)
5454
5455         d.addErrback(self.explain_web_error)
5456         return d
5457
5458     def test_deep_add_lease(self):
5459         self.basedir = "web/Grid/deep_add_lease"
5460         self.set_up_grid(num_clients=2)
5461         c0 = self.g.clients[0]
5462         self.uris = {}
5463         self.fileurls = {}
5464         DATA = "data" * 100
5465         d = c0.create_dirnode()
5466         def _stash_root_and_create_file(n):
5467             self.rootnode = n
5468             self.uris["root"] = n.get_uri()
5469             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5470             return n.add_file(u"one", upload.Data(DATA, convergence=""))
5471         d.addCallback(_stash_root_and_create_file)
5472         def _stash_uri(fn, which):
5473             self.uris[which] = fn.get_uri()
5474         d.addCallback(_stash_uri, "one")
5475         d.addCallback(lambda ign:
5476                       self.rootnode.add_file(u"small",
5477                                              upload.Data("literal",
5478                                                         convergence="")))
5479         d.addCallback(_stash_uri, "small")
5480
5481         d.addCallback(lambda ign:
5482             c0.create_mutable_file(publish.MutableData("mutable")))
5483         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
5484         d.addCallback(_stash_uri, "mutable")
5485
5486         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
5487         def _done(res):
5488             units = [simplejson.loads(line)
5489                      for line in res.splitlines()
5490                      if line]
5491             # root, one, small, mutable,   stats
5492             self.failUnlessReallyEqual(len(units), 4+1)
5493         d.addCallback(_done)
5494
5495         d.addCallback(self._count_leases, "root")
5496         d.addCallback(self._assert_leasecount, 1)
5497         d.addCallback(self._count_leases, "one")
5498         d.addCallback(self._assert_leasecount, 1)
5499         d.addCallback(self._count_leases, "mutable")
5500         d.addCallback(self._assert_leasecount, 1)
5501
5502         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
5503         d.addCallback(_done)
5504
5505         d.addCallback(self._count_leases, "root")
5506         d.addCallback(self._assert_leasecount, 1)
5507         d.addCallback(self._count_leases, "one")
5508         d.addCallback(self._assert_leasecount, 1)
5509         d.addCallback(self._count_leases, "mutable")
5510         d.addCallback(self._assert_leasecount, 1)
5511
5512         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
5513                       clientnum=1)
5514         d.addCallback(_done)
5515
5516         d.addCallback(self._count_leases, "root")
5517         d.addCallback(self._assert_leasecount, 2)
5518         d.addCallback(self._count_leases, "one")
5519         d.addCallback(self._assert_leasecount, 2)
5520         d.addCallback(self._count_leases, "mutable")
5521         d.addCallback(self._assert_leasecount, 2)
5522
5523         d.addErrback(self.explain_web_error)
5524         return d
5525
5526
5527     def test_exceptions(self):
5528         self.basedir = "web/Grid/exceptions"
5529         self.set_up_grid(num_clients=1, num_servers=2)
5530         c0 = self.g.clients[0]
5531         c0.encoding_params['happy'] = 2
5532         self.fileurls = {}
5533         DATA = "data" * 100
5534         d = c0.create_dirnode()
5535         def _stash_root(n):
5536             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5537             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
5538             return n
5539         d.addCallback(_stash_root)
5540         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
5541         def _stash_bad(ur):
5542             self.fileurls["1share"] = "uri/" + urllib.quote(ur.get_uri())
5543             self.delete_shares_numbered(ur.get_uri(), range(1,10))
5544
5545             u = uri.from_string(ur.get_uri())
5546             u.key = testutil.flip_bit(u.key, 0)
5547             baduri = u.to_string()
5548             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
5549         d.addCallback(_stash_bad)
5550         d.addCallback(lambda ign: c0.create_dirnode())
5551         def _mangle_dirnode_1share(n):
5552             u = n.get_uri()
5553             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
5554             self.fileurls["dir-1share-json"] = url + "?t=json"
5555             self.delete_shares_numbered(u, range(1,10))
5556         d.addCallback(_mangle_dirnode_1share)
5557         d.addCallback(lambda ign: c0.create_dirnode())
5558         def _mangle_dirnode_0share(n):
5559             u = n.get_uri()
5560             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
5561             self.fileurls["dir-0share-json"] = url + "?t=json"
5562             self.delete_shares_numbered(u, range(0,10))
5563         d.addCallback(_mangle_dirnode_0share)
5564
5565         # NotEnoughSharesError should be reported sensibly, with a
5566         # text/plain explanation of the problem, and perhaps some
5567         # information on which shares *could* be found.
5568
5569         d.addCallback(lambda ignored:
5570                       self.shouldHTTPError("GET unrecoverable",
5571                                            410, "Gone", "NoSharesError",
5572                                            self.GET, self.fileurls["0shares"]))
5573         def _check_zero_shares(body):
5574             self.failIfIn("<html>", body)
5575             body = " ".join(body.strip().split())
5576             exp = ("NoSharesError: no shares could be found. "
5577                    "Zero shares usually indicates a corrupt URI, or that "
5578                    "no servers were connected, but it might also indicate "
5579                    "severe corruption. You should perform a filecheck on "
5580                    "this object to learn more. The full error message is: "
5581                    "no shares (need 3). Last failure: None")
5582             self.failUnlessReallyEqual(exp, body)
5583         d.addCallback(_check_zero_shares)
5584
5585
5586         d.addCallback(lambda ignored:
5587                       self.shouldHTTPError("GET 1share",
5588                                            410, "Gone", "NotEnoughSharesError",
5589                                            self.GET, self.fileurls["1share"]))
5590         def _check_one_share(body):
5591             self.failIfIn("<html>", body)
5592             body = " ".join(body.strip().split())
5593             msgbase = ("NotEnoughSharesError: This indicates that some "
5594                        "servers were unavailable, or that shares have been "
5595                        "lost to server departure, hard drive failure, or disk "
5596                        "corruption. You should perform a filecheck on "
5597                        "this object to learn more. The full error message is:"
5598                        )
5599             msg1 = msgbase + (" ran out of shares:"
5600                               " complete=sh0"
5601                               " pending="
5602                               " overdue= unused= need 3. Last failure: None")
5603             msg2 = msgbase + (" ran out of shares:"
5604                               " complete="
5605                               " pending=Share(sh0-on-xgru5)"
5606                               " overdue= unused= need 3. Last failure: None")
5607             self.failUnless(body == msg1 or body == msg2, body)
5608         d.addCallback(_check_one_share)
5609
5610         d.addCallback(lambda ignored:
5611                       self.shouldHTTPError("GET imaginary",
5612                                            404, "Not Found", None,
5613                                            self.GET, self.fileurls["imaginary"]))
5614         def _missing_child(body):
5615             self.failUnlessIn("No such child: imaginary", body)
5616         d.addCallback(_missing_child)
5617
5618         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
5619         def _check_0shares_dir_html(body):
5620             self.failUnlessIn(DIR_HTML_TAG, body)
5621             # we should see the regular page, but without the child table or
5622             # the dirops forms
5623             body = " ".join(body.strip().split())
5624             self.failUnlessIn('href="?t=info">More info on this directory',
5625                               body)
5626             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5627                    "could not be retrieved, because there were insufficient "
5628                    "good shares. This might indicate that no servers were "
5629                    "connected, insufficient servers were connected, the URI "
5630                    "was corrupt, or that shares have been lost due to server "
5631                    "departure, hard drive failure, or disk corruption. You "
5632                    "should perform a filecheck on this object to learn more.")
5633             self.failUnlessIn(exp, body)
5634             self.failUnlessIn("No upload forms: directory is unreadable", body)
5635         d.addCallback(_check_0shares_dir_html)
5636
5637         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
5638         def _check_1shares_dir_html(body):
5639             # at some point, we'll split UnrecoverableFileError into 0-shares
5640             # and some-shares like we did for immutable files (since there
5641             # are different sorts of advice to offer in each case). For now,
5642             # they present the same way.
5643             self.failUnlessIn(DIR_HTML_TAG, body)
5644             body = " ".join(body.strip().split())
5645             self.failUnlessIn('href="?t=info">More info on this directory',
5646                               body)
5647             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5648                    "could not be retrieved, because there were insufficient "
5649                    "good shares. This might indicate that no servers were "
5650                    "connected, insufficient servers were connected, the URI "
5651                    "was corrupt, or that shares have been lost due to server "
5652                    "departure, hard drive failure, or disk corruption. You "
5653                    "should perform a filecheck on this object to learn more.")
5654             self.failUnlessIn(exp, body)
5655             self.failUnlessIn("No upload forms: directory is unreadable", body)
5656         d.addCallback(_check_1shares_dir_html)
5657
5658         d.addCallback(lambda ignored:
5659                       self.shouldHTTPError("GET dir-0share-json",
5660                                            410, "Gone", "UnrecoverableFileError",
5661                                            self.GET,
5662                                            self.fileurls["dir-0share-json"]))
5663         def _check_unrecoverable_file(body):
5664             self.failIfIn("<html>", body)
5665             body = " ".join(body.strip().split())
5666             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5667                    "could not be retrieved, because there were insufficient "
5668                    "good shares. This might indicate that no servers were "
5669                    "connected, insufficient servers were connected, the URI "
5670                    "was corrupt, or that shares have been lost due to server "
5671                    "departure, hard drive failure, or disk corruption. You "
5672                    "should perform a filecheck on this object to learn more.")
5673             self.failUnlessReallyEqual(exp, body)
5674         d.addCallback(_check_unrecoverable_file)
5675
5676         d.addCallback(lambda ignored:
5677                       self.shouldHTTPError("GET dir-1share-json",
5678                                            410, "Gone", "UnrecoverableFileError",
5679                                            self.GET,
5680                                            self.fileurls["dir-1share-json"]))
5681         d.addCallback(_check_unrecoverable_file)
5682
5683         d.addCallback(lambda ignored:
5684                       self.shouldHTTPError("GET imaginary",
5685                                            404, "Not Found", None,
5686                                            self.GET, self.fileurls["imaginary"]))
5687
5688         # attach a webapi child that throws a random error, to test how it
5689         # gets rendered.
5690         w = c0.getServiceNamed("webish")
5691         w.root.putChild("ERRORBOOM", ErrorBoom())
5692
5693         # "Accept: */*" :        should get a text/html stack trace
5694         # "Accept: text/plain" : should get a text/plain stack trace
5695         # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
5696         # no Accept header:      should get a text/html stack trace
5697
5698         d.addCallback(lambda ignored:
5699                       self.shouldHTTPError("GET errorboom_html",
5700                                            500, "Internal Server Error", None,
5701                                            self.GET, "ERRORBOOM",
5702                                            headers={"accept": "*/*"}))
5703         def _internal_error_html1(body):
5704             self.failUnlessIn("<html>", "expected HTML, not '%s'" % body)
5705         d.addCallback(_internal_error_html1)
5706
5707         d.addCallback(lambda ignored:
5708                       self.shouldHTTPError("GET errorboom_text",
5709                                            500, "Internal Server Error", None,
5710                                            self.GET, "ERRORBOOM",
5711                                            headers={"accept": "text/plain"}))
5712         def _internal_error_text2(body):
5713             self.failIfIn("<html>", body)
5714             self.failUnless(body.startswith("Traceback "), body)
5715         d.addCallback(_internal_error_text2)
5716
5717         CLI_accepts = "text/plain, application/octet-stream"
5718         d.addCallback(lambda ignored:
5719                       self.shouldHTTPError("GET errorboom_text",
5720                                            500, "Internal Server Error", None,
5721                                            self.GET, "ERRORBOOM",
5722                                            headers={"accept": CLI_accepts}))
5723         def _internal_error_text3(body):
5724             self.failIfIn("<html>", body)
5725             self.failUnless(body.startswith("Traceback "), body)
5726         d.addCallback(_internal_error_text3)
5727
5728         d.addCallback(lambda ignored:
5729                       self.shouldHTTPError("GET errorboom_text",
5730                                            500, "Internal Server Error", None,
5731                                            self.GET, "ERRORBOOM"))
5732         def _internal_error_html4(body):
5733             self.failUnlessIn("<html>", body)
5734         d.addCallback(_internal_error_html4)
5735
5736         def _flush_errors(res):
5737             # Trial: please ignore the CompletelyUnhandledError in the logs
5738             self.flushLoggedErrors(CompletelyUnhandledError)
5739             return res
5740         d.addBoth(_flush_errors)
5741
5742         return d
5743
5744     def test_blacklist(self):
5745         # download from a blacklisted URI, get an error
5746         self.basedir = "web/Grid/blacklist"
5747         self.set_up_grid()
5748         c0 = self.g.clients[0]
5749         c0_basedir = c0.basedir
5750         fn = os.path.join(c0_basedir, "access.blacklist")
5751         self.uris = {}
5752         DATA = "off-limits " * 50
5753
5754         d = c0.upload(upload.Data(DATA, convergence=""))
5755         def _stash_uri_and_create_dir(ur):
5756             self.uri = ur.get_uri()
5757             self.url = "uri/"+self.uri
5758             u = uri.from_string_filenode(self.uri)
5759             self.si = u.get_storage_index()
5760             childnode = c0.create_node_from_uri(self.uri, None)
5761             return c0.create_dirnode({u"blacklisted.txt": (childnode,{}) })
5762         d.addCallback(_stash_uri_and_create_dir)
5763         def _stash_dir(node):
5764             self.dir_node = node
5765             self.dir_uri = node.get_uri()
5766             self.dir_url = "uri/"+self.dir_uri
5767         d.addCallback(_stash_dir)
5768         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5769         def _check_dir_html(body):
5770             self.failUnlessIn(DIR_HTML_TAG, body)
5771             self.failUnlessIn("blacklisted.txt</a>", body)
5772         d.addCallback(_check_dir_html)
5773         d.addCallback(lambda ign: self.GET(self.url))
5774         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5775
5776         def _blacklist(ign):
5777             f = open(fn, "w")
5778             f.write(" # this is a comment\n")
5779             f.write(" \n")
5780             f.write("\n") # also exercise blank lines
5781             f.write("%s %s\n" % (base32.b2a(self.si), "off-limits to you"))
5782             f.close()
5783             # clients should be checking the blacklist each time, so we don't
5784             # need to restart the client
5785         d.addCallback(_blacklist)
5786         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_uri",
5787                                                        403, "Forbidden",
5788                                                        "Access Prohibited: off-limits",
5789                                                        self.GET, self.url))
5790
5791         # We should still be able to list the parent directory, in HTML...
5792         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5793         def _check_dir_html2(body):
5794             self.failUnlessIn(DIR_HTML_TAG, body)
5795             self.failUnlessIn("blacklisted.txt</strike>", body)
5796         d.addCallback(_check_dir_html2)
5797
5798         # ... and in JSON (used by CLI).
5799         d.addCallback(lambda ign: self.GET(self.dir_url+"?t=json", followRedirect=True))
5800         def _check_dir_json(res):
5801             data = simplejson.loads(res)
5802             self.failUnless(isinstance(data, list), data)
5803             self.failUnlessEqual(data[0], "dirnode")
5804             self.failUnless(isinstance(data[1], dict), data)
5805             self.failUnlessIn("children", data[1])
5806             self.failUnlessIn("blacklisted.txt", data[1]["children"])
5807             childdata = data[1]["children"]["blacklisted.txt"]
5808             self.failUnless(isinstance(childdata, list), data)
5809             self.failUnlessEqual(childdata[0], "filenode")
5810             self.failUnless(isinstance(childdata[1], dict), data)
5811         d.addCallback(_check_dir_json)
5812
5813         def _unblacklist(ign):
5814             open(fn, "w").close()
5815             # the Blacklist object watches mtime to tell when the file has
5816             # changed, but on windows this test will run faster than the
5817             # filesystem's mtime resolution. So we edit Blacklist.last_mtime
5818             # to force a reload.
5819             self.g.clients[0].blacklist.last_mtime -= 2.0
5820         d.addCallback(_unblacklist)
5821
5822         # now a read should work
5823         d.addCallback(lambda ign: self.GET(self.url))
5824         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5825
5826         # read again to exercise the blacklist-is-unchanged logic
5827         d.addCallback(lambda ign: self.GET(self.url))
5828         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5829
5830         # now add a blacklisted directory, and make sure files under it are
5831         # refused too
5832         def _add_dir(ign):
5833             childnode = c0.create_node_from_uri(self.uri, None)
5834             return c0.create_dirnode({u"child": (childnode,{}) })
5835         d.addCallback(_add_dir)
5836         def _get_dircap(dn):
5837             self.dir_si_b32 = base32.b2a(dn.get_storage_index())
5838             self.dir_url_base = "uri/"+dn.get_write_uri()
5839             self.dir_url_json1 = "uri/"+dn.get_write_uri()+"?t=json"
5840             self.dir_url_json2 = "uri/"+dn.get_write_uri()+"/?t=json"
5841             self.dir_url_json_ro = "uri/"+dn.get_readonly_uri()+"/?t=json"
5842             self.child_url = "uri/"+dn.get_readonly_uri()+"/child"
5843         d.addCallback(_get_dircap)
5844         d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True))
5845         d.addCallback(lambda body: self.failUnlessIn(DIR_HTML_TAG, body))
5846         d.addCallback(lambda ign: self.GET(self.dir_url_json1))
5847         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5848         d.addCallback(lambda ign: self.GET(self.dir_url_json2))
5849         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5850         d.addCallback(lambda ign: self.GET(self.dir_url_json_ro))
5851         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5852         d.addCallback(lambda ign: self.GET(self.child_url))
5853         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5854
5855         def _block_dir(ign):
5856             f = open(fn, "w")
5857             f.write("%s %s\n" % (self.dir_si_b32, "dir-off-limits to you"))
5858             f.close()
5859             self.g.clients[0].blacklist.last_mtime -= 2.0
5860         d.addCallback(_block_dir)
5861         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir base",
5862                                                        403, "Forbidden",
5863                                                        "Access Prohibited: dir-off-limits",
5864                                                        self.GET, self.dir_url_base))
5865         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json1",
5866                                                        403, "Forbidden",
5867                                                        "Access Prohibited: dir-off-limits",
5868                                                        self.GET, self.dir_url_json1))
5869         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json2",
5870                                                        403, "Forbidden",
5871                                                        "Access Prohibited: dir-off-limits",
5872                                                        self.GET, self.dir_url_json2))
5873         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json_ro",
5874                                                        403, "Forbidden",
5875                                                        "Access Prohibited: dir-off-limits",
5876                                                        self.GET, self.dir_url_json_ro))
5877         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir child",
5878                                                        403, "Forbidden",
5879                                                        "Access Prohibited: dir-off-limits",
5880                                                        self.GET, self.child_url))
5881         return d
5882
5883
5884 class CompletelyUnhandledError(Exception):
5885     pass
5886 class ErrorBoom(rend.Page):
5887     def beforeRender(self, ctx):
5888         raise CompletelyUnhandledError("whoops")