src/allmydata/test/test_web.py

   1 import os.path, re, urllib, time, cgi
   2 import simplejson
   3 from StringIO import StringIO
   4
   5 from twisted.application import service
   6 from twisted.trial import unittest
   7 from twisted.internet import defer, reactor
   8 from twisted.internet.task import Clock
   9 from twisted.web import client, error, http
  10 from twisted.python import failure, log
  11
  12 from foolscap.api import fireEventually, flushEventualQueue
  13
  14 from nevow.util import escapeToXML
  15 from nevow import rend
  16
  17 from allmydata import interfaces, uri, webish, dirnode
  18 from allmydata.storage.shares import get_share_file
  19 from allmydata.storage_client import StorageFarmBroker, StubServer
  20 from allmydata.immutable import upload
  21 from allmydata.immutable.downloader.status import DownloadStatus
  22 from allmydata.dirnode import DirectoryNode
  23 from allmydata.nodemaker import NodeMaker
  24 from allmydata.unknown import UnknownNode
  25 from allmydata.web import status, common
  26 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  27 from allmydata.util import fileutil, base32, hashutil
  28 from allmydata.util.consumer import download_to_data
  29 from allmydata.util.netstring import split_netstring
  30 from allmydata.util.encodingutil import to_str
  31 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  32      create_chk_filenode, WebErrorMixin, ShouldFailMixin, \
  33      make_mutable_file_uri, create_mutable_filenode
  34 from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
  35 from allmydata.mutable import servermap, publish, retrieve
  36 import allmydata.test.common_util as testutil
  37 from allmydata.test.no_network import GridTestMixin
  38 from allmydata.test.common_web import HTTPClientGETFactory, \
  39      HTTPClientHEADFactory
  40 from allmydata.client import Client, SecretHolder
  41 from allmydata.introducer import IntroducerNode
  42
  43 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  44 # create a webserver that works against them
  45
  46 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  47
  48 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
  49 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
  50 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
  51
  52 FAVICON_MARKUP = '<link href="/icon.png" rel="shortcut icon" />'
  53 DIR_HTML_TAG = '<html lang="en">'
  54
  55 class FakeStatsProvider:
  56     def get_stats(self):
  57         stats = {'stats': {}, 'counters': {}}
  58         return stats
  59
  60 class FakeNodeMaker(NodeMaker):
  61     encoding_params = {
  62         'k': 3,
  63         'n': 10,
  64         'happy': 7,
  65         'max_segment_size':128*1024 # 1024=KiB
  66     }
  67     def _create_lit(self, cap):
  68         return FakeCHKFileNode(cap, self.all_contents)
  69     def _create_immutable(self, cap):
  70         return FakeCHKFileNode(cap, self.all_contents)
  71     def _create_mutable(self, cap):
  72         return FakeMutableFileNode(None, None,
  73                                    self.encoding_params, None,
  74                                    self.all_contents).init_from_cap(cap)
  75     def create_mutable_file(self, contents="", keysize=None,
  76                             version=SDMF_VERSION):
  77         n = FakeMutableFileNode(None, None, self.encoding_params, None,
  78                                 self.all_contents)
  79         return n.create(contents, version=version)
  80
  81 class FakeUploader(service.Service):
  82     name = "uploader"
  83     helper_furl = None
  84     helper_connected = False
  85
  86     def upload(self, uploadable):
  87         d = uploadable.get_size()
  88         d.addCallback(lambda size: uploadable.read(size))
  89         def _got_data(datav):
  90             data = "".join(datav)
  91             n = create_chk_filenode(data, self.all_contents)
  92             ur = upload.UploadResults(file_size=len(data),
  93                                       ciphertext_fetched=0,
  94                                       preexisting_shares=0,
  95                                       pushed_shares=10,
  96                                       sharemap={},
  97                                       servermap={},
  98                                       timings={},
  99                                       uri_extension_data={},
 100                                       uri_extension_hash="fake",
 101                                       verifycapstr="fakevcap")
 102             ur.set_uri(n.get_uri())
 103             return ur
 104         d.addCallback(_got_data)
 105         return d
 106
 107     def get_helper_info(self):
 108         return (self.helper_furl, self.helper_connected)
 109
 110
 111 def build_one_ds():
 112     ds = DownloadStatus("storage_index", 1234)
 113     now = time.time()
 114
 115     serverA = StubServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
 116     serverB = StubServer(hashutil.tagged_hash("foo", "serverid_b")[:20])
 117     storage_index = hashutil.storage_index_hash("SI")
 118     e0 = ds.add_segment_request(0, now)
 119     e0.activate(now+0.5)
 120     e0.deliver(now+1, 0, 100, 0.5) # when, start,len, decodetime
 121     e1 = ds.add_segment_request(1, now+2)
 122     e1.error(now+3)
 123     # two outstanding requests
 124     e2 = ds.add_segment_request(2, now+4)
 125     e3 = ds.add_segment_request(3, now+5)
 126     del e2,e3 # hush pyflakes
 127
 128     # simulate a segment which gets delivered faster than a system clock tick (ticket #1166)
 129     e = ds.add_segment_request(4, now)
 130     e.activate(now)
 131     e.deliver(now, 0, 140, 0.5)
 132
 133     e = ds.add_dyhb_request(serverA, now)
 134     e.finished([1,2], now+1)
 135     e = ds.add_dyhb_request(serverB, now+2) # left unfinished
 136
 137     e = ds.add_read_event(0, 120, now)
 138     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 139     e.finished(now+1)
 140     e = ds.add_read_event(120, 30, now+2) # left unfinished
 141
 142     e = ds.add_block_request(serverA, 1, 100, 20, now)
 143     e.finished(20, now+1)
 144     e = ds.add_block_request(serverB, 1, 120, 30, now+1) # left unfinished
 145
 146     # make sure that add_read_event() can come first too
 147     ds1 = DownloadStatus(storage_index, 1234)
 148     e = ds1.add_read_event(0, 120, now)
 149     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 150     e.finished(now+1)
 151
 152     return ds
 153
 154 class FakeHistory:
 155     _all_upload_status = [upload.UploadStatus()]
 156     _all_download_status = [build_one_ds()]
 157     _all_mapupdate_statuses = [servermap.UpdateStatus()]
 158     _all_publish_statuses = [publish.PublishStatus()]
 159     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
 160
 161     def list_all_upload_statuses(self):
 162         return self._all_upload_status
 163     def list_all_download_statuses(self):
 164         return self._all_download_status
 165     def list_all_mapupdate_statuses(self):
 166         return self._all_mapupdate_statuses
 167     def list_all_publish_statuses(self):
 168         return self._all_publish_statuses
 169     def list_all_retrieve_statuses(self):
 170         return self._all_retrieve_statuses
 171     def list_all_helper_statuses(self):
 172         return []
 173
 174 class FakeDisplayableServer(StubServer):
 175     def __init__(self, serverid, nickname, connected,
 176                             last_connect_time, last_lost_time, last_rx):
 177         StubServer.__init__(self, serverid)
 178         self.announcement = {"my-version": "allmydata-tahoe-fake",
 179                              "service-name": "storage",
 180                              "nickname": nickname}
 181         self.connected = connected
 182         self.last_lost_time = last_lost_time
 183         self.last_rx = last_rx
 184         self.last_connect_time = last_connect_time
 185     def is_connected(self):
 186         return self.connected
 187     def get_permutation_seed(self):
 188         return ""
 189     def get_remote_host(self):
 190         return ""
 191     def get_last_loss_time(self):
 192         return self.last_lost_time
 193     def get_last_received_data_time(self):
 194         return self.last_rx
 195     def get_last_connect_time(self):
 196         return self.last_connect_time
 197     def get_announcement(self):
 198         return self.announcement
 199     def get_nickname(self):
 200         return self.announcement["nickname"]
 201     def get_available_space(self):
 202         return 123456
 203
 204 class FakeBucketCounter(object):
 205     def get_state(self):
 206         return {"last-complete-bucket-count": 0}
 207     def get_progress(self):
 208         return {"estimated-time-per-cycle": 0,
 209                 "cycle-in-progress": False,
 210                 "remaining-wait-time": 0}
 211
 212 class FakeLeaseChecker(object):
 213     def __init__(self):
 214         self.expiration_enabled = False
 215         self.mode = "age"
 216         self.override_lease_duration = None
 217         self.sharetypes_to_expire = {}
 218     def get_state(self):
 219         return {"history": None}
 220     def get_progress(self):
 221         return {"estimated-time-per-cycle": 0,
 222                 "cycle-in-progress": False,
 223                 "remaining-wait-time": 0}
 224
 225 class FakeStorageServer(service.MultiService):
 226     name = 'storage'
 227     def __init__(self, nodeid, nickname):
 228         service.MultiService.__init__(self)
 229         self.my_nodeid = nodeid
 230         self.nickname = nickname
 231         self.bucket_counter = FakeBucketCounter()
 232         self.lease_checker = FakeLeaseChecker()
 233     def get_stats(self):
 234         return {"storage_server.accepting_immutable_shares": False}
 235
 236 class FakeClient(Client):
 237     def __init__(self):
 238         # don't upcall to Client.__init__, since we only want to initialize a
 239         # minimal subset
 240         service.MultiService.__init__(self)
 241         self.all_contents = {}
 242         self.nodeid = "fake_nodeid"
 243         self.nickname = u"fake_nickname \u263A"
 244         self.introducer_furl = "None"
 245         self.stats_provider = FakeStatsProvider()
 246         self._secret_holder = SecretHolder("lease secret", "convergence secret")
 247         self.helper = None
 248         self.convergence = "some random string"
 249         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 250         # fake knowledge of another server
 251         self.storage_broker.test_add_server("other_nodeid",
 252             FakeDisplayableServer(
 253                 serverid="other_nodeid", nickname=u"other_nickname \u263B", connected = True,
 254                 last_connect_time = 10, last_lost_time = 20, last_rx = 30))
 255         self.storage_broker.test_add_server("disconnected_nodeid",
 256             FakeDisplayableServer(
 257                 serverid="other_nodeid", nickname=u"disconnected_nickname \u263B", connected = False,
 258                 last_connect_time = 15, last_lost_time = 25, last_rx = 35))
 259         self.introducer_client = None
 260         self.history = FakeHistory()
 261         self.uploader = FakeUploader()
 262         self.uploader.all_contents = self.all_contents
 263         self.uploader.setServiceParent(self)
 264         self.blacklist = None
 265         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 266                                        self.uploader, None,
 267                                        None, None, None)
 268         self.nodemaker.all_contents = self.all_contents
 269         self.mutable_file_default = SDMF_VERSION
 270         self.addService(FakeStorageServer(self.nodeid, self.nickname))
 271
 272     def get_long_nodeid(self):
 273         return "v0-nodeid"
 274     def get_long_tubid(self):
 275         return "tubid"
 276
 277     def startService(self):
 278         return service.MultiService.startService(self)
 279     def stopService(self):
 280         return service.MultiService.stopService(self)
 281
 282     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 283
 284 class WebMixin(object):
 285     def setUp(self):
 286         self.s = FakeClient()
 287         self.s.startService()
 288         self.staticdir = self.mktemp()
 289         self.clock = Clock()
 290         self.fakeTime = 86460 # 1d 0h 1m 0s
 291         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
 292                                       clock=self.clock, now=lambda:self.fakeTime)
 293         self.ws.setServiceParent(self.s)
 294         self.webish_port = self.ws.getPortnum()
 295         self.webish_url = self.ws.getURL()
 296         assert self.webish_url.endswith("/")
 297         self.webish_url = self.webish_url[:-1] # these tests add their own /
 298
 299         l = [ self.s.create_dirnode() for x in range(6) ]
 300         d = defer.DeferredList(l)
 301         def _then(res):
 302             self.public_root = res[0][1]
 303             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 304             self.public_url = "/uri/" + self.public_root.get_uri()
 305             self.private_root = res[1][1]
 306
 307             foo = res[2][1]
 308             self._foo_node = foo
 309             self._foo_uri = foo.get_uri()
 310             self._foo_readonly_uri = foo.get_readonly_uri()
 311             self._foo_verifycap = foo.get_verify_cap().to_string()
 312             # NOTE: we ignore the deferred on all set_uri() calls, because we
 313             # know the fake nodes do these synchronously
 314             self.public_root.set_uri(u"foo", foo.get_uri(),
 315                                      foo.get_readonly_uri())
 316
 317             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 318             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 319             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 320
 321             # sdmf
 322             # XXX: Do we ever use this?
 323             self.BAZ_CONTENTS, n, self._baz_txt_uri, self._baz_txt_readonly_uri = self.makefile_mutable(0)
 324
 325             foo.set_uri(u"baz.txt", self._baz_txt_uri, self._baz_txt_readonly_uri)
 326
 327             # mdmf
 328             self.QUUX_CONTENTS, n, self._quux_txt_uri, self._quux_txt_readonly_uri = self.makefile_mutable(0, mdmf=True)
 329             assert self._quux_txt_uri.startswith("URI:MDMF")
 330             foo.set_uri(u"quux.txt", self._quux_txt_uri, self._quux_txt_readonly_uri)
 331
 332             foo.set_uri(u"empty", res[3][1].get_uri(),
 333                         res[3][1].get_readonly_uri())
 334             sub_uri = res[4][1].get_uri()
 335             self._sub_uri = sub_uri
 336             foo.set_uri(u"sub", sub_uri, sub_uri)
 337             sub = self.s.create_node_from_uri(sub_uri)
 338             self._sub_node = sub
 339
 340             _ign, n, blocking_uri = self.makefile(1)
 341             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 342
 343             # filenode to test for html encoding issues
 344             self._htmlname_unicode = u"<&weirdly'named\"file>>>_<iframe />.txt"
 345             self._htmlname_raw = self._htmlname_unicode.encode('utf-8')
 346             self._htmlname_urlencoded = urllib.quote(self._htmlname_raw, '')
 347             self._htmlname_escaped = escapeToXML(self._htmlname_raw)
 348             self._htmlname_escaped_attr = cgi.escape(self._htmlname_raw, quote=True)
 349             self._htmlname_escaped_double = escapeToXML(cgi.escape(self._htmlname_raw, quote=True))
 350             self.HTMLNAME_CONTENTS, n, self._htmlname_txt_uri = self.makefile(0)
 351             foo.set_uri(self._htmlname_unicode, self._htmlname_txt_uri, self._htmlname_txt_uri)
 352
 353             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 354             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 355             # still think of it as an umlaut
 356             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 357
 358             self.SUBBAZ_CONTENTS, n, baz_file = self.makefile(2)
 359             self._baz_file_uri = baz_file
 360             sub.set_uri(u"baz.txt", baz_file, baz_file)
 361
 362             _ign, n, self._bad_file_uri = self.makefile(3)
 363             # this uri should not be downloadable
 364             del self.s.all_contents[self._bad_file_uri]
 365
 366             rodir = res[5][1]
 367             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 368                                      rodir.get_readonly_uri())
 369             rodir.set_uri(u"nor", baz_file, baz_file)
 370
 371             # public/
 372             # public/foo/
 373             # public/foo/bar.txt
 374             # public/foo/baz.txt
 375             # public/foo/quux.txt
 376             # public/foo/blockingfile
 377             # public/foo/<&weirdly'named\"file>>>_<iframe />.txt
 378             # public/foo/empty/
 379             # public/foo/sub/
 380             # public/foo/sub/baz.txt
 381             # public/reedownlee/
 382             # public/reedownlee/nor
 383             self.NEWFILE_CONTENTS = "newfile contents\n"
 384
 385             return foo.get_metadata_for(u"bar.txt")
 386         d.addCallback(_then)
 387         def _got_metadata(metadata):
 388             self._bar_txt_metadata = metadata
 389         d.addCallback(_got_metadata)
 390         return d
 391
 392     def get_all_contents(self):
 393         return self.s.all_contents
 394
 395     def makefile(self, number):
 396         contents = "contents of file %s\n" % number
 397         n = create_chk_filenode(contents, self.get_all_contents())
 398         return contents, n, n.get_uri()
 399
 400     def makefile_mutable(self, number, mdmf=False):
 401         contents = "contents of mutable file %s\n" % number
 402         n = create_mutable_filenode(contents, mdmf, self.s.all_contents)
 403         return contents, n, n.get_uri(), n.get_readonly_uri()
 404
 405     def tearDown(self):
 406         return self.s.stopService()
 407
 408     def failUnlessIsBarDotTxt(self, res):
 409         self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
 410
 411     def failUnlessIsQuuxDotTxt(self, res):
 412         self.failUnlessReallyEqual(res, self.QUUX_CONTENTS, res)
 413
 414     def failUnlessIsBazDotTxt(self, res):
 415         self.failUnlessReallyEqual(res, self.BAZ_CONTENTS, res)
 416
 417     def failUnlessIsSubBazDotTxt(self, res):
 418         self.failUnlessReallyEqual(res, self.SUBBAZ_CONTENTS, res)
 419
 420     def failUnlessIsBarJSON(self, res):
 421         data = simplejson.loads(res)
 422         self.failUnless(isinstance(data, list))
 423         self.failUnlessEqual(data[0], "filenode")
 424         self.failUnless(isinstance(data[1], dict))
 425         self.failIf(data[1]["mutable"])
 426         self.failIfIn("rw_uri", data[1]) # immutable
 427         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
 428         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
 429         self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
 430
 431     def failUnlessIsQuuxJSON(self, res, readonly=False):
 432         data = simplejson.loads(res)
 433         self.failUnless(isinstance(data, list))
 434         self.failUnlessEqual(data[0], "filenode")
 435         self.failUnless(isinstance(data[1], dict))
 436         metadata = data[1]
 437         return self.failUnlessIsQuuxDotTxtMetadata(metadata, readonly)
 438
 439     def failUnlessIsQuuxDotTxtMetadata(self, metadata, readonly):
 440         self.failUnless(metadata['mutable'])
 441         if readonly:
 442             self.failIfIn("rw_uri", metadata)
 443         else:
 444             self.failUnlessIn("rw_uri", metadata)
 445             self.failUnlessEqual(metadata['rw_uri'], self._quux_txt_uri)
 446         self.failUnlessIn("ro_uri", metadata)
 447         self.failUnlessEqual(metadata['ro_uri'], self._quux_txt_readonly_uri)
 448         self.failUnlessReallyEqual(metadata['size'], len(self.QUUX_CONTENTS))
 449
 450     def failUnlessIsFooJSON(self, res):
 451         data = simplejson.loads(res)
 452         self.failUnless(isinstance(data, list))
 453         self.failUnlessEqual(data[0], "dirnode", res)
 454         self.failUnless(isinstance(data[1], dict))
 455         self.failUnless(data[1]["mutable"])
 456         self.failUnlessIn("rw_uri", data[1]) # mutable
 457         self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
 458         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
 459         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
 460
 461         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 462         self.failUnlessEqual(kidnames,
 463                              [self._htmlname_unicode, u"bar.txt", u"baz.txt",
 464                               u"blockingfile", u"empty", u"n\u00fc.txt", u"quux.txt", u"sub"])
 465         kids = dict( [(unicode(name),value)
 466                       for (name,value)
 467                       in data[1]["children"].iteritems()] )
 468         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 469         self.failUnlessIn("metadata", kids[u"sub"][1])
 470         self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
 471         tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
 472         self.failUnlessIn("linkcrtime", tahoe_md)
 473         self.failUnlessIn("linkmotime", tahoe_md)
 474         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 475         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 476         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri)
 477         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]),
 478                                    self._bar_txt_verifycap)
 479         self.failUnlessIn("metadata", kids[u"bar.txt"][1])
 480         self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
 481         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
 482                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
 483         self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
 484                                    self._bar_txt_uri)
 485         self.failUnlessIn("quux.txt", kids)
 486         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["rw_uri"]),
 487                                    self._quux_txt_uri)
 488         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["ro_uri"]),
 489                                    self._quux_txt_readonly_uri)
 490
 491     def GET(self, urlpath, followRedirect=False, return_response=False,
 492             **kwargs):
 493         # if return_response=True, this fires with (data, statuscode,
 494         # respheaders) instead of just data.
 495         assert not isinstance(urlpath, unicode)
 496         url = self.webish_url + urlpath
 497         factory = HTTPClientGETFactory(url, method="GET",
 498                                        followRedirect=followRedirect, **kwargs)
 499         reactor.connectTCP("localhost", self.webish_port, factory)
 500         d = factory.deferred
 501         def _got_data(data):
 502             return (data, factory.status, factory.response_headers)
 503         if return_response:
 504             d.addCallback(_got_data)
 505         return factory.deferred
 506
 507     def HEAD(self, urlpath, return_response=False, **kwargs):
 508         # this requires some surgery, because twisted.web.client doesn't want
 509         # to give us back the response headers.
 510         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 511         reactor.connectTCP("localhost", self.webish_port, factory)
 512         d = factory.deferred
 513         def _got_data(data):
 514             return (data, factory.status, factory.response_headers)
 515         if return_response:
 516             d.addCallback(_got_data)
 517         return factory.deferred
 518
 519     def PUT(self, urlpath, data, **kwargs):
 520         url = self.webish_url + urlpath
 521         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 522
 523     def DELETE(self, urlpath):
 524         url = self.webish_url + urlpath
 525         return client.getPage(url, method="DELETE")
 526
 527     def POST(self, urlpath, followRedirect=False, **fields):
 528         sepbase = "boogabooga"
 529         sep = "--" + sepbase
 530         form = []
 531         form.append(sep)
 532         form.append('Content-Disposition: form-data; name="_charset"')
 533         form.append('')
 534         form.append('UTF-8')
 535         form.append(sep)
 536         for name, value in fields.iteritems():
 537             if isinstance(value, tuple):
 538                 filename, value = value
 539                 form.append('Content-Disposition: form-data; name="%s"; '
 540                             'filename="%s"' % (name, filename.encode("utf-8")))
 541             else:
 542                 form.append('Content-Disposition: form-data; name="%s"' % name)
 543             form.append('')
 544             if isinstance(value, unicode):
 545                 value = value.encode("utf-8")
 546             else:
 547                 value = str(value)
 548             assert isinstance(value, str)
 549             form.append(value)
 550             form.append(sep)
 551         form[-1] += "--"
 552         body = ""
 553         headers = {}
 554         if fields:
 555             body = "\r\n".join(form) + "\r\n"
 556             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 557         return self.POST2(urlpath, body, headers, followRedirect)
 558
 559     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 560         url = self.webish_url + urlpath
 561         return client.getPage(url, method="POST", postdata=body,
 562                               headers=headers, followRedirect=followRedirect)
 563
 564     def shouldFail(self, res, expected_failure, which,
 565                    substring=None, response_substring=None):
 566         if isinstance(res, failure.Failure):
 567             res.trap(expected_failure)
 568             if substring:
 569                 self.failUnlessIn(substring, str(res), which)
 570             if response_substring:
 571                 self.failUnlessIn(response_substring, res.value.response, which)
 572         else:
 573             self.fail("%s was supposed to raise %s, not get '%s'" %
 574                       (which, expected_failure, res))
 575
 576     def shouldFail2(self, expected_failure, which, substring,
 577                     response_substring,
 578                     callable, *args, **kwargs):
 579         assert substring is None or isinstance(substring, str)
 580         assert response_substring is None or isinstance(response_substring, str)
 581         d = defer.maybeDeferred(callable, *args, **kwargs)
 582         def done(res):
 583             if isinstance(res, failure.Failure):
 584                 res.trap(expected_failure)
 585                 if substring:
 586                     self.failUnlessIn(substring, str(res),
 587                                       "'%s' not in '%s' (response is '%s') for test '%s'" % \
 588                                       (substring, str(res),
 589                                        getattr(res.value, "response", ""),
 590                                        which))
 591                 if response_substring:
 592                     self.failUnlessIn(response_substring, res.value.response,
 593                                       "'%s' not in '%s' for test '%s'" % \
 594                                       (response_substring, res.value.response,
 595                                        which))
 596             else:
 597                 self.fail("%s was supposed to raise %s, not get '%s'" %
 598                           (which, expected_failure, res))
 599         d.addBoth(done)
 600         return d
 601
 602     def should404(self, res, which):
 603         if isinstance(res, failure.Failure):
 604             res.trap(error.Error)
 605             self.failUnlessReallyEqual(res.value.status, "404")
 606         else:
 607             self.fail("%s was supposed to Error(404), not get '%s'" %
 608                       (which, res))
 609
 610     def should302(self, res, which):
 611         if isinstance(res, failure.Failure):
 612             res.trap(error.Error)
 613             self.failUnlessReallyEqual(res.value.status, "302")
 614         else:
 615             self.fail("%s was supposed to Error(302), not get '%s'" %
 616                       (which, res))
 617
 618
 619 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
 620
 621     def test_create(self):
 622         pass
 623
 624     def test_welcome(self):
 625         d = self.GET("/")
 626         def _check(res):
 627             self.failUnlessIn('<title>Tahoe-LAFS - Welcome</title>', res)
 628             self.failUnlessIn(FAVICON_MARKUP, res)
 629             self.failUnlessIn('<a href="status">Recent and Active Operations</a>', res)
 630             self.failUnlessIn('<a href="statistics">Operational Statistics</a>', res)
 631             self.failUnlessIn('<input type="hidden" name="t" value="report-incident" />', res)
 632             self.failUnlessIn('Page rendered at', res)
 633             self.failUnlessIn('Tahoe-LAFS code imported from:', res)
 634             res_u = res.decode('utf-8')
 635             self.failUnlessIn(u'<td>fake_nickname \u263A</td>', res_u)
 636             self.failUnlessIn(u'<div class="nickname">other_nickname \u263B</div>', res_u)
 637             self.failUnlessIn('Connected to <span>1</span>\n              of <span>2</span> known storage servers', res_u)
 638             self.failUnlessIn('<div class="status-indicator service-Connected"></div>\n<div class="status-description">Connected<br /><a class="timestamp" title="00:00:10 01-Jan-1970">1d0h0m50s</a></div></td>', res_u)
 639             self.failUnlessIn('<div class="status-indicator service-Disconnected"></div>\n<div class="status-description">Disconnected<br /><a class="timestamp" title="00:00:25 01-Jan-1970">1d0h0m35s</a></div></td>', res_u)
 640             self.failUnlessIn('<td class="service-last-received-data"><a class="timestamp" title="00:00:30 01-Jan-1970">1d0h0m30s</a></td>', res)
 641             self.failUnlessIn('<td class="service-last-received-data"><a class="timestamp" title="00:00:35 01-Jan-1970">1d0h0m25s</a></td>', res)
 642             self.failUnlessIn(u'\u00A9 <a href="https://tahoe-lafs.org/">Tahoe-LAFS Software Foundation', res_u)
 643             self.failUnlessIn('<td><h3>Available</h3></td>', res)
 644             self.failUnlessIn('123.5kB', res)
 645
 646             self.s.basedir = 'web/test_welcome'
 647             fileutil.make_dirs("web/test_welcome")
 648             fileutil.make_dirs("web/test_welcome/private")
 649             return self.GET("/")
 650         d.addCallback(_check)
 651         return d
 652
 653     def test_introducer_status(self):
 654         class MockIntroducerClient(object):
 655             def __init__(self, connected):
 656                 self.connected = connected
 657             def connected_to_introducer(self):
 658                 return self.connected
 659
 660         d = defer.succeed(None)
 661
 662         # introducer not connected, unguessable furl
 663         def _set_introducer_not_connected_unguessable(ign):
 664             self.s.introducer_furl = "pb://someIntroducer/secret"
 665             self.s.introducer_client = MockIntroducerClient(False)
 666             return self.GET("/")
 667         d.addCallback(_set_introducer_not_connected_unguessable)
 668         def _check_introducer_not_connected_unguessable(res):
 669             html = res.replace('\n', ' ')
 670             self.failUnlessIn('<div class="furl">pb://someIntroducer/[censored]</div>', html)
 671             self.failIfIn('pb://someIntroducer/secret', html)
 672             self.failUnless(re.search('<div class="status-indicator connected-no"></div>[ ]*<div>Introducer not connected</div>', html), res)
 673         d.addCallback(_check_introducer_not_connected_unguessable)
 674
 675         # introducer connected, unguessable furl
 676         def _set_introducer_connected_unguessable(ign):
 677             self.s.introducer_furl = "pb://someIntroducer/secret"
 678             self.s.introducer_client = MockIntroducerClient(True)
 679             return self.GET("/")
 680         d.addCallback(_set_introducer_connected_unguessable)
 681         def _check_introducer_connected_unguessable(res):
 682             html = res.replace('\n', ' ')
 683             self.failUnlessIn('<div class="furl">pb://someIntroducer/[censored]</div>', html)
 684             self.failIfIn('pb://someIntroducer/secret', html)
 685             self.failUnless(re.search('<div class="status-indicator connected-yes"></div>[ ]*<div>Introducer</div>', html), res)
 686         d.addCallback(_check_introducer_connected_unguessable)
 687
 688         # introducer connected, guessable furl
 689         def _set_introducer_connected_guessable(ign):
 690             self.s.introducer_furl = "pb://someIntroducer/introducer"
 691             self.s.introducer_client = MockIntroducerClient(True)
 692             return self.GET("/")
 693         d.addCallback(_set_introducer_connected_guessable)
 694         def _check_introducer_connected_guessable(res):
 695             html = res.replace('\n', ' ')
 696             self.failUnlessIn('<div class="furl">pb://someIntroducer/introducer</div>', html)
 697             self.failUnless(re.search('<div class="status-indicator connected-yes"></div>[ ]*<div>Introducer</div>', html), res)
 698         d.addCallback(_check_introducer_connected_guessable)
 699         return d
 700
 701     def test_helper_status(self):
 702         d = defer.succeed(None)
 703
 704         # set helper furl to None
 705         def _set_no_helper(ign):
 706             self.s.uploader.helper_furl = None
 707             return self.GET("/")
 708         d.addCallback(_set_no_helper)
 709         def _check_no_helper(res):
 710             html = res.replace('\n', ' ')
 711             self.failUnless(re.search('<div class="status-indicator connected-not-configured"></div>[ ]*<div>Helper</div>', html), res)
 712         d.addCallback(_check_no_helper)
 713
 714         # enable helper, not connected
 715         def _set_helper_not_connected(ign):
 716             self.s.uploader.helper_furl = "pb://someHelper/secret"
 717             self.s.uploader.helper_connected = False
 718             return self.GET("/")
 719         d.addCallback(_set_helper_not_connected)
 720         def _check_helper_not_connected(res):
 721             html = res.replace('\n', ' ')
 722             self.failUnlessIn('<div class="furl">pb://someHelper/[censored]</div>', html)
 723             self.failIfIn('pb://someHelper/secret', html)
 724             self.failUnless(re.search('<div class="status-indicator connected-no"></div>[ ]*<div>Helper not connected</div>', html), res)
 725         d.addCallback(_check_helper_not_connected)
 726
 727         # enable helper, connected
 728         def _set_helper_connected(ign):
 729             self.s.uploader.helper_furl = "pb://someHelper/secret"
 730             self.s.uploader.helper_connected = True
 731             return self.GET("/")
 732         d.addCallback(_set_helper_connected)
 733         def _check_helper_connected(res):
 734             html = res.replace('\n', ' ')
 735             self.failUnlessIn('<div class="furl">pb://someHelper/[censored]</div>', html)
 736             self.failIfIn('pb://someHelper/secret', html)
 737             self.failUnless(re.search('<div class="status-indicator connected-yes"></div>[ ]*<div>Helper</div>', html), res)
 738         d.addCallback(_check_helper_connected)
 739         return d
 740
 741     def test_storage(self):
 742         d = self.GET("/storage")
 743         def _check(res):
 744             self.failUnlessIn('Storage Server Status', res)
 745             self.failUnlessIn(FAVICON_MARKUP, res)
 746             res_u = res.decode('utf-8')
 747             self.failUnlessIn(u'<li>Server Nickname: <span class="nickname mine">fake_nickname \u263A</span></li>', res_u)
 748         d.addCallback(_check)
 749         return d
 750
 751     def test_status(self):
 752         h = self.s.get_history()
 753         dl_num = h.list_all_download_statuses()[0].get_counter()
 754         ul_num = h.list_all_upload_statuses()[0].get_counter()
 755         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 756         pub_num = h.list_all_publish_statuses()[0].get_counter()
 757         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 758         d = self.GET("/status", followRedirect=True)
 759         def _check(res):
 760             self.failUnlessIn('Recent and Active Operations', res)
 761             self.failUnlessIn('"down-%d"' % dl_num, res)
 762             self.failUnlessIn('"up-%d"' % ul_num, res)
 763             self.failUnlessIn('"mapupdate-%d"' % mu_num, res)
 764             self.failUnlessIn('"publish-%d"' % pub_num, res)
 765             self.failUnlessIn('"retrieve-%d"' % ret_num, res)
 766         d.addCallback(_check)
 767         d.addCallback(lambda res: self.GET("/status/?t=json"))
 768         def _check_json(res):
 769             data = simplejson.loads(res)
 770             self.failUnless(isinstance(data, dict))
 771             #active = data["active"]
 772             # TODO: test more. We need a way to fake an active operation
 773             # here.
 774         d.addCallback(_check_json)
 775
 776         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 777         def _check_dl(res):
 778             self.failUnlessIn("File Download Status", res)
 779         d.addCallback(_check_dl)
 780         d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
 781         def _check_dl_json(res):
 782             data = simplejson.loads(res)
 783             self.failUnless(isinstance(data, dict))
 784             self.failUnlessIn("read", data)
 785             self.failUnlessEqual(data["read"][0]["length"], 120)
 786             self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
 787             self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
 788             self.failUnlessEqual(data["segment"][2]["finish_time"], None)
 789             phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
 790             cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
 791             # serverids[] keys are strings, since that's what JSON does, but
 792             # we'd really like them to be ints
 793             self.failUnlessEqual(data["serverids"]["0"], "phwrsjte")
 794             self.failUnless(data["serverids"].has_key("1"),
 795                             str(data["serverids"]))
 796             self.failUnlessEqual(data["serverids"]["1"], "cmpuvkjm",
 797                                  str(data["serverids"]))
 798             self.failUnlessEqual(data["server_info"][phwr_id]["short"],
 799                                  "phwrsjte")
 800             self.failUnlessEqual(data["server_info"][cmpu_id]["short"],
 801                                  "cmpuvkjm")
 802             self.failUnlessIn("dyhb", data)
 803             self.failUnlessIn("misc", data)
 804         d.addCallback(_check_dl_json)
 805         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 806         def _check_ul(res):
 807             self.failUnlessIn("File Upload Status", res)
 808         d.addCallback(_check_ul)
 809         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 810         def _check_mapupdate(res):
 811             self.failUnlessIn("Mutable File Servermap Update Status", res)
 812         d.addCallback(_check_mapupdate)
 813         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 814         def _check_publish(res):
 815             self.failUnlessIn("Mutable File Publish Status", res)
 816         d.addCallback(_check_publish)
 817         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 818         def _check_retrieve(res):
 819             self.failUnlessIn("Mutable File Retrieve Status", res)
 820         d.addCallback(_check_retrieve)
 821
 822         return d
 823
 824     def test_status_numbers(self):
 825         drrm = status.DownloadResultsRendererMixin()
 826         self.failUnlessReallyEqual(drrm.render_time(None, None), "")
 827         self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
 828         self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
 829         self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
 830         self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
 831         self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
 832         self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 833         self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
 834         self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
 835
 836         urrm = status.UploadResultsRendererMixin()
 837         self.failUnlessReallyEqual(urrm.render_time(None, None), "")
 838         self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
 839         self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
 840         self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
 841         self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
 842         self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
 843         self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 844         self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
 845         self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
 846
 847     def test_GET_FILEURL(self):
 848         d = self.GET(self.public_url + "/foo/bar.txt")
 849         d.addCallback(self.failUnlessIsBarDotTxt)
 850         return d
 851
 852     def test_GET_FILEURL_range(self):
 853         headers = {"range": "bytes=1-10"}
 854         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 855                      return_response=True)
 856         def _got((res, status, headers)):
 857             self.failUnlessReallyEqual(int(status), 206)
 858             self.failUnless(headers.has_key("content-range"))
 859             self.failUnlessReallyEqual(headers["content-range"][0],
 860                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 861             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
 862         d.addCallback(_got)
 863         return d
 864
 865     def test_GET_FILEURL_partial_range(self):
 866         headers = {"range": "bytes=5-"}
 867         length  = len(self.BAR_CONTENTS)
 868         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 869                      return_response=True)
 870         def _got((res, status, headers)):
 871             self.failUnlessReallyEqual(int(status), 206)
 872             self.failUnless(headers.has_key("content-range"))
 873             self.failUnlessReallyEqual(headers["content-range"][0],
 874                                        "bytes 5-%d/%d" % (length-1, length))
 875             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
 876         d.addCallback(_got)
 877         return d
 878
 879     def test_GET_FILEURL_partial_end_range(self):
 880         headers = {"range": "bytes=-5"}
 881         length  = len(self.BAR_CONTENTS)
 882         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 883                      return_response=True)
 884         def _got((res, status, headers)):
 885             self.failUnlessReallyEqual(int(status), 206)
 886             self.failUnless(headers.has_key("content-range"))
 887             self.failUnlessReallyEqual(headers["content-range"][0],
 888                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 889             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
 890         d.addCallback(_got)
 891         return d
 892
 893     def test_GET_FILEURL_partial_range_overrun(self):
 894         headers = {"range": "bytes=100-200"}
 895         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
 896                              "416 Requested Range not satisfiable",
 897                              "First beyond end of file",
 898                              self.GET, self.public_url + "/foo/bar.txt",
 899                              headers=headers)
 900         return d
 901
 902     def test_HEAD_FILEURL_range(self):
 903         headers = {"range": "bytes=1-10"}
 904         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 905                      return_response=True)
 906         def _got((res, status, headers)):
 907             self.failUnlessReallyEqual(res, "")
 908             self.failUnlessReallyEqual(int(status), 206)
 909             self.failUnless(headers.has_key("content-range"))
 910             self.failUnlessReallyEqual(headers["content-range"][0],
 911                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 912         d.addCallback(_got)
 913         return d
 914
 915     def test_HEAD_FILEURL_partial_range(self):
 916         headers = {"range": "bytes=5-"}
 917         length  = len(self.BAR_CONTENTS)
 918         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 919                      return_response=True)
 920         def _got((res, status, headers)):
 921             self.failUnlessReallyEqual(int(status), 206)
 922             self.failUnless(headers.has_key("content-range"))
 923             self.failUnlessReallyEqual(headers["content-range"][0],
 924                                        "bytes 5-%d/%d" % (length-1, length))
 925         d.addCallback(_got)
 926         return d
 927
 928     def test_HEAD_FILEURL_partial_end_range(self):
 929         headers = {"range": "bytes=-5"}
 930         length  = len(self.BAR_CONTENTS)
 931         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 932                      return_response=True)
 933         def _got((res, status, headers)):
 934             self.failUnlessReallyEqual(int(status), 206)
 935             self.failUnless(headers.has_key("content-range"))
 936             self.failUnlessReallyEqual(headers["content-range"][0],
 937                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 938         d.addCallback(_got)
 939         return d
 940
 941     def test_HEAD_FILEURL_partial_range_overrun(self):
 942         headers = {"range": "bytes=100-200"}
 943         d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
 944                              "416 Requested Range not satisfiable",
 945                              "",
 946                              self.HEAD, self.public_url + "/foo/bar.txt",
 947                              headers=headers)
 948         return d
 949
 950     def test_GET_FILEURL_range_bad(self):
 951         headers = {"range": "BOGUS=fizbop-quarnak"}
 952         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 953                      return_response=True)
 954         def _got((res, status, headers)):
 955             self.failUnlessReallyEqual(int(status), 200)
 956             self.failUnless(not headers.has_key("content-range"))
 957             self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
 958         d.addCallback(_got)
 959         return d
 960
 961     def test_HEAD_FILEURL(self):
 962         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 963         def _got((res, status, headers)):
 964             self.failUnlessReallyEqual(res, "")
 965             self.failUnlessReallyEqual(headers["content-length"][0],
 966                                        str(len(self.BAR_CONTENTS)))
 967             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
 968         d.addCallback(_got)
 969         return d
 970
 971     def test_GET_FILEURL_named(self):
 972         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 973         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 974         d = self.GET(base + "/@@name=/blah.txt")
 975         d.addCallback(self.failUnlessIsBarDotTxt)
 976         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 977         d.addCallback(self.failUnlessIsBarDotTxt)
 978         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 979         d.addCallback(self.failUnlessIsBarDotTxt)
 980         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 981         d.addCallback(self.failUnlessIsBarDotTxt)
 982         save_url = base + "?save=true&filename=blah.txt"
 983         d.addCallback(lambda res: self.GET(save_url))
 984         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 985         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 986         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 987         u_url = base + "?save=true&filename=" + u_fn_e
 988         d.addCallback(lambda res: self.GET(u_url))
 989         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 990         return d
 991
 992     def test_PUT_FILEURL_named_bad(self):
 993         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 994         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 995                              "400 Bad Request",
 996                              "/file can only be used with GET or HEAD",
 997                              self.PUT, base + "/@@name=/blah.txt", "")
 998         return d
 999
1000
1001     def test_GET_DIRURL_named_bad(self):
1002         base = "/file/%s" % urllib.quote(self._foo_uri)
1003         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
1004                              "400 Bad Request",
1005                              "is not a file-cap",
1006                              self.GET, base + "/@@name=/blah.txt")
1007         return d
1008
1009     def test_GET_slash_file_bad(self):
1010         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
1011                              "404 Not Found",
1012                              "/file must be followed by a file-cap and a name",
1013                              self.GET, "/file")
1014         return d
1015
1016     def test_GET_unhandled_URI_named(self):
1017         contents, n, newuri = self.makefile(12)
1018         verifier_cap = n.get_verify_cap().to_string()
1019         base = "/file/%s" % urllib.quote(verifier_cap)
1020         # client.create_node_from_uri() can't handle verify-caps
1021         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
1022                              "400 Bad Request", "is not a file-cap",
1023                              self.GET, base)
1024         return d
1025
1026     def test_GET_unhandled_URI(self):
1027         contents, n, newuri = self.makefile(12)
1028         verifier_cap = n.get_verify_cap().to_string()
1029         base = "/uri/%s" % urllib.quote(verifier_cap)
1030         # client.create_node_from_uri() can't handle verify-caps
1031         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
1032                              "400 Bad Request",
1033                              "GET unknown URI type: can only do t=info",
1034                              self.GET, base)
1035         return d
1036
1037     def test_GET_FILE_URI(self):
1038         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
1039         d = self.GET(base)
1040         d.addCallback(self.failUnlessIsBarDotTxt)
1041         return d
1042
1043     def test_GET_FILE_URI_mdmf(self):
1044         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
1045         d = self.GET(base)
1046         d.addCallback(self.failUnlessIsQuuxDotTxt)
1047         return d
1048
1049     def test_GET_FILE_URI_mdmf_extensions(self):
1050         base = "/uri/%s" % urllib.quote("%s:RANDOMSTUFF" % self._quux_txt_uri)
1051         d = self.GET(base)
1052         d.addCallback(self.failUnlessIsQuuxDotTxt)
1053         return d
1054
1055     def test_GET_FILE_URI_mdmf_readonly(self):
1056         base = "/uri/%s" % urllib.quote(self._quux_txt_readonly_uri)
1057         d = self.GET(base)
1058         d.addCallback(self.failUnlessIsQuuxDotTxt)
1059         return d
1060
1061     def test_GET_FILE_URI_badchild(self):
1062         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
1063         errmsg = "Files have no children, certainly not named 'boguschild'"
1064         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
1065                              "400 Bad Request", errmsg,
1066                              self.GET, base)
1067         return d
1068
1069     def test_PUT_FILE_URI_badchild(self):
1070         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
1071         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
1072         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
1073                              "400 Bad Request", errmsg,
1074                              self.PUT, base, "")
1075         return d
1076
1077     def test_PUT_FILE_URI_mdmf(self):
1078         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
1079         self._quux_new_contents = "new_contents"
1080         d = self.GET(base)
1081         d.addCallback(lambda res:
1082             self.failUnlessIsQuuxDotTxt(res))
1083         d.addCallback(lambda ignored:
1084             self.PUT(base, self._quux_new_contents))
1085         d.addCallback(lambda ignored:
1086             self.GET(base))
1087         d.addCallback(lambda res:
1088             self.failUnlessReallyEqual(res, self._quux_new_contents))
1089         return d
1090
1091     def test_PUT_FILE_URI_mdmf_extensions(self):
1092         base = "/uri/%s" % urllib.quote("%s:EXTENSIONSTUFF" % self._quux_txt_uri)
1093         self._quux_new_contents = "new_contents"
1094         d = self.GET(base)
1095         d.addCallback(lambda res: self.failUnlessIsQuuxDotTxt(res))
1096         d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
1097         d.addCallback(lambda ignored: self.GET(base))
1098         d.addCallback(lambda res: self.failUnlessEqual(self._quux_new_contents,
1099                                                        res))
1100         return d
1101
1102     def test_PUT_FILE_URI_mdmf_readonly(self):
1103         # We're not allowed to PUT things to a readonly cap.
1104         base = "/uri/%s" % self._quux_txt_readonly_uri
1105         d = self.GET(base)
1106         d.addCallback(lambda res:
1107             self.failUnlessIsQuuxDotTxt(res))
1108         # What should we get here? We get a 500 error now; that's not right.
1109         d.addCallback(lambda ignored:
1110             self.shouldFail2(error.Error, "test_PUT_FILE_URI_mdmf_readonly",
1111                              "400 Bad Request", "read-only cap",
1112                              self.PUT, base, "new data"))
1113         return d
1114
1115     def test_PUT_FILE_URI_sdmf_readonly(self):
1116         # We're not allowed to put things to a readonly cap.
1117         base = "/uri/%s" % self._baz_txt_readonly_uri
1118         d = self.GET(base)
1119         d.addCallback(lambda res:
1120             self.failUnlessIsBazDotTxt(res))
1121         d.addCallback(lambda ignored:
1122             self.shouldFail2(error.Error, "test_PUT_FILE_URI_sdmf_readonly",
1123                              "400 Bad Request", "read-only cap",
1124                              self.PUT, base, "new_data"))
1125         return d
1126
1127     def test_GET_etags(self):
1128
1129         def _check_etags(uri):
1130             d1 = _get_etag(uri)
1131             d2 = _get_etag(uri, 'json')
1132             d = defer.DeferredList([d1, d2], consumeErrors=True)
1133             def _check(results):
1134                 # All deferred must succeed
1135                 self.failUnless(all([r[0] for r in results]))
1136                 # the etag for the t=json form should be just like the etag
1137                 # fo the default t='' form, but with a 'json' suffix
1138                 self.failUnlessEqual(results[0][1] + 'json', results[1][1])
1139             d.addCallback(_check)
1140             return d
1141
1142         def _get_etag(uri, t=''):
1143             targetbase = "/uri/%s?t=%s" % (urllib.quote(uri.strip()), t)
1144             d = self.GET(targetbase, return_response=True, followRedirect=True)
1145             def _just_the_etag(result):
1146                 data, response, headers = result
1147                 etag = headers['etag'][0]
1148                 if uri.startswith('URI:DIR'):
1149                     self.failUnless(etag.startswith('DIR:'), etag)
1150                 return etag
1151             return d.addCallback(_just_the_etag)
1152
1153         # Check that etags work with immutable directories
1154         (newkids, caps) = self._create_immutable_children()
1155         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1156                       simplejson.dumps(newkids))
1157         def _stash_immdir_uri(uri):
1158             self._immdir_uri = uri
1159             return uri
1160         d.addCallback(_stash_immdir_uri)
1161         d.addCallback(_check_etags)
1162
1163         # Check that etags work with immutable files
1164         d.addCallback(lambda _: _check_etags(self._bar_txt_uri))
1165
1166         # use the ETag on GET
1167         def _check_match(ign):
1168             uri = "/uri/%s" % self._bar_txt_uri
1169             d = self.GET(uri, return_response=True)
1170             # extract the ETag
1171             d.addCallback(lambda (data, code, headers):
1172                           headers['etag'][0])
1173             # do a GET that's supposed to match the ETag
1174             d.addCallback(lambda etag:
1175                           self.GET(uri, return_response=True,
1176                                    headers={"If-None-Match": etag}))
1177             # make sure it short-circuited (304 instead of 200)
1178             d.addCallback(lambda (data, code, headers):
1179                           self.failUnlessEqual(int(code), http.NOT_MODIFIED))
1180             return d
1181         d.addCallback(_check_match)
1182
1183         def _no_etag(uri, t):
1184             target = "/uri/%s?t=%s" % (uri, t)
1185             d = self.GET(target, return_response=True, followRedirect=True)
1186             d.addCallback(lambda (data, code, headers):
1187                           self.failIf("etag" in headers, target))
1188             return d
1189         def _yes_etag(uri, t):
1190             target = "/uri/%s?t=%s" % (uri, t)
1191             d = self.GET(target, return_response=True, followRedirect=True)
1192             d.addCallback(lambda (data, code, headers):
1193                           self.failUnless("etag" in headers, target))
1194             return d
1195
1196         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, ""))
1197         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "json"))
1198         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "uri"))
1199         d.addCallback(lambda ign: _yes_etag(self._bar_txt_uri, "readonly-uri"))
1200         d.addCallback(lambda ign: _no_etag(self._bar_txt_uri, "info"))
1201
1202         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, ""))
1203         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "json"))
1204         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "uri"))
1205         d.addCallback(lambda ign: _yes_etag(self._immdir_uri, "readonly-uri"))
1206         d.addCallback(lambda ign: _no_etag(self._immdir_uri, "info"))
1207         d.addCallback(lambda ign: _no_etag(self._immdir_uri, "rename-form"))
1208
1209         return d
1210
1211     # TODO: version of this with a Unicode filename
1212     def test_GET_FILEURL_save(self):
1213         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
1214                      return_response=True)
1215         def _got((res, statuscode, headers)):
1216             content_disposition = headers["content-disposition"][0]
1217             self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
1218             self.failUnlessIsBarDotTxt(res)
1219         d.addCallback(_got)
1220         return d
1221
1222     def test_GET_FILEURL_missing(self):
1223         d = self.GET(self.public_url + "/foo/missing")
1224         d.addBoth(self.should404, "test_GET_FILEURL_missing")
1225         return d
1226
1227     def test_GET_FILEURL_info_mdmf(self):
1228         d = self.GET("/uri/%s?t=info" % self._quux_txt_uri)
1229         def _got(res):
1230             self.failUnlessIn("mutable file (mdmf)", res)
1231             self.failUnlessIn(self._quux_txt_uri, res)
1232             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1233         d.addCallback(_got)
1234         return d
1235
1236     def test_GET_FILEURL_info_mdmf_readonly(self):
1237         d = self.GET("/uri/%s?t=info" % self._quux_txt_readonly_uri)
1238         def _got(res):
1239             self.failUnlessIn("mutable file (mdmf)", res)
1240             self.failIfIn(self._quux_txt_uri, res)
1241             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1242         d.addCallback(_got)
1243         return d
1244
1245     def test_GET_FILEURL_info_sdmf(self):
1246         d = self.GET("/uri/%s?t=info" % self._baz_txt_uri)
1247         def _got(res):
1248             self.failUnlessIn("mutable file (sdmf)", res)
1249             self.failUnlessIn(self._baz_txt_uri, res)
1250         d.addCallback(_got)
1251         return d
1252
1253     def test_GET_FILEURL_info_mdmf_extensions(self):
1254         d = self.GET("/uri/%s:STUFF?t=info" % self._quux_txt_uri)
1255         def _got(res):
1256             self.failUnlessIn("mutable file (mdmf)", res)
1257             self.failUnlessIn(self._quux_txt_uri, res)
1258             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1259         d.addCallback(_got)
1260         return d
1261
1262     def test_PUT_overwrite_only_files(self):
1263         # create a directory, put a file in that directory.
1264         contents, n, filecap = self.makefile(8)
1265         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
1266         d.addCallback(lambda res:
1267             self.PUT(self.public_url + "/foo/dir/file1.txt",
1268                      self.NEWFILE_CONTENTS))
1269         # try to overwrite the file with replace=only-files
1270         # (this should work)
1271         d.addCallback(lambda res:
1272             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
1273                      filecap))
1274         d.addCallback(lambda res:
1275             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
1276                  "There was already a child by that name, and you asked me "
1277                  "to not replace it",
1278                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
1279                  filecap))
1280         return d
1281
1282     def test_PUT_NEWFILEURL(self):
1283         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
1284         # TODO: we lose the response code, so we can't check this
1285         #self.failUnlessReallyEqual(responsecode, 201)
1286         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1287         d.addCallback(lambda res:
1288                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1289                                                       self.NEWFILE_CONTENTS))
1290         return d
1291
1292     def test_PUT_NEWFILEURL_not_mutable(self):
1293         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
1294                      self.NEWFILE_CONTENTS)
1295         # TODO: we lose the response code, so we can't check this
1296         #self.failUnlessReallyEqual(responsecode, 201)
1297         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1298         d.addCallback(lambda res:
1299                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1300                                                       self.NEWFILE_CONTENTS))
1301         return d
1302
1303     def test_PUT_NEWFILEURL_unlinked_mdmf(self):
1304         # this should get us a few segments of an MDMF mutable file,
1305         # which we can then test for.
1306         contents = self.NEWFILE_CONTENTS * 300000
1307         d = self.PUT("/uri?format=mdmf",
1308                      contents)
1309         def _got_filecap(filecap):
1310             self.failUnless(filecap.startswith("URI:MDMF"))
1311             return filecap
1312         d.addCallback(_got_filecap)
1313         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1314         d.addCallback(lambda json: self.failUnlessIn("MDMF", json))
1315         return d
1316
1317     def test_PUT_NEWFILEURL_unlinked_sdmf(self):
1318         contents = self.NEWFILE_CONTENTS * 300000
1319         d = self.PUT("/uri?format=sdmf",
1320                      contents)
1321         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1322         d.addCallback(lambda json: self.failUnlessIn("SDMF", json))
1323         return d
1324
1325     def test_PUT_NEWFILEURL_unlinked_bad_format(self):
1326         contents = self.NEWFILE_CONTENTS * 300000
1327         return self.shouldHTTPError("PUT_NEWFILEURL_unlinked_bad_format",
1328                                     400, "Bad Request", "Unknown format: foo",
1329                                     self.PUT, "/uri?format=foo",
1330                                     contents)
1331
1332     def test_PUT_NEWFILEURL_range_bad(self):
1333         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
1334         target = self.public_url + "/foo/new.txt"
1335         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
1336                              "501 Not Implemented",
1337                              "Content-Range in PUT not yet supported",
1338                              # (and certainly not for immutable files)
1339                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
1340                              headers=headers)
1341         d.addCallback(lambda res:
1342                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
1343         return d
1344
1345     def test_PUT_NEWFILEURL_mutable(self):
1346         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1347                      self.NEWFILE_CONTENTS)
1348         # TODO: we lose the response code, so we can't check this
1349         #self.failUnlessReallyEqual(responsecode, 201)
1350         def _check_uri(res):
1351             u = uri.from_string_mutable_filenode(res)
1352             self.failUnless(u.is_mutable())
1353             self.failIf(u.is_readonly())
1354             return res
1355         d.addCallback(_check_uri)
1356         d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
1357         d.addCallback(lambda res:
1358                       self.failUnlessMutableChildContentsAre(self._foo_node,
1359                                                              u"new.txt",
1360                                                              self.NEWFILE_CONTENTS))
1361         return d
1362
1363     def test_PUT_NEWFILEURL_mutable_toobig(self):
1364         # It is okay to upload large mutable files, so we should be able
1365         # to do that.
1366         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1367                      "b" * (self.s.MUTABLE_SIZELIMIT + 1))
1368         return d
1369
1370     def test_PUT_NEWFILEURL_replace(self):
1371         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
1372         # TODO: we lose the response code, so we can't check this
1373         #self.failUnlessReallyEqual(responsecode, 200)
1374         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
1375         d.addCallback(lambda res:
1376                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1377                                                       self.NEWFILE_CONTENTS))
1378         return d
1379
1380     def test_PUT_NEWFILEURL_bad_t(self):
1381         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
1382                              "PUT to a file: bad t=bogus",
1383                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
1384                              "contents")
1385         return d
1386
1387     def test_PUT_NEWFILEURL_no_replace(self):
1388         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
1389                      self.NEWFILE_CONTENTS)
1390         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
1391                   "409 Conflict",
1392                   "There was already a child by that name, and you asked me "
1393                   "to not replace it")
1394         return d
1395
1396     def test_PUT_NEWFILEURL_mkdirs(self):
1397         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
1398         fn = self._foo_node
1399         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
1400         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
1401         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
1402         d.addCallback(lambda res:
1403                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
1404                                                       self.NEWFILE_CONTENTS))
1405         return d
1406
1407     def test_PUT_NEWFILEURL_blocked(self):
1408         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
1409                      self.NEWFILE_CONTENTS)
1410         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
1411                   "409 Conflict",
1412                   "Unable to create directory 'blockingfile': a file was in the way")
1413         return d
1414
1415     def test_PUT_NEWFILEURL_emptyname(self):
1416         # an empty pathname component (i.e. a double-slash) is disallowed
1417         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
1418                              "400 Bad Request",
1419                              "The webapi does not allow empty pathname components",
1420                              self.PUT, self.public_url + "/foo//new.txt", "")
1421         return d
1422
1423     def test_DELETE_FILEURL(self):
1424         d = self.DELETE(self.public_url + "/foo/bar.txt")
1425         d.addCallback(lambda res:
1426                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1427         return d
1428
1429     def test_DELETE_FILEURL_missing(self):
1430         d = self.DELETE(self.public_url + "/foo/missing")
1431         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
1432         return d
1433
1434     def test_DELETE_FILEURL_missing2(self):
1435         d = self.DELETE(self.public_url + "/missing/missing")
1436         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
1437         return d
1438
1439     def failUnlessHasBarDotTxtMetadata(self, res):
1440         data = simplejson.loads(res)
1441         self.failUnless(isinstance(data, list))
1442         self.failUnlessIn("metadata", data[1])
1443         self.failUnlessIn("tahoe", data[1]["metadata"])
1444         self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
1445         self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
1446         self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
1447                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
1448
1449     def test_GET_FILEURL_json(self):
1450         # twisted.web.http.parse_qs ignores any query args without an '=', so
1451         # I can't do "GET /path?json", I have to do "GET /path/t=json"
1452         # instead. This may make it tricky to emulate the S3 interface
1453         # completely.
1454         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
1455         def _check1(data):
1456             self.failUnlessIsBarJSON(data)
1457             self.failUnlessHasBarDotTxtMetadata(data)
1458             return
1459         d.addCallback(_check1)
1460         return d
1461
1462     def test_GET_FILEURL_json_mutable_type(self):
1463         # The JSON should include format, which says whether the
1464         # file is SDMF or MDMF
1465         d = self.PUT("/uri?format=mdmf",
1466                      self.NEWFILE_CONTENTS * 300000)
1467         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1468         def _got_json(json, version):
1469             data = simplejson.loads(json)
1470             assert "filenode" == data[0]
1471             data = data[1]
1472             assert isinstance(data, dict)
1473
1474             self.failUnlessIn("format", data)
1475             self.failUnlessEqual(data["format"], version)
1476
1477         d.addCallback(_got_json, "MDMF")
1478         # Now make an SDMF file and check that it is reported correctly.
1479         d.addCallback(lambda ignored:
1480             self.PUT("/uri?format=sdmf",
1481                       self.NEWFILE_CONTENTS * 300000))
1482         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1483         d.addCallback(_got_json, "SDMF")
1484         return d
1485
1486     def test_GET_FILEURL_json_mdmf(self):
1487         d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
1488         d.addCallback(self.failUnlessIsQuuxJSON)
1489         return d
1490
1491     def test_GET_FILEURL_json_missing(self):
1492         d = self.GET(self.public_url + "/foo/missing?json")
1493         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
1494         return d
1495
1496     def test_GET_FILEURL_uri(self):
1497         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
1498         def _check(res):
1499             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1500         d.addCallback(_check)
1501         d.addCallback(lambda res:
1502                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
1503         def _check2(res):
1504             # for now, for files, uris and readonly-uris are the same
1505             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1506         d.addCallback(_check2)
1507         return d
1508
1509     def test_GET_FILEURL_badtype(self):
1510         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
1511                                  "bad t=bogus",
1512                                  self.GET,
1513                                  self.public_url + "/foo/bar.txt?t=bogus")
1514         return d
1515
1516     def test_CSS_FILE(self):
1517         d = self.GET("/tahoe.css", followRedirect=True)
1518         def _check(res):
1519             CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
1520             self.failUnless(CSS_STYLE.search(res), res)
1521         d.addCallback(_check)
1522         return d
1523
1524     def test_GET_FILEURL_uri_missing(self):
1525         d = self.GET(self.public_url + "/foo/missing?t=uri")
1526         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
1527         return d
1528
1529     def _check_upload_and_mkdir_forms(self, html):
1530         # We should have a form to create a file, with radio buttons that allow
1531         # the user to toggle whether it is a CHK/LIT (default), SDMF, or MDMF file.
1532         self.failUnlessIn('name="t" value="upload"', html)
1533         self.failUnless(re.search('<input [^/]*id="upload-chk"', html), html)
1534         self.failUnless(re.search('<input [^/]*id="upload-sdmf"', html), html)
1535         self.failUnless(re.search('<input [^/]*id="upload-mdmf"', html), html)
1536
1537         # We should also have the ability to create a mutable directory, with
1538         # radio buttons that allow the user to toggle whether it is an SDMF (default)
1539         # or MDMF directory.
1540         self.failUnlessIn('name="t" value="mkdir"', html)
1541         self.failUnless(re.search('<input [^/]*id="mkdir-sdmf"', html), html)
1542         self.failUnless(re.search('<input [^/]*id="mkdir-mdmf"', html), html)
1543
1544         self.failUnlessIn(FAVICON_MARKUP, html)
1545
1546     def test_GET_DIRECTORY_html(self):
1547         d = self.GET(self.public_url + "/foo", followRedirect=True)
1548         def _check(html):
1549             self.failUnlessIn('<li class="toolbar-item"><a href="../../..">Return to Welcome page</a></li>', html)
1550             self._check_upload_and_mkdir_forms(html)
1551             self.failUnlessIn("quux", html)
1552         d.addCallback(_check)
1553         return d
1554
1555     def test_GET_DIRECTORY_html_filenode_encoding(self):
1556         d = self.GET(self.public_url + "/foo", followRedirect=True)
1557         def _check(html):
1558             # Check if encoded entries are there
1559             self.failUnlessIn('@@named=/' + self._htmlname_urlencoded + '">'
1560                               + self._htmlname_escaped + '</a>', html)
1561             self.failUnlessIn('value="' + self._htmlname_escaped_attr + '"', html)
1562             self.failIfIn(self._htmlname_escaped_double, html)
1563             # Make sure that Nevow escaping actually works by checking for unsafe characters
1564             # and that '&' is escaped.
1565             for entity in '<>':
1566                 self.failUnlessIn(entity, self._htmlname_raw)
1567                 self.failIfIn(entity, self._htmlname_escaped)
1568             self.failUnlessIn('&', re.sub(r'&(amp|lt|gt|quot|apos);', '', self._htmlname_raw))
1569             self.failIfIn('&', re.sub(r'&(amp|lt|gt|quot|apos);', '', self._htmlname_escaped))
1570         d.addCallback(_check)
1571         return d
1572
1573     def test_GET_root_html(self):
1574         d = self.GET("/")
1575         d.addCallback(self._check_upload_and_mkdir_forms)
1576         return d
1577
1578     def test_GET_DIRURL(self):
1579         # the addSlash means we get a redirect here
1580         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1581         ROOT = "../../.."
1582         d = self.GET(self.public_url + "/foo", followRedirect=True)
1583         def _check(res):
1584             self.failUnlessIn('<a href="%s">Return to Welcome page' % ROOT, res)
1585
1586             # the FILE reference points to a URI, but it should end in bar.txt
1587             bar_url = ("%s/file/%s/@@named=/bar.txt" %
1588                        (ROOT, urllib.quote(self._bar_txt_uri)))
1589             get_bar = "".join([r'<td>FILE</td>',
1590                                r'\s+<td>',
1591                                r'<a href="%s">bar.txt</a>' % bar_url,
1592                                r'</td>',
1593                                r'\s+<td align="right">%d</td>' % len(self.BAR_CONTENTS),
1594                                ])
1595             self.failUnless(re.search(get_bar, res), res)
1596             for label in ['unlink', 'rename/relink']:
1597                 for line in res.split("\n"):
1598                     # find the line that contains the relevant button for bar.txt
1599                     if ("form action" in line and
1600                         ('value="%s"' % (label,)) in line and
1601                         'value="bar.txt"' in line):
1602                         # the form target should use a relative URL
1603                         foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1604                         self.failUnlessIn('action="%s"' % foo_url, line)
1605                         # and the when_done= should too
1606                         #done_url = urllib.quote(???)
1607                         #self.failUnlessIn('name="when_done" value="%s"' % done_url, line)
1608
1609                         # 'unlink' needs to use POST because it directly has a side effect
1610                         if label == 'unlink':
1611                             self.failUnlessIn('method="post"', line)
1612                         break
1613                 else:
1614                     self.fail("unable to find '%s bar.txt' line" % (label,))
1615
1616             # the DIR reference just points to a URI
1617             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1618             get_sub = ((r'<td>DIR</td>')
1619                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1620             self.failUnless(re.search(get_sub, res), res)
1621         d.addCallback(_check)
1622
1623         # look at a readonly directory
1624         d.addCallback(lambda res:
1625                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
1626         def _check2(res):
1627             self.failUnlessIn("(read-only)", res)
1628             self.failIfIn("Upload a file", res)
1629         d.addCallback(_check2)
1630
1631         # and at a directory that contains a readonly directory
1632         d.addCallback(lambda res:
1633                       self.GET(self.public_url, followRedirect=True))
1634         def _check3(res):
1635             self.failUnless(re.search('<td>DIR-RO</td>'
1636                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1637         d.addCallback(_check3)
1638
1639         # and an empty directory
1640         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1641         def _check4(res):
1642             self.failUnlessIn("directory is empty", res)
1643             MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" class="btn" value="Create" />', re.I)
1644             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1645         d.addCallback(_check4)
1646
1647         # and at a literal directory
1648         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1649         d.addCallback(lambda res:
1650                       self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1651         def _check5(res):
1652             self.failUnlessIn('(immutable)', res)
1653             self.failUnless(re.search('<td>FILE</td>'
1654                                       r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1655         d.addCallback(_check5)
1656         return d
1657
1658     def test_GET_DIRURL_badtype(self):
1659         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1660                                  400, "Bad Request",
1661                                  "bad t=bogus",
1662                                  self.GET,
1663                                  self.public_url + "/foo?t=bogus")
1664         return d
1665
1666     def test_GET_DIRURL_json(self):
1667         d = self.GET(self.public_url + "/foo?t=json")
1668         d.addCallback(self.failUnlessIsFooJSON)
1669         return d
1670
1671     def test_GET_DIRURL_json_format(self):
1672         d = self.PUT(self.public_url + \
1673                      "/foo/sdmf.txt?format=sdmf",
1674                      self.NEWFILE_CONTENTS * 300000)
1675         d.addCallback(lambda ignored:
1676             self.PUT(self.public_url + \
1677                      "/foo/mdmf.txt?format=mdmf",
1678                      self.NEWFILE_CONTENTS * 300000))
1679         # Now we have an MDMF and SDMF file in the directory. If we GET
1680         # its JSON, we should see their encodings.
1681         d.addCallback(lambda ignored:
1682             self.GET(self.public_url + "/foo?t=json"))
1683         def _got_json(json):
1684             data = simplejson.loads(json)
1685             assert data[0] == "dirnode"
1686
1687             data = data[1]
1688             kids = data['children']
1689
1690             mdmf_data = kids['mdmf.txt'][1]
1691             self.failUnlessIn("format", mdmf_data)
1692             self.failUnlessEqual(mdmf_data["format"], "MDMF")
1693
1694             sdmf_data = kids['sdmf.txt'][1]
1695             self.failUnlessIn("format", sdmf_data)
1696             self.failUnlessEqual(sdmf_data["format"], "SDMF")
1697         d.addCallback(_got_json)
1698         return d
1699
1700
1701     def test_POST_DIRURL_manifest_no_ophandle(self):
1702         d = self.shouldFail2(error.Error,
1703                              "test_POST_DIRURL_manifest_no_ophandle",
1704                              "400 Bad Request",
1705                              "slow operation requires ophandle=",
1706                              self.POST, self.public_url, t="start-manifest")
1707         return d
1708
1709     def test_POST_DIRURL_manifest(self):
1710         d = defer.succeed(None)
1711         def getman(ignored, output):
1712             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1713                           followRedirect=True)
1714             d.addCallback(self.wait_for_operation, "125")
1715             d.addCallback(self.get_operation_results, "125", output)
1716             return d
1717         d.addCallback(getman, None)
1718         def _got_html(manifest):
1719             self.failUnlessIn("Manifest of SI=", manifest)
1720             self.failUnlessIn("<td>sub</td>", manifest)
1721             self.failUnlessIn(self._sub_uri, manifest)
1722             self.failUnlessIn("<td>sub/baz.txt</td>", manifest)
1723             self.failUnlessIn(FAVICON_MARKUP, manifest)
1724         d.addCallback(_got_html)
1725
1726         # both t=status and unadorned GET should be identical
1727         d.addCallback(lambda res: self.GET("/operations/125"))
1728         d.addCallback(_got_html)
1729
1730         d.addCallback(getman, "html")
1731         d.addCallback(_got_html)
1732         d.addCallback(getman, "text")
1733         def _got_text(manifest):
1734             self.failUnlessIn("\nsub " + self._sub_uri + "\n", manifest)
1735             self.failUnlessIn("\nsub/baz.txt URI:CHK:", manifest)
1736         d.addCallback(_got_text)
1737         d.addCallback(getman, "JSON")
1738         def _got_json(res):
1739             data = res["manifest"]
1740             got = {}
1741             for (path_list, cap) in data:
1742                 got[tuple(path_list)] = cap
1743             self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
1744             self.failUnlessIn((u"sub", u"baz.txt"), got)
1745             self.failUnlessIn("finished", res)
1746             self.failUnlessIn("origin", res)
1747             self.failUnlessIn("storage-index", res)
1748             self.failUnlessIn("verifycaps", res)
1749             self.failUnlessIn("stats", res)
1750         d.addCallback(_got_json)
1751         return d
1752
1753     def test_POST_DIRURL_deepsize_no_ophandle(self):
1754         d = self.shouldFail2(error.Error,
1755                              "test_POST_DIRURL_deepsize_no_ophandle",
1756                              "400 Bad Request",
1757                              "slow operation requires ophandle=",
1758                              self.POST, self.public_url, t="start-deep-size")
1759         return d
1760
1761     def test_POST_DIRURL_deepsize(self):
1762         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1763                       followRedirect=True)
1764         d.addCallback(self.wait_for_operation, "126")
1765         d.addCallback(self.get_operation_results, "126", "json")
1766         def _got_json(data):
1767             self.failUnlessReallyEqual(data["finished"], True)
1768             size = data["size"]
1769             self.failUnless(size > 1000)
1770         d.addCallback(_got_json)
1771         d.addCallback(self.get_operation_results, "126", "text")
1772         def _got_text(res):
1773             mo = re.search(r'^size: (\d+)$', res, re.M)
1774             self.failUnless(mo, res)
1775             size = int(mo.group(1))
1776             # with directories, the size varies.
1777             self.failUnless(size > 1000)
1778         d.addCallback(_got_text)
1779         return d
1780
1781     def test_POST_DIRURL_deepstats_no_ophandle(self):
1782         d = self.shouldFail2(error.Error,
1783                              "test_POST_DIRURL_deepstats_no_ophandle",
1784                              "400 Bad Request",
1785                              "slow operation requires ophandle=",
1786                              self.POST, self.public_url, t="start-deep-stats")
1787         return d
1788
1789     def test_POST_DIRURL_deepstats(self):
1790         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1791                       followRedirect=True)
1792         d.addCallback(self.wait_for_operation, "127")
1793         d.addCallback(self.get_operation_results, "127", "json")
1794         def _got_json(stats):
1795             expected = {"count-immutable-files": 4,
1796                         "count-mutable-files": 2,
1797                         "count-literal-files": 0,
1798                         "count-files": 6,
1799                         "count-directories": 3,
1800                         "size-immutable-files": 76,
1801                         "size-literal-files": 0,
1802                         #"size-directories": 1912, # varies
1803                         #"largest-directory": 1590,
1804                         "largest-directory-children": 8,
1805                         "largest-immutable-file": 19,
1806                         }
1807             for k,v in expected.iteritems():
1808                 self.failUnlessReallyEqual(stats[k], v,
1809                                            "stats[%s] was %s, not %s" %
1810                                            (k, stats[k], v))
1811             self.failUnlessReallyEqual(stats["size-files-histogram"],
1812                                        [ [11, 31, 4] ])
1813         d.addCallback(_got_json)
1814         return d
1815
1816     def test_POST_DIRURL_stream_manifest(self):
1817         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1818         def _check(res):
1819             self.failUnless(res.endswith("\n"))
1820             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1821             self.failUnlessReallyEqual(len(units), 10)
1822             self.failUnlessEqual(units[-1]["type"], "stats")
1823             first = units[0]
1824             self.failUnlessEqual(first["path"], [])
1825             self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri)
1826             self.failUnlessEqual(first["type"], "directory")
1827             baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0]
1828             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1829             self.failIfEqual(baz["storage-index"], None)
1830             self.failIfEqual(baz["verifycap"], None)
1831             self.failIfEqual(baz["repaircap"], None)
1832             # XXX: Add quux and baz to this test.
1833             return
1834         d.addCallback(_check)
1835         return d
1836
1837     def test_GET_DIRURL_uri(self):
1838         d = self.GET(self.public_url + "/foo?t=uri")
1839         def _check(res):
1840             self.failUnlessReallyEqual(to_str(res), self._foo_uri)
1841         d.addCallback(_check)
1842         return d
1843
1844     def test_GET_DIRURL_readonly_uri(self):
1845         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1846         def _check(res):
1847             self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri)
1848         d.addCallback(_check)
1849         return d
1850
1851     def test_PUT_NEWDIRURL(self):
1852         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1853         d.addCallback(lambda res:
1854                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1855         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1856         d.addCallback(self.failUnlessNodeKeysAre, [])
1857         return d
1858
1859     def test_PUT_NEWDIRURL_mdmf(self):
1860         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1861         d.addCallback(lambda res:
1862                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1863         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1864         d.addCallback(lambda node:
1865             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1866         return d
1867
1868     def test_PUT_NEWDIRURL_sdmf(self):
1869         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=sdmf",
1870                      "")
1871         d.addCallback(lambda res:
1872                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1873         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1874         d.addCallback(lambda node:
1875             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1876         return d
1877
1878     def test_PUT_NEWDIRURL_bad_format(self):
1879         return self.shouldHTTPError("PUT_NEWDIRURL_bad_format",
1880                                     400, "Bad Request", "Unknown format: foo",
1881                                     self.PUT, self.public_url +
1882                                     "/foo/newdir=?t=mkdir&format=foo", "")
1883
1884     def test_POST_NEWDIRURL(self):
1885         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1886         d.addCallback(lambda res:
1887                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1888         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1889         d.addCallback(self.failUnlessNodeKeysAre, [])
1890         return d
1891
1892     def test_POST_NEWDIRURL_mdmf(self):
1893         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1894         d.addCallback(lambda res:
1895                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1896         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1897         d.addCallback(lambda node:
1898             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1899         return d
1900
1901     def test_POST_NEWDIRURL_sdmf(self):
1902         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=sdmf", "")
1903         d.addCallback(lambda res:
1904             self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1905         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1906         d.addCallback(lambda node:
1907             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1908         return d
1909
1910     def test_POST_NEWDIRURL_bad_format(self):
1911         return self.shouldHTTPError("POST_NEWDIRURL_bad_format",
1912                                     400, "Bad Request", "Unknown format: foo",
1913                                     self.POST2, self.public_url + \
1914                                     "/foo/newdir?t=mkdir&format=foo", "")
1915
1916     def test_POST_NEWDIRURL_emptyname(self):
1917         # an empty pathname component (i.e. a double-slash) is disallowed
1918         d = self.shouldFail2(error.Error, "POST_NEWDIRURL_emptyname",
1919                              "400 Bad Request",
1920                              "The webapi does not allow empty pathname components, i.e. a double slash",
1921                              self.POST, self.public_url + "//?t=mkdir")
1922         return d
1923
1924     def _do_POST_NEWDIRURL_initial_children_test(self, version=None):
1925         (newkids, caps) = self._create_initial_children()
1926         query = "/foo/newdir?t=mkdir-with-children"
1927         if version == MDMF_VERSION:
1928             query += "&format=mdmf"
1929         elif version == SDMF_VERSION:
1930             query += "&format=sdmf"
1931         else:
1932             version = SDMF_VERSION # for later
1933         d = self.POST2(self.public_url + query,
1934                        simplejson.dumps(newkids))
1935         def _check(uri):
1936             n = self.s.create_node_from_uri(uri.strip())
1937             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1938             self.failUnlessEqual(n._node.get_version(), version)
1939             d2.addCallback(lambda ign:
1940                            self.failUnlessROChildURIIs(n, u"child-imm",
1941                                                        caps['filecap1']))
1942             d2.addCallback(lambda ign:
1943                            self.failUnlessRWChildURIIs(n, u"child-mutable",
1944                                                        caps['filecap2']))
1945             d2.addCallback(lambda ign:
1946                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1947                                                        caps['filecap3']))
1948             d2.addCallback(lambda ign:
1949                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1950                                                        caps['unknown_rocap']))
1951             d2.addCallback(lambda ign:
1952                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1953                                                        caps['unknown_rwcap']))
1954             d2.addCallback(lambda ign:
1955                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1956                                                        caps['unknown_immcap']))
1957             d2.addCallback(lambda ign:
1958                            self.failUnlessRWChildURIIs(n, u"dirchild",
1959                                                        caps['dircap']))
1960             d2.addCallback(lambda ign:
1961                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1962                                                        caps['litdircap']))
1963             d2.addCallback(lambda ign:
1964                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1965                                                        caps['emptydircap']))
1966             return d2
1967         d.addCallback(_check)
1968         d.addCallback(lambda res:
1969                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1970         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1971         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1972         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1973         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1974         return d
1975
1976     def test_POST_NEWDIRURL_initial_children(self):
1977         return self._do_POST_NEWDIRURL_initial_children_test()
1978
1979     def test_POST_NEWDIRURL_initial_children_mdmf(self):
1980         return self._do_POST_NEWDIRURL_initial_children_test(MDMF_VERSION)
1981
1982     def test_POST_NEWDIRURL_initial_children_sdmf(self):
1983         return self._do_POST_NEWDIRURL_initial_children_test(SDMF_VERSION)
1984
1985     def test_POST_NEWDIRURL_initial_children_bad_format(self):
1986         (newkids, caps) = self._create_initial_children()
1987         return self.shouldHTTPError("POST_NEWDIRURL_initial_children_bad_format",
1988                                     400, "Bad Request", "Unknown format: foo",
1989                                     self.POST2, self.public_url + \
1990                                     "/foo/newdir?t=mkdir-with-children&format=foo",
1991                                     simplejson.dumps(newkids))
1992
1993     def test_POST_NEWDIRURL_immutable(self):
1994         (newkids, caps) = self._create_immutable_children()
1995         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1996                        simplejson.dumps(newkids))
1997         def _check(uri):
1998             n = self.s.create_node_from_uri(uri.strip())
1999             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2000             d2.addCallback(lambda ign:
2001                            self.failUnlessROChildURIIs(n, u"child-imm",
2002                                                        caps['filecap1']))
2003             d2.addCallback(lambda ign:
2004                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2005                                                        caps['unknown_immcap']))
2006             d2.addCallback(lambda ign:
2007                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
2008                                                        caps['immdircap']))
2009             d2.addCallback(lambda ign:
2010                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
2011                                                        caps['litdircap']))
2012             d2.addCallback(lambda ign:
2013                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
2014                                                        caps['emptydircap']))
2015             return d2
2016         d.addCallback(_check)
2017         d.addCallback(lambda res:
2018                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2019         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2020         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2021         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2022         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2023         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2024         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2025         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2026         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2027         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2028         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2029         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2030         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2031         d.addErrback(self.explain_web_error)
2032         return d
2033
2034     def test_POST_NEWDIRURL_immutable_bad(self):
2035         (newkids, caps) = self._create_initial_children()
2036         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
2037                              "400 Bad Request",
2038                              "needed to be immutable but was not",
2039                              self.POST2,
2040                              self.public_url + "/foo/newdir?t=mkdir-immutable",
2041                              simplejson.dumps(newkids))
2042         return d
2043
2044     def test_PUT_NEWDIRURL_exists(self):
2045         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
2046         d.addCallback(lambda res:
2047                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
2048         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2049         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2050         return d
2051
2052     def test_PUT_NEWDIRURL_blocked(self):
2053         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
2054                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
2055                              self.PUT,
2056                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
2057         d.addCallback(lambda res:
2058                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
2059         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2060         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2061         return d
2062
2063     def test_PUT_NEWDIRURL_mkdirs(self):
2064         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
2065         d.addCallback(lambda res:
2066                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
2067         d.addCallback(lambda res:
2068                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
2069         d.addCallback(lambda res:
2070                       self._foo_node.get_child_at_path(u"subdir/newdir"))
2071         d.addCallback(self.failUnlessNodeKeysAre, [])
2072         return d
2073
2074     def test_PUT_NEWDIRURL_mkdirs_mdmf(self):
2075         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=mdmf", "")
2076         d.addCallback(lambda ignored:
2077             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
2078         d.addCallback(lambda ignored:
2079             self.failIfNodeHasChild(self._foo_node, u"newdir"))
2080         d.addCallback(lambda ignored:
2081             self._foo_node.get_child_at_path(u"subdir"))
2082         def _got_subdir(subdir):
2083             # XXX: What we want?
2084             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
2085             self.failUnlessNodeHasChild(subdir, u"newdir")
2086             return subdir.get_child_at_path(u"newdir")
2087         d.addCallback(_got_subdir)
2088         d.addCallback(lambda newdir:
2089             self.failUnlessEqual(newdir._node.get_version(), MDMF_VERSION))
2090         return d
2091
2092     def test_PUT_NEWDIRURL_mkdirs_sdmf(self):
2093         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=sdmf", "")
2094         d.addCallback(lambda ignored:
2095             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
2096         d.addCallback(lambda ignored:
2097             self.failIfNodeHasChild(self._foo_node, u"newdir"))
2098         d.addCallback(lambda ignored:
2099             self._foo_node.get_child_at_path(u"subdir"))
2100         def _got_subdir(subdir):
2101             # XXX: What we want?
2102             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
2103             self.failUnlessNodeHasChild(subdir, u"newdir")
2104             return subdir.get_child_at_path(u"newdir")
2105         d.addCallback(_got_subdir)
2106         d.addCallback(lambda newdir:
2107             self.failUnlessEqual(newdir._node.get_version(), SDMF_VERSION))
2108         return d
2109
2110     def test_PUT_NEWDIRURL_mkdirs_bad_format(self):
2111         return self.shouldHTTPError("PUT_NEWDIRURL_mkdirs_bad_format",
2112                                     400, "Bad Request", "Unknown format: foo",
2113                                     self.PUT, self.public_url + \
2114                                     "/foo/subdir/newdir?t=mkdir&format=foo",
2115                                     "")
2116
2117     def test_DELETE_DIRURL(self):
2118         d = self.DELETE(self.public_url + "/foo")
2119         d.addCallback(lambda res:
2120                       self.failIfNodeHasChild(self.public_root, u"foo"))
2121         return d
2122
2123     def test_DELETE_DIRURL_missing(self):
2124         d = self.DELETE(self.public_url + "/foo/missing")
2125         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
2126         d.addCallback(lambda res:
2127                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
2128         return d
2129
2130     def test_DELETE_DIRURL_missing2(self):
2131         d = self.DELETE(self.public_url + "/missing")
2132         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
2133         return d
2134
2135     def dump_root(self):
2136         print "NODEWALK"
2137         w = webish.DirnodeWalkerMixin()
2138         def visitor(childpath, childnode, metadata):
2139             print childpath
2140         d = w.walk(self.public_root, visitor)
2141         return d
2142
2143     def failUnlessNodeKeysAre(self, node, expected_keys):
2144         for k in expected_keys:
2145             assert isinstance(k, unicode)
2146         d = node.list()
2147         def _check(children):
2148             self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
2149         d.addCallback(_check)
2150         return d
2151     def failUnlessNodeHasChild(self, node, name):
2152         assert isinstance(name, unicode)
2153         d = node.list()
2154         def _check(children):
2155             self.failUnlessIn(name, children)
2156         d.addCallback(_check)
2157         return d
2158     def failIfNodeHasChild(self, node, name):
2159         assert isinstance(name, unicode)
2160         d = node.list()
2161         def _check(children):
2162             self.failIfIn(name, children)
2163         d.addCallback(_check)
2164         return d
2165
2166     def failUnlessChildContentsAre(self, node, name, expected_contents):
2167         assert isinstance(name, unicode)
2168         d = node.get_child_at_path(name)
2169         d.addCallback(lambda node: download_to_data(node))
2170         def _check(contents):
2171             self.failUnlessReallyEqual(contents, expected_contents)
2172         d.addCallback(_check)
2173         return d
2174
2175     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
2176         assert isinstance(name, unicode)
2177         d = node.get_child_at_path(name)
2178         d.addCallback(lambda node: node.download_best_version())
2179         def _check(contents):
2180             self.failUnlessReallyEqual(contents, expected_contents)
2181         d.addCallback(_check)
2182         return d
2183
2184     def failUnlessRWChildURIIs(self, node, name, expected_uri):
2185         assert isinstance(name, unicode)
2186         d = node.get_child_at_path(name)
2187         def _check(child):
2188             self.failUnless(child.is_unknown() or not child.is_readonly())
2189             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
2190             self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
2191             expected_ro_uri = self._make_readonly(expected_uri)
2192             if expected_ro_uri:
2193                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
2194         d.addCallback(_check)
2195         return d
2196
2197     def failUnlessROChildURIIs(self, node, name, expected_uri):
2198         assert isinstance(name, unicode)
2199         d = node.get_child_at_path(name)
2200         def _check(child):
2201             self.failUnless(child.is_unknown() or child.is_readonly())
2202             self.failUnlessReallyEqual(child.get_write_uri(), None)
2203             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
2204             self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
2205         d.addCallback(_check)
2206         return d
2207
2208     def failUnlessURIMatchesRWChild(self, got_uri, node, name):
2209         assert isinstance(name, unicode)
2210         d = node.get_child_at_path(name)
2211         def _check(child):
2212             self.failUnless(child.is_unknown() or not child.is_readonly())
2213             self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
2214             self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
2215             expected_ro_uri = self._make_readonly(got_uri)
2216             if expected_ro_uri:
2217                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
2218         d.addCallback(_check)
2219         return d
2220
2221     def failUnlessURIMatchesROChild(self, got_uri, node, name):
2222         assert isinstance(name, unicode)
2223         d = node.get_child_at_path(name)
2224         def _check(child):
2225             self.failUnless(child.is_unknown() or child.is_readonly())
2226             self.failUnlessReallyEqual(child.get_write_uri(), None)
2227             self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
2228             self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
2229         d.addCallback(_check)
2230         return d
2231
2232     def failUnlessCHKURIHasContents(self, got_uri, contents):
2233         self.failUnless(self.get_all_contents()[got_uri] == contents)
2234
2235     def test_POST_upload(self):
2236         d = self.POST(self.public_url + "/foo", t="upload",
2237                       file=("new.txt", self.NEWFILE_CONTENTS))
2238         fn = self._foo_node
2239         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2240         d.addCallback(lambda res:
2241                       self.failUnlessChildContentsAre(fn, u"new.txt",
2242                                                       self.NEWFILE_CONTENTS))
2243         return d
2244
2245     def test_POST_upload_unicode(self):
2246         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2247         d = self.POST(self.public_url + "/foo", t="upload",
2248                       file=(filename, self.NEWFILE_CONTENTS))
2249         fn = self._foo_node
2250         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2251         d.addCallback(lambda res:
2252                       self.failUnlessChildContentsAre(fn, filename,
2253                                                       self.NEWFILE_CONTENTS))
2254         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2255         d.addCallback(lambda res: self.GET(target_url))
2256         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2257                                                                   self.NEWFILE_CONTENTS,
2258                                                                   contents))
2259         return d
2260
2261     def test_POST_upload_unicode_named(self):
2262         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2263         d = self.POST(self.public_url + "/foo", t="upload",
2264                       name=filename,
2265                       file=("overridden", self.NEWFILE_CONTENTS))
2266         fn = self._foo_node
2267         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2268         d.addCallback(lambda res:
2269                       self.failUnlessChildContentsAre(fn, filename,
2270                                                       self.NEWFILE_CONTENTS))
2271         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2272         d.addCallback(lambda res: self.GET(target_url))
2273         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2274                                                                   self.NEWFILE_CONTENTS,
2275                                                                   contents))
2276         return d
2277
2278     def test_POST_upload_no_link(self):
2279         d = self.POST("/uri", t="upload",
2280                       file=("new.txt", self.NEWFILE_CONTENTS))
2281         def _check_upload_results(page):
2282             # this should be a page which describes the results of the upload
2283             # that just finished.
2284             self.failUnlessIn("Upload Results:", page)
2285             self.failUnlessIn("URI:", page)
2286             uri_re = re.compile("URI: <tt><span>(.*)</span>")
2287             mo = uri_re.search(page)
2288             self.failUnless(mo, page)
2289             new_uri = mo.group(1)
2290             return new_uri
2291         d.addCallback(_check_upload_results)
2292         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
2293         return d
2294
2295     def test_POST_upload_no_link_whendone(self):
2296         d = self.POST("/uri", t="upload", when_done="/",
2297                       file=("new.txt", self.NEWFILE_CONTENTS))
2298         d.addBoth(self.shouldRedirect, "/")
2299         return d
2300
2301     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
2302         d = defer.maybeDeferred(callable, *args, **kwargs)
2303         def done(res):
2304             if isinstance(res, failure.Failure):
2305                 res.trap(error.PageRedirect)
2306                 statuscode = res.value.status
2307                 target = res.value.location
2308                 return checker(statuscode, target)
2309             self.fail("%s: callable was supposed to redirect, not return '%s'"
2310                       % (which, res))
2311         d.addBoth(done)
2312         return d
2313
2314     def test_POST_upload_no_link_whendone_results(self):
2315         def check(statuscode, target):
2316             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2317             self.failUnless(target.startswith(self.webish_url), target)
2318             return client.getPage(target, method="GET")
2319         # We encode "uri" as "%75ri" to exercise a case affected by ticket #1860.
2320         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
2321                                  check,
2322                                  self.POST, "/uri", t="upload",
2323                                  when_done="/%75ri/%(uri)s",
2324                                  file=("new.txt", self.NEWFILE_CONTENTS))
2325         d.addCallback(lambda res:
2326                       self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
2327         return d
2328
2329     def test_POST_upload_no_link_mutable(self):
2330         d = self.POST("/uri", t="upload", mutable="true",
2331                       file=("new.txt", self.NEWFILE_CONTENTS))
2332         def _check(filecap):
2333             filecap = filecap.strip()
2334             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2335             self.filecap = filecap
2336             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2337             self.failUnlessIn(u.get_storage_index(), self.get_all_contents())
2338             n = self.s.create_node_from_uri(filecap)
2339             return n.download_best_version()
2340         d.addCallback(_check)
2341         def _check2(data):
2342             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2343             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2344         d.addCallback(_check2)
2345         def _check3(data):
2346             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2347             return self.GET("/file/%s" % urllib.quote(self.filecap))
2348         d.addCallback(_check3)
2349         def _check4(data):
2350             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2351         d.addCallback(_check4)
2352         return d
2353
2354     def test_POST_upload_no_link_mutable_toobig(self):
2355         # The SDMF size limit is no longer in place, so we should be
2356         # able to upload mutable files that are as large as we want them
2357         # to be.
2358         d = self.POST("/uri", t="upload", mutable="true",
2359                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2360         return d
2361
2362
2363     def test_POST_upload_format_unlinked(self):
2364         def _check_upload_unlinked(ign, format, uri_prefix):
2365             filename = format + ".txt"
2366             d = self.POST("/uri?t=upload&format=" + format,
2367                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2368             def _got_results(results):
2369                 if format.upper() in ("SDMF", "MDMF"):
2370                     # webapi.rst says this returns a filecap
2371                     filecap = results
2372                 else:
2373                     # for immutable, it returns an "upload results page", and
2374                     # the filecap is buried inside
2375                     line = [l for l in results.split("\n") if "URI: " in l][0]
2376                     mo = re.search(r'<span>([^<]+)</span>', line)
2377                     filecap = mo.group(1)
2378                 self.failUnless(filecap.startswith(uri_prefix),
2379                                 (uri_prefix, filecap))
2380                 return self.GET("/uri/%s?t=json" % filecap)
2381             d.addCallback(_got_results)
2382             def _got_json(json):
2383                 data = simplejson.loads(json)
2384                 data = data[1]
2385                 self.failUnlessIn("format", data)
2386                 self.failUnlessEqual(data["format"], format.upper())
2387             d.addCallback(_got_json)
2388             return d
2389         d = defer.succeed(None)
2390         d.addCallback(_check_upload_unlinked, "chk", "URI:CHK")
2391         d.addCallback(_check_upload_unlinked, "CHK", "URI:CHK")
2392         d.addCallback(_check_upload_unlinked, "sdmf", "URI:SSK")
2393         d.addCallback(_check_upload_unlinked, "mdmf", "URI:MDMF")
2394         return d
2395
2396     def test_POST_upload_bad_format_unlinked(self):
2397         return self.shouldHTTPError("POST_upload_bad_format_unlinked",
2398                                     400, "Bad Request", "Unknown format: foo",
2399                                     self.POST,
2400                                     "/uri?t=upload&format=foo",
2401                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2402
2403     def test_POST_upload_format(self):
2404         def _check_upload(ign, format, uri_prefix, fn=None):
2405             filename = format + ".txt"
2406             d = self.POST(self.public_url +
2407                           "/foo?t=upload&format=" + format,
2408                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2409             def _got_filecap(filecap):
2410                 if fn is not None:
2411                     filenameu = unicode(filename)
2412                     self.failUnlessURIMatchesRWChild(filecap, fn, filenameu)
2413                 self.failUnless(filecap.startswith(uri_prefix))
2414                 return self.GET(self.public_url + "/foo/%s?t=json" % filename)
2415             d.addCallback(_got_filecap)
2416             def _got_json(json):
2417                 data = simplejson.loads(json)
2418                 data = data[1]
2419                 self.failUnlessIn("format", data)
2420                 self.failUnlessEqual(data["format"], format.upper())
2421             d.addCallback(_got_json)
2422             return d
2423
2424         d = defer.succeed(None)
2425         d.addCallback(_check_upload, "chk", "URI:CHK")
2426         d.addCallback(_check_upload, "sdmf", "URI:SSK", self._foo_node)
2427         d.addCallback(_check_upload, "mdmf", "URI:MDMF")
2428         d.addCallback(_check_upload, "MDMF", "URI:MDMF")
2429         return d
2430
2431     def test_POST_upload_bad_format(self):
2432         return self.shouldHTTPError("POST_upload_bad_format",
2433                                     400, "Bad Request", "Unknown format: foo",
2434                                     self.POST, self.public_url + \
2435                                     "/foo?t=upload&format=foo",
2436                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2437
2438     def test_POST_upload_mutable(self):
2439         # this creates a mutable file
2440         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
2441                       file=("new.txt", self.NEWFILE_CONTENTS))
2442         fn = self._foo_node
2443         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2444         d.addCallback(lambda res:
2445                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2446                                                              self.NEWFILE_CONTENTS))
2447         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2448         def _got(newnode):
2449             self.failUnless(IMutableFileNode.providedBy(newnode))
2450             self.failUnless(newnode.is_mutable())
2451             self.failIf(newnode.is_readonly())
2452             self._mutable_node = newnode
2453             self._mutable_uri = newnode.get_uri()
2454         d.addCallback(_got)
2455
2456         # now upload it again and make sure that the URI doesn't change
2457         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
2458         d.addCallback(lambda res:
2459                       self.POST(self.public_url + "/foo", t="upload",
2460                                 mutable="true",
2461                                 file=("new.txt", NEWER_CONTENTS)))
2462         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2463         d.addCallback(lambda res:
2464                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2465                                                              NEWER_CONTENTS))
2466         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2467         def _got2(newnode):
2468             self.failUnless(IMutableFileNode.providedBy(newnode))
2469             self.failUnless(newnode.is_mutable())
2470             self.failIf(newnode.is_readonly())
2471             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2472         d.addCallback(_got2)
2473
2474         # upload a second time, using PUT instead of POST
2475         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
2476         d.addCallback(lambda res:
2477                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
2478         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2479         d.addCallback(lambda res:
2480                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2481                                                              NEW2_CONTENTS))
2482
2483         # finally list the directory, since mutable files are displayed
2484         # slightly differently
2485
2486         d.addCallback(lambda res:
2487                       self.GET(self.public_url + "/foo/",
2488                                followRedirect=True))
2489         def _check_page(res):
2490             # TODO: assert more about the contents
2491             self.failUnlessIn("SSK", res)
2492             return res
2493         d.addCallback(_check_page)
2494
2495         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2496         def _got3(newnode):
2497             self.failUnless(IMutableFileNode.providedBy(newnode))
2498             self.failUnless(newnode.is_mutable())
2499             self.failIf(newnode.is_readonly())
2500             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2501         d.addCallback(_got3)
2502
2503         # look at the JSON form of the enclosing directory
2504         d.addCallback(lambda res:
2505                       self.GET(self.public_url + "/foo/?t=json",
2506                                followRedirect=True))
2507         def _check_page_json(res):
2508             parsed = simplejson.loads(res)
2509             self.failUnlessEqual(parsed[0], "dirnode")
2510             children = dict( [(unicode(name),value)
2511                               for (name,value)
2512                               in parsed[1]["children"].iteritems()] )
2513             self.failUnlessIn(u"new.txt", children)
2514             new_json = children[u"new.txt"]
2515             self.failUnlessEqual(new_json[0], "filenode")
2516             self.failUnless(new_json[1]["mutable"])
2517             self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri)
2518             ro_uri = self._mutable_node.get_readonly().to_string()
2519             self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri)
2520         d.addCallback(_check_page_json)
2521
2522         # and the JSON form of the file
2523         d.addCallback(lambda res:
2524                       self.GET(self.public_url + "/foo/new.txt?t=json"))
2525         def _check_file_json(res):
2526             parsed = simplejson.loads(res)
2527             self.failUnlessEqual(parsed[0], "filenode")
2528             self.failUnless(parsed[1]["mutable"])
2529             self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri)
2530             ro_uri = self._mutable_node.get_readonly().to_string()
2531             self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri)
2532         d.addCallback(_check_file_json)
2533
2534         # and look at t=uri and t=readonly-uri
2535         d.addCallback(lambda res:
2536                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
2537         d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
2538         d.addCallback(lambda res:
2539                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
2540         def _check_ro_uri(res):
2541             ro_uri = self._mutable_node.get_readonly().to_string()
2542             self.failUnlessReallyEqual(res, ro_uri)
2543         d.addCallback(_check_ro_uri)
2544
2545         # make sure we can get to it from /uri/URI
2546         d.addCallback(lambda res:
2547                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
2548         d.addCallback(lambda res:
2549                       self.failUnlessReallyEqual(res, NEW2_CONTENTS))
2550
2551         # and that HEAD computes the size correctly
2552         d.addCallback(lambda res:
2553                       self.HEAD(self.public_url + "/foo/new.txt",
2554                                 return_response=True))
2555         def _got_headers((res, status, headers)):
2556             self.failUnlessReallyEqual(res, "")
2557             self.failUnlessReallyEqual(headers["content-length"][0],
2558                                        str(len(NEW2_CONTENTS)))
2559             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
2560         d.addCallback(_got_headers)
2561
2562         # make sure that outdated size limits aren't enforced anymore.
2563         d.addCallback(lambda ignored:
2564             self.POST(self.public_url + "/foo", t="upload",
2565                       mutable="true",
2566                       file=("new.txt",
2567                             "b" * (self.s.MUTABLE_SIZELIMIT+1))))
2568         d.addErrback(self.dump_error)
2569         return d
2570
2571     def test_POST_upload_mutable_toobig(self):
2572         # SDMF had a size limti that was removed a while ago. MDMF has
2573         # never had a size limit. Test to make sure that we do not
2574         # encounter errors when trying to upload large mutable files,
2575         # since there should be no coded prohibitions regarding large
2576         # mutable files.
2577         d = self.POST(self.public_url + "/foo",
2578                       t="upload", mutable="true",
2579                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2580         return d
2581
2582     def dump_error(self, f):
2583         # if the web server returns an error code (like 400 Bad Request),
2584         # web.client.getPage puts the HTTP response body into the .response
2585         # attribute of the exception object that it gives back. It does not
2586         # appear in the Failure's repr(), so the ERROR that trial displays
2587         # will be rather terse and unhelpful. addErrback this method to the
2588         # end of your chain to get more information out of these errors.
2589         if f.check(error.Error):
2590             print "web.error.Error:"
2591             print f
2592             print f.value.response
2593         return f
2594
2595     def test_POST_upload_replace(self):
2596         d = self.POST(self.public_url + "/foo", t="upload",
2597                       file=("bar.txt", self.NEWFILE_CONTENTS))
2598         fn = self._foo_node
2599         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
2600         d.addCallback(lambda res:
2601                       self.failUnlessChildContentsAre(fn, u"bar.txt",
2602                                                       self.NEWFILE_CONTENTS))
2603         return d
2604
2605     def test_POST_upload_no_replace_ok(self):
2606         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2607                       file=("new.txt", self.NEWFILE_CONTENTS))
2608         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
2609         d.addCallback(lambda res: self.failUnlessReallyEqual(res,
2610                                                              self.NEWFILE_CONTENTS))
2611         return d
2612
2613     def test_POST_upload_no_replace_queryarg(self):
2614         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2615                       file=("bar.txt", self.NEWFILE_CONTENTS))
2616         d.addBoth(self.shouldFail, error.Error,
2617                   "POST_upload_no_replace_queryarg",
2618                   "409 Conflict",
2619                   "There was already a child by that name, and you asked me "
2620                   "to not replace it")
2621         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2622         d.addCallback(self.failUnlessIsBarDotTxt)
2623         return d
2624
2625     def test_POST_upload_no_replace_field(self):
2626         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
2627                       file=("bar.txt", self.NEWFILE_CONTENTS))
2628         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
2629                   "409 Conflict",
2630                   "There was already a child by that name, and you asked me "
2631                   "to not replace it")
2632         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2633         d.addCallback(self.failUnlessIsBarDotTxt)
2634         return d
2635
2636     def test_POST_upload_whendone(self):
2637         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
2638                       file=("new.txt", self.NEWFILE_CONTENTS))
2639         d.addBoth(self.shouldRedirect, "/THERE")
2640         fn = self._foo_node
2641         d.addCallback(lambda res:
2642                       self.failUnlessChildContentsAre(fn, u"new.txt",
2643                                                       self.NEWFILE_CONTENTS))
2644         return d
2645
2646     def test_POST_upload_named(self):
2647         fn = self._foo_node
2648         d = self.POST(self.public_url + "/foo", t="upload",
2649                       name="new.txt", file=self.NEWFILE_CONTENTS)
2650         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2651         d.addCallback(lambda res:
2652                       self.failUnlessChildContentsAre(fn, u"new.txt",
2653                                                       self.NEWFILE_CONTENTS))
2654         return d
2655
2656     def test_POST_upload_named_badfilename(self):
2657         d = self.POST(self.public_url + "/foo", t="upload",
2658                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
2659         d.addBoth(self.shouldFail, error.Error,
2660                   "test_POST_upload_named_badfilename",
2661                   "400 Bad Request",
2662                   "name= may not contain a slash",
2663                   )
2664         # make sure that nothing was added
2665         d.addCallback(lambda res:
2666                       self.failUnlessNodeKeysAre(self._foo_node,
2667                                                  [self._htmlname_unicode,
2668                                                   u"bar.txt", u"baz.txt", u"blockingfile",
2669                                                   u"empty", u"n\u00fc.txt", u"quux.txt",
2670                                                   u"sub"]))
2671         return d
2672
2673     def test_POST_FILEURL_check(self):
2674         bar_url = self.public_url + "/foo/bar.txt"
2675         d = self.POST(bar_url, t="check")
2676         def _check(res):
2677             self.failUnlessIn("Healthy :", res)
2678         d.addCallback(_check)
2679         redir_url = "http://allmydata.org/TARGET"
2680         def _check2(statuscode, target):
2681             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2682             self.failUnlessReallyEqual(target, redir_url)
2683         d.addCallback(lambda res:
2684                       self.shouldRedirect2("test_POST_FILEURL_check",
2685                                            _check2,
2686                                            self.POST, bar_url,
2687                                            t="check",
2688                                            when_done=redir_url))
2689         d.addCallback(lambda res:
2690                       self.POST(bar_url, t="check", return_to=redir_url))
2691         def _check3(res):
2692             self.failUnlessIn("Healthy :", res)
2693             self.failUnlessIn("Return to file", res)
2694             self.failUnlessIn(redir_url, res)
2695         d.addCallback(_check3)
2696
2697         d.addCallback(lambda res:
2698                       self.POST(bar_url, t="check", output="JSON"))
2699         def _check_json(res):
2700             data = simplejson.loads(res)
2701             self.failUnlessIn("storage-index", data)
2702             self.failUnless(data["results"]["healthy"])
2703         d.addCallback(_check_json)
2704
2705         return d
2706
2707     def test_POST_FILEURL_check_and_repair(self):
2708         bar_url = self.public_url + "/foo/bar.txt"
2709         d = self.POST(bar_url, t="check", repair="true")
2710         def _check(res):
2711             self.failUnlessIn("Healthy :", res)
2712         d.addCallback(_check)
2713         redir_url = "http://allmydata.org/TARGET"
2714         def _check2(statuscode, target):
2715             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2716             self.failUnlessReallyEqual(target, redir_url)
2717         d.addCallback(lambda res:
2718                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
2719                                            _check2,
2720                                            self.POST, bar_url,
2721                                            t="check", repair="true",
2722                                            when_done=redir_url))
2723         d.addCallback(lambda res:
2724                       self.POST(bar_url, t="check", return_to=redir_url))
2725         def _check3(res):
2726             self.failUnlessIn("Healthy :", res)
2727             self.failUnlessIn("Return to file", res)
2728             self.failUnlessIn(redir_url, res)
2729         d.addCallback(_check3)
2730         return d
2731
2732     def test_POST_DIRURL_check(self):
2733         foo_url = self.public_url + "/foo/"
2734         d = self.POST(foo_url, t="check")
2735         def _check(res):
2736             self.failUnlessIn("Healthy :", res)
2737         d.addCallback(_check)
2738         redir_url = "http://allmydata.org/TARGET"
2739         def _check2(statuscode, target):
2740             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2741             self.failUnlessReallyEqual(target, redir_url)
2742         d.addCallback(lambda res:
2743                       self.shouldRedirect2("test_POST_DIRURL_check",
2744                                            _check2,
2745                                            self.POST, foo_url,
2746                                            t="check",
2747                                            when_done=redir_url))
2748         d.addCallback(lambda res:
2749                       self.POST(foo_url, t="check", return_to=redir_url))
2750         def _check3(res):
2751             self.failUnlessIn("Healthy :", res)
2752             self.failUnlessIn("Return to file/directory", res)
2753             self.failUnlessIn(redir_url, res)
2754         d.addCallback(_check3)
2755
2756         d.addCallback(lambda res:
2757                       self.POST(foo_url, t="check", output="JSON"))
2758         def _check_json(res):
2759             data = simplejson.loads(res)
2760             self.failUnlessIn("storage-index", data)
2761             self.failUnless(data["results"]["healthy"])
2762         d.addCallback(_check_json)
2763
2764         return d
2765
2766     def test_POST_DIRURL_check_and_repair(self):
2767         foo_url = self.public_url + "/foo/"
2768         d = self.POST(foo_url, t="check", repair="true")
2769         def _check(res):
2770             self.failUnlessIn("Healthy :", res)
2771         d.addCallback(_check)
2772         redir_url = "http://allmydata.org/TARGET"
2773         def _check2(statuscode, target):
2774             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2775             self.failUnlessReallyEqual(target, redir_url)
2776         d.addCallback(lambda res:
2777                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2778                                            _check2,
2779                                            self.POST, foo_url,
2780                                            t="check", repair="true",
2781                                            when_done=redir_url))
2782         d.addCallback(lambda res:
2783                       self.POST(foo_url, t="check", return_to=redir_url))
2784         def _check3(res):
2785             self.failUnlessIn("Healthy :", res)
2786             self.failUnlessIn("Return to file/directory", res)
2787             self.failUnlessIn(redir_url, res)
2788         d.addCallback(_check3)
2789         return d
2790
2791     def test_POST_FILEURL_mdmf_check(self):
2792         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2793         d = self.POST(quux_url, t="check")
2794         def _check(res):
2795             self.failUnlessIn("Healthy", res)
2796         d.addCallback(_check)
2797         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2798         d.addCallback(lambda ignored:
2799                       self.POST(quux_extension_url, t="check"))
2800         d.addCallback(_check)
2801         return d
2802
2803     def test_POST_FILEURL_mdmf_check_and_repair(self):
2804         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2805         d = self.POST(quux_url, t="check", repair="true")
2806         def _check(res):
2807             self.failUnlessIn("Healthy", res)
2808         d.addCallback(_check)
2809         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2810         d.addCallback(lambda ignored:
2811                       self.POST(quux_extension_url, t="check", repair="true"))
2812         d.addCallback(_check)
2813         return d
2814
2815     def wait_for_operation(self, ignored, ophandle):
2816         url = "/operations/" + ophandle
2817         url += "?t=status&output=JSON"
2818         d = self.GET(url)
2819         def _got(res):
2820             data = simplejson.loads(res)
2821             if not data["finished"]:
2822                 d = self.stall(delay=1.0)
2823                 d.addCallback(self.wait_for_operation, ophandle)
2824                 return d
2825             return data
2826         d.addCallback(_got)
2827         return d
2828
2829     def get_operation_results(self, ignored, ophandle, output=None):
2830         url = "/operations/" + ophandle
2831         url += "?t=status"
2832         if output:
2833             url += "&output=" + output
2834         d = self.GET(url)
2835         def _got(res):
2836             if output and output.lower() == "json":
2837                 return simplejson.loads(res)
2838             return res
2839         d.addCallback(_got)
2840         return d
2841
2842     def test_POST_DIRURL_deepcheck_no_ophandle(self):
2843         d = self.shouldFail2(error.Error,
2844                              "test_POST_DIRURL_deepcheck_no_ophandle",
2845                              "400 Bad Request",
2846                              "slow operation requires ophandle=",
2847                              self.POST, self.public_url, t="start-deep-check")
2848         return d
2849
2850     def test_POST_DIRURL_deepcheck(self):
2851         def _check_redirect(statuscode, target):
2852             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2853             self.failUnless(target.endswith("/operations/123"))
2854         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2855                                  self.POST, self.public_url,
2856                                  t="start-deep-check", ophandle="123")
2857         d.addCallback(self.wait_for_operation, "123")
2858         def _check_json(data):
2859             self.failUnlessReallyEqual(data["finished"], True)
2860             self.failUnlessReallyEqual(data["count-objects-checked"], 11)
2861             self.failUnlessReallyEqual(data["count-objects-healthy"], 11)
2862         d.addCallback(_check_json)
2863         d.addCallback(self.get_operation_results, "123", "html")
2864         def _check_html(res):
2865             self.failUnlessIn("Objects Checked: <span>11</span>", res)
2866             self.failUnlessIn("Objects Healthy: <span>11</span>", res)
2867             self.failUnlessIn(FAVICON_MARKUP, res)
2868         d.addCallback(_check_html)
2869
2870         d.addCallback(lambda res:
2871                       self.GET("/operations/123/"))
2872         d.addCallback(_check_html) # should be the same as without the slash
2873
2874         d.addCallback(lambda res:
2875                       self.shouldFail2(error.Error, "one", "404 Not Found",
2876                                        "No detailed results for SI bogus",
2877                                        self.GET, "/operations/123/bogus"))
2878
2879         foo_si = self._foo_node.get_storage_index()
2880         foo_si_s = base32.b2a(foo_si)
2881         d.addCallback(lambda res:
2882                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2883         def _check_foo_json(res):
2884             data = simplejson.loads(res)
2885             self.failUnlessEqual(data["storage-index"], foo_si_s)
2886             self.failUnless(data["results"]["healthy"])
2887         d.addCallback(_check_foo_json)
2888         return d
2889
2890     def test_POST_DIRURL_deepcheck_and_repair(self):
2891         d = self.POST(self.public_url, t="start-deep-check", repair="true",
2892                       ophandle="124", output="json", followRedirect=True)
2893         d.addCallback(self.wait_for_operation, "124")
2894         def _check_json(data):
2895             self.failUnlessReallyEqual(data["finished"], True)
2896             self.failUnlessReallyEqual(data["count-objects-checked"], 11)
2897             self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 11)
2898             self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2899             self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2900             self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2901             self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2902             self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2903             self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 11)
2904             self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2905             self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2906         d.addCallback(_check_json)
2907         d.addCallback(self.get_operation_results, "124", "html")
2908         def _check_html(res):
2909             self.failUnlessIn("Objects Checked: <span>11</span>", res)
2910
2911             self.failUnlessIn("Objects Healthy (before repair): <span>11</span>", res)
2912             self.failUnlessIn("Objects Unhealthy (before repair): <span>0</span>", res)
2913             self.failUnlessIn("Corrupt Shares (before repair): <span>0</span>", res)
2914
2915             self.failUnlessIn("Repairs Attempted: <span>0</span>", res)
2916             self.failUnlessIn("Repairs Successful: <span>0</span>", res)
2917             self.failUnlessIn("Repairs Unsuccessful: <span>0</span>", res)
2918
2919             self.failUnlessIn("Objects Healthy (after repair): <span>11</span>", res)
2920             self.failUnlessIn("Objects Unhealthy (after repair): <span>0</span>", res)
2921             self.failUnlessIn("Corrupt Shares (after repair): <span>0</span>", res)
2922
2923             self.failUnlessIn(FAVICON_MARKUP, res)
2924         d.addCallback(_check_html)
2925         return d
2926
2927     def test_POST_FILEURL_bad_t(self):
2928         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2929                              "POST to file: bad t=bogus",
2930                              self.POST, self.public_url + "/foo/bar.txt",
2931                              t="bogus")
2932         return d
2933
2934     def test_POST_mkdir(self): # return value?
2935         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2936         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2937         d.addCallback(self.failUnlessNodeKeysAre, [])
2938         return d
2939
2940     def test_POST_mkdir_mdmf(self):
2941         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=mdmf")
2942         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2943         d.addCallback(lambda node:
2944             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2945         return d
2946
2947     def test_POST_mkdir_sdmf(self):
2948         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=sdmf")
2949         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2950         d.addCallback(lambda node:
2951             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2952         return d
2953
2954     def test_POST_mkdir_bad_format(self):
2955         return self.shouldHTTPError("POST_mkdir_bad_format",
2956                                     400, "Bad Request", "Unknown format: foo",
2957                                     self.POST, self.public_url +
2958                                     "/foo?t=mkdir&name=newdir&format=foo")
2959
2960     def test_POST_mkdir_initial_children(self):
2961         (newkids, caps) = self._create_initial_children()
2962         d = self.POST2(self.public_url +
2963                        "/foo?t=mkdir-with-children&name=newdir",
2964                        simplejson.dumps(newkids))
2965         d.addCallback(lambda res:
2966                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2967         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2968         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2969         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2970         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2971         return d
2972
2973     def test_POST_mkdir_initial_children_mdmf(self):
2974         (newkids, caps) = self._create_initial_children()
2975         d = self.POST2(self.public_url +
2976                        "/foo?t=mkdir-with-children&name=newdir&format=mdmf",
2977                        simplejson.dumps(newkids))
2978         d.addCallback(lambda res:
2979                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2980         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2981         d.addCallback(lambda node:
2982             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2983         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2984         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2985                        caps['filecap1'])
2986         return d
2987
2988     # XXX: Duplication.
2989     def test_POST_mkdir_initial_children_sdmf(self):
2990         (newkids, caps) = self._create_initial_children()
2991         d = self.POST2(self.public_url +
2992                        "/foo?t=mkdir-with-children&name=newdir&format=sdmf",
2993                        simplejson.dumps(newkids))
2994         d.addCallback(lambda res:
2995                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2996         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2997         d.addCallback(lambda node:
2998             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2999         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3000         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
3001                        caps['filecap1'])
3002         return d
3003
3004     def test_POST_mkdir_initial_children_bad_format(self):
3005         (newkids, caps) = self._create_initial_children()
3006         return self.shouldHTTPError("POST_mkdir_initial_children_bad_format",
3007                                     400, "Bad Request", "Unknown format: foo",
3008                                     self.POST, self.public_url + \
3009                                     "/foo?t=mkdir-with-children&name=newdir&format=foo",
3010                                     simplejson.dumps(newkids))
3011
3012     def test_POST_mkdir_immutable(self):
3013         (newkids, caps) = self._create_immutable_children()
3014         d = self.POST2(self.public_url +
3015                        "/foo?t=mkdir-immutable&name=newdir",
3016                        simplejson.dumps(newkids))
3017         d.addCallback(lambda res:
3018                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
3019         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3020         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
3021         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3022         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
3023         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3024         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
3025         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3026         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
3027         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3028         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
3029         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3030         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
3031         return d
3032
3033     def test_POST_mkdir_immutable_bad(self):
3034         (newkids, caps) = self._create_initial_children()
3035         d = self.shouldFail2(error.Error, "POST_mkdir_immutable_bad",
3036                              "400 Bad Request",
3037                              "needed to be immutable but was not",
3038                              self.POST2,
3039                              self.public_url +
3040                              "/foo?t=mkdir-immutable&name=newdir",
3041                              simplejson.dumps(newkids))
3042         return d
3043
3044     def test_POST_mkdir_2(self):
3045         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
3046         d.addCallback(lambda res:
3047                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
3048         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3049         d.addCallback(self.failUnlessNodeKeysAre, [])
3050         return d
3051
3052     def test_POST_mkdirs_2(self):
3053         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
3054         d.addCallback(lambda res:
3055                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
3056         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
3057         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
3058         d.addCallback(self.failUnlessNodeKeysAre, [])
3059         return d
3060
3061     def test_POST_mkdir_no_parentdir_noredirect(self):
3062         d = self.POST("/uri?t=mkdir")
3063         def _after_mkdir(res):
3064             uri.DirectoryURI.init_from_string(res)
3065         d.addCallback(_after_mkdir)
3066         return d
3067
3068     def test_POST_mkdir_no_parentdir_noredirect_mdmf(self):
3069         d = self.POST("/uri?t=mkdir&format=mdmf")
3070         def _after_mkdir(res):
3071             u = uri.from_string(res)
3072             # Check that this is an MDMF writecap
3073             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
3074         d.addCallback(_after_mkdir)
3075         return d
3076
3077     def test_POST_mkdir_no_parentdir_noredirect_sdmf(self):
3078         d = self.POST("/uri?t=mkdir&format=sdmf")
3079         def _after_mkdir(res):
3080             u = uri.from_string(res)
3081             self.failUnlessIsInstance(u, uri.DirectoryURI)
3082         d.addCallback(_after_mkdir)
3083         return d
3084
3085     def test_POST_mkdir_no_parentdir_noredirect_bad_format(self):
3086         return self.shouldHTTPError("POST_mkdir_no_parentdir_noredirect_bad_format",
3087                                     400, "Bad Request", "Unknown format: foo",
3088                                     self.POST, self.public_url +
3089                                     "/uri?t=mkdir&format=foo")
3090
3091     def test_POST_mkdir_no_parentdir_noredirect2(self):
3092         # make sure form-based arguments (as on the welcome page) still work
3093         d = self.POST("/uri", t="mkdir")
3094         def _after_mkdir(res):
3095             uri.DirectoryURI.init_from_string(res)
3096         d.addCallback(_after_mkdir)
3097         d.addErrback(self.explain_web_error)
3098         return d
3099
3100     def test_POST_mkdir_no_parentdir_redirect(self):
3101         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
3102         d.addBoth(self.shouldRedirect, None, statuscode='303')
3103         def _check_target(target):
3104             target = urllib.unquote(target)
3105             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
3106         d.addCallback(_check_target)
3107         return d
3108
3109     def test_POST_mkdir_no_parentdir_redirect2(self):
3110         d = self.POST("/uri", t="mkdir", redirect_to_result="true")
3111         d.addBoth(self.shouldRedirect, None, statuscode='303')
3112         def _check_target(target):
3113             target = urllib.unquote(target)
3114             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
3115         d.addCallback(_check_target)
3116         d.addErrback(self.explain_web_error)
3117         return d
3118
3119     def _make_readonly(self, u):
3120         ro_uri = uri.from_string(u).get_readonly()
3121         if ro_uri is None:
3122             return None
3123         return ro_uri.to_string()
3124
3125     def _create_initial_children(self):
3126         contents, n, filecap1 = self.makefile(12)
3127         md1 = {"metakey1": "metavalue1"}
3128         filecap2 = make_mutable_file_uri()
3129         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
3130         filecap3 = node3.get_readonly_uri()
3131         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
3132         dircap = DirectoryNode(node4, None, None).get_uri()
3133         mdmfcap = make_mutable_file_uri(mdmf=True)
3134         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
3135         emptydircap = "URI:DIR2-LIT:"
3136         newkids = {u"child-imm":        ["filenode", {"rw_uri": filecap1,
3137                                                       "ro_uri": self._make_readonly(filecap1),
3138                                                       "metadata": md1, }],
3139                    u"child-mutable":    ["filenode", {"rw_uri": filecap2,
3140                                                       "ro_uri": self._make_readonly(filecap2)}],
3141                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
3142                    u"unknownchild-rw":  ["unknown",  {"rw_uri": unknown_rwcap,
3143                                                       "ro_uri": unknown_rocap}],
3144                    u"unknownchild-ro":  ["unknown",  {"ro_uri": unknown_rocap}],
3145                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
3146                    u"dirchild":         ["dirnode",  {"rw_uri": dircap,
3147                                                       "ro_uri": self._make_readonly(dircap)}],
3148                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
3149                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
3150                    u"child-mutable-mdmf": ["filenode", {"rw_uri": mdmfcap,
3151                                                         "ro_uri": self._make_readonly(mdmfcap)}],
3152                    }
3153         return newkids, {'filecap1': filecap1,
3154                          'filecap2': filecap2,
3155                          'filecap3': filecap3,
3156                          'unknown_rwcap': unknown_rwcap,
3157                          'unknown_rocap': unknown_rocap,
3158                          'unknown_immcap': unknown_immcap,
3159                          'dircap': dircap,
3160                          'litdircap': litdircap,
3161                          'emptydircap': emptydircap,
3162                          'mdmfcap': mdmfcap}
3163
3164     def _create_immutable_children(self):
3165         contents, n, filecap1 = self.makefile(12)
3166         md1 = {"metakey1": "metavalue1"}
3167         tnode = create_chk_filenode("immutable directory contents\n"*10,
3168                                     self.get_all_contents())
3169         dnode = DirectoryNode(tnode, None, None)
3170         assert not dnode.is_mutable()
3171         immdircap = dnode.get_uri()
3172         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
3173         emptydircap = "URI:DIR2-LIT:"
3174         newkids = {u"child-imm":        ["filenode", {"ro_uri": filecap1,
3175                                                       "metadata": md1, }],
3176                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
3177                    u"dirchild-imm":     ["dirnode",  {"ro_uri": immdircap}],
3178                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
3179                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
3180                    }
3181         return newkids, {'filecap1': filecap1,
3182                          'unknown_immcap': unknown_immcap,
3183                          'immdircap': immdircap,
3184                          'litdircap': litdircap,
3185                          'emptydircap': emptydircap}
3186
3187     def test_POST_mkdir_no_parentdir_initial_children(self):
3188         (newkids, caps) = self._create_initial_children()
3189         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
3190         def _after_mkdir(res):
3191             self.failUnless(res.startswith("URI:DIR"), res)
3192             n = self.s.create_node_from_uri(res)
3193             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3194             d2.addCallback(lambda ign:
3195                            self.failUnlessROChildURIIs(n, u"child-imm",
3196                                                        caps['filecap1']))
3197             d2.addCallback(lambda ign:
3198                            self.failUnlessRWChildURIIs(n, u"child-mutable",
3199                                                        caps['filecap2']))
3200             d2.addCallback(lambda ign:
3201                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
3202                                                        caps['filecap3']))
3203             d2.addCallback(lambda ign:
3204                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
3205                                                        caps['unknown_rwcap']))
3206             d2.addCallback(lambda ign:
3207                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
3208                                                        caps['unknown_rocap']))
3209             d2.addCallback(lambda ign:
3210                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3211                                                        caps['unknown_immcap']))
3212             d2.addCallback(lambda ign:
3213                            self.failUnlessRWChildURIIs(n, u"dirchild",
3214                                                        caps['dircap']))
3215             return d2
3216         d.addCallback(_after_mkdir)
3217         return d
3218
3219     def test_POST_mkdir_no_parentdir_unexpected_children(self):
3220         # the regular /uri?t=mkdir operation is specified to ignore its body.
3221         # Only t=mkdir-with-children pays attention to it.
3222         (newkids, caps) = self._create_initial_children()
3223         d = self.shouldHTTPError("POST_mkdir_no_parentdir_unexpected_children",
3224                                  400, "Bad Request",
3225                                  "t=mkdir does not accept children=, "
3226                                  "try t=mkdir-with-children instead",
3227                                  self.POST2, "/uri?t=mkdir", # without children
3228                                  simplejson.dumps(newkids))
3229         return d
3230
3231     def test_POST_noparent_bad(self):
3232         d = self.shouldHTTPError("POST_noparent_bad",
3233                                  400, "Bad Request",
3234                                  "/uri accepts only PUT, PUT?t=mkdir, "
3235                                  "POST?t=upload, and POST?t=mkdir",
3236                                  self.POST, "/uri?t=bogus")
3237         return d
3238
3239     def test_POST_mkdir_no_parentdir_immutable(self):
3240         (newkids, caps) = self._create_immutable_children()
3241         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
3242         def _after_mkdir(res):
3243             self.failUnless(res.startswith("URI:DIR"), res)
3244             n = self.s.create_node_from_uri(res)
3245             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3246             d2.addCallback(lambda ign:
3247                            self.failUnlessROChildURIIs(n, u"child-imm",
3248                                                           caps['filecap1']))
3249             d2.addCallback(lambda ign:
3250                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3251                                                           caps['unknown_immcap']))
3252             d2.addCallback(lambda ign:
3253                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
3254                                                           caps['immdircap']))
3255             d2.addCallback(lambda ign:
3256                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
3257                                                           caps['litdircap']))
3258             d2.addCallback(lambda ign:
3259                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
3260                                                           caps['emptydircap']))
3261             return d2
3262         d.addCallback(_after_mkdir)
3263         return d
3264
3265     def test_POST_mkdir_no_parentdir_immutable_bad(self):
3266         (newkids, caps) = self._create_initial_children()
3267         d = self.shouldFail2(error.Error,
3268                              "test_POST_mkdir_no_parentdir_immutable_bad",
3269                              "400 Bad Request",
3270                              "needed to be immutable but was not",
3271                              self.POST2,
3272                              "/uri?t=mkdir-immutable",
3273                              simplejson.dumps(newkids))
3274         return d
3275
3276     def test_welcome_page_mkdir_button(self):
3277         # Fetch the welcome page.
3278         d = self.GET("/")
3279         def _after_get_welcome_page(res):
3280             MKDIR_BUTTON_RE = re.compile(
3281                 '<form action="([^"]*)" method="post".*'
3282                 '<input type="hidden" name="t" value="([^"]*)" />[ ]*'
3283                 '<input type="hidden" name="([^"]*)" value="([^"]*)" />[ ]*'
3284                 '<input type="submit" class="btn" value="Create a directory[^"]*" />')
3285             html = res.replace('\n', ' ')
3286             mo = MKDIR_BUTTON_RE.search(html)
3287             self.failUnless(mo, html)
3288             formaction = mo.group(1)
3289             formt = mo.group(2)
3290             formaname = mo.group(3)
3291             formavalue = mo.group(4)
3292             return (formaction, formt, formaname, formavalue)
3293         d.addCallback(_after_get_welcome_page)
3294         def _after_parse_form(res):
3295             (formaction, formt, formaname, formavalue) = res
3296             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
3297         d.addCallback(_after_parse_form)
3298         d.addBoth(self.shouldRedirect, None, statuscode='303')
3299         return d
3300
3301     def test_POST_mkdir_replace(self): # return value?
3302         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
3303         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3304         d.addCallback(self.failUnlessNodeKeysAre, [])
3305         return d
3306
3307     def test_POST_mkdir_no_replace_queryarg(self): # return value?
3308         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
3309         d.addBoth(self.shouldFail, error.Error,
3310                   "POST_mkdir_no_replace_queryarg",
3311                   "409 Conflict",
3312                   "There was already a child by that name, and you asked me "
3313                   "to not replace it")
3314         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3315         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3316         return d
3317
3318     def test_POST_mkdir_no_replace_field(self): # return value?
3319         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
3320                       replace="false")
3321         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
3322                   "409 Conflict",
3323                   "There was already a child by that name, and you asked me "
3324                   "to not replace it")
3325         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3326         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3327         return d
3328
3329     def test_POST_mkdir_whendone_field(self):
3330         d = self.POST(self.public_url + "/foo",
3331                       t="mkdir", name="newdir", when_done="/THERE")
3332         d.addBoth(self.shouldRedirect, "/THERE")
3333         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3334         d.addCallback(self.failUnlessNodeKeysAre, [])
3335         return d
3336
3337     def test_POST_mkdir_whendone_queryarg(self):
3338         d = self.POST(self.public_url + "/foo?when_done=/THERE",
3339                       t="mkdir", name="newdir")
3340         d.addBoth(self.shouldRedirect, "/THERE")
3341         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3342         d.addCallback(self.failUnlessNodeKeysAre, [])
3343         return d
3344
3345     def test_POST_bad_t(self):
3346         d = self.shouldFail2(error.Error, "POST_bad_t",
3347                              "400 Bad Request",
3348                              "POST to a directory with bad t=BOGUS",
3349                              self.POST, self.public_url + "/foo", t="BOGUS")
3350         return d
3351
3352     def test_POST_set_children(self, command_name="set_children"):
3353         contents9, n9, newuri9 = self.makefile(9)
3354         contents10, n10, newuri10 = self.makefile(10)
3355         contents11, n11, newuri11 = self.makefile(11)
3356
3357         reqbody = """{
3358                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
3359                                                 "size": 0,
3360                                                 "metadata": {
3361                                                   "ctime": 1002777696.7564139,
3362                                                   "mtime": 1002777696.7564139
3363                                                  }
3364                                                } ],
3365                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
3366                                                 "size": 1,
3367                                                 "metadata": {
3368                                                   "ctime": 1002777696.7564139,
3369                                                   "mtime": 1002777696.7564139
3370                                                  }
3371                                                } ],
3372                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
3373                                                 "size": 2,
3374                                                 "metadata": {
3375                                                   "ctime": 1002777696.7564139,
3376                                                   "mtime": 1002777696.7564139
3377                                                  }
3378                                                } ]
3379                     }""" % (newuri9, newuri10, newuri11)
3380
3381         url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
3382
3383         d = client.getPage(url, method="POST", postdata=reqbody)
3384         def _then(res):
3385             self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
3386             self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
3387             self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
3388
3389         d.addCallback(_then)
3390         d.addErrback(self.dump_error)
3391         return d
3392
3393     def test_POST_set_children_with_hyphen(self):
3394         return self.test_POST_set_children(command_name="set-children")
3395
3396     def test_POST_link_uri(self):
3397         contents, n, newuri = self.makefile(8)
3398         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
3399         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
3400         d.addCallback(lambda res:
3401                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3402                                                       contents))
3403         return d
3404
3405     def test_POST_link_uri_replace(self):
3406         contents, n, newuri = self.makefile(8)
3407         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
3408         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
3409         d.addCallback(lambda res:
3410                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3411                                                       contents))
3412         return d
3413
3414     def test_POST_link_uri_unknown_bad(self):
3415         d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
3416         d.addBoth(self.shouldFail, error.Error,
3417                   "POST_link_uri_unknown_bad",
3418                   "400 Bad Request",
3419                   "unknown cap in a write slot")
3420         return d
3421
3422     def test_POST_link_uri_unknown_ro_good(self):
3423         d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
3424         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
3425         return d
3426
3427     def test_POST_link_uri_unknown_imm_good(self):
3428         d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
3429         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
3430         return d
3431
3432     def test_POST_link_uri_no_replace_queryarg(self):
3433         contents, n, newuri = self.makefile(8)
3434         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
3435                       name="bar.txt", uri=newuri)
3436         d.addBoth(self.shouldFail, error.Error,
3437                   "POST_link_uri_no_replace_queryarg",
3438                   "409 Conflict",
3439                   "There was already a child by that name, and you asked me "
3440                   "to not replace it")
3441         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3442         d.addCallback(self.failUnlessIsBarDotTxt)
3443         return d
3444
3445     def test_POST_link_uri_no_replace_field(self):
3446         contents, n, newuri = self.makefile(8)
3447         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
3448                       name="bar.txt", uri=newuri)
3449         d.addBoth(self.shouldFail, error.Error,
3450                   "POST_link_uri_no_replace_field",
3451                   "409 Conflict",
3452                   "There was already a child by that name, and you asked me "
3453                   "to not replace it")
3454         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3455         d.addCallback(self.failUnlessIsBarDotTxt)
3456         return d
3457
3458     def test_POST_delete(self, command_name='delete'):
3459         d = self._foo_node.list()
3460         def _check_before(children):
3461             self.failUnlessIn(u"bar.txt", children)
3462         d.addCallback(_check_before)
3463         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t=command_name, name="bar.txt"))
3464         d.addCallback(lambda res: self._foo_node.list())
3465         def _check_after(children):
3466             self.failIfIn(u"bar.txt", children)
3467         d.addCallback(_check_after)
3468         return d
3469
3470     def test_POST_unlink(self):
3471         return self.test_POST_delete(command_name='unlink')
3472
3473     def test_POST_rename_file(self):
3474         d = self.POST(self.public_url + "/foo", t="rename",
3475                       from_name="bar.txt", to_name='wibble.txt')
3476         d.addCallback(lambda res:
3477                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3478         d.addCallback(lambda res:
3479                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
3480         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
3481         d.addCallback(self.failUnlessIsBarDotTxt)
3482         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
3483         d.addCallback(self.failUnlessIsBarJSON)
3484         return d
3485
3486     def test_POST_rename_file_redundant(self):
3487         d = self.POST(self.public_url + "/foo", t="rename",
3488                       from_name="bar.txt", to_name='bar.txt')
3489         d.addCallback(lambda res:
3490                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3491         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3492         d.addCallback(self.failUnlessIsBarDotTxt)
3493         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3494         d.addCallback(self.failUnlessIsBarJSON)
3495         return d
3496
3497     def test_POST_rename_file_replace(self):
3498         # rename a file and replace a directory with it
3499         d = self.POST(self.public_url + "/foo", t="rename",
3500                       from_name="bar.txt", to_name='empty')
3501         d.addCallback(lambda res:
3502                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3503         d.addCallback(lambda res:
3504                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
3505         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
3506         d.addCallback(self.failUnlessIsBarDotTxt)
3507         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3508         d.addCallback(self.failUnlessIsBarJSON)
3509         return d
3510
3511     def test_POST_rename_file_no_replace_queryarg(self):
3512         # rename a file and replace a directory with it
3513         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
3514                       from_name="bar.txt", to_name='empty')
3515         d.addBoth(self.shouldFail, error.Error,
3516                   "POST_rename_file_no_replace_queryarg",
3517                   "409 Conflict",
3518                   "There was already a child by that name, and you asked me "
3519                   "to not replace it")
3520         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3521         d.addCallback(self.failUnlessIsEmptyJSON)
3522         return d
3523
3524     def test_POST_rename_file_no_replace_field(self):
3525         # rename a file and replace a directory with it
3526         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
3527                       from_name="bar.txt", to_name='empty')
3528         d.addBoth(self.shouldFail, error.Error,
3529                   "POST_rename_file_no_replace_field",
3530                   "409 Conflict",
3531                   "There was already a child by that name, and you asked me "
3532                   "to not replace it")
3533         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3534         d.addCallback(self.failUnlessIsEmptyJSON)
3535         return d
3536
3537     def test_POST_rename_file_no_replace_same_link(self):
3538         d = self.POST(self.public_url + "/foo", t="rename",
3539                       replace="false", from_name="bar.txt", to_name="bar.txt")
3540         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3541         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3542         d.addCallback(self.failUnlessIsBarDotTxt)
3543         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3544         d.addCallback(self.failUnlessIsBarJSON)
3545         return d
3546
3547     def test_POST_rename_file_replace_only_files(self):
3548         d = self.POST(self.public_url + "/foo", t="rename",
3549                       replace="only-files", from_name="bar.txt",
3550                       to_name="baz.txt")
3551         d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3552         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3553         d.addCallback(self.failUnlessIsBarDotTxt)
3554         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt?t=json"))
3555         d.addCallback(self.failUnlessIsBarJSON)
3556         return d
3557
3558     def test_POST_rename_file_replace_only_files_conflict(self):
3559         d = self.shouldFail2(error.Error, "POST_relink_file_replace_only_files_conflict",
3560                              "409 Conflict",
3561                              "There was already a child by that name, and you asked me to not replace it.",
3562                              self.POST, self.public_url + "/foo", t="relink",
3563                              replace="only-files", from_name="bar.txt",
3564                              to_name="empty")
3565         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3566         d.addCallback(self.failUnlessIsBarDotTxt)
3567         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3568         d.addCallback(self.failUnlessIsBarJSON)
3569         return d
3570
3571     def failUnlessIsEmptyJSON(self, res):
3572         data = simplejson.loads(res)
3573         self.failUnlessEqual(data[0], "dirnode", data)
3574         self.failUnlessReallyEqual(len(data[1]["children"]), 0)
3575
3576     def test_POST_rename_file_to_slash_fail(self):
3577         d = self.POST(self.public_url + "/foo", t="rename",
3578                       from_name="bar.txt", to_name='kirk/spock.txt')
3579         d.addBoth(self.shouldFail, error.Error,
3580                   "test_POST_rename_file_to_slash_fail",
3581                   "400 Bad Request",
3582                   "to_name= may not contain a slash",
3583                   )
3584         d.addCallback(lambda res:
3585                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3586         return d
3587
3588     def test_POST_rename_file_from_slash_fail(self):
3589         d = self.POST(self.public_url + "/foo", t="rename",
3590                       from_name="sub/bar.txt", to_name='spock.txt')
3591         d.addBoth(self.shouldFail, error.Error,
3592                   "test_POST_rename_from_file_slash_fail",
3593                   "400 Bad Request",
3594                   "from_name= may not contain a slash",
3595                   )
3596         d.addCallback(lambda res:
3597                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3598         return d
3599
3600     def test_POST_rename_dir(self):
3601         d = self.POST(self.public_url, t="rename",
3602                       from_name="foo", to_name='plunk')
3603         d.addCallback(lambda res:
3604                       self.failIfNodeHasChild(self.public_root, u"foo"))
3605         d.addCallback(lambda res:
3606                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
3607         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
3608         d.addCallback(self.failUnlessIsFooJSON)
3609         return d
3610
3611     def test_POST_relink_file(self):
3612         d = self.POST(self.public_url + "/foo", t="relink",
3613                       from_name="bar.txt",
3614                       to_dir=self.public_root.get_uri() + "/foo/sub")
3615         d.addCallback(lambda res:
3616                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3617         d.addCallback(lambda res:
3618                       self.failUnlessNodeHasChild(self._sub_node, u"bar.txt"))
3619         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3620         d.addCallback(self.failUnlessIsBarDotTxt)
3621         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3622         d.addCallback(self.failUnlessIsBarJSON)
3623         return d
3624
3625     def test_POST_relink_file_new_name(self):
3626         d = self.POST(self.public_url + "/foo", t="relink",
3627                       from_name="bar.txt",
3628                       to_name="wibble.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3629         d.addCallback(lambda res:
3630                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3631         d.addCallback(lambda res:
3632                       self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3633         d.addCallback(lambda res:
3634                       self.failUnlessNodeHasChild(self._sub_node, u"wibble.txt"))
3635         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt"))
3636         d.addCallback(self.failUnlessIsBarDotTxt)
3637         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt?t=json"))
3638         d.addCallback(self.failUnlessIsBarJSON)
3639         return d
3640
3641     def test_POST_relink_file_replace(self):
3642         d = self.POST(self.public_url + "/foo", t="relink",
3643                       from_name="bar.txt",
3644                       to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3645         d.addCallback(lambda res:
3646                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3647         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3648         d.addCallback(self.failUnlessIsBarDotTxt)
3649         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt?t=json"))
3650         d.addCallback(self.failUnlessIsBarJSON)
3651         return d
3652
3653     def test_POST_relink_file_no_replace(self):
3654         d = self.shouldFail2(error.Error, "POST_relink_file_no_replace",
3655                              "409 Conflict",
3656                              "There was already a child by that name, and you asked me to not replace it",
3657                              self.POST, self.public_url + "/foo", t="relink",
3658                              replace="false", from_name="bar.txt",
3659                              to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3660         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3661         d.addCallback(self.failUnlessIsBarDotTxt)
3662         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3663         d.addCallback(self.failUnlessIsBarJSON)
3664         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3665         d.addCallback(self.failUnlessIsSubBazDotTxt)
3666         return d
3667
3668     def test_POST_relink_file_no_replace_explicitly_same_link(self):
3669         d = self.POST(self.public_url + "/foo", t="relink",
3670                       replace="false", from_name="bar.txt",
3671                       to_name="bar.txt", to_dir=self.public_root.get_uri() + "/foo")
3672         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3673         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3674         d.addCallback(self.failUnlessIsBarDotTxt)
3675         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3676         d.addCallback(self.failUnlessIsBarJSON)
3677         return d
3678
3679     def test_POST_relink_file_replace_only_files(self):
3680         d = self.POST(self.public_url + "/foo", t="relink",
3681                       replace="only-files", from_name="bar.txt",
3682                       to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3683         d.addCallback(lambda res:
3684                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3685         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3686         d.addCallback(self.failUnlessIsBarDotTxt)
3687         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt?t=json"))
3688         d.addCallback(self.failUnlessIsBarJSON)
3689         return d
3690
3691     def test_POST_relink_file_replace_only_files_conflict(self):
3692         d = self.shouldFail2(error.Error, "POST_relink_file_replace_only_files_conflict",
3693                              "409 Conflict",
3694                              "There was already a child by that name, and you asked me to not replace it.",
3695                              self.POST, self.public_url + "/foo", t="relink",
3696                              replace="only-files", from_name="bar.txt",
3697                              to_name="sub", to_dir=self.public_root.get_uri() + "/foo")
3698         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3699         d.addCallback(self.failUnlessIsBarDotTxt)
3700         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3701         d.addCallback(self.failUnlessIsBarJSON)
3702         return d
3703
3704     def test_POST_relink_file_to_slash_fail(self):
3705         d = self.shouldFail2(error.Error, "test_POST_rename_file_slash_fail",
3706                              "400 Bad Request",
3707                              "to_name= may not contain a slash",
3708                              self.POST, self.public_url + "/foo", t="relink",
3709                              from_name="bar.txt",
3710                              to_name="slash/fail.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
3711         d.addCallback(lambda res:
3712                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3713         d.addCallback(lambda res:
3714                       self.failIfNodeHasChild(self._sub_node, u"slash/fail.txt"))
3715         d.addCallback(lambda ign:
3716                       self.shouldFail2(error.Error,
3717                                        "test_POST_rename_file_slash_fail2",
3718                                        "400 Bad Request",
3719                                        "from_name= may not contain a slash",
3720                                        self.POST, self.public_url + "/foo",
3721                                        t="relink",
3722                                        from_name="nope/bar.txt",
3723                                        to_name="fail.txt",
3724                                        to_dir=self.public_root.get_uri() + "/foo/sub"))
3725         return d
3726
3727     def test_POST_relink_file_explicitly_same_link(self):
3728         d = self.POST(self.public_url + "/foo", t="relink",
3729                       from_name="bar.txt",
3730                       to_name="bar.txt", to_dir=self.public_root.get_uri() + "/foo")
3731         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3732         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3733         d.addCallback(self.failUnlessIsBarDotTxt)
3734         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3735         d.addCallback(self.failUnlessIsBarJSON)
3736         return d
3737
3738     def test_POST_relink_file_implicitly_same_link(self):
3739         d = self.POST(self.public_url + "/foo", t="relink",
3740                       from_name="bar.txt")
3741         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3742         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3743         d.addCallback(self.failUnlessIsBarDotTxt)
3744         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3745         d.addCallback(self.failUnlessIsBarJSON)
3746         return d
3747
3748     def test_POST_relink_file_same_dir(self):
3749         d = self.POST(self.public_url + "/foo", t="relink",
3750                       from_name="bar.txt",
3751                       to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo")
3752         d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3753         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._sub_node, u"baz.txt"))
3754         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3755         d.addCallback(self.failUnlessIsBarDotTxt)
3756         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt?t=json"))
3757         d.addCallback(self.failUnlessIsBarJSON)
3758         return d
3759
3760     def test_POST_relink_file_bad_replace(self):
3761         d = self.shouldFail2(error.Error, "test_POST_relink_file_bad_replace",
3762                              "400 Bad Request", "invalid replace= argument: 'boogabooga'",
3763                              self.POST,
3764                              self.public_url + "/foo", t="relink",
3765                              replace="boogabooga", from_name="bar.txt",
3766                              to_dir=self.public_root.get_uri() + "/foo/sub")
3767         return d
3768
3769     def test_POST_relink_file_multi_level(self):
3770         d = self.POST(self.public_url + "/foo/sub/level2?t=mkdir", "")
3771         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t="relink",
3772                       from_name="bar.txt", to_dir=self.public_root.get_uri() + "/foo/sub/level2"))
3773         d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3774         d.addCallback(lambda res: self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3775         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt"))
3776         d.addCallback(self.failUnlessIsBarDotTxt)
3777         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt?t=json"))
3778         d.addCallback(self.failUnlessIsBarJSON)
3779         return d
3780
3781     def test_POST_relink_file_to_uri(self):
3782         d = self.POST(self.public_url + "/foo", t="relink", target_type="uri",
3783                       from_name="bar.txt", to_dir=self._sub_uri)
3784         d.addCallback(lambda res:
3785                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3786         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3787         d.addCallback(self.failUnlessIsBarDotTxt)
3788         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3789         d.addCallback(self.failUnlessIsBarJSON)
3790         return d
3791
3792     def test_POST_relink_file_to_nonexistent_dir(self):
3793         d = self.shouldFail2(error.Error, "POST_relink_file_to_nonexistent_dir",
3794                             "404 Not Found", "No such child: nopechucktesta",
3795                             self.POST, self.public_url + "/foo", t="relink",
3796                             from_name="bar.txt",
3797                             to_dir=self.public_root.get_uri() + "/nopechucktesta")
3798         return d
3799
3800     def test_POST_relink_file_into_file(self):
3801         d = self.shouldFail2(error.Error, "POST_relink_file_into_file",
3802                              "400 Bad Request", "to_dir is not a directory",
3803                              self.POST, self.public_url + "/foo", t="relink",
3804                              from_name="bar.txt",
3805                              to_dir=self.public_root.get_uri() + "/foo/baz.txt")
3806         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3807         d.addCallback(self.failUnlessIsBazDotTxt)
3808         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3809         d.addCallback(self.failUnlessIsBarDotTxt)
3810         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3811         d.addCallback(self.failUnlessIsBarJSON)
3812         return d
3813
3814     def test_POST_relink_file_to_bad_uri(self):
3815         d =  self.shouldFail2(error.Error, "POST_relink_file_to_bad_uri",
3816                               "400 Bad Request", "to_dir is not a directory",
3817                               self.POST, self.public_url + "/foo", t="relink",
3818                               from_name="bar.txt",
3819                               to_dir="URI:DIR2:mn5jlyjnrjeuydyswlzyui72i:rmneifcj6k6sycjljjhj3f6majsq2zqffydnnul5hfa4j577arma")
3820         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3821         d.addCallback(self.failUnlessIsBarDotTxt)
3822         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3823         d.addCallback(self.failUnlessIsBarJSON)
3824         return d
3825
3826     def test_POST_relink_dir(self):
3827         d = self.POST(self.public_url + "/foo", t="relink",
3828                       from_name="bar.txt",
3829                       to_dir=self.public_root.get_uri() + "/foo/empty")
3830         d.addCallback(lambda res: self.POST(self.public_url + "/foo",
3831                       t="relink", from_name="empty",
3832                       to_dir=self.public_root.get_uri() + "/foo/sub"))
3833         d.addCallback(lambda res:
3834                       self.failIfNodeHasChild(self._foo_node, u"empty"))
3835         d.addCallback(lambda res:
3836                       self.failUnlessNodeHasChild(self._sub_node, u"empty"))
3837         d.addCallback(lambda res:
3838                       self._sub_node.get_child_at_path(u"empty"))
3839         d.addCallback(lambda node:
3840                       self.failUnlessNodeHasChild(node, u"bar.txt"))
3841         d.addCallback(lambda res:
3842                       self.GET(self.public_url + "/foo/sub/empty/bar.txt"))
3843         d.addCallback(self.failUnlessIsBarDotTxt)
3844         return d
3845
3846     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
3847         """ If target is not None then the redirection has to go to target.  If
3848         statuscode is not None then the redirection has to be accomplished with
3849         that HTTP status code."""
3850         if not isinstance(res, failure.Failure):
3851             to_where = (target is None) and "somewhere" or ("to " + target)
3852             self.fail("%s: we were expecting to get redirected %s, not get an"
3853                       " actual page: %s" % (which, to_where, res))
3854         res.trap(error.PageRedirect)
3855         if statuscode is not None:
3856             self.failUnlessReallyEqual(res.value.status, statuscode,
3857                                        "%s: not a redirect" % which)
3858         if target is not None:
3859             # the PageRedirect does not seem to capture the uri= query arg
3860             # properly, so we can't check for it.
3861             realtarget = self.webish_url + target
3862             self.failUnlessReallyEqual(res.value.location, realtarget,
3863                                        "%s: wrong target" % which)
3864         return res.value.location
3865
3866     def test_GET_URI_form(self):
3867         base = "/uri?uri=%s" % self._bar_txt_uri
3868         # this is supposed to give us a redirect to /uri/$URI, plus arguments
3869         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
3870         d = self.GET(base)
3871         d.addBoth(self.shouldRedirect, targetbase)
3872         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
3873         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
3874         d.addCallback(lambda res: self.GET(base+"&t=json"))
3875         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
3876         d.addCallback(self.log, "about to get file by uri")
3877         d.addCallback(lambda res: self.GET(base, followRedirect=True))
3878         d.addCallback(self.failUnlessIsBarDotTxt)
3879         d.addCallback(self.log, "got file by uri, about to get dir by uri")
3880         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
3881                                            followRedirect=True))
3882         d.addCallback(self.failUnlessIsFooJSON)
3883         d.addCallback(self.log, "got dir by uri")
3884
3885         return d
3886
3887     def test_GET_URI_form_bad(self):
3888         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
3889                              "400 Bad Request", "GET /uri requires uri=",
3890                              self.GET, "/uri")
3891         return d
3892
3893     def test_GET_rename_form(self):
3894         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
3895                      followRedirect=True)
3896         def _check(res):
3897             self.failUnlessIn('name="when_done" value="."', res)
3898             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
3899             self.failUnlessIn(FAVICON_MARKUP, res)
3900         d.addCallback(_check)
3901         return d
3902
3903     def log(self, res, msg):
3904         #print "MSG: %s  RES: %s" % (msg, res)
3905         log.msg(msg)
3906         return res
3907
3908     def test_GET_URI_URL(self):
3909         base = "/uri/%s" % self._bar_txt_uri
3910         d = self.GET(base)
3911         d.addCallback(self.failUnlessIsBarDotTxt)
3912         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
3913         d.addCallback(self.failUnlessIsBarDotTxt)
3914         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
3915         d.addCallback(self.failUnlessIsBarDotTxt)
3916         return d
3917
3918     def test_GET_URI_URL_dir(self):
3919         base = "/uri/%s?t=json" % self._foo_uri
3920         d = self.GET(base)
3921         d.addCallback(self.failUnlessIsFooJSON)
3922         return d
3923
3924     def test_GET_URI_URL_missing(self):
3925         base = "/uri/%s" % self._bad_file_uri
3926         d = self.shouldHTTPError("test_GET_URI_URL_missing",
3927                                  http.GONE, None, "NotEnoughSharesError",
3928                                  self.GET, base)
3929         # TODO: how can we exercise both sides of WebDownloadTarget.fail
3930         # here? we must arrange for a download to fail after target.open()
3931         # has been called, and then inspect the response to see that it is
3932         # shorter than we expected.
3933         return d
3934
3935     def test_PUT_DIRURL_uri(self):
3936         d = self.s.create_dirnode()
3937         def _made_dir(dn):
3938             new_uri = dn.get_uri()
3939             # replace /foo with a new (empty) directory
3940             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
3941             d.addCallback(lambda res:
3942                           self.failUnlessReallyEqual(res.strip(), new_uri))
3943             d.addCallback(lambda res:
3944                           self.failUnlessRWChildURIIs(self.public_root,
3945                                                       u"foo",
3946                                                       new_uri))
3947             return d
3948         d.addCallback(_made_dir)
3949         return d
3950
3951     def test_PUT_DIRURL_uri_noreplace(self):
3952         d = self.s.create_dirnode()
3953         def _made_dir(dn):
3954             new_uri = dn.get_uri()
3955             # replace /foo with a new (empty) directory, but ask that
3956             # replace=false, so it should fail
3957             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
3958                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
3959                                  self.PUT,
3960                                  self.public_url + "/foo?t=uri&replace=false",
3961                                  new_uri)
3962             d.addCallback(lambda res:
3963                           self.failUnlessRWChildURIIs(self.public_root,
3964                                                       u"foo",
3965                                                       self._foo_uri))
3966             return d
3967         d.addCallback(_made_dir)
3968         return d
3969
3970     def test_PUT_DIRURL_bad_t(self):
3971         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
3972                              "400 Bad Request", "PUT to a directory",
3973                              self.PUT, self.public_url + "/foo?t=BOGUS", "")
3974         d.addCallback(lambda res:
3975                       self.failUnlessRWChildURIIs(self.public_root,
3976                                                   u"foo",
3977                                                   self._foo_uri))
3978         return d
3979
3980     def test_PUT_NEWFILEURL_uri(self):
3981         contents, n, new_uri = self.makefile(8)
3982         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
3983         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3984         d.addCallback(lambda res:
3985                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3986                                                       contents))
3987         return d
3988
3989     def test_PUT_NEWFILEURL_mdmf(self):
3990         new_contents = self.NEWFILE_CONTENTS * 300000
3991         d = self.PUT(self.public_url + \
3992                      "/foo/mdmf.txt?format=mdmf",
3993                      new_contents)
3994         d.addCallback(lambda ignored:
3995             self.GET(self.public_url + "/foo/mdmf.txt?t=json"))
3996         def _got_json(json):
3997             data = simplejson.loads(json)
3998             data = data[1]
3999             self.failUnlessIn("format", data)
4000             self.failUnlessEqual(data["format"], "MDMF")
4001             self.failUnless(data['rw_uri'].startswith("URI:MDMF"))
4002             self.failUnless(data['ro_uri'].startswith("URI:MDMF"))
4003         d.addCallback(_got_json)
4004         return d
4005
4006     def test_PUT_NEWFILEURL_sdmf(self):
4007         new_contents = self.NEWFILE_CONTENTS * 300000
4008         d = self.PUT(self.public_url + \
4009                      "/foo/sdmf.txt?format=sdmf",
4010                      new_contents)
4011         d.addCallback(lambda ignored:
4012             self.GET(self.public_url + "/foo/sdmf.txt?t=json"))
4013         def _got_json(json):
4014             data = simplejson.loads(json)
4015             data = data[1]
4016             self.failUnlessIn("format", data)
4017             self.failUnlessEqual(data["format"], "SDMF")
4018         d.addCallback(_got_json)
4019         return d
4020
4021     def test_PUT_NEWFILEURL_bad_format(self):
4022         new_contents = self.NEWFILE_CONTENTS * 300000
4023         return self.shouldHTTPError("PUT_NEWFILEURL_bad_format",
4024                                     400, "Bad Request", "Unknown format: foo",
4025                                     self.PUT, self.public_url + \
4026                                     "/foo/foo.txt?format=foo",
4027                                     new_contents)
4028
4029     def test_PUT_NEWFILEURL_uri_replace(self):
4030         contents, n, new_uri = self.makefile(8)
4031         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
4032         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
4033         d.addCallback(lambda res:
4034                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
4035                                                       contents))
4036         return d
4037
4038     def test_PUT_NEWFILEURL_uri_no_replace(self):
4039         contents, n, new_uri = self.makefile(8)
4040         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
4041         d.addBoth(self.shouldFail, error.Error,
4042                   "PUT_NEWFILEURL_uri_no_replace",
4043                   "409 Conflict",
4044                   "There was already a child by that name, and you asked me "
4045                   "to not replace it")
4046         return d
4047
4048     def test_PUT_NEWFILEURL_uri_unknown_bad(self):
4049         d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
4050         d.addBoth(self.shouldFail, error.Error,
4051                   "POST_put_uri_unknown_bad",
4052                   "400 Bad Request",
4053                   "unknown cap in a write slot")
4054         return d
4055
4056     def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
4057         d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
4058         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
4059                       u"put-future-ro.txt")
4060         return d
4061
4062     def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
4063         d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
4064         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
4065                       u"put-future-imm.txt")
4066         return d
4067
4068     def test_PUT_NEWFILE_URI(self):
4069         file_contents = "New file contents here\n"
4070         d = self.PUT("/uri", file_contents)
4071         def _check(uri):
4072             assert isinstance(uri, str), uri
4073             self.failUnlessIn(uri, self.get_all_contents())
4074             self.failUnlessReallyEqual(self.get_all_contents()[uri],
4075                                        file_contents)
4076             return self.GET("/uri/%s" % uri)
4077         d.addCallback(_check)
4078         def _check2(res):
4079             self.failUnlessReallyEqual(res, file_contents)
4080         d.addCallback(_check2)
4081         return d
4082
4083     def test_PUT_NEWFILE_URI_not_mutable(self):
4084         file_contents = "New file contents here\n"
4085         d = self.PUT("/uri?mutable=false", file_contents)
4086         def _check(uri):
4087             assert isinstance(uri, str), uri
4088             self.failUnlessIn(uri, self.get_all_contents())
4089             self.failUnlessReallyEqual(self.get_all_contents()[uri],
4090                                        file_contents)
4091             return self.GET("/uri/%s" % uri)
4092         d.addCallback(_check)
4093         def _check2(res):
4094             self.failUnlessReallyEqual(res, file_contents)
4095         d.addCallback(_check2)
4096         return d
4097
4098     def test_PUT_NEWFILE_URI_only_PUT(self):
4099         d = self.PUT("/uri?t=bogus", "")
4100         d.addBoth(self.shouldFail, error.Error,
4101                   "PUT_NEWFILE_URI_only_PUT",
4102                   "400 Bad Request",
4103                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
4104         return d
4105
4106     def test_PUT_NEWFILE_URI_mutable(self):
4107         file_contents = "New file contents here\n"
4108         d = self.PUT("/uri?mutable=true", file_contents)
4109         def _check1(filecap):
4110             filecap = filecap.strip()
4111             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
4112             self.filecap = filecap
4113             u = uri.WriteableSSKFileURI.init_from_string(filecap)
4114             self.failUnlessIn(u.get_storage_index(), self.get_all_contents())
4115             n = self.s.create_node_from_uri(filecap)
4116             return n.download_best_version()
4117         d.addCallback(_check1)
4118         def _check2(data):
4119             self.failUnlessReallyEqual(data, file_contents)
4120             return self.GET("/uri/%s" % urllib.quote(self.filecap))
4121         d.addCallback(_check2)
4122         def _check3(res):
4123             self.failUnlessReallyEqual(res, file_contents)
4124         d.addCallback(_check3)
4125         return d
4126
4127     def test_PUT_mkdir(self):
4128         d = self.PUT("/uri?t=mkdir", "")
4129         def _check(uri):
4130             n = self.s.create_node_from_uri(uri.strip())
4131             d2 = self.failUnlessNodeKeysAre(n, [])
4132             d2.addCallback(lambda res:
4133                            self.GET("/uri/%s?t=json" % uri))
4134             return d2
4135         d.addCallback(_check)
4136         d.addCallback(self.failUnlessIsEmptyJSON)
4137         return d
4138
4139     def test_PUT_mkdir_mdmf(self):
4140         d = self.PUT("/uri?t=mkdir&format=mdmf", "")
4141         def _got(res):
4142             u = uri.from_string(res)
4143             # Check that this is an MDMF writecap
4144             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
4145         d.addCallback(_got)
4146         return d
4147
4148     def test_PUT_mkdir_sdmf(self):
4149         d = self.PUT("/uri?t=mkdir&format=sdmf", "")
4150         def _got(res):
4151             u = uri.from_string(res)
4152             self.failUnlessIsInstance(u, uri.DirectoryURI)
4153         d.addCallback(_got)
4154         return d
4155
4156     def test_PUT_mkdir_bad_format(self):
4157         return self.shouldHTTPError("PUT_mkdir_bad_format",
4158                                     400, "Bad Request", "Unknown format: foo",
4159                                     self.PUT, "/uri?t=mkdir&format=foo",
4160                                     "")
4161
4162     def test_POST_check(self):
4163         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
4164         def _done(res):
4165             # this returns a string form of the results, which are probably
4166             # None since we're using fake filenodes.
4167             # TODO: verify that the check actually happened, by changing
4168             # FakeCHKFileNode to count how many times .check() has been
4169             # called.
4170             pass
4171         d.addCallback(_done)
4172         return d
4173
4174
4175     def test_PUT_update_at_offset(self):
4176         file_contents = "test file" * 100000 # about 900 KiB
4177         d = self.PUT("/uri?mutable=true", file_contents)
4178         def _then(filecap):
4179             self.filecap = filecap
4180             new_data = file_contents[:100]
4181             new = "replaced and so on"
4182             new_data += new
4183             new_data += file_contents[len(new_data):]
4184             assert len(new_data) == len(file_contents)
4185             self.new_data = new_data
4186         d.addCallback(_then)
4187         d.addCallback(lambda ignored:
4188             self.PUT("/uri/%s?replace=True&offset=100" % self.filecap,
4189                      "replaced and so on"))
4190         def _get_data(filecap):
4191             n = self.s.create_node_from_uri(filecap)
4192             return n.download_best_version()
4193         d.addCallback(_get_data)
4194         d.addCallback(lambda results:
4195             self.failUnlessEqual(results, self.new_data))
4196         # Now try appending things to the file
4197         d.addCallback(lambda ignored:
4198             self.PUT("/uri/%s?offset=%d" % (self.filecap, len(self.new_data)),
4199                      "puppies" * 100))
4200         d.addCallback(_get_data)
4201         d.addCallback(lambda results:
4202             self.failUnlessEqual(results, self.new_data + ("puppies" * 100)))
4203         # and try replacing the beginning of the file
4204         d.addCallback(lambda ignored:
4205             self.PUT("/uri/%s?offset=0" % self.filecap, "begin"))
4206         d.addCallback(_get_data)
4207         d.addCallback(lambda results:
4208             self.failUnlessEqual(results, "begin"+self.new_data[len("begin"):]+("puppies"*100)))
4209         return d
4210
4211     def test_PUT_update_at_invalid_offset(self):
4212         file_contents = "test file" * 100000 # about 900 KiB
4213         d = self.PUT("/uri?mutable=true", file_contents)
4214         def _then(filecap):
4215             self.filecap = filecap
4216         d.addCallback(_then)
4217         # Negative offsets should cause an error.
4218         d.addCallback(lambda ignored:
4219             self.shouldHTTPError("PUT_update_at_invalid_offset",
4220                                  400, "Bad Request",
4221                                  "Invalid offset",
4222                                  self.PUT,
4223                                  "/uri/%s?offset=-1" % self.filecap,
4224                                  "foo"))
4225         return d
4226
4227     def test_PUT_update_at_offset_immutable(self):
4228         file_contents = "Test file" * 100000
4229         d = self.PUT("/uri", file_contents)
4230         def _then(filecap):
4231             self.filecap = filecap
4232         d.addCallback(_then)
4233         d.addCallback(lambda ignored:
4234             self.shouldHTTPError("PUT_update_at_offset_immutable",
4235                                  400, "Bad Request",
4236                                  "immutable",
4237                                  self.PUT,
4238                                  "/uri/%s?offset=50" % self.filecap,
4239                                  "foo"))
4240         return d
4241
4242
4243     def test_bad_method(self):
4244         url = self.webish_url + self.public_url + "/foo/bar.txt"
4245         d = self.shouldHTTPError("bad_method",
4246                                  501, "Not Implemented",
4247                                  "I don't know how to treat a BOGUS request.",
4248                                  client.getPage, url, method="BOGUS")
4249         return d
4250
4251     def test_short_url(self):
4252         url = self.webish_url + "/uri"
4253         d = self.shouldHTTPError("short_url", 501, "Not Implemented",
4254                                  "I don't know how to treat a DELETE request.",
4255                                  client.getPage, url, method="DELETE")
4256         return d
4257
4258     def test_ophandle_bad(self):
4259         url = self.webish_url + "/operations/bogus?t=status"
4260         d = self.shouldHTTPError("ophandle_bad", 404, "404 Not Found",
4261                                  "unknown/expired handle 'bogus'",
4262                                  client.getPage, url)
4263         return d
4264
4265     def test_ophandle_cancel(self):
4266         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
4267                       followRedirect=True)
4268         d.addCallback(lambda ignored:
4269                       self.GET("/operations/128?t=status&output=JSON"))
4270         def _check1(res):
4271             data = simplejson.loads(res)
4272             self.failUnless("finished" in data, res)
4273             monitor = self.ws.root.child_operations.handles["128"][0]
4274             d = self.POST("/operations/128?t=cancel&output=JSON")
4275             def _check2(res):
4276                 data = simplejson.loads(res)
4277                 self.failUnless("finished" in data, res)
4278                 # t=cancel causes the handle to be forgotten
4279                 self.failUnless(monitor.is_cancelled())
4280             d.addCallback(_check2)
4281             return d
4282         d.addCallback(_check1)
4283         d.addCallback(lambda ignored:
4284                       self.shouldHTTPError("ophandle_cancel",
4285                                            404, "404 Not Found",
4286                                            "unknown/expired handle '128'",
4287                                            self.GET,
4288                                            "/operations/128?t=status&output=JSON"))
4289         return d
4290
4291     def test_ophandle_retainfor(self):
4292         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
4293                       followRedirect=True)
4294         d.addCallback(lambda ignored:
4295                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
4296         def _check1(res):
4297             data = simplejson.loads(res)
4298             self.failUnless("finished" in data, res)
4299         d.addCallback(_check1)
4300         # the retain-for=0 will cause the handle to be expired very soon
4301         d.addCallback(lambda ign:
4302             self.clock.advance(2.0))
4303         d.addCallback(lambda ignored:
4304                       self.shouldHTTPError("ophandle_retainfor",
4305                                            404, "404 Not Found",
4306                                            "unknown/expired handle '129'",
4307                                            self.GET,
4308                                            "/operations/129?t=status&output=JSON"))
4309         return d
4310
4311     def test_ophandle_release_after_complete(self):
4312         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
4313                       followRedirect=True)
4314         d.addCallback(self.wait_for_operation, "130")
4315         d.addCallback(lambda ignored:
4316                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
4317         # the release-after-complete=true will cause the handle to be expired
4318         d.addCallback(lambda ignored:
4319                       self.shouldHTTPError("ophandle_release_after_complete",
4320                                            404, "404 Not Found",
4321                                            "unknown/expired handle '130'",
4322                                            self.GET,
4323                                            "/operations/130?t=status&output=JSON"))
4324         return d
4325
4326     def test_uncollected_ophandle_expiration(self):
4327         # uncollected ophandles should expire after 4 days
4328         def _make_uncollected_ophandle(ophandle):
4329             d = self.POST(self.public_url +
4330                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
4331                           followRedirect=False)
4332             # When we start the operation, the webapi server will want
4333             # to redirect us to the page for the ophandle, so we get
4334             # confirmation that the operation has started. If the
4335             # manifest operation has finished by the time we get there,
4336             # following that redirect (by setting followRedirect=True
4337             # above) has the side effect of collecting the ophandle that
4338             # we've just created, which means that we can't use the
4339             # ophandle to test the uncollected timeout anymore. So,
4340             # instead, catch the 302 here and don't follow it.
4341             d.addBoth(self.should302, "uncollected_ophandle_creation")
4342             return d
4343         # Create an ophandle, don't collect it, then advance the clock by
4344         # 4 days - 1 second and make sure that the ophandle is still there.
4345         d = _make_uncollected_ophandle(131)
4346         d.addCallback(lambda ign:
4347             self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
4348         d.addCallback(lambda ign:
4349             self.GET("/operations/131?t=status&output=JSON"))
4350         def _check1(res):
4351             data = simplejson.loads(res)
4352             self.failUnless("finished" in data, res)
4353         d.addCallback(_check1)
4354         # Create an ophandle, don't collect it, then try to collect it
4355         # after 4 days. It should be gone.
4356         d.addCallback(lambda ign:
4357             _make_uncollected_ophandle(132))
4358         d.addCallback(lambda ign:
4359             self.clock.advance(96*60*60))
4360         d.addCallback(lambda ign:
4361             self.shouldHTTPError("uncollected_ophandle_expired_after_100_hours",
4362                                  404, "404 Not Found",
4363                                  "unknown/expired handle '132'",
4364                                  self.GET,
4365                                  "/operations/132?t=status&output=JSON"))
4366         return d
4367
4368     def test_collected_ophandle_expiration(self):
4369         # collected ophandles should expire after 1 day
4370         def _make_collected_ophandle(ophandle):
4371             d = self.POST(self.public_url +
4372                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
4373                           followRedirect=True)
4374             # By following the initial redirect, we collect the ophandle
4375             # we've just created.
4376             return d
4377         # Create a collected ophandle, then collect it after 23 hours
4378         # and 59 seconds to make sure that it is still there.
4379         d = _make_collected_ophandle(133)
4380         d.addCallback(lambda ign:
4381             self.clock.advance((24*60*60) - 1))
4382         d.addCallback(lambda ign:
4383             self.GET("/operations/133?t=status&output=JSON"))
4384         def _check1(res):
4385             data = simplejson.loads(res)
4386             self.failUnless("finished" in data, res)
4387         d.addCallback(_check1)
4388         # Create another uncollected ophandle, then try to collect it
4389         # after 24 hours to make sure that it is gone.
4390         d.addCallback(lambda ign:
4391             _make_collected_ophandle(134))
4392         d.addCallback(lambda ign:
4393             self.clock.advance(24*60*60))
4394         d.addCallback(lambda ign:
4395             self.shouldHTTPError("collected_ophandle_expired_after_1_day",
4396                                  404, "404 Not Found",
4397                                  "unknown/expired handle '134'",
4398                                  self.GET,
4399                                  "/operations/134?t=status&output=JSON"))
4400         return d
4401
4402     def test_incident(self):
4403         d = self.POST("/report_incident", details="eek")
4404         def _done(res):
4405             self.failIfIn("<html>", res)
4406             self.failUnlessIn("An incident report has been saved", res)
4407         d.addCallback(_done)
4408         return d
4409
4410     def test_static(self):
4411         webdir = os.path.join(self.staticdir, "subdir")
4412         fileutil.make_dirs(webdir)
4413         f = open(os.path.join(webdir, "hello.txt"), "wb")
4414         f.write("hello")
4415         f.close()
4416
4417         d = self.GET("/static/subdir/hello.txt")
4418         def _check(res):
4419             self.failUnlessReallyEqual(res, "hello")
4420         d.addCallback(_check)
4421         return d
4422
4423
4424 class IntroducerWeb(unittest.TestCase):
4425     def setUp(self):
4426         self.node = None
4427
4428     def tearDown(self):
4429         d = defer.succeed(None)
4430         if self.node:
4431             d.addCallback(lambda ign: self.node.stopService())
4432         d.addCallback(flushEventualQueue)
4433         return d
4434
4435     def test_welcome(self):
4436         basedir = "web.IntroducerWeb.test_welcome"
4437         os.mkdir(basedir)
4438         fileutil.write(os.path.join(basedir, "tahoe.cfg"), "[node]\nweb.port = tcp:0\n")
4439         self.node = IntroducerNode(basedir)
4440         self.ws = self.node.getServiceNamed("webish")
4441
4442         d = fireEventually(None)
4443         d.addCallback(lambda ign: self.node.startService())
4444         d.addCallback(lambda ign: self.node.when_tub_ready())
4445
4446         d.addCallback(lambda ign: self.GET("/"))
4447         def _check(res):
4448             self.failUnlessIn('Welcome to the Tahoe-LAFS Introducer', res)
4449             self.failUnlessIn(FAVICON_MARKUP, res)
4450             self.failUnlessIn('Page rendered at', res)
4451             self.failUnlessIn('Tahoe-LAFS code imported from:', res)
4452         d.addCallback(_check)
4453         return d
4454
4455     def GET(self, urlpath, followRedirect=False, return_response=False,
4456             **kwargs):
4457         # if return_response=True, this fires with (data, statuscode,
4458         # respheaders) instead of just data.
4459         assert not isinstance(urlpath, unicode)
4460         url = self.ws.getURL().rstrip('/') + urlpath
4461         factory = HTTPClientGETFactory(url, method="GET",
4462                                        followRedirect=followRedirect, **kwargs)
4463         reactor.connectTCP("localhost", self.ws.getPortnum(), factory)
4464         d = factory.deferred
4465         def _got_data(data):
4466             return (data, factory.status, factory.response_headers)
4467         if return_response:
4468             d.addCallback(_got_data)
4469         return factory.deferred
4470
4471
4472 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4473     def test_load_file(self):
4474         # This will raise an exception unless a well-formed XML file is found under that name.
4475         common.getxmlfile('directory.xhtml').load()
4476
4477     def test_parse_replace_arg(self):
4478         self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
4479         self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
4480         self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
4481                                    "only-files")
4482         self.failUnlessRaises(common.WebError, common.parse_replace_arg, "only_fles")
4483
4484     def test_abbreviate_time(self):
4485         self.failUnlessReallyEqual(common.abbreviate_time(None), "")
4486         self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
4487         self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
4488         self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
4489         self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
4490         self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
4491
4492     def test_compute_rate(self):
4493         self.failUnlessReallyEqual(common.compute_rate(None, None), None)
4494         self.failUnlessReallyEqual(common.compute_rate(None, 1), None)
4495         self.failUnlessReallyEqual(common.compute_rate(250000, None), None)
4496         self.failUnlessReallyEqual(common.compute_rate(250000, 0), None)
4497         self.failUnlessReallyEqual(common.compute_rate(250000, 10), 25000.0)
4498         self.failUnlessReallyEqual(common.compute_rate(0, 10), 0.0)
4499         self.shouldFail(AssertionError, "test_compute_rate", "",
4500                         common.compute_rate, -100, 10)
4501         self.shouldFail(AssertionError, "test_compute_rate", "",
4502                         common.compute_rate, 100, -10)
4503
4504         # Sanity check
4505         rate = common.compute_rate(10*1000*1000, 1)
4506         self.failUnlessReallyEqual(common.abbreviate_rate(rate), "10.00MBps")
4507
4508     def test_abbreviate_rate(self):
4509         self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
4510         self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
4511         self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
4512         self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
4513
4514     def test_abbreviate_size(self):
4515         self.failUnlessReallyEqual(common.abbreviate_size(None), "")
4516         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
4517         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
4518         self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
4519         self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
4520
4521     def test_plural(self):
4522         def convert(s):
4523             return "%d second%s" % (s, status.plural(s))
4524         self.failUnlessReallyEqual(convert(0), "0 seconds")
4525         self.failUnlessReallyEqual(convert(1), "1 second")
4526         self.failUnlessReallyEqual(convert(2), "2 seconds")
4527         def convert2(s):
4528             return "has share%s: %s" % (status.plural(s), ",".join(s))
4529         self.failUnlessReallyEqual(convert2([]), "has shares: ")
4530         self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
4531         self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
4532
4533
4534 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4535
4536     def CHECK(self, ign, which, args, clientnum=0):
4537         fileurl = self.fileurls[which]
4538         url = fileurl + "?" + args
4539         return self.GET(url, method="POST", clientnum=clientnum)
4540
4541     def test_filecheck(self):
4542         self.basedir = "web/Grid/filecheck"
4543         self.set_up_grid()
4544         c0 = self.g.clients[0]
4545         self.uris = {}
4546         DATA = "data" * 100
4547         d = c0.upload(upload.Data(DATA, convergence=""))
4548         def _stash_uri(ur, which):
4549             self.uris[which] = ur.get_uri()
4550         d.addCallback(_stash_uri, "good")
4551         d.addCallback(lambda ign:
4552                       c0.upload(upload.Data(DATA+"1", convergence="")))
4553         d.addCallback(_stash_uri, "sick")
4554         d.addCallback(lambda ign:
4555                       c0.upload(upload.Data(DATA+"2", convergence="")))
4556         d.addCallback(_stash_uri, "dead")
4557         def _stash_mutable_uri(n, which):
4558             self.uris[which] = n.get_uri()
4559             assert isinstance(self.uris[which], str)
4560         d.addCallback(lambda ign:
4561             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4562         d.addCallback(_stash_mutable_uri, "corrupt")
4563         d.addCallback(lambda ign:
4564                       c0.upload(upload.Data("literal", convergence="")))
4565         d.addCallback(_stash_uri, "small")
4566         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
4567         d.addCallback(_stash_mutable_uri, "smalldir")
4568
4569         def _compute_fileurls(ignored):
4570             self.fileurls = {}
4571             for which in self.uris:
4572                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4573         d.addCallback(_compute_fileurls)
4574
4575         def _clobber_shares(ignored):
4576             good_shares = self.find_uri_shares(self.uris["good"])
4577             self.failUnlessReallyEqual(len(good_shares), 10)
4578             sick_shares = self.find_uri_shares(self.uris["sick"])
4579             os.unlink(sick_shares[0][2])
4580             dead_shares = self.find_uri_shares(self.uris["dead"])
4581             for i in range(1, 10):
4582                 os.unlink(dead_shares[i][2])
4583             c_shares = self.find_uri_shares(self.uris["corrupt"])
4584             cso = CorruptShareOptions()
4585             cso.stdout = StringIO()
4586             cso.parseOptions([c_shares[0][2]])
4587             corrupt_share(cso)
4588         d.addCallback(_clobber_shares)
4589
4590         d.addCallback(self.CHECK, "good", "t=check")
4591         def _got_html_good(res):
4592             self.failUnlessIn("Healthy", res)
4593             self.failIfIn("Not Healthy", res)
4594             self.failUnlessIn(FAVICON_MARKUP, res)
4595         d.addCallback(_got_html_good)
4596         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
4597         def _got_html_good_return_to(res):
4598             self.failUnlessIn("Healthy", res)
4599             self.failIfIn("Not Healthy", res)
4600             self.failUnlessIn('<a href="somewhere">Return to file', res)
4601         d.addCallback(_got_html_good_return_to)
4602         d.addCallback(self.CHECK, "good", "t=check&output=json")
4603         def _got_json_good(res):
4604             r = simplejson.loads(res)
4605             self.failUnlessEqual(r["summary"], "Healthy")
4606             self.failUnless(r["results"]["healthy"])
4607             self.failIfIn("needs-rebalancing", r["results"])
4608             self.failUnless(r["results"]["recoverable"])
4609         d.addCallback(_got_json_good)
4610
4611         d.addCallback(self.CHECK, "small", "t=check")
4612         def _got_html_small(res):
4613             self.failUnlessIn("Literal files are always healthy", res)
4614             self.failIfIn("Not Healthy", res)
4615         d.addCallback(_got_html_small)
4616         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
4617         def _got_html_small_return_to(res):
4618             self.failUnlessIn("Literal files are always healthy", res)
4619             self.failIfIn("Not Healthy", res)
4620             self.failUnlessIn('<a href="somewhere">Return to file', res)
4621         d.addCallback(_got_html_small_return_to)
4622         d.addCallback(self.CHECK, "small", "t=check&output=json")
4623         def _got_json_small(res):
4624             r = simplejson.loads(res)
4625             self.failUnlessEqual(r["storage-index"], "")
4626             self.failUnless(r["results"]["healthy"])
4627         d.addCallback(_got_json_small)
4628
4629         d.addCallback(self.CHECK, "smalldir", "t=check")
4630         def _got_html_smalldir(res):
4631             self.failUnlessIn("Literal files are always healthy", res)
4632             self.failIfIn("Not Healthy", res)
4633         d.addCallback(_got_html_smalldir)
4634         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
4635         def _got_json_smalldir(res):
4636             r = simplejson.loads(res)
4637             self.failUnlessEqual(r["storage-index"], "")
4638             self.failUnless(r["results"]["healthy"])
4639         d.addCallback(_got_json_smalldir)
4640
4641         d.addCallback(self.CHECK, "sick", "t=check")
4642         def _got_html_sick(res):
4643             self.failUnlessIn("Not Healthy", res)
4644         d.addCallback(_got_html_sick)
4645         d.addCallback(self.CHECK, "sick", "t=check&output=json")
4646         def _got_json_sick(res):
4647             r = simplejson.loads(res)
4648             self.failUnlessEqual(r["summary"],
4649                                  "Not Healthy: 9 shares (enc 3-of-10)")
4650             self.failIf(r["results"]["healthy"])
4651             self.failUnless(r["results"]["recoverable"])
4652             self.failIfIn("needs-rebalancing", r["results"])
4653         d.addCallback(_got_json_sick)
4654
4655         d.addCallback(self.CHECK, "dead", "t=check")
4656         def _got_html_dead(res):
4657             self.failUnlessIn("Not Healthy", res)
4658         d.addCallback(_got_html_dead)
4659         d.addCallback(self.CHECK, "dead", "t=check&output=json")
4660         def _got_json_dead(res):
4661             r = simplejson.loads(res)
4662             self.failUnlessEqual(r["summary"],
4663                                  "Not Healthy: 1 shares (enc 3-of-10)")
4664             self.failIf(r["results"]["healthy"])
4665             self.failIf(r["results"]["recoverable"])
4666             self.failIfIn("needs-rebalancing", r["results"])
4667         d.addCallback(_got_json_dead)
4668
4669         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
4670         def _got_html_corrupt(res):
4671             self.failUnlessIn("Not Healthy! : Unhealthy", res)
4672         d.addCallback(_got_html_corrupt)
4673         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
4674         def _got_json_corrupt(res):
4675             r = simplejson.loads(res)
4676             self.failUnlessIn("Unhealthy: 9 shares (enc 3-of-10)", r["summary"])
4677             self.failIf(r["results"]["healthy"])
4678             self.failUnless(r["results"]["recoverable"])
4679             self.failIfIn("needs-rebalancing", r["results"])
4680             self.failUnlessReallyEqual(r["results"]["count-happiness"], 9)
4681             self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
4682             self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
4683         d.addCallback(_got_json_corrupt)
4684
4685         d.addErrback(self.explain_web_error)
4686         return d
4687
4688     def test_repair_html(self):
4689         self.basedir = "web/Grid/repair_html"
4690         self.set_up_grid()
4691         c0 = self.g.clients[0]
4692         self.uris = {}
4693         DATA = "data" * 100
4694         d = c0.upload(upload.Data(DATA, convergence=""))
4695         def _stash_uri(ur, which):
4696             self.uris[which] = ur.get_uri()
4697         d.addCallback(_stash_uri, "good")
4698         d.addCallback(lambda ign:
4699                       c0.upload(upload.Data(DATA+"1", convergence="")))
4700         d.addCallback(_stash_uri, "sick")
4701         d.addCallback(lambda ign:
4702                       c0.upload(upload.Data(DATA+"2", convergence="")))
4703         d.addCallback(_stash_uri, "dead")
4704         def _stash_mutable_uri(n, which):
4705             self.uris[which] = n.get_uri()
4706             assert isinstance(self.uris[which], str)
4707         d.addCallback(lambda ign:
4708             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4709         d.addCallback(_stash_mutable_uri, "corrupt")
4710
4711         def _compute_fileurls(ignored):
4712             self.fileurls = {}
4713             for which in self.uris:
4714                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4715         d.addCallback(_compute_fileurls)
4716
4717         def _clobber_shares(ignored):
4718             good_shares = self.find_uri_shares(self.uris["good"])
4719             self.failUnlessReallyEqual(len(good_shares), 10)
4720             sick_shares = self.find_uri_shares(self.uris["sick"])
4721             os.unlink(sick_shares[0][2])
4722             dead_shares = self.find_uri_shares(self.uris["dead"])
4723             for i in range(1, 10):
4724                 os.unlink(dead_shares[i][2])
4725             c_shares = self.find_uri_shares(self.uris["corrupt"])
4726             cso = CorruptShareOptions()
4727             cso.stdout = StringIO()
4728             cso.parseOptions([c_shares[0][2]])
4729             corrupt_share(cso)
4730         d.addCallback(_clobber_shares)
4731
4732         d.addCallback(self.CHECK, "good", "t=check&repair=true")
4733         def _got_html_good(res):
4734             self.failUnlessIn("Healthy", res)
4735             self.failIfIn("Not Healthy", res)
4736             self.failUnlessIn("No repair necessary", res)
4737             self.failUnlessIn(FAVICON_MARKUP, res)
4738         d.addCallback(_got_html_good)
4739
4740         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
4741         def _got_html_sick(res):
4742             self.failUnlessIn("Healthy : healthy", res)
4743             self.failIfIn("Not Healthy", res)
4744             self.failUnlessIn("Repair successful", res)
4745         d.addCallback(_got_html_sick)
4746
4747         # repair of a dead file will fail, of course, but it isn't yet
4748         # clear how this should be reported. Right now it shows up as
4749         # a "410 Gone".
4750         #
4751         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
4752         #def _got_html_dead(res):
4753         #    print res
4754         #    self.failUnlessIn("Healthy : healthy", res)
4755         #    self.failIfIn("Not Healthy", res)
4756         #    self.failUnlessIn("No repair necessary", res)
4757         #d.addCallback(_got_html_dead)
4758
4759         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
4760         def _got_html_corrupt(res):
4761             self.failUnlessIn("Healthy : Healthy", res)
4762             self.failIfIn("Not Healthy", res)
4763             self.failUnlessIn("Repair successful", res)
4764         d.addCallback(_got_html_corrupt)
4765
4766         d.addErrback(self.explain_web_error)
4767         return d
4768
4769     def test_repair_json(self):
4770         self.basedir = "web/Grid/repair_json"
4771         self.set_up_grid()
4772         c0 = self.g.clients[0]
4773         self.uris = {}
4774         DATA = "data" * 100
4775         d = c0.upload(upload.Data(DATA+"1", convergence=""))
4776         def _stash_uri(ur, which):
4777             self.uris[which] = ur.get_uri()
4778         d.addCallback(_stash_uri, "sick")
4779
4780         def _compute_fileurls(ignored):
4781             self.fileurls = {}
4782             for which in self.uris:
4783                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4784         d.addCallback(_compute_fileurls)
4785
4786         def _clobber_shares(ignored):
4787             sick_shares = self.find_uri_shares(self.uris["sick"])
4788             os.unlink(sick_shares[0][2])
4789         d.addCallback(_clobber_shares)
4790
4791         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
4792         def _got_json_sick(res):
4793             r = simplejson.loads(res)
4794             self.failUnlessReallyEqual(r["repair-attempted"], True)
4795             self.failUnlessReallyEqual(r["repair-successful"], True)
4796             self.failUnlessEqual(r["pre-repair-results"]["summary"],
4797                                  "Not Healthy: 9 shares (enc 3-of-10)")
4798             self.failIf(r["pre-repair-results"]["results"]["healthy"])
4799             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
4800             self.failUnless(r["post-repair-results"]["results"]["healthy"])
4801         d.addCallback(_got_json_sick)
4802
4803         d.addErrback(self.explain_web_error)
4804         return d
4805
4806     def test_unknown(self, immutable=False):
4807         self.basedir = "web/Grid/unknown"
4808         if immutable:
4809             self.basedir = "web/Grid/unknown-immutable"
4810
4811         self.set_up_grid()
4812         c0 = self.g.clients[0]
4813         self.uris = {}
4814         self.fileurls = {}
4815
4816         # the future cap format may contain slashes, which must be tolerated
4817         expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
4818                                                            safe="")
4819
4820         if immutable:
4821             name = u"future-imm"
4822             future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
4823             d = c0.create_immutable_dirnode({name: (future_node, {})})
4824         else:
4825             name = u"future"
4826             future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4827             d = c0.create_dirnode()
4828
4829         def _stash_root_and_create_file(n):
4830             self.rootnode = n
4831             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
4832             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
4833             if not immutable:
4834                 return self.rootnode.set_node(name, future_node)
4835         d.addCallback(_stash_root_and_create_file)
4836
4837         # make sure directory listing tolerates unknown nodes
4838         d.addCallback(lambda ign: self.GET(self.rooturl))
4839         def _check_directory_html(res, expected_type_suffix):
4840             pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
4841                                   '<td>%s</td>' % (expected_type_suffix, str(name)),
4842                                  re.DOTALL)
4843             self.failUnless(re.search(pattern, res), res)
4844             # find the More Info link for name, should be relative
4845             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4846             info_url = mo.group(1)
4847             self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
4848         if immutable:
4849             d.addCallback(_check_directory_html, "-IMM")
4850         else:
4851             d.addCallback(_check_directory_html, "")
4852
4853         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4854         def _check_directory_json(res, expect_rw_uri):
4855             data = simplejson.loads(res)
4856             self.failUnlessEqual(data[0], "dirnode")
4857             f = data[1]["children"][name]
4858             self.failUnlessEqual(f[0], "unknown")
4859             if expect_rw_uri:
4860                 self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data)
4861             else:
4862                 self.failIfIn("rw_uri", f[1])
4863             if immutable:
4864                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
4865             else:
4866                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
4867             self.failUnlessIn("metadata", f[1])
4868         d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
4869
4870         def _check_info(res, expect_rw_uri, expect_ro_uri):
4871             self.failUnlessIn("Object Type: <span>unknown</span>", res)
4872             if expect_rw_uri:
4873                 self.failUnlessIn(unknown_rwcap, res)
4874             if expect_ro_uri:
4875                 if immutable:
4876                     self.failUnlessIn(unknown_immcap, res)
4877                 else:
4878                     self.failUnlessIn(unknown_rocap, res)
4879             else:
4880                 self.failIfIn(unknown_rocap, res)
4881             self.failIfIn("Raw data as", res)
4882             self.failIfIn("Directory writecap", res)
4883             self.failIfIn("Checker Operations", res)
4884             self.failIfIn("Mutable File Operations", res)
4885             self.failIfIn("Directory Operations", res)
4886
4887         # FIXME: these should have expect_rw_uri=not immutable; I don't know
4888         # why they fail. Possibly related to ticket #922.
4889
4890         d.addCallback(lambda ign: self.GET(expected_info_url))
4891         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
4892         d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
4893         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
4894
4895         def _check_json(res, expect_rw_uri):
4896             data = simplejson.loads(res)
4897             self.failUnlessEqual(data[0], "unknown")
4898             if expect_rw_uri:
4899                 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data)
4900             else:
4901                 self.failIfIn("rw_uri", data[1])
4902
4903             if immutable:
4904                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data)
4905                 self.failUnlessReallyEqual(data[1]["mutable"], False)
4906             elif expect_rw_uri:
4907                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4908                 self.failUnlessReallyEqual(data[1]["mutable"], True)
4909             else:
4910                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4911                 self.failIfIn("mutable", data[1])
4912
4913             # TODO: check metadata contents
4914             self.failUnlessIn("metadata", data[1])
4915
4916         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
4917         d.addCallback(_check_json, expect_rw_uri=not immutable)
4918
4919         # and make sure that a read-only version of the directory can be
4920         # rendered too. This version will not have unknown_rwcap, whether
4921         # or not future_node was immutable.
4922         d.addCallback(lambda ign: self.GET(self.rourl))
4923         if immutable:
4924             d.addCallback(_check_directory_html, "-IMM")
4925         else:
4926             d.addCallback(_check_directory_html, "-RO")
4927
4928         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
4929         d.addCallback(_check_directory_json, expect_rw_uri=False)
4930
4931         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
4932         d.addCallback(_check_json, expect_rw_uri=False)
4933
4934         # TODO: check that getting t=info from the Info link in the ro directory
4935         # works, and does not include the writecap URI.
4936         return d
4937
4938     def test_immutable_unknown(self):
4939         return self.test_unknown(immutable=True)
4940
4941     def test_mutant_dirnodes_are_omitted(self):
4942         self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
4943
4944         self.set_up_grid()
4945         c = self.g.clients[0]
4946         nm = c.nodemaker
4947         self.uris = {}
4948         self.fileurls = {}
4949
4950         lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
4951         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
4952         mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
4953
4954         # This method tests mainly dirnode, but we'd have to duplicate code in order to
4955         # test the dirnode and web layers separately.
4956
4957         # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
4958         # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
4959         # When the directory is read, the mutants should be silently disposed of, leaving
4960         # their lonely sibling.
4961         # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
4962         # because immutable directories don't have a writecap and therefore that field
4963         # isn't (and can't be) decrypted.
4964         # TODO: The field still exists in the netstring. Technically we should check what
4965         # happens if something is put there (_unpack_contents should raise ValueError),
4966         # but that can wait.
4967
4968         lonely_child = nm.create_from_cap(lonely_uri)
4969         mutant_ro_child = nm.create_from_cap(mut_read_uri)
4970         mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
4971
4972         def _by_hook_or_by_crook():
4973             return True
4974         for n in [mutant_ro_child, mutant_write_in_ro_child]:
4975             n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
4976
4977         mutant_write_in_ro_child.get_write_uri    = lambda: None
4978         mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
4979
4980         kids = {u"lonely":      (lonely_child, {}),
4981                 u"ro":          (mutant_ro_child, {}),
4982                 u"write-in-ro": (mutant_write_in_ro_child, {}),
4983                 }
4984         d = c.create_immutable_dirnode(kids)
4985
4986         def _created(dn):
4987             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
4988             self.failIf(dn.is_mutable())
4989             self.failUnless(dn.is_readonly())
4990             # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
4991             self.failIf(hasattr(dn._node, 'get_writekey'))
4992             rep = str(dn)
4993             self.failUnlessIn("RO-IMM", rep)
4994             cap = dn.get_cap()
4995             self.failUnlessIn("CHK", cap.to_string())
4996             self.cap = cap
4997             self.rootnode = dn
4998             self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
4999             return download_to_data(dn._node)
5000         d.addCallback(_created)
5001
5002         def _check_data(data):
5003             # Decode the netstring representation of the directory to check that all children
5004             # are present. This is a bit of an abstraction violation, but there's not really
5005             # any other way to do it given that the real DirectoryNode._unpack_contents would
5006             # strip the mutant children out (which is what we're trying to test, later).
5007             position = 0
5008             numkids = 0
5009             while position < len(data):
5010                 entries, position = split_netstring(data, 1, position)
5011                 entry = entries[0]
5012                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
5013                 name = name_utf8.decode("utf-8")
5014                 self.failUnlessEqual(rwcapdata, "")
5015                 self.failUnlessIn(name, kids)
5016                 (expected_child, ign) = kids[name]
5017                 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
5018                 numkids += 1
5019
5020             self.failUnlessReallyEqual(numkids, 3)
5021             return self.rootnode.list()
5022         d.addCallback(_check_data)
5023
5024         # Now when we use the real directory listing code, the mutants should be absent.
5025         def _check_kids(children):
5026             self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
5027             lonely_node, lonely_metadata = children[u"lonely"]
5028
5029             self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
5030             self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
5031         d.addCallback(_check_kids)
5032
5033         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
5034         d.addCallback(lambda n: n.list())
5035         d.addCallback(_check_kids)  # again with dirnode recreated from cap
5036
5037         # Make sure the lonely child can be listed in HTML...
5038         d.addCallback(lambda ign: self.GET(self.rooturl))
5039         def _check_html(res):
5040             self.failIfIn("URI:SSK", res)
5041             get_lonely = "".join([r'<td>FILE</td>',
5042                                   r'\s+<td>',
5043                                   r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
5044                                   r'</td>',
5045                                   r'\s+<td align="right">%d</td>' % len("one"),
5046                                   ])
5047             self.failUnless(re.search(get_lonely, res), res)
5048
5049             # find the More Info link for name, should be relative
5050             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
5051             info_url = mo.group(1)
5052             self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
5053         d.addCallback(_check_html)
5054
5055         # ... and in JSON.
5056         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
5057         def _check_json(res):
5058             data = simplejson.loads(res)
5059             self.failUnlessEqual(data[0], "dirnode")
5060             listed_children = data[1]["children"]
5061             self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
5062             ll_type, ll_data = listed_children[u"lonely"]
5063             self.failUnlessEqual(ll_type, "filenode")
5064             self.failIfIn("rw_uri", ll_data)
5065             self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
5066         d.addCallback(_check_json)
5067         return d
5068
5069     def test_deep_check(self):
5070         self.basedir = "web/Grid/deep_check"
5071         self.set_up_grid()
5072         c0 = self.g.clients[0]
5073         self.uris = {}
5074         self.fileurls = {}
5075         DATA = "data" * 100
5076         d = c0.create_dirnode()
5077         def _stash_root_and_create_file(n):
5078             self.rootnode = n
5079             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5080             return n.add_file(u"good", upload.Data(DATA, convergence=""))
5081         d.addCallback(_stash_root_and_create_file)
5082         def _stash_uri(fn, which):
5083             self.uris[which] = fn.get_uri()
5084             return fn
5085         d.addCallback(_stash_uri, "good")
5086         d.addCallback(lambda ign:
5087                       self.rootnode.add_file(u"small",
5088                                              upload.Data("literal",
5089                                                         convergence="")))
5090         d.addCallback(_stash_uri, "small")
5091         d.addCallback(lambda ign:
5092                       self.rootnode.add_file(u"sick",
5093                                              upload.Data(DATA+"1",
5094                                                         convergence="")))
5095         d.addCallback(_stash_uri, "sick")
5096
5097         # this tests that deep-check and stream-manifest will ignore
5098         # UnknownNode instances. Hopefully this will also cover deep-stats.
5099         future_node = UnknownNode(unknown_rwcap, unknown_rocap)
5100         d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
5101
5102         def _clobber_shares(ignored):
5103             self.delete_shares_numbered(self.uris["sick"], [0,1])
5104         d.addCallback(_clobber_shares)
5105
5106         # root
5107         # root/good
5108         # root/small
5109         # root/sick
5110         # root/future
5111
5112         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
5113         def _done(res):
5114             try:
5115                 units = [simplejson.loads(line)
5116                          for line in res.splitlines()
5117                          if line]
5118             except ValueError:
5119                 print "response is:", res
5120                 print "undecodeable line was '%s'" % line
5121                 raise
5122             self.failUnlessReallyEqual(len(units), 5+1)
5123             # should be parent-first
5124             u0 = units[0]
5125             self.failUnlessEqual(u0["path"], [])
5126             self.failUnlessEqual(u0["type"], "directory")
5127             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
5128             u0cr = u0["check-results"]
5129             self.failUnlessReallyEqual(u0cr["results"]["count-happiness"], 10)
5130             self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
5131
5132             ugood = [u for u in units
5133                      if u["type"] == "file" and u["path"] == [u"good"]][0]
5134             self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
5135             ugoodcr = ugood["check-results"]
5136             self.failUnlessReallyEqual(ugoodcr["results"]["count-happiness"], 10)
5137             self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
5138
5139             stats = units[-1]
5140             self.failUnlessEqual(stats["type"], "stats")
5141             s = stats["stats"]
5142             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
5143             self.failUnlessReallyEqual(s["count-literal-files"], 1)
5144             self.failUnlessReallyEqual(s["count-directories"], 1)
5145             self.failUnlessReallyEqual(s["count-unknown"], 1)
5146         d.addCallback(_done)
5147
5148         d.addCallback(self.CHECK, "root", "t=stream-manifest")
5149         def _check_manifest(res):
5150             self.failUnless(res.endswith("\n"))
5151             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
5152             self.failUnlessReallyEqual(len(units), 5+1)
5153             self.failUnlessEqual(units[-1]["type"], "stats")
5154             first = units[0]
5155             self.failUnlessEqual(first["path"], [])
5156             self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri())
5157             self.failUnlessEqual(first["type"], "directory")
5158             stats = units[-1]["stats"]
5159             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
5160             self.failUnlessReallyEqual(stats["count-literal-files"], 1)
5161             self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
5162             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
5163             self.failUnlessReallyEqual(stats["count-unknown"], 1)
5164         d.addCallback(_check_manifest)
5165
5166         # now add root/subdir and root/subdir/grandchild, then make subdir
5167         # unrecoverable, then see what happens
5168
5169         d.addCallback(lambda ign:
5170                       self.rootnode.create_subdirectory(u"subdir"))
5171         d.addCallback(_stash_uri, "subdir")
5172         d.addCallback(lambda subdir_node:
5173                       subdir_node.add_file(u"grandchild",
5174                                            upload.Data(DATA+"2",
5175                                                        convergence="")))
5176         d.addCallback(_stash_uri, "grandchild")
5177
5178         d.addCallback(lambda ign:
5179                       self.delete_shares_numbered(self.uris["subdir"],
5180                                                   range(1, 10)))
5181
5182         # root
5183         # root/good
5184         # root/small
5185         # root/sick
5186         # root/future
5187         # root/subdir [unrecoverable]
5188         # root/subdir/grandchild
5189
5190         # how should a streaming-JSON API indicate fatal error?
5191         # answer: emit ERROR: instead of a JSON string
5192
5193         d.addCallback(self.CHECK, "root", "t=stream-manifest")
5194         def _check_broken_manifest(res):
5195             lines = res.splitlines()
5196             error_lines = [i
5197                            for (i,line) in enumerate(lines)
5198                            if line.startswith("ERROR:")]
5199             if not error_lines:
5200                 self.fail("no ERROR: in output: %s" % (res,))
5201             first_error = error_lines[0]
5202             error_line = lines[first_error]
5203             error_msg = lines[first_error+1:]
5204             error_msg_s = "\n".join(error_msg) + "\n"
5205             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
5206                               error_line)
5207             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
5208             units = [simplejson.loads(line) for line in lines[:first_error]]
5209             self.failUnlessReallyEqual(len(units), 6) # includes subdir
5210             last_unit = units[-1]
5211             self.failUnlessEqual(last_unit["path"], ["subdir"])
5212         d.addCallback(_check_broken_manifest)
5213
5214         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
5215         def _check_broken_deepcheck(res):
5216             lines = res.splitlines()
5217             error_lines = [i
5218                            for (i,line) in enumerate(lines)
5219                            if line.startswith("ERROR:")]
5220             if not error_lines:
5221                 self.fail("no ERROR: in output: %s" % (res,))
5222             first_error = error_lines[0]
5223             error_line = lines[first_error]
5224             error_msg = lines[first_error+1:]
5225             error_msg_s = "\n".join(error_msg) + "\n"
5226             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
5227                               error_line)
5228             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
5229             units = [simplejson.loads(line) for line in lines[:first_error]]
5230             self.failUnlessReallyEqual(len(units), 6) # includes subdir
5231             last_unit = units[-1]
5232             self.failUnlessEqual(last_unit["path"], ["subdir"])
5233             r = last_unit["check-results"]["results"]
5234             self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
5235             self.failUnlessReallyEqual(r["count-happiness"], 1)
5236             self.failUnlessReallyEqual(r["count-shares-good"], 1)
5237             self.failUnlessReallyEqual(r["recoverable"], False)
5238         d.addCallback(_check_broken_deepcheck)
5239
5240         d.addErrback(self.explain_web_error)
5241         return d
5242
5243     def test_deep_check_and_repair(self):
5244         self.basedir = "web/Grid/deep_check_and_repair"
5245         self.set_up_grid()
5246         c0 = self.g.clients[0]
5247         self.uris = {}
5248         self.fileurls = {}
5249         DATA = "data" * 100
5250         d = c0.create_dirnode()
5251         def _stash_root_and_create_file(n):
5252             self.rootnode = n
5253             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5254             return n.add_file(u"good", upload.Data(DATA, convergence=""))
5255         d.addCallback(_stash_root_and_create_file)
5256         def _stash_uri(fn, which):
5257             self.uris[which] = fn.get_uri()
5258         d.addCallback(_stash_uri, "good")
5259         d.addCallback(lambda ign:
5260                       self.rootnode.add_file(u"small",
5261                                              upload.Data("literal",
5262                                                         convergence="")))
5263         d.addCallback(_stash_uri, "small")
5264         d.addCallback(lambda ign:
5265                       self.rootnode.add_file(u"sick",
5266                                              upload.Data(DATA+"1",
5267                                                         convergence="")))
5268         d.addCallback(_stash_uri, "sick")
5269         #d.addCallback(lambda ign:
5270         #              self.rootnode.add_file(u"dead",
5271         #                                     upload.Data(DATA+"2",
5272         #                                                convergence="")))
5273         #d.addCallback(_stash_uri, "dead")
5274
5275         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
5276         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
5277         #d.addCallback(_stash_uri, "corrupt")
5278
5279         def _clobber_shares(ignored):
5280             good_shares = self.find_uri_shares(self.uris["good"])
5281             self.failUnlessReallyEqual(len(good_shares), 10)
5282             sick_shares = self.find_uri_shares(self.uris["sick"])
5283             os.unlink(sick_shares[0][2])
5284             #dead_shares = self.find_uri_shares(self.uris["dead"])
5285             #for i in range(1, 10):
5286             #    os.unlink(dead_shares[i][2])
5287
5288             #c_shares = self.find_uri_shares(self.uris["corrupt"])
5289             #cso = CorruptShareOptions()
5290             #cso.stdout = StringIO()
5291             #cso.parseOptions([c_shares[0][2]])
5292             #corrupt_share(cso)
5293         d.addCallback(_clobber_shares)
5294
5295         # root
5296         # root/good   CHK, 10 shares
5297         # root/small  LIT
5298         # root/sick   CHK, 9 shares
5299
5300         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
5301         def _done(res):
5302             units = [simplejson.loads(line)
5303                      for line in res.splitlines()
5304                      if line]
5305             self.failUnlessReallyEqual(len(units), 4+1)
5306             # should be parent-first
5307             u0 = units[0]
5308             self.failUnlessEqual(u0["path"], [])
5309             self.failUnlessEqual(u0["type"], "directory")
5310             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
5311             u0crr = u0["check-and-repair-results"]
5312             self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
5313             self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-happiness"], 10)
5314             self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
5315
5316             ugood = [u for u in units
5317                      if u["type"] == "file" and u["path"] == [u"good"]][0]
5318             self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
5319             ugoodcrr = ugood["check-and-repair-results"]
5320             self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
5321             self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-happiness"], 10)
5322             self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
5323
5324             usick = [u for u in units
5325                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
5326             self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"])
5327             usickcrr = usick["check-and-repair-results"]
5328             self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
5329             self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
5330             self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-happiness"], 9)
5331             self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
5332             self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-happiness"], 10)
5333             self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
5334
5335             stats = units[-1]
5336             self.failUnlessEqual(stats["type"], "stats")
5337             s = stats["stats"]
5338             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
5339             self.failUnlessReallyEqual(s["count-literal-files"], 1)
5340             self.failUnlessReallyEqual(s["count-directories"], 1)
5341         d.addCallback(_done)
5342
5343         d.addErrback(self.explain_web_error)
5344         return d
5345
5346     def _count_leases(self, ignored, which):
5347         u = self.uris[which]
5348         shares = self.find_uri_shares(u)
5349         lease_counts = []
5350         for shnum, serverid, fn in shares:
5351             sf = get_share_file(fn)
5352             num_leases = len(list(sf.get_leases()))
5353             lease_counts.append( (fn, num_leases) )
5354         return lease_counts
5355
5356     def _assert_leasecount(self, lease_counts, expected):
5357         for (fn, num_leases) in lease_counts:
5358             if num_leases != expected:
5359                 self.fail("expected %d leases, have %d, on %s" %
5360                           (expected, num_leases, fn))
5361
5362     def test_add_lease(self):
5363         self.basedir = "web/Grid/add_lease"
5364         self.set_up_grid(num_clients=2)
5365         c0 = self.g.clients[0]
5366         self.uris = {}
5367         DATA = "data" * 100
5368         d = c0.upload(upload.Data(DATA, convergence=""))
5369         def _stash_uri(ur, which):
5370             self.uris[which] = ur.get_uri()
5371         d.addCallback(_stash_uri, "one")
5372         d.addCallback(lambda ign:
5373                       c0.upload(upload.Data(DATA+"1", convergence="")))
5374         d.addCallback(_stash_uri, "two")
5375         def _stash_mutable_uri(n, which):
5376             self.uris[which] = n.get_uri()
5377             assert isinstance(self.uris[which], str)
5378         d.addCallback(lambda ign:
5379             c0.create_mutable_file(publish.MutableData(DATA+"2")))
5380         d.addCallback(_stash_mutable_uri, "mutable")
5381
5382         def _compute_fileurls(ignored):
5383             self.fileurls = {}
5384             for which in self.uris:
5385                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
5386         d.addCallback(_compute_fileurls)
5387
5388         d.addCallback(self._count_leases, "one")
5389         d.addCallback(self._assert_leasecount, 1)
5390         d.addCallback(self._count_leases, "two")
5391         d.addCallback(self._assert_leasecount, 1)
5392         d.addCallback(self._count_leases, "mutable")
5393         d.addCallback(self._assert_leasecount, 1)
5394
5395         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
5396         def _got_html_good(res):
5397             self.failUnlessIn("Healthy", res)
5398             self.failIfIn("Not Healthy", res)
5399         d.addCallback(_got_html_good)
5400
5401         d.addCallback(self._count_leases, "one")
5402         d.addCallback(self._assert_leasecount, 1)
5403         d.addCallback(self._count_leases, "two")
5404         d.addCallback(self._assert_leasecount, 1)
5405         d.addCallback(self._count_leases, "mutable")
5406         d.addCallback(self._assert_leasecount, 1)
5407
5408         # this CHECK uses the original client, which uses the same
5409         # lease-secrets, so it will just renew the original lease
5410         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
5411         d.addCallback(_got_html_good)
5412
5413         d.addCallback(self._count_leases, "one")
5414         d.addCallback(self._assert_leasecount, 1)
5415         d.addCallback(self._count_leases, "two")
5416         d.addCallback(self._assert_leasecount, 1)
5417         d.addCallback(self._count_leases, "mutable")
5418         d.addCallback(self._assert_leasecount, 1)
5419
5420         # this CHECK uses an alternate client, which adds a second lease
5421         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
5422         d.addCallback(_got_html_good)
5423
5424         d.addCallback(self._count_leases, "one")
5425         d.addCallback(self._assert_leasecount, 2)
5426         d.addCallback(self._count_leases, "two")
5427         d.addCallback(self._assert_leasecount, 1)
5428         d.addCallback(self._count_leases, "mutable")
5429         d.addCallback(self._assert_leasecount, 1)
5430
5431         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
5432         d.addCallback(_got_html_good)
5433
5434         d.addCallback(self._count_leases, "one")
5435         d.addCallback(self._assert_leasecount, 2)
5436         d.addCallback(self._count_leases, "two")
5437         d.addCallback(self._assert_leasecount, 1)
5438         d.addCallback(self._count_leases, "mutable")
5439         d.addCallback(self._assert_leasecount, 1)
5440
5441         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
5442                       clientnum=1)
5443         d.addCallback(_got_html_good)
5444
5445         d.addCallback(self._count_leases, "one")
5446         d.addCallback(self._assert_leasecount, 2)
5447         d.addCallback(self._count_leases, "two")
5448         d.addCallback(self._assert_leasecount, 1)
5449         d.addCallback(self._count_leases, "mutable")
5450         d.addCallback(self._assert_leasecount, 2)
5451
5452         d.addErrback(self.explain_web_error)
5453         return d
5454
5455     def test_deep_add_lease(self):
5456         self.basedir = "web/Grid/deep_add_lease"
5457         self.set_up_grid(num_clients=2)
5458         c0 = self.g.clients[0]
5459         self.uris = {}
5460         self.fileurls = {}
5461         DATA = "data" * 100
5462         d = c0.create_dirnode()
5463         def _stash_root_and_create_file(n):
5464             self.rootnode = n
5465             self.uris["root"] = n.get_uri()
5466             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5467             return n.add_file(u"one", upload.Data(DATA, convergence=""))
5468         d.addCallback(_stash_root_and_create_file)
5469         def _stash_uri(fn, which):
5470             self.uris[which] = fn.get_uri()
5471         d.addCallback(_stash_uri, "one")
5472         d.addCallback(lambda ign:
5473                       self.rootnode.add_file(u"small",
5474                                              upload.Data("literal",
5475                                                         convergence="")))
5476         d.addCallback(_stash_uri, "small")
5477
5478         d.addCallback(lambda ign:
5479             c0.create_mutable_file(publish.MutableData("mutable")))
5480         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
5481         d.addCallback(_stash_uri, "mutable")
5482
5483         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
5484         def _done(res):
5485             units = [simplejson.loads(line)
5486                      for line in res.splitlines()
5487                      if line]
5488             # root, one, small, mutable,   stats
5489             self.failUnlessReallyEqual(len(units), 4+1)
5490         d.addCallback(_done)
5491
5492         d.addCallback(self._count_leases, "root")
5493         d.addCallback(self._assert_leasecount, 1)
5494         d.addCallback(self._count_leases, "one")
5495         d.addCallback(self._assert_leasecount, 1)
5496         d.addCallback(self._count_leases, "mutable")
5497         d.addCallback(self._assert_leasecount, 1)
5498
5499         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
5500         d.addCallback(_done)
5501
5502         d.addCallback(self._count_leases, "root")
5503         d.addCallback(self._assert_leasecount, 1)
5504         d.addCallback(self._count_leases, "one")
5505         d.addCallback(self._assert_leasecount, 1)
5506         d.addCallback(self._count_leases, "mutable")
5507         d.addCallback(self._assert_leasecount, 1)
5508
5509         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
5510                       clientnum=1)
5511         d.addCallback(_done)
5512
5513         d.addCallback(self._count_leases, "root")
5514         d.addCallback(self._assert_leasecount, 2)
5515         d.addCallback(self._count_leases, "one")
5516         d.addCallback(self._assert_leasecount, 2)
5517         d.addCallback(self._count_leases, "mutable")
5518         d.addCallback(self._assert_leasecount, 2)
5519
5520         d.addErrback(self.explain_web_error)
5521         return d
5522
5523
5524     def test_exceptions(self):
5525         self.basedir = "web/Grid/exceptions"
5526         self.set_up_grid(num_clients=1, num_servers=2)
5527         c0 = self.g.clients[0]
5528         c0.encoding_params['happy'] = 2
5529         self.fileurls = {}
5530         DATA = "data" * 100
5531         d = c0.create_dirnode()
5532         def _stash_root(n):
5533             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5534             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
5535             return n
5536         d.addCallback(_stash_root)
5537         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
5538         def _stash_bad(ur):
5539             self.fileurls["1share"] = "uri/" + urllib.quote(ur.get_uri())
5540             self.delete_shares_numbered(ur.get_uri(), range(1,10))
5541
5542             u = uri.from_string(ur.get_uri())
5543             u.key = testutil.flip_bit(u.key, 0)
5544             baduri = u.to_string()
5545             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
5546         d.addCallback(_stash_bad)
5547         d.addCallback(lambda ign: c0.create_dirnode())
5548         def _mangle_dirnode_1share(n):
5549             u = n.get_uri()
5550             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
5551             self.fileurls["dir-1share-json"] = url + "?t=json"
5552             self.delete_shares_numbered(u, range(1,10))
5553         d.addCallback(_mangle_dirnode_1share)
5554         d.addCallback(lambda ign: c0.create_dirnode())
5555         def _mangle_dirnode_0share(n):
5556             u = n.get_uri()
5557             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
5558             self.fileurls["dir-0share-json"] = url + "?t=json"
5559             self.delete_shares_numbered(u, range(0,10))
5560         d.addCallback(_mangle_dirnode_0share)
5561
5562         # NotEnoughSharesError should be reported sensibly, with a
5563         # text/plain explanation of the problem, and perhaps some
5564         # information on which shares *could* be found.
5565
5566         d.addCallback(lambda ignored:
5567                       self.shouldHTTPError("GET unrecoverable",
5568                                            410, "Gone", "NoSharesError",
5569                                            self.GET, self.fileurls["0shares"]))
5570         def _check_zero_shares(body):
5571             self.failIfIn("<html>", body)
5572             body = " ".join(body.strip().split())
5573             exp = ("NoSharesError: no shares could be found. "
5574                    "Zero shares usually indicates a corrupt URI, or that "
5575                    "no servers were connected, but it might also indicate "
5576                    "severe corruption. You should perform a filecheck on "
5577                    "this object to learn more. The full error message is: "
5578                    "no shares (need 3). Last failure: None")
5579             self.failUnlessReallyEqual(exp, body)
5580         d.addCallback(_check_zero_shares)
5581
5582
5583         d.addCallback(lambda ignored:
5584                       self.shouldHTTPError("GET 1share",
5585                                            410, "Gone", "NotEnoughSharesError",
5586                                            self.GET, self.fileurls["1share"]))
5587         def _check_one_share(body):
5588             self.failIfIn("<html>", body)
5589             body = " ".join(body.strip().split())
5590             msgbase = ("NotEnoughSharesError: This indicates that some "
5591                        "servers were unavailable, or that shares have been "
5592                        "lost to server departure, hard drive failure, or disk "
5593                        "corruption. You should perform a filecheck on "
5594                        "this object to learn more. The full error message is:"
5595                        )
5596             msg1 = msgbase + (" ran out of shares:"
5597                               " complete=sh0"
5598                               " pending="
5599                               " overdue= unused= need 3. Last failure: None")
5600             msg2 = msgbase + (" ran out of shares:"
5601                               " complete="
5602                               " pending=Share(sh0-on-xgru5)"
5603                               " overdue= unused= need 3. Last failure: None")
5604             self.failUnless(body == msg1 or body == msg2, body)
5605         d.addCallback(_check_one_share)
5606
5607         d.addCallback(lambda ignored:
5608                       self.shouldHTTPError("GET imaginary",
5609                                            404, "Not Found", None,
5610                                            self.GET, self.fileurls["imaginary"]))
5611         def _missing_child(body):
5612             self.failUnlessIn("No such child: imaginary", body)
5613         d.addCallback(_missing_child)
5614
5615         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
5616         def _check_0shares_dir_html(body):
5617             self.failUnlessIn(DIR_HTML_TAG, body)
5618             # we should see the regular page, but without the child table or
5619             # the dirops forms
5620             body = " ".join(body.strip().split())
5621             self.failUnlessIn('href="?t=info">More info on this directory',
5622                               body)
5623             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5624                    "could not be retrieved, because there were insufficient "
5625                    "good shares. This might indicate that no servers were "
5626                    "connected, insufficient servers were connected, the URI "
5627                    "was corrupt, or that shares have been lost due to server "
5628                    "departure, hard drive failure, or disk corruption. You "
5629                    "should perform a filecheck on this object to learn more.")
5630             self.failUnlessIn(exp, body)
5631             self.failUnlessIn("No upload forms: directory is unreadable", body)
5632         d.addCallback(_check_0shares_dir_html)
5633
5634         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
5635         def _check_1shares_dir_html(body):
5636             # at some point, we'll split UnrecoverableFileError into 0-shares
5637             # and some-shares like we did for immutable files (since there
5638             # are different sorts of advice to offer in each case). For now,
5639             # they present the same way.
5640             self.failUnlessIn(DIR_HTML_TAG, body)
5641             body = " ".join(body.strip().split())
5642             self.failUnlessIn('href="?t=info">More info on this directory',
5643                               body)
5644             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5645                    "could not be retrieved, because there were insufficient "
5646                    "good shares. This might indicate that no servers were "
5647                    "connected, insufficient servers were connected, the URI "
5648                    "was corrupt, or that shares have been lost due to server "
5649                    "departure, hard drive failure, or disk corruption. You "
5650                    "should perform a filecheck on this object to learn more.")
5651             self.failUnlessIn(exp, body)
5652             self.failUnlessIn("No upload forms: directory is unreadable", body)
5653         d.addCallback(_check_1shares_dir_html)
5654
5655         d.addCallback(lambda ignored:
5656                       self.shouldHTTPError("GET dir-0share-json",
5657                                            410, "Gone", "UnrecoverableFileError",
5658                                            self.GET,
5659                                            self.fileurls["dir-0share-json"]))
5660         def _check_unrecoverable_file(body):
5661             self.failIfIn("<html>", body)
5662             body = " ".join(body.strip().split())
5663             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5664                    "could not be retrieved, because there were insufficient "
5665                    "good shares. This might indicate that no servers were "
5666                    "connected, insufficient servers were connected, the URI "
5667                    "was corrupt, or that shares have been lost due to server "
5668                    "departure, hard drive failure, or disk corruption. You "
5669                    "should perform a filecheck on this object to learn more.")
5670             self.failUnlessReallyEqual(exp, body)
5671         d.addCallback(_check_unrecoverable_file)
5672
5673         d.addCallback(lambda ignored:
5674                       self.shouldHTTPError("GET dir-1share-json",
5675                                            410, "Gone", "UnrecoverableFileError",
5676                                            self.GET,
5677                                            self.fileurls["dir-1share-json"]))
5678         d.addCallback(_check_unrecoverable_file)
5679
5680         d.addCallback(lambda ignored:
5681                       self.shouldHTTPError("GET imaginary",
5682                                            404, "Not Found", None,
5683                                            self.GET, self.fileurls["imaginary"]))
5684
5685         # attach a webapi child that throws a random error, to test how it
5686         # gets rendered.
5687         w = c0.getServiceNamed("webish")
5688         w.root.putChild("ERRORBOOM", ErrorBoom())
5689
5690         # "Accept: */*" :        should get a text/html stack trace
5691         # "Accept: text/plain" : should get a text/plain stack trace
5692         # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
5693         # no Accept header:      should get a text/html stack trace
5694
5695         d.addCallback(lambda ignored:
5696                       self.shouldHTTPError("GET errorboom_html",
5697                                            500, "Internal Server Error", None,
5698                                            self.GET, "ERRORBOOM",
5699                                            headers={"accept": "*/*"}))
5700         def _internal_error_html1(body):
5701             self.failUnlessIn("<html>", "expected HTML, not '%s'" % body)
5702         d.addCallback(_internal_error_html1)
5703
5704         d.addCallback(lambda ignored:
5705                       self.shouldHTTPError("GET errorboom_text",
5706                                            500, "Internal Server Error", None,
5707                                            self.GET, "ERRORBOOM",
5708                                            headers={"accept": "text/plain"}))
5709         def _internal_error_text2(body):
5710             self.failIfIn("<html>", body)
5711             self.failUnless(body.startswith("Traceback "), body)
5712         d.addCallback(_internal_error_text2)
5713
5714         CLI_accepts = "text/plain, application/octet-stream"
5715         d.addCallback(lambda ignored:
5716                       self.shouldHTTPError("GET errorboom_text",
5717                                            500, "Internal Server Error", None,
5718                                            self.GET, "ERRORBOOM",
5719                                            headers={"accept": CLI_accepts}))
5720         def _internal_error_text3(body):
5721             self.failIfIn("<html>", body)
5722             self.failUnless(body.startswith("Traceback "), body)
5723         d.addCallback(_internal_error_text3)
5724
5725         d.addCallback(lambda ignored:
5726                       self.shouldHTTPError("GET errorboom_text",
5727                                            500, "Internal Server Error", None,
5728                                            self.GET, "ERRORBOOM"))
5729         def _internal_error_html4(body):
5730             self.failUnlessIn("<html>", body)
5731         d.addCallback(_internal_error_html4)
5732
5733         def _flush_errors(res):
5734             # Trial: please ignore the CompletelyUnhandledError in the logs
5735             self.flushLoggedErrors(CompletelyUnhandledError)
5736             return res
5737         d.addBoth(_flush_errors)
5738
5739         return d
5740
5741     def test_blacklist(self):
5742         # download from a blacklisted URI, get an error
5743         self.basedir = "web/Grid/blacklist"
5744         self.set_up_grid()
5745         c0 = self.g.clients[0]
5746         c0_basedir = c0.basedir
5747         fn = os.path.join(c0_basedir, "access.blacklist")
5748         self.uris = {}
5749         DATA = "off-limits " * 50
5750
5751         d = c0.upload(upload.Data(DATA, convergence=""))
5752         def _stash_uri_and_create_dir(ur):
5753             self.uri = ur.get_uri()
5754             self.url = "uri/"+self.uri
5755             u = uri.from_string_filenode(self.uri)
5756             self.si = u.get_storage_index()
5757             childnode = c0.create_node_from_uri(self.uri, None)
5758             return c0.create_dirnode({u"blacklisted.txt": (childnode,{}) })
5759         d.addCallback(_stash_uri_and_create_dir)
5760         def _stash_dir(node):
5761             self.dir_node = node
5762             self.dir_uri = node.get_uri()
5763             self.dir_url = "uri/"+self.dir_uri
5764         d.addCallback(_stash_dir)
5765         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5766         def _check_dir_html(body):
5767             self.failUnlessIn(DIR_HTML_TAG, body)
5768             self.failUnlessIn("blacklisted.txt</a>", body)
5769         d.addCallback(_check_dir_html)
5770         d.addCallback(lambda ign: self.GET(self.url))
5771         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5772
5773         def _blacklist(ign):
5774             f = open(fn, "w")
5775             f.write(" # this is a comment\n")
5776             f.write(" \n")
5777             f.write("\n") # also exercise blank lines
5778             f.write("%s %s\n" % (base32.b2a(self.si), "off-limits to you"))
5779             f.close()
5780             # clients should be checking the blacklist each time, so we don't
5781             # need to restart the client
5782         d.addCallback(_blacklist)
5783         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_uri",
5784                                                        403, "Forbidden",
5785                                                        "Access Prohibited: off-limits",
5786                                                        self.GET, self.url))
5787
5788         # We should still be able to list the parent directory, in HTML...
5789         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5790         def _check_dir_html2(body):
5791             self.failUnlessIn(DIR_HTML_TAG, body)
5792             self.failUnlessIn("blacklisted.txt</strike>", body)
5793         d.addCallback(_check_dir_html2)
5794
5795         # ... and in JSON (used by CLI).
5796         d.addCallback(lambda ign: self.GET(self.dir_url+"?t=json", followRedirect=True))
5797         def _check_dir_json(res):
5798             data = simplejson.loads(res)
5799             self.failUnless(isinstance(data, list), data)
5800             self.failUnlessEqual(data[0], "dirnode")
5801             self.failUnless(isinstance(data[1], dict), data)
5802             self.failUnlessIn("children", data[1])
5803             self.failUnlessIn("blacklisted.txt", data[1]["children"])
5804             childdata = data[1]["children"]["blacklisted.txt"]
5805             self.failUnless(isinstance(childdata, list), data)
5806             self.failUnlessEqual(childdata[0], "filenode")
5807             self.failUnless(isinstance(childdata[1], dict), data)
5808         d.addCallback(_check_dir_json)
5809
5810         def _unblacklist(ign):
5811             open(fn, "w").close()
5812             # the Blacklist object watches mtime to tell when the file has
5813             # changed, but on windows this test will run faster than the
5814             # filesystem's mtime resolution. So we edit Blacklist.last_mtime
5815             # to force a reload.
5816             self.g.clients[0].blacklist.last_mtime -= 2.0
5817         d.addCallback(_unblacklist)
5818
5819         # now a read should work
5820         d.addCallback(lambda ign: self.GET(self.url))
5821         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5822
5823         # read again to exercise the blacklist-is-unchanged logic
5824         d.addCallback(lambda ign: self.GET(self.url))
5825         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5826
5827         # now add a blacklisted directory, and make sure files under it are
5828         # refused too
5829         def _add_dir(ign):
5830             childnode = c0.create_node_from_uri(self.uri, None)
5831             return c0.create_dirnode({u"child": (childnode,{}) })
5832         d.addCallback(_add_dir)
5833         def _get_dircap(dn):
5834             self.dir_si_b32 = base32.b2a(dn.get_storage_index())
5835             self.dir_url_base = "uri/"+dn.get_write_uri()
5836             self.dir_url_json1 = "uri/"+dn.get_write_uri()+"?t=json"
5837             self.dir_url_json2 = "uri/"+dn.get_write_uri()+"/?t=json"
5838             self.dir_url_json_ro = "uri/"+dn.get_readonly_uri()+"/?t=json"
5839             self.child_url = "uri/"+dn.get_readonly_uri()+"/child"
5840         d.addCallback(_get_dircap)
5841         d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True))
5842         d.addCallback(lambda body: self.failUnlessIn(DIR_HTML_TAG, body))
5843         d.addCallback(lambda ign: self.GET(self.dir_url_json1))
5844         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5845         d.addCallback(lambda ign: self.GET(self.dir_url_json2))
5846         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5847         d.addCallback(lambda ign: self.GET(self.dir_url_json_ro))
5848         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5849         d.addCallback(lambda ign: self.GET(self.child_url))
5850         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5851
5852         def _block_dir(ign):
5853             f = open(fn, "w")
5854             f.write("%s %s\n" % (self.dir_si_b32, "dir-off-limits to you"))
5855             f.close()
5856             self.g.clients[0].blacklist.last_mtime -= 2.0
5857         d.addCallback(_block_dir)
5858         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir base",
5859                                                        403, "Forbidden",
5860                                                        "Access Prohibited: dir-off-limits",
5861                                                        self.GET, self.dir_url_base))
5862         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json1",
5863                                                        403, "Forbidden",
5864                                                        "Access Prohibited: dir-off-limits",
5865                                                        self.GET, self.dir_url_json1))
5866         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json2",
5867                                                        403, "Forbidden",
5868                                                        "Access Prohibited: dir-off-limits",
5869                                                        self.GET, self.dir_url_json2))
5870         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json_ro",
5871                                                        403, "Forbidden",
5872                                                        "Access Prohibited: dir-off-limits",
5873                                                        self.GET, self.dir_url_json_ro))
5874         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir child",
5875                                                        403, "Forbidden",
5876                                                        "Access Prohibited: dir-off-limits",
5877                                                        self.GET, self.child_url))
5878         return d
5879
5880
5881 class CompletelyUnhandledError(Exception):
5882     pass
5883 class ErrorBoom(rend.Page):
5884     def beforeRender(self, ctx):
5885         raise CompletelyUnhandledError("whoops")